Switch SMT node hashing to the v6a construction

internal/smt/disk/hash.go

@@ -46,26 +46,40 @@ func HashLeaf(key Key, value []byte) Hash {
 	return out
 }
 
-// HashNode implements the yellowpaper SMT internal-node hash:
-// H(0x01 || depth_1B || left_hash_32B || right_hash_32B).
-func HashNode(left, right Hash, depth uint8) Hash {
+// HashNode implements the yellowpaper v6a (Appendix C.3.2.2) internal-node
+// hash: H(0x01 || depth_1B || region_32B || left_hash_32B || right_hash_32B).
+// The region is the node's absolute key prefix at its bifurcation depth,
+// packed LSB-in-byte with all bits at positions >= depth cleared.
+func HashNode(left, right Hash, depth uint8, region PrefixBits) Hash {
 	h := sha256.New()
 	_, _ = h.Write([]byte{0x01, depth})
+	_, _ = h.Write(region[:])
 	_, _ = h.Write(left[:])
 	_, _ = h.Write(right[:])
 	var out Hash
 	h.Sum(out[:0])
 	return out
 }
 
-// EmptyRootHash matches the current Go v2 memory SMT empty-root rule:
-// H(0x01 || 0x00). Unary roots with one child use the child hash directly.
+// RegionFromKey packs the depth-bit prefix of a descendant key into the
+// canonical v6a region encoding: key bits 0..depth-1 in place, all bits at
+// positions >= depth cleared.
+func RegionFromKey(key Key, depth uint8) PrefixBits {
+	var region PrefixBits
+	d := int(depth)
+	byteLen := (d + 7) / 8
+	copy(region[:byteLen], key[:byteLen])
+	if rem := d % 8; rem != 0 {
+		region[byteLen-1] &= byte(1<<uint(rem)) - 1
+	}
+	return region
+}
+
+// EmptyRootHash is the v6a empty-tree root (spec: root is ⊥): the all-zero
+// hash, matching the JS/Java SDK v6a implementations. Unary roots with one
+// child use the child hash directly.
 func EmptyRootHash() Hash {
-	h := sha256.New()
-	_, _ = h.Write([]byte{0x01, 0x00})
-	var out Hash
-	h.Sum(out[:0])
-	return out
+	return Hash{}
 }
 
 // KeyBit returns bit d of key using the yellowpaper/Go v2 LSB-first key layout.

internal/smt/disk/model_test.go

@@ -125,7 +125,7 @@ func TestHashNodeMatchesMemoryAndGoldenRoot(t *testing.T) {
 		left, right = l2, l1
 	}
 
-	root, err := NewInternal(EmptyPath(), 0, left, right)
+	root, err := NewInternal(EmptyPath(), 0, PrefixBits{}, left, right)
 	require.NoError(t, err)
 	got, err := root.HashValue()
 	require.NoError(t, err)
@@ -135,7 +135,7 @@ func TestHashNodeMatchesMemoryAndGoldenRoot(t *testing.T) {
 		leafInput{key: k1, value: v1},
 		leafInput{key: k2, value: v2},
 	), got)
-	require.Equal(t, mustHash(t, "20563433422d651813394a07697b9c09f9c2ab2ddb95eaa8ed2dc3211de3e869"), got)
+	require.Equal(t, mustHash(t, "fb0b8b6efbb9861202b4f49ca9f2d596f6698d5645f7545b74caf9d8b5161fcc"), got)
 }
 
 func TestLeafSerializationRoundTrip(t *testing.T) {
@@ -164,7 +164,7 @@ func TestInternalSerializationRoundTrip(t *testing.T) {
 
 	left := NewLeaf(mustKey(t, "0000000000000000000000000000000000000000000000000000000000000000"), []byte("left"))
 	right := NewLeaf(mustKey(t, "0100000000000000000000000000000000000000000000000000000000000000"), []byte("right"))
-	branch, err := NewInternal(path, 13, left, right)
+	branch, err := NewInternal(path, 13, RegionFromKey(key, 13), left, right)
 	require.NoError(t, err)
 
 	encoded, err := MarshalInternal(branch.Internal)
@@ -179,7 +179,7 @@ func TestInternalSerializationRoundTrip(t *testing.T) {
 	expected = append(expected, rightHash[:]...)
 	require.Equal(t, expected, encoded)
 
-	decoded, err := UnmarshalInternal(encoded)
+	decoded, err := UnmarshalInternal(encoded, RootNodeKey())
 	require.NoError(t, err)
 	require.Equal(t, branch.Internal.Depth, decoded.Depth)
 	require.True(t, branch.Internal.Path.Equal(decoded.Path))
@@ -192,37 +192,37 @@ func TestInternalSerializationRoundTrip(t *testing.T) {
 	require.Equal(t, rightHash, decodedRightHash)
 
 	withTrailing := append(append([]byte(nil), encoded...), branch.Internal.Hash[:]...)
-	_, err = UnmarshalInternal(withTrailing)
+	_, err = UnmarshalInternal(withTrailing, RootNodeKey())
 	require.Error(t, err)
 }
 
 func TestUnmarshalInternalRejectsMalformedData(t *testing.T) {
 	left := NewLeaf(mustKey(t, "0000000000000000000000000000000000000000000000000000000000000000"), []byte("left"))
 	right := NewLeaf(mustKey(t, "0100000000000000000000000000000000000000000000000000000000000000"), []byte("right"))
-	branch, err := NewInternal(EmptyPath(), 0, left, right)
+	branch, err := NewInternal(EmptyPath(), 0, PrefixBits{}, left, right)
 	require.NoError(t, err)
 
 	encoded, err := MarshalInternal(branch.Internal)
 	require.NoError(t, err)
 
-	_, err = UnmarshalInternal(append([]byte(nil), encoded[:len(encoded)-1]...))
+	_, err = UnmarshalInternal(append([]byte(nil), encoded[:len(encoded)-1]...), RootNodeKey())
 	require.Error(t, err)
 
 	withTrailing := append(append([]byte(nil), encoded...), 0x00)
-	_, err = UnmarshalInternal(withTrailing)
+	_, err = UnmarshalInternal(withTrailing, RootNodeKey())
 	require.Error(t, err)
 
 	withWrongTag := append([]byte(nil), encoded...)
 	withWrongTag[0] = TagLeaf
-	_, err = UnmarshalInternal(withWrongTag)
+	_, err = UnmarshalInternal(withWrongTag, RootNodeKey())
 	require.Error(t, err)
 
 	nonCanonicalPath := []byte{TagInternal, 13, 13, 0xa5, 0xe1}
 	nonCanonicalPath = append(nonCanonicalPath, make([]byte, 2*HashSize)...)
-	_, err = UnmarshalInternal(nonCanonicalPath)
+	_, err = UnmarshalInternal(nonCanonicalPath, RootNodeKey())
 	require.Error(t, err)
 
-	_, err = UnmarshalInternal([]byte{TagInternal, 0, 1})
+	_, err = UnmarshalInternal([]byte{TagInternal, 0, 1}, RootNodeKey())
 	require.Error(t, err)
 }
 

internal/smt/disk/node.go

@@ -26,9 +26,15 @@ type LeafNode struct {
 type InternalNode struct {
 	Path  CompressedPath
 	Depth uint8
-	Left  *Branch
-	Right *Branch
-	Hash  Hash
+	// Region is the node's absolute key prefix at Depth (v6a hash operand).
+	// It is derived, not persisted: from a descendant key at creation and
+	// from the storage NodeKey prefix at load. Splitting an edge above a
+	// node changes neither its Depth nor its Region, so preserved subtrees
+	// keep their Region (and therefore their hash) across restructures.
+	Region PrefixBits
+	Left   *Branch
+	Right  *Branch
+	Hash   Hash
 }
 
 func NewLeaf(key Key, value []byte) *Branch {
@@ -43,7 +49,7 @@ func NewLeaf(key Key, value []byte) *Branch {
 	}
 }
 
-func NewInternal(path CompressedPath, depth uint8, left, right *Branch) (*Branch, error) {
+func NewInternal(path CompressedPath, depth uint8, region PrefixBits, left, right *Branch) (*Branch, error) {
 	if left == nil || right == nil {
 		return nil, fmt.Errorf("disk smt: internal node requires both children")
 	}
@@ -58,11 +64,12 @@ func NewInternal(path CompressedPath, depth uint8, left, right *Branch) (*Branch
 	return &Branch{
 		Kind: BranchKindInternal,
 		Internal: &InternalNode{
-			Path:  path,
-			Depth: depth,
-			Left:  left,
-			Right: right,
-			Hash:  HashNode(leftHash, rightHash, depth),
+			Path:   path,
+			Depth:  depth,
+			Region: region,
+			Left:   left,
+			Right:  right,
+			Hash:   HashNode(leftHash, rightHash, depth, region),
 		},
 	}, nil
 }

internal/smt/disk/node_key.go

@@ -72,6 +72,12 @@ func (k NodeKey) DepthBits() int {
 	return int(k.depth)
 }
 
+// Prefix returns the node's absolute routing prefix with unused bits cleared.
+// For non-root keys this is exactly the v6a region operand at DepthBits.
+func (k NodeKey) Prefix() PrefixBits {
+	return k.prefix
+}
+
 func (k NodeKey) Bytes() []byte {
 	return k.AppendBytes(nil)
 }

internal/smt/disk/persist/tree.go

@@ -484,7 +484,7 @@ func loadProofBranch(reader storage.ReadStore, key disk.NodeKey, expectedHash di
 	if !ok {
 		return nil, fmt.Errorf("disk SMT persist: node %x missing from store", key.Bytes())
 	}
-	branch, err := decodeBranch(encoded)
+	branch, err := decodeBranch(key, encoded)
 	if err != nil {
 		return nil, err
 	}
@@ -499,7 +499,7 @@ func loadProofBranch(reader storage.ReadStore, key disk.NodeKey, expectedHash di
 }
 
 func decodeAndValidateProofBranch(key disk.NodeKey, encoded []byte, expectedHash disk.Hash) (*disk.Branch, error) {
-	branch, err := decodeBranch(encoded)
+	branch, err := decodeBranch(key, encoded)
 	if err != nil {
 		return nil, err
 	}
@@ -1174,7 +1174,7 @@ func (s *Snapshot) loadBranchParallel(key disk.NodeKey, expectedHash disk.Hash)
 		return nil, fmt.Errorf("disk SMT persist: node %x missing from store", key.Bytes())
 	}
 	decodeStart := time.Now()
-	branch, err := decodeBranch(encoded)
+	branch, err := decodeBranch(key, encoded)
 	decodeDuration := time.Since(decodeStart)
 	if err != nil {
 		return nil, err
@@ -1208,7 +1208,7 @@ func (s *Snapshot) loadBranchFromOverlay(key disk.NodeKey, expectedHash disk.Has
 	if entry.delete {
 		return nil, true, fmt.Errorf("disk SMT persist: node %x is tombstoned in snapshot overlay", key.Bytes())
 	}
-	branch, err := decodeBranch(entry.value)
+	branch, err := decodeBranch(key, entry.value)
 	if err != nil {
 		return nil, true, err
 	}
@@ -1526,7 +1526,7 @@ func (s *Snapshot) loadFrontierParallel(loads []frontierLoadReq) error {
 					results[idx].err = fmt.Errorf("disk SMT persist: node %x missing from store", miss.req.nodeKey.Bytes())
 					continue
 				}
-				branch, err := decodeBranch(encoded)
+				branch, err := decodeBranch(miss.req.nodeKey, encoded)
 				if err != nil {
 					results[idx].err = err
 					continue
@@ -1579,7 +1579,7 @@ func (s *Snapshot) attachLoadedBranch(req frontierReq, branch *disk.Branch) {
 
 func (s *Snapshot) decodeAndValidateLoadedBranch(key disk.NodeKey, encoded []byte, expectedHash disk.Hash) (*disk.Branch, disk.Hash, error) {
 	decodeStart := time.Now()
-	branch, err := decodeBranch(encoded)
+	branch, err := decodeBranch(key, encoded)
 	s.stats.MaterializeDecodeDuration += time.Since(decodeStart)
 	if err != nil {
 		return nil, disk.Hash{}, err
@@ -1794,7 +1794,7 @@ func (s *Snapshot) loadBranch(key disk.NodeKey, expectedHash disk.Hash) (*disk.B
 	return branch, nil
 }
 
-func decodeBranch(encoded []byte) (*disk.Branch, error) {
+func decodeBranch(key disk.NodeKey, encoded []byte) (*disk.Branch, error) {
 	tag, err := disk.SerializedTag(encoded)
 	if err != nil {
 		return nil, err
@@ -1807,7 +1807,7 @@ func decodeBranch(encoded []byte) (*disk.Branch, error) {
 		}
 		return &disk.Branch{Kind: disk.BranchKindLeaf, Leaf: leaf}, nil
 	case disk.TagInternal:
-		internal, err := disk.UnmarshalInternal(encoded)
+		internal, err := disk.UnmarshalInternal(encoded, key)
 		if err != nil {
 			return nil, err
 		}
@@ -1918,11 +1918,12 @@ func cacheBranchForPersistedNode(branch *disk.Branch) (*disk.Branch, disk.Hash,
 		return &disk.Branch{
 			Kind: disk.BranchKindInternal,
 			Internal: &disk.InternalNode{
-				Path:  branch.Internal.Path,
-				Depth: branch.Internal.Depth,
-				Left:  disk.NewStub(leftHash),
-				Right: disk.NewStub(rightHash),
-				Hash:  branch.Internal.Hash,
+				Path:   branch.Internal.Path,
+				Depth:  branch.Internal.Depth,
+				Region: branch.Internal.Region,
+				Left:   disk.NewStub(leftHash),
+				Right:  disk.NewStub(rightHash),
+				Hash:   branch.Internal.Hash,
 			},
 		}, hash, nil
 	default:
@@ -1991,11 +1992,12 @@ func cloneBranch(branch *disk.Branch) *disk.Branch {
 	}
 	if branch.Internal != nil {
 		next.Internal = &disk.InternalNode{
-			Path:  branch.Internal.Path,
-			Depth: branch.Internal.Depth,
-			Left:  cloneBranch(branch.Internal.Left),
-			Right: cloneBranch(branch.Internal.Right),
-			Hash:  branch.Internal.Hash,
+			Path:   branch.Internal.Path,
+			Depth:  branch.Internal.Depth,
+			Region: branch.Internal.Region,
+			Left:   cloneBranch(branch.Internal.Left),
+			Right:  cloneBranch(branch.Internal.Right),
+			Hash:   branch.Internal.Hash,
 		}
 	}
 	return next

internal/smt/disk/persist/tree_test.go

@@ -350,7 +350,7 @@ func TestSnapshotCommitPersistsNonRootLoadedNodeMovement(t *testing.T) {
 	rightKey, rightPrefix, rightStart, err := childNodeKey(disk.PrefixBits{}, 0, root.Path, 1)
 	require.NoError(t, err)
 	rightBefore := requireNodeExists(t, store, rightKey)
-	rightNodeBefore, err := disk.UnmarshalInternal(rightBefore)
+	rightNodeBefore, err := disk.UnmarshalInternal(rightBefore, rightKey)
 	require.NoError(t, err)
 	require.Equal(t, uint8(3), rightNodeBefore.Depth)
 
@@ -374,14 +374,14 @@ func TestSnapshotCommitPersistsNonRootLoadedNodeMovement(t *testing.T) {
 	require.NoError(t, snapshot.Commit(api.NewBigIntFromUint64(2)))
 
 	rightAfter := requireNodeExists(t, store, rightKey)
-	rightNodeAfter, err := disk.UnmarshalInternal(rightAfter)
+	rightNodeAfter, err := disk.UnmarshalInternal(rightAfter, rightKey)
 	require.NoError(t, err)
 	require.Equal(t, uint8(1), rightNodeAfter.Depth)
 
 	movedOldKey, _, _, err := childNodeKey(rightPrefix, rightStart, rightNodeAfter.Path, 1)
 	require.NoError(t, err)
 	movedOld := requireNodeExists(t, store, movedOldKey)
-	movedOldNode, err := disk.UnmarshalInternal(movedOld)
+	movedOldNode, err := disk.UnmarshalInternal(movedOld, movedOldKey)
 	require.NoError(t, err)
 	require.Equal(t, uint8(3), movedOldNode.Depth)
 	require.Equal(t, 1, movedOldNode.Path.Len())
@@ -800,7 +800,7 @@ func treeRootNode(t *testing.T, store *rocksstore.Store) *disk.InternalNode {
 	tag, err := disk.SerializedTag(encoded)
 	require.NoError(t, err)
 	require.Equal(t, disk.TagInternal, tag)
-	node, err := disk.UnmarshalInternal(encoded)
+	node, err := disk.UnmarshalInternal(encoded, disk.RootNodeKey())
 	require.NoError(t, err)
 	return node
 }

internal/smt/disk/rocksstore/store.go

@@ -64,7 +64,7 @@ import (
 
 const (
 	SchemaVersion = "1"
-	TreeLayout    = "yellowpaper-rsmt-sha256-v1"
+	TreeLayout    = "yellowpaper-rsmt-sha256-v6a"
 	KeyBits       = "256"
 
 	maxCInt = int(^uint32(0) >> 1)