diff --git a/internal/smt/disk/hash.go b/internal/smt/disk/hash.go index 02c3630d..b3e0c55a 100644 --- a/internal/smt/disk/hash.go +++ b/internal/smt/disk/hash.go @@ -46,11 +46,14 @@ func HashLeaf(key Key, value []byte) Hash { return out } -// HashNode implements the yellowpaper SMT internal-node hash: -// H(0x01 || depth_1B || left_hash_32B || right_hash_32B). -func HashNode(left, right Hash, depth uint8) Hash { +// HashNode implements the yellowpaper v6a (Appendix C.3.2.2) internal-node +// hash: H(0x01 || depth_1B || region_32B || left_hash_32B || right_hash_32B). +// The region is the node's absolute key prefix at its bifurcation depth, +// packed LSB-in-byte with all bits at positions >= depth cleared. +func HashNode(left, right Hash, depth uint8, region PrefixBits) Hash { h := sha256.New() _, _ = h.Write([]byte{0x01, depth}) + _, _ = h.Write(region[:]) _, _ = h.Write(left[:]) _, _ = h.Write(right[:]) var out Hash @@ -58,14 +61,25 @@ func HashNode(left, right Hash, depth uint8) Hash { return out } -// EmptyRootHash matches the current Go v2 memory SMT empty-root rule: -// H(0x01 || 0x00). Unary roots with one child use the child hash directly. +// RegionFromKey packs the depth-bit prefix of a descendant key into the +// canonical v6a region encoding: key bits 0..depth-1 in place, all bits at +// positions >= depth cleared. +func RegionFromKey(key Key, depth uint8) PrefixBits { + var region PrefixBits + d := int(depth) + byteLen := (d + 7) / 8 + copy(region[:byteLen], key[:byteLen]) + if rem := d % 8; rem != 0 { + region[byteLen-1] &= byte(1<> 1) diff --git a/internal/smt/disk/serde.go b/internal/smt/disk/serde.go index ff6a750a..f95452fa 100644 --- a/internal/smt/disk/serde.go +++ b/internal/smt/disk/serde.go @@ -97,7 +97,30 @@ func MarshalInternal(node *InternalNode) ([]byte, error) { return out, nil } -func UnmarshalInternal(data []byte) (*InternalNode, error) { +// RegionForNode derives an internal node's absolute v6a region from its +// storage key and its serialized compressed path. The storage key covers key +// bits [0, key.DepthBits()) — the branch slot: parent split plus direction +// bit — and the compressed path extends it to the node's bifurcation depth, +// so region = key prefix bits followed by path bits at offset DepthBits(). +func RegionForNode(key NodeKey, path CompressedPath) (PrefixBits, error) { + start := key.DepthBits() + if start+path.Len() > KeyBits { + return PrefixBits{}, fmt.Errorf("disk smt: node region exceeds key width: start=%d pathLen=%d", start, path.Len()) + } + region := key.Prefix() + for i := 0; i < path.Len(); i++ { + if path.BitAt(i) != 0 { + region[(start+i)/8] |= 1 << (uint(start+i) % 8) + } + } + return region, nil +} + +// UnmarshalInternal decodes a serialized internal node stored under the given +// storage key. The node's absolute v6a region is derived from the key prefix +// plus the serialized compressed path (see RegionForNode); it is a hash +// operand and is not part of the serialized payload. +func UnmarshalInternal(data []byte, key NodeKey) (*InternalNode, error) { if len(data) < 1+1+1+2*HashSize { return nil, fmt.Errorf("disk smt: serialized internal node too short: %d", len(data)) } @@ -133,13 +156,22 @@ func UnmarshalInternal(data []byte) (*InternalNode, error) { if len(data) != pos { return nil, fmt.Errorf("disk smt: serialized internal node has trailing bytes: got %d, consumed %d", len(data), pos) } - hash := HashNode(leftHash, rightHash, depth) + if key.DepthBits()+path.Len() != int(depth) { + return nil, fmt.Errorf("disk smt: node depth mismatch: key depth %d + path len %d != stored depth %d", + key.DepthBits(), path.Len(), depth) + } + region, err := RegionForNode(key, path) + if err != nil { + return nil, err + } + hash := HashNode(leftHash, rightHash, depth, region) return &InternalNode{ - Path: path, - Depth: depth, - Left: NewStub(leftHash), - Right: NewStub(rightHash), - Hash: hash, + Path: path, + Depth: depth, + Region: region, + Left: NewStub(leftHash), + Right: NewStub(rightHash), + Hash: hash, }, nil } diff --git a/internal/smt/disk/tree.go b/internal/smt/disk/tree.go index f336c10b..8528e5c5 100644 --- a/internal/smt/disk/tree.go +++ b/internal/smt/disk/tree.go @@ -346,7 +346,7 @@ func batchInsert(branch *Branch, items []batchItem, start, end, startBit int, ct if rightErr != nil { return nil, rightErr } - next, err := NewInternal(node.Path, node.Depth, left, right) + next, err := NewInternal(node.Path, node.Depth, node.Region, left, right) if err != nil { return nil, err } @@ -405,7 +405,7 @@ func buildSubtree(items []batchItem, start, end, startBit int, ctx *applyContext if rightErr != nil { return nil, rightErr } - node, err := NewInternal(path, uint8(split), left, right) + node, err := NewInternal(path, uint8(split), RegionFromKey(items[start].Key, uint8(split)), left, right) if err != nil { return nil, err } @@ -425,7 +425,7 @@ func batchSplitInternal(node *InternalNode, items []batchItem, start, end, start if err != nil { return nil, err } - oldNode, err := NewInternal(oldPath, node.Depth, node.Left, node.Right) + oldNode, err := NewInternal(oldPath, node.Depth, node.Region, node.Left, node.Right) if err != nil { return nil, err } @@ -471,7 +471,7 @@ func batchSplitInternal(node *InternalNode, items []batchItem, start, end, start if rightErr != nil { return nil, rightErr } - next, err := NewInternal(newPath, uint8(newSplit), left, right) + next, err := NewInternal(newPath, uint8(newSplit), RegionFromKey(items[start].Key, uint8(newSplit)), left, right) if err != nil { return nil, err } @@ -584,7 +584,7 @@ func insertBranch(branch *Branch, key Key, value []byte, startBit int, ctx *appl } else { left, right = newLeaf, oldLeaf } - node, err := NewInternal(path, uint8(div), left, right) + node, err := NewInternal(path, uint8(div), RegionFromKey(key, uint8(div)), left, right) if err != nil { return nil, insertAccepted, err } @@ -633,7 +633,7 @@ func insertBranch(branch *Branch, key Key, value []byte, startBit int, ctx *appl left, right = nextLeft, node.Right } - next, err := NewInternal(node.Path, node.Depth, left, right) + next, err := NewInternal(node.Path, node.Depth, node.Region, left, right) if err != nil { return nil, insertAccepted, err } @@ -658,7 +658,7 @@ func splitInternalNode(node *InternalNode, key Key, value []byte, startBit, firs if err != nil { return nil, err } - oldNode, err := NewInternal(oldPath, node.Depth, node.Left, node.Right) + oldNode, err := NewInternal(oldPath, node.Depth, node.Region, node.Left, node.Right) if err != nil { return nil, err } @@ -678,7 +678,7 @@ func splitInternalNode(node *InternalNode, key Key, value []byte, startBit, firs } else { left, right = oldNode, newLeaf } - next, err := NewInternal(newPath, uint8(newSplit), left, right) + next, err := NewInternal(newPath, uint8(newSplit), RegionFromKey(key, uint8(newSplit)), left, right) if err != nil { return nil, err } diff --git a/internal/smt/golden_vectors_test.go b/internal/smt/golden_vectors_test.go index 2173f47a..7b61c03d 100644 --- a/internal/smt/golden_vectors_test.go +++ b/internal/smt/golden_vectors_test.go @@ -20,7 +20,7 @@ func TestGoldenVector_RootMatches(t *testing.T) { require.NoError(t, tree.AddLeaf(k1, []byte("value-one"))) require.NoError(t, tree.AddLeaf(k2, []byte("value-two"))) - const expectedRoot = "20563433422d651813394a07697b9c09f9c2ab2ddb95eaa8ed2dc3211de3e869" + const expectedRoot = "fb0b8b6efbb9861202b4f49ca9f2d596f6698d5645f7545b74caf9d8b5161fcc" require.Equal(t, expectedRoot, tree.GetRootHashHex()) } @@ -39,7 +39,7 @@ func TestGoldenVector_ProofBitmapAndSiblingsMatch(t *testing.T) { require.NoError(t, tree.AddLeaf(k2, v2)) require.NoError(t, tree.AddLeaf(k3, v3)) - const expectedRoot = "b08cae8f98a168a4b39dced99fc3ea2833291c8c53a0eb447e0056044dee598a" + const expectedRoot = "5dd3c11610f053b31a8e1e42b51a4b92940ce0ddf019bbb89e2f27d44e33c0bd" require.Equal(t, expectedRoot, tree.GetRootHashHex()) path, err := tree.GetPath(k2) diff --git a/internal/smt/inclusion_cert_test.go b/internal/smt/inclusion_cert_test.go index 973ffcff..5cb9f4d8 100644 --- a/internal/smt/inclusion_cert_test.go +++ b/internal/smt/inclusion_cert_test.go @@ -91,7 +91,7 @@ func TestGetInclusionCert_GoldenVector(t *testing.T) { addLeaf(t, tree, k2, v2) addLeaf(t, tree, k3, v3) - const expectedRoot = "b08cae8f98a168a4b39dced99fc3ea2833291c8c53a0eb447e0056044dee598a" + const expectedRoot = "5dd3c11610f053b31a8e1e42b51a4b92940ce0ddf019bbb89e2f27d44e33c0bd" require.Equal(t, expectedRoot, tree.GetRootHashHex()) cert, err := tree.GetInclusionCert(k2) diff --git a/internal/smt/smt.go b/internal/smt/smt.go index 358adac9..24ab651d 100644 --- a/internal/smt/smt.go +++ b/internal/smt/smt.go @@ -129,8 +129,10 @@ func NewParentSparseMerkleTree(algorithm api.HashAlgorithm, keyLength int) *Spar // better to ensure all the leaves exist; otherwise the hash values // of siblings of the missing nodes would not match the structure of // the tree and the corresponding inclusion proofs would fail to verify - tree.root.Left = populate(0b10, keyLength, 1) - tree.root.Right = populate(0b11, keyLength, 1) + tree.root.Left = populate(0b10, keyLength, 1, make([]byte, 32)) + rightRegion := make([]byte, 32) + rightRegion[0] |= 1 + tree.root.Right = populate(0b11, keyLength, 1, rightRegion) // Mutation above invalidated the root hash primed by NewSparseMerkleTree. // We reset and re-prime. @@ -140,13 +142,19 @@ func NewParentSparseMerkleTree(algorithm api.HashAlgorithm, keyLength int) *Spar return tree } -func populate(path, levels, depth int) branch { +// populate builds the fully-populated parent-mode tree. region carries the +// node's absolute prefix (v6a); a child extends it with its direction bit at +// position depth. +func populate(path, levels, depth int, region []byte) branch { if levels == 1 { return newChildLeafBranch(big.NewInt(int64(path)), nil) } - left := populate(0b10, levels-1, depth+1) - right := populate(0b11, levels-1, depth+1) - return newNodeBranchWithDepth(big.NewInt(int64(path)), left, right, depth) + leftRegion := append([]byte(nil), region...) + rightRegion := append([]byte(nil), region...) + rightRegion[depth/8] |= 1 << (uint(depth) % 8) + left := populate(0b10, levels-1, depth+1, leftRegion) + right := populate(0b11, levels-1, depth+1, rightRegion) + return newNodeBranchWithDepth(big.NewInt(int64(path)), left, right, depth, region) } // CreateSnapshot creates a snapshot of the current SMT state @@ -235,7 +243,7 @@ func (smt *SparseMerkleTree) cloneBranch(branch branch) branch { return cloned } else { nodeBranch := branch.(*NodeBranch) - return newNodeBranchWithDepth(nodeBranch.Path, nodeBranch.Left, nodeBranch.Right, int(nodeBranch.Depth)) + return newNodeBranchWithDepth(nodeBranch.Path, nodeBranch.Left, nodeBranch.Right, int(nodeBranch.Depth), nodeBranch.Region) } } @@ -258,8 +266,14 @@ type LeafBranch struct { // NodeBranch represents an internal node type NodeBranch struct { - Path *big.Int - Depth uint8 + Path *big.Int + Depth uint8 + // Region is the node's absolute Depth-bit key prefix packed in the + // canonical v6a 32-byte encoding. Derived at construction (from the + // inserted leaf key for new junctions, preserved for restructured + // nodes); splitting an edge above a node changes neither Depth nor + // Region, so cached hashes stay valid across restructures. + Region []byte Left branch Right branch rawHash [smtCachedHashBytes]byte // inline hash cache; valid when hashSet == true @@ -340,16 +354,18 @@ func (l *LeafBranch) isLeaf() bool { // NewNodeBranch creates a regular node branch func newNodeBranch(path *big.Int, left, right branch) *NodeBranch { - return newNodeBranchWithDepth(path, left, right, path.BitLen()-1) + // Absolute-path variant: the region is derivable from the path itself. + return newNodeBranchWithDepth(path, left, right, path.BitLen()-1, regionFromPath(path, uint8(path.BitLen()-1))) } -func newNodeBranchWithDepth(path *big.Int, left, right branch, depth int) *NodeBranch { +func newNodeBranchWithDepth(path *big.Int, left, right branch, depth int, region []byte) *NodeBranch { if depth < 0 || depth > 255 { panic(fmt.Sprintf("smt: node depth %d out of uint8 range [0, 255]", depth)) } return &NodeBranch{ Path: new(big.Int).Set(path), Depth: uint8(depth), + Region: region, Left: left, Right: right, isRoot: false, @@ -364,6 +380,7 @@ func newRootBranch(path *big.Int, left, right branch, depth int) *NodeBranch { return &NodeBranch{ Path: new(big.Int).Set(path), Depth: uint8(depth), + Region: regionFromPath(path, uint8(depth)), Left: left, Right: right, isRoot: true, @@ -396,11 +413,27 @@ func (n *NodeBranch) calculateHash(hasher *api.DataHasher) []byte { return n.rawHash[:] } - // Keep root hash stable for empty trees by hashing domain+level when both - // children are empty. + // v6a empty-tree rule: the empty root is the all-zero hash (spec: root + // is bottom), matching the JS/Java SDK implementations. + if leftHash == nil && rightHash == nil { + for i := range n.rawHash { + n.rawHash[i] = 0 + } + n.hashSet = true + return n.rawHash[:] + } + + // v6a internal-node hash (yellowpaper C.3.2.2): + // H(0x01 || depth || region || left || right), where region is the + // node's absolute depth-bit key prefix packed LSB-in-byte into 32 bytes. + region := n.Region + if region == nil { + region = make([]byte, 32) + } hasher.Reset(). AddData([]byte{0x01}). - AddData([]byte{n.Depth}) + AddData([]byte{n.Depth}). + AddData(region) if leftHash != nil { hasher.AddData(leftHash) } @@ -413,6 +446,28 @@ func (n *NodeBranch) calculateHash(hasher *api.DataHasher) []byte { return n.rawHash[:] } +// regionFromPath packs the low depth bits of an absolute sentinel-prefixed +// path into the canonical v6a 32-byte region encoding: path bit i lands at +// bit (i mod 8) of byte (i / 8); all bits at positions >= depth are zero. +// RegionFromKeyBytes is the canonical v6a region packing of an LSB-first key +// prefix (see api.RegionFromKeyBytes). +func RegionFromKeyBytes(key []byte, depth int) []byte { + return api.RegionFromKeyBytes(key, depth) +} + +func regionFromPath(path *big.Int, depth uint8) []byte { + region := make([]byte, 32) + if path == nil { + return region + } + for i := 0; i < int(depth); i++ { + if path.Bit(i) != 0 { + region[i/8] |= 1 << (uint(i) % 8) + } + } + return region +} + func (n *NodeBranch) getPath() *big.Int { return n.Path } @@ -765,11 +820,12 @@ func (smt *SparseMerkleTree) buildTree(branch branch, remainingPath *big.Int, le newBranch := newLeafBranchWithKey(newBranchPath, leafKey, value) nodeDepth := depthOffset + (commonPath.BitLen() - 1) + region := RegionFromKeyBytes(leafKey, nodeDepth) if isRight { - return newNodeBranchWithDepth(commonPath, oldBranch, newBranch, nodeDepth), nil + return newNodeBranchWithDepth(commonPath, oldBranch, newBranch, nodeDepth, region), nil } else { - return newNodeBranchWithDepth(commonPath, newBranch, oldBranch, nodeDepth), nil + return newNodeBranchWithDepth(commonPath, newBranch, oldBranch, nodeDepth, region), nil } } @@ -780,14 +836,17 @@ func (smt *SparseMerkleTree) buildTree(branch branch, remainingPath *big.Int, le newBranch := newLeafBranchWithKey(newBranchPath, leafKey, value) oldBranchPath := new(big.Int).Rsh(nodeBranch.Path, uint(commonPath.BitLen()-1)) - oldBranch := newNodeBranchWithDepth(oldBranchPath, nodeBranch.Left, nodeBranch.Right, int(nodeBranch.Depth)) + // Preserved node: splitting the edge above it changes neither its + // depth nor its region (v6a), so its hash is unchanged. + oldBranch := newNodeBranchWithDepth(oldBranchPath, nodeBranch.Left, nodeBranch.Right, int(nodeBranch.Depth), nodeBranch.Region) nodeDepth := depthOffset + (commonPath.BitLen() - 1) + region := RegionFromKeyBytes(leafKey, nodeDepth) if isRight { - return newNodeBranchWithDepth(commonPath, oldBranch, newBranch, nodeDepth), nil + return newNodeBranchWithDepth(commonPath, oldBranch, newBranch, nodeDepth, region), nil } else { - return newNodeBranchWithDepth(commonPath, newBranch, oldBranch, nodeDepth), nil + return newNodeBranchWithDepth(commonPath, newBranch, oldBranch, nodeDepth, region), nil } } @@ -797,13 +856,13 @@ func (smt *SparseMerkleTree) buildTree(branch branch, remainingPath *big.Int, le if err != nil { return nil, err } - return newNodeBranchWithDepth(nodeBranch.Path, nodeBranch.Left, newRight, int(nodeBranch.Depth)), nil + return newNodeBranchWithDepth(nodeBranch.Path, nodeBranch.Left, newRight, int(nodeBranch.Depth), nodeBranch.Region), nil } else { newLeft, err := smt.buildTree(nodeBranch.Left, new(big.Int).Rsh(remainingPath, uint(commonPath.BitLen()-1)), leafKey, value, nextDepthOffset) if err != nil { return nil, err } - return newNodeBranchWithDepth(nodeBranch.Path, newLeft, nodeBranch.Right, int(nodeBranch.Depth)), nil + return newNodeBranchWithDepth(nodeBranch.Path, newLeft, nodeBranch.Right, int(nodeBranch.Depth), nodeBranch.Region), nil } } diff --git a/internal/smt/smt_test.go b/internal/smt/smt_test.go index f454ca85..604e26cf 100644 --- a/internal/smt/smt_test.go +++ b/internal/smt/smt_test.go @@ -19,7 +19,7 @@ func TestSMTGetRoot(t *testing.T) { // v2 reference values for basic tree shapes. t.Run("EmptyTree", func(t *testing.T) { smt := NewSparseMerkleTree(api.SHA256, 2) - expected := "47dc540c94ceb704a23875c11273e16bb0b8a87aed84de911f2133568115f254" + expected := "0000000000000000000000000000000000000000000000000000000000000000" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -44,7 +44,7 @@ func TestSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b100), []byte{0x61}) smt.AddLeaf(big.NewInt(0b111), []byte{0x62}) - expected := "f0698f0230044b700c1e5e433f7776b8af113199905b6122b19504274dd77111" + expected := "737f21207992db605e9f894154720f5fc433ca7f5861dba576720633ad948dbd" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -55,7 +55,7 @@ func TestSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b1011), []byte{0x63}) smt.AddLeaf(big.NewInt(0b1111), []byte{0x64}) - expected := "728a4e5f71d239df87b57bdf1e3bd5ca3383d2b0d16758a9b3f2aedff02e4c24" + expected := "2a937ba2bf8c934fa6474c3aea4d20f88a145425bd4874089ca298d2c10d7907" require.Equal(t, expected, smt.GetRootHashHex()) }) } @@ -82,7 +82,7 @@ func TestChildSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b10010), []byte{0x61}) smt.AddLeaf(big.NewInt(0b11010), []byte{0x62}) - expected := "564b213cf6cee27badc130c7b9c7f06c27b76e8bbe25149e1412646d24027d2d" + expected := "9cd027f96658b917f35ebab75d1051853c1cdb3adca3fe980ecb1db40cae6cbc" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -91,7 +91,7 @@ func TestChildSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b10101), []byte{0x63}) smt.AddLeaf(big.NewInt(0b11101), []byte{0x64}) - expected := "c5f0538e97bb172a7e423848673faa84141b2201cc803b328f1824299f24dd7f" + expected := "b2f48e753b5e65d184354f831ed64e182d0f5d66ac9340c5541d5c9e15e453ce" require.Equal(t, expected, smt.GetRootHashHex()) }) } @@ -104,7 +104,7 @@ func TestParentSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b10), left) smt.AddLeaf(big.NewInt(0b11), right) - expected := "245915b6e866e0dfa36eb5c1323325c6663bd0ea7fe9ea7c60efe54700901577" + expected := "33e2a95b21f38f6e0caaded9d09ddeaf04812cd259faa9a9c3dc5990fbecf5d3" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -115,7 +115,7 @@ func TestParentSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b110), left) smt.AddLeaf(big.NewInt(0b101), right) - expected := "1f52283972b0b30de79673b0a889357af74859504f70a657c7516ab77b698302" + expected := "c3c148e3509bb9543801fde91e32e0784ec68ae4da06553f522d84cae2fcf6ab" require.Equal(t, expected, smt.GetRootHashHex()) }) } @@ -227,7 +227,7 @@ func TestSMTBatchOperations(t *testing.T) { // TestSMTRootHashRegressionFixture pins an implementation reference root hash // for a fixed leaf set, so refactors cannot accidentally change hash behavior. func TestSMTRootHashRegressionFixture(t *testing.T) { - const expectedRoot = "8f12d069a0a8d02649dae4485d97ea1d98f2742b5c22de64a4f331b6f0b7b7dd" + const expectedRoot = "55470bd5b8f6a8a6ecb1bd669e87d8aedaae19c8b5aaed54aaced61704db7012" leaves := []*Leaf{ NewLeaf(big.NewInt(0b110010000), []byte("value00010000")), // 400 @@ -254,9 +254,9 @@ func TestSMTRootHashRegressionFixture(t *testing.T) { // deterministic child-root inputs. func TestParentSMTRootHashRegressionFixture(t *testing.T) { const ( - expectedEmpty = "cd123cd6893ea82539bbce16cd69f196ad3770a1a16806acd624061684f04c22" - expectedOneUpdate = "b3bf509ebc9114647fd69f72b817b257b07b8bd32ed82f6b85b8f5b19dedcfc8" - expectedTwoUpdates = "0eb669e4b5572cd9c2cf3b4a18b491354f67c0591d549cc2879a63defe0a7759" + expectedEmpty = "816c274f74eadcb829a94d5b91d9c49fbede03f4defde62a841a8d11f1045e89" + expectedOneUpdate = "dc850f58f28646650f89a57cf1adee968e8643cf3b8283183975291137d8b6be" + expectedTwoUpdates = "c73c220c13ce846b64ad5b03431cbfb10495c0ec85488f9aee2dae5fedb59d50" ) make32 := func(start byte) []byte { diff --git a/internal/smt/v6a_interop_vectors_test.go b/internal/smt/v6a_interop_vectors_test.go new file mode 100644 index 00000000..26d86599 --- /dev/null +++ b/internal/smt/v6a_interop_vectors_test.go @@ -0,0 +1,174 @@ +package smt + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/unicitynetwork/aggregator-go/pkg/api" +) + +// v6a interop vectors (issue #167). The constructions below are the shared +// cross-SDK acceptance cases: empty tree, one leaf, shallow split, deep +// split, and a multi-leaf tree. Roots are the v6a construction +// (yellowpaper Appendix C.3.2): +// +// leaf: H(0x00 || key || value) +// node: H(0x01 || depth_1B || region_32B || left_32B || right_32B) +// region: key bits 0..depth-1 packed LSB-in-byte into 32 bytes, rest zero +// empty tree root: all-zero 32 bytes +// +// TestV6AInterop_JSSDKParity asserts the JS SDK's published v6a root for the +// same tree — a cross-implementation anchor in the reverse direction +// (JS-generated, Go-reproduced). + +func interopKey(b ...byte) []byte { + key := make([]byte, 32) + copy(key, b) + return key +} + +func interopAddLeaf(t *testing.T, tree *SparseMerkleTree, key []byte, value []byte, keyBits int) { + t.Helper() + path, err := api.FixedBytesToPath(key[:keyBits/8], keyBits) + require.NoError(t, err) + require.NoError(t, tree.AddLeaf(path, value)) +} + +func interopRoot(t *testing.T, tree *SparseMerkleTree) string { + t.Helper() + return tree.GetRootHashHex() +} + +func interopLeafHash(key, value []byte) string { + h := sha256.New() + h.Write([]byte{0x00}) + h.Write(key) + h.Write(value) + return hex.EncodeToString(h.Sum(nil)) +} + +// TestV6AInterop_JSSDKParity builds the identical sparse tree used by the JS +// SDK v6a test suite (32-byte keys with only the first byte set) and asserts +// the JS SDK's expected root, proving cross-implementation agreement. +func TestV6AInterop_JSSDKParity(t *testing.T) { + // Keys and values mirror the JS SDK's SparseMerkleTree v6a test + // ("should verify the tree"). + firstBytes := []byte{0b10010000, 0b00000000, 0b00010000, 0b10000000, 0b01100000, 0b00010100} + + tree := NewSparseMerkleTree(api.SHA256, 256) + for i, b := range firstBytes { + interopAddLeaf(t, tree, interopKey(b), []byte(fmt.Sprintf("value%d", i)), 256) + } + + // Expected root published by the JS SDK v6a test (imprint prefix stripped). + require.Equal(t, + "cd23fc1265484a7173323cd862b85a61796b8e0af31149944a828e6c1734b846", + interopRoot(t, tree), + "Go v6a root must match the JS SDK v6a root for the identical tree") + + for i, b := range firstBytes { + requireCertRoundTrip(t, tree, interopKey(b), []byte(fmt.Sprintf("value%d", i))) + } +} + +func TestV6AInterop_EmptyTree(t *testing.T) { + tree := NewSparseMerkleTree(api.SHA256, 256) + require.Equal(t, + "0000000000000000000000000000000000000000000000000000000000000000", + interopRoot(t, tree), + "v6a empty tree root is the all-zero hash") +} + +func TestV6AInterop_OneLeaf(t *testing.T) { + key := interopKey(0xB2) // 0b10110010, mirrors the JS single-leaf case shape + value := []byte{9, 9, 9} + + tree := NewSparseMerkleTree(api.SHA256, 256) + interopAddLeaf(t, tree, key, value, 256) + + // Single leaf: the root is the leaf hash itself (unary passthrough, + // no interior node, no region involved). + require.Equal(t, interopLeafHash(key, value), interopRoot(t, tree)) +} + +func TestV6AInterop_ShallowSplit(t *testing.T) { + // Two keys differing at bit 0: one junction at depth 0 with a zero region. + a := interopKey(0x00) + b := interopKey(0x01) + + tree := NewSparseMerkleTree(api.SHA256, 256) + interopAddLeaf(t, tree, a, []byte("left"), 256) + interopAddLeaf(t, tree, b, []byte("right"), 256) + + require.Equal(t, shallowSplitExpectedRoot, interopRoot(t, tree)) + requireCertRoundTrip(t, tree, a, []byte("left")) + requireCertRoundTrip(t, tree, b, []byte("right")) +} + +func TestV6AInterop_DeepSplit(t *testing.T) { + // Two keys identical except bit 255 (the high bit of the last byte): + // a single junction at depth 255 whose region is 255 shared zero bits. + a := interopKey() // all zeros + b := interopKey() + b[31] = 0x80 + + valueA := interopKey() + valueA[0] = 1 + valueB := interopKey() + valueB[0] = 2 + + tree := NewSparseMerkleTree(api.SHA256, 256) + interopAddLeaf(t, tree, a, valueA, 256) + interopAddLeaf(t, tree, b, valueB, 256) + + require.Equal(t, deepSplitExpectedRoot, interopRoot(t, tree)) + requireCertRoundTrip(t, tree, a, valueA) + requireCertRoundTrip(t, tree, b, valueB) +} + +func TestV6AInterop_MultiLeaf(t *testing.T) { + // Five keys producing junctions at several depths, including byte + // boundaries (bits 0, 2, 8, 16). + keys := [][]byte{ + interopKey(0x00), // bits: all zero + interopKey(0x04), // diverges at bit 2 + interopKey(0x01), // diverges at bit 0 + interopKey(0x00, 0x01), // diverges at bit 8 + interopKey(0x00, 0x00, 0x01), // diverges at bit 16 + } + + tree := NewSparseMerkleTree(api.SHA256, 256) + for i, k := range keys { + interopAddLeaf(t, tree, k, []byte(fmt.Sprintf("value%d", i)), 256) + } + + require.Equal(t, multiLeafExpectedRoot, interopRoot(t, tree)) + for i, k := range keys { + requireCertRoundTrip(t, tree, k, []byte(fmt.Sprintf("value%d", i))) + } +} + +func requireCertRoundTrip(t *testing.T, tree *SparseMerkleTree, key, value []byte) { + t.Helper() + cert, err := tree.GetInclusionCert(key) + require.NoError(t, err) + root := tree.GetRootHashRaw() + require.NoError(t, cert.Verify(key, value, root, api.SHA256), + "v6a cert must verify for key %x", key) +} + +// Shared cross-implementation interop vector roots for the v6a construction +// (issue #167): the canonical values the Go, JS, Java, and Rust SMTs must all +// reproduce for these trees. The deep-split and multi-leaf cases are the ones +// that exercise multi-byte region packing, so they are the load-bearing +// cross-implementation checks. Per-SDK cross-verification status is tracked on +// the issue, not here. +const ( + shallowSplitExpectedRoot = "8cc069f48345d8117664a31590eea28dae79cac066a4c457cd467c4d4d2648e2" + deepSplitExpectedRoot = "789f3ba1c3b31402bef371ad3cb8a7a176589648d898cb792303a0e3fe128611" + multiLeafExpectedRoot = "7fe744edd3bfe7e973675d773c49e159fccb54e27aa59972deb703fe466bbf2e" +) diff --git a/internal/smt/yellowpaper_hash_semantics_test.go b/internal/smt/yellowpaper_hash_semantics_test.go index d0efa462..64ebe607 100644 --- a/internal/smt/yellowpaper_hash_semantics_test.go +++ b/internal/smt/yellowpaper_hash_semantics_test.go @@ -46,6 +46,7 @@ func TestNodeHash_BinaryDomainSeparated(t *testing.T) { expectedHasher := api.NewDataHasher(api.SHA256) expectedHasher.Reset(). AddData([]byte{0x01, node.Depth}). + AddData(regionFromPath(node.Path, node.Depth)). AddData(leftLeaf.calculateHash(api.NewDataHasher(api.SHA256))). AddData(rightLeaf.calculateHash(api.NewDataHasher(api.SHA256))) expected := expectedHasher.GetHash().RawHash diff --git a/pkg/api/inclusion_cert.go b/pkg/api/inclusion_cert.go index 7ad3ec96..c65e1cd5 100644 --- a/pkg/api/inclusion_cert.go +++ b/pkg/api/inclusion_cert.go @@ -149,7 +149,7 @@ func verifyBitmapPath(bitmap *[BitmapSize]byte, siblings [][SiblingSize]byte, ke j-- sibling := siblings[j][:] - hasher.Reset().AddData([]byte{0x01, byte(d)}) + hasher.Reset().AddData([]byte{0x01, byte(d)}).AddData(RegionFromKeyBytes(key, d)) if keyBitAt(key, d) == 1 { // Descent went right at depth d → sibling is the left child. hasher.AddData(sibling).AddData(h) @@ -248,3 +248,30 @@ func bitmapPopcount(b *[BitmapSize]byte) int { func keyBitAt(key []byte, d int) byte { return (key[d/8] >> (uint(d) % 8)) & 1 } + +// RegionFromKeyBytes packs the depth-bit prefix of an LSB-first SMT key into +// the canonical v6a 32-byte region encoding: key bits 0..depth-1 in place, +// all bits at positions >= depth cleared. +func RegionFromKeyBytes(key []byte, depth int) []byte { + region := make([]byte, StateTreeKeyLengthBytes) + if depth <= 0 { + return region + } + if depth > StateTreeKeyLengthBits { + depth = StateTreeKeyLengthBits + } + byteLen := (depth + 7) / 8 + if byteLen > len(key) { + byteLen = len(key) + } + copy(region[:byteLen], key[:byteLen]) + // Mask the byte containing the depth boundary. When the key is shorter + // than the depth's byte span, every copied bit is below depth and no + // masking applies. + if rem := depth % 8; rem != 0 { + if maskByte := (depth - 1) / 8; maskByte < byteLen { + region[maskByte] &= byte(1<= depth zero). The low bits of a path are +// absolutely aligned: bit i is the routing decision at tree depth i. +func RegionFromPathBits(path *big.Int, depth int) []byte { + region := make([]byte, StateTreeKeyLengthBytes) + if path == nil || depth <= 0 { + return region + } + if depth > StateTreeKeyLengthBits { + depth = StateTreeKeyLengthBits + } + for i := 0; i < depth; i++ { + if path.Bit(i) != 0 { + region[i/8] |= 1 << (uint(i) % 8) + } + } + return region +}