diff --git a/ledger/complete/payloadless/equivalence_test.go b/ledger/complete/payloadless/equivalence_test.go new file mode 100644 index 00000000000..7281ef9be3e --- /dev/null +++ b/ledger/complete/payloadless/equivalence_test.go @@ -0,0 +1,520 @@ +package payloadless_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/hash" + "github.com/onflow/flow-go/ledger/common/testutils" + "github.com/onflow/flow-go/ledger/complete/mtrie/trie" + "github.com/onflow/flow-go/ledger/complete/payloadless" + "github.com/onflow/flow-go/utils/unittest" +) + +// These tests build a regular mtrie and a payloadless mtrie from the same +// inputs and assert they agree on observable outputs. The payloadless trie is +// designed to be hash-equivalent to the full mtrie when given matching +// path/value pairs, so any divergence in root hash, leaf hash, or proof +// interim hashes is a bug. + +// 1. TestEquivalence_EmptyTrie — empty tries report the same root hash. +// 2. TestEquivalence_SingleRegister — one allocated register; root hash + register count match. +// 3. TestEquivalence_ManyRegisters — 5000 deduplicated random registers, both with and without pruning. +// 4. TestEquivalence_IncrementalUpdates — 10 rounds of updates over a growing trie, both with and without pruning, asserting after every round. +// 5. TestEquivalence_Unallocation — allocate 200 registers, then unallocate them all; root hash must return to the empty-trie default and AllocatedRegCount to 0 in both implementations. +// 6. TestEquivalence_ReadSinglePath — for every allocated path, asserts payloadless.ReadSingleLeafHash(p) == HashLeaf(p, mtrie.ReadSinglePayload(p).Value()). For unallocated paths, asserts payloadless returns nil while the full mtrie returns +// an empty payload. +// 7. TestEquivalence_UnsafeRead — batched version of the above, with a mix of allocated and unallocated paths. Indexes results by path since both implementations permute their inputs in place independently. +// 8. TestEquivalence_UnsafeProofs — proves the structural equivalence of TrieBatchProof vs. PayloadlessTrieBatchProof: Inclusion, Steps, Flags, Interims, and Path must match exactly. For inclusion proofs, additionally checks +// payloadless.LeafHash == HashLeaf(mtrie.Payload.Value()). +// 9. TestEquivalence_RandomWalk — 50-step random walk performing both allocations and unallocations per step, asserting root hash and register count agree after every step. + +// applyToBoth applies the same register updates to a regular mtrie and a +// payloadless mtrie. Returns both updated tries. +// +// `mtrieParent` and `plParent` are the parent tries to update; pass empty +// tries for a fresh build. Each call deep-copies its slice inputs because +// both implementations permute paths/values in place. +func applyToBoth( + t *testing.T, + mtrieParent *trie.MTrie, + plParent *payloadless.MTrie, + paths []ledger.Path, + values [][]byte, + prune bool, +) (*trie.MTrie, *payloadless.MTrie) { + t.Helper() + + // Build payloads for the full mtrie from (path, value) pairs. The key + // portion is irrelevant for hashing — only the value bytes are hashed — + // so we use a constant key for every register. + mtriePaths := make([]ledger.Path, len(paths)) + copy(mtriePaths, paths) + mtriePayloads := make([]ledger.Payload, len(values)) + for i, v := range values { + mtriePayloads[i] = *ledger.NewPayload(constantKey, ledger.Value(v)) + } + + plPaths := make([]ledger.Path, len(paths)) + copy(plPaths, paths) + plValues := make([][]byte, len(values)) + copy(plValues, values) + + mtrieUpdated, _, err := trie.NewTrieWithUpdatedRegisters(mtrieParent, mtriePaths, mtriePayloads, prune) + require.NoError(t, err) + + plUpdated, _, err := payloadless.NewTrieWithUpdatedRegisters(plParent, plPaths, plValues, prune) + require.NoError(t, err) + + return mtrieUpdated, plUpdated +} + +// constantKey is reused for every payload built from a raw value, since the +// payloadless trie only sees the value bytes and the full mtrie's key is +// not part of the hash. +var constantKey = ledger.NewKey([]ledger.KeyPart{{Type: 0, Value: []byte("k")}}) + +// TestEquivalence_EmptyTrie verifies that empty tries match. +func TestEquivalence_EmptyTrie(t *testing.T) { + m := trie.NewEmptyMTrie() + pl := payloadless.NewEmptyMTrie() + require.Equal(t, m.RootHash(), pl.RootHash()) +} + +// TestEquivalence_SingleRegister verifies a trie with one allocated register. +func TestEquivalence_SingleRegister(t *testing.T) { + path := testutils.PathByUint16(56809) + value := payloadValue(testutils.LightPayload(56810, 59656)) + + m, pl := applyToBoth(t, + trie.NewEmptyMTrie(), payloadless.NewEmptyMTrie(), + []ledger.Path{path}, [][]byte{value}, true, + ) + + require.Equal(t, m.RootHash(), pl.RootHash()) + require.Equal(t, m.AllocatedRegCount(), pl.AllocatedRegCount()) +} + +// TestEquivalence_ManyRegisters verifies the trie root hash agrees over many +// registers, both with pruning enabled and disabled. +func TestEquivalence_ManyRegisters(t *testing.T) { + for _, prune := range []bool{false, true} { + t.Run(prefixForPrune(prune), func(t *testing.T) { + rng := &LinearCongruentialGenerator{seed: 0} + paths, values := deduplicateWrites(sampleRandomRegisterWrites(rng, 5000)) + + m, pl := applyToBoth(t, + trie.NewEmptyMTrie(), payloadless.NewEmptyMTrie(), + paths, values, prune, + ) + + require.Equal(t, m.RootHash(), pl.RootHash()) + require.Equal(t, m.AllocatedRegCount(), pl.AllocatedRegCount()) + }) + } +} + +// TestEquivalence_IncrementalUpdates verifies that root hashes agree after +// each round of updates over multiple update rounds. +func TestEquivalence_IncrementalUpdates(t *testing.T) { + for _, prune := range []bool{false, true} { + t.Run(prefixForPrune(prune), func(t *testing.T) { + rng := &LinearCongruentialGenerator{seed: 0} + m := trie.NewEmptyMTrie() + pl := payloadless.NewEmptyMTrie() + + for round := 1; round <= 10; round++ { + paths, values := deduplicateWrites(sampleRandomRegisterWrites(rng, round*50)) + m, pl = applyToBoth(t, m, pl, paths, values, prune) + require.Equalf(t, m.RootHash(), pl.RootHash(), "root hashes diverged after round %d", round) + require.Equalf(t, m.AllocatedRegCount(), pl.AllocatedRegCount(), "register counts diverged after round %d", round) + } + }) + } +} + +// TestEquivalence_Unallocation verifies the root hashes match after allocating +// registers and subsequently unallocating them. +func TestEquivalence_Unallocation(t *testing.T) { + rng := &LinearCongruentialGenerator{seed: 0} + paths, values := deduplicateWrites(sampleRandomRegisterWrites(rng, 200)) + + m, pl := applyToBoth(t, + trie.NewEmptyMTrie(), payloadless.NewEmptyMTrie(), + paths, values, true, + ) + require.Equal(t, m.RootHash(), pl.RootHash()) + + // Unallocate all registers by writing nil values for the same paths. + emptyValues := make([][]byte, len(paths)) + m, pl = applyToBoth(t, m, pl, paths, emptyValues, true) + require.Equal(t, m.RootHash(), pl.RootHash()) + require.Equal(t, uint64(0), pl.AllocatedRegCount()) + require.Equal(t, uint64(0), m.AllocatedRegCount()) + require.Equal(t, m.RootHash(), pl.RootHash()) + require.Equal(t, ledger.RootHash(ledger.GetDefaultHashForHeight(ledger.NodeMaxHeight)), pl.RootHash()) +} + +// TestEquivalence_ReadSinglePath verifies that for every path in the trie, +// the payloadless ReadSingleLeafHash matches HashLeaf(path, fullPayloadValue) +// retrieved from the full mtrie. Also checks that non-existent paths return +// nil from payloadless and an empty payload from the full mtrie. +func TestEquivalence_ReadSinglePath(t *testing.T) { + rng := &LinearCongruentialGenerator{seed: 0} + paths, values := deduplicateWrites(sampleRandomRegisterWrites(rng, 500)) + + m, pl := applyToBoth(t, + trie.NewEmptyMTrie(), payloadless.NewEmptyMTrie(), + paths, values, true, + ) + require.Equal(t, m.RootHash(), pl.RootHash()) + + // Allocated paths: both implementations report the value, and the + // payloadless leaf hash equals HashLeaf(path, mtrie-payload-value). + for i, p := range paths { + mPayload := m.ReadSinglePayload(p) + plLeafHash := pl.ReadSingleLeafHash(p) + + require.False(t, mPayload.IsEmpty(), "mtrie should have a payload for allocated path %d", i) + require.NotNil(t, plLeafHash, "payloadless should have a leaf hash for allocated path %d", i) + + expected := hash.HashLeaf(hash.Hash(p), []byte(mPayload.Value())) + require.Equalf(t, expected, *plLeafHash, "leaf hash mismatch for path index %d", i) + } + + // Non-existent paths: mtrie returns empty, payloadless returns nil. + for i := 0; i < 50; i++ { + var p ledger.Path + // Choose a path that almost certainly isn't in the trie. + p[0] = byte(0xff) + p[1] = byte(i) + mPayload := m.ReadSinglePayload(p) + plLeafHash := pl.ReadSingleLeafHash(p) + require.True(t, mPayload.IsEmpty(), "mtrie should not have a payload for unallocated path") + require.Nil(t, plLeafHash, "payloadless should not have a leaf hash for unallocated path") + } +} + +// TestEquivalence_UnsafeRead verifies that the batched UnsafeRead from both +// implementations agrees on a slice of paths (existent and non-existent). +func TestEquivalence_UnsafeRead(t *testing.T) { + rng := &LinearCongruentialGenerator{seed: 0} + paths, values := deduplicateWrites(sampleRandomRegisterWrites(rng, 300)) + + m, pl := applyToBoth(t, + trie.NewEmptyMTrie(), payloadless.NewEmptyMTrie(), + paths, values, true, + ) + + // Build a query: half of the allocated paths plus some unallocated paths. + queryPaths := make([]ledger.Path, 0, len(paths)/2+50) + queryPaths = append(queryPaths, paths[:len(paths)/2]...) + for i := 0; i < 50; i++ { + var p ledger.Path + p[0] = byte(0xff) + p[31] = byte(i) + queryPaths = append(queryPaths, p) + } + + // Both implementations permute their `paths` argument in place, so use + // independent copies. + mPaths := make([]ledger.Path, len(queryPaths)) + copy(mPaths, queryPaths) + plPaths := make([]ledger.Path, len(queryPaths)) + copy(plPaths, queryPaths) + + mPayloads := m.UnsafeRead(mPaths) + plLeafHashes := pl.UnsafeRead(plPaths) + + require.Equal(t, len(mPayloads), len(plLeafHashes)) + + // Each read returns results in a permuted order matching its own paths + // argument. Compare value-by-path via maps. + mByPath := make(map[ledger.Path]*ledger.Payload, len(mPaths)) + for i, p := range mPaths { + mByPath[p] = mPayloads[i] + } + plByPath := make(map[ledger.Path]*hash.Hash, len(plPaths)) + for i, p := range plPaths { + plByPath[p] = plLeafHashes[i] + } + + for _, p := range queryPaths { + mp := mByPath[p] + plh := plByPath[p] + if mp.IsEmpty() { + require.Nil(t, plh, "payloadless should report nil for unallocated path") + continue + } + require.NotNil(t, plh, "payloadless should report a leaf hash for allocated path") + expected := hash.HashLeaf(hash.Hash(p), []byte(mp.Value())) + require.Equal(t, expected, *plh) + } +} + +// TestEquivalence_UnsafeProofs verifies that the proof interim hashes and +// structure (Flags, Steps, Inclusion) match between the two implementations. +// The payload-vs-leafHash field differs by design. +func TestEquivalence_UnsafeProofs(t *testing.T) { + rng := &LinearCongruentialGenerator{seed: 0} + paths, values := deduplicateWrites(sampleRandomRegisterWrites(rng, 300)) + + m, pl := applyToBoth(t, + trie.NewEmptyMTrie(), payloadless.NewEmptyMTrie(), + paths, values, true, + ) + + // Query a mix of allocated and unallocated paths. + queryPaths := make([]ledger.Path, 0, len(paths)/4+25) + queryPaths = append(queryPaths, paths[:len(paths)/4]...) + for i := 0; i < 25; i++ { + var p ledger.Path + p[0] = byte(0xfe) + p[31] = byte(i) + queryPaths = append(queryPaths, p) + } + + mPaths := make([]ledger.Path, len(queryPaths)) + copy(mPaths, queryPaths) + plPaths := make([]ledger.Path, len(queryPaths)) + copy(plPaths, queryPaths) + + mBatch := m.UnsafeProofs(mPaths) + plBatch := pl.UnsafeProofs(plPaths) + + require.Equal(t, mBatch.Size(), plBatch.Size()) + + // Both implementations permute their `paths` argument in place, possibly + // in different orders. Index proofs by path so comparisons stay aligned. + mByPath := make(map[ledger.Path]*ledger.TrieProof, len(mPaths)) + for i, p := range mPaths { + mByPath[p] = mBatch.Proofs[i] + } + plByPath := make(map[ledger.Path]*ledger.PayloadlessTrieProof, len(plPaths)) + for i, p := range plPaths { + plByPath[p] = plBatch.Proofs[i] + } + + for _, p := range queryPaths { + mp := mByPath[p] + plp := plByPath[p] + + require.Equalf(t, mp.Inclusion, plp.Inclusion, "Inclusion mismatch for path %x", p[:]) + require.Equalf(t, mp.Steps, plp.Steps, "Steps mismatch for path %x", p[:]) + require.Equalf(t, mp.Flags, plp.Flags, "Flags mismatch for path %x", p[:]) + require.Equalf(t, mp.Interims, plp.Interims, "Interims mismatch for path %x", p[:]) + require.Equal(t, mp.Path, plp.Path) + + if mp.Inclusion { + // On inclusion, the full proof carries the payload and the + // payloadless proof carries HashLeaf(path, value). + require.NotNil(t, plp.LeafHash) + expected := hash.HashLeaf(hash.Hash(mp.Path), []byte(mp.Payload.Value())) + require.Equal(t, expected, *plp.LeafHash) + } + } +} + +// TestEquivalence_UnsafeProofs_NoPruning verifies structural proof equivalence for an +// unpruned trie (prune=false). With pruning disabled, unallocated registers are kept as +// explicit default leaves; their proofs carry Inclusion=true but an empty payload (mtrie) +// or nil LeafHash (payloadless). +func TestEquivalence_UnsafeProofs_NoPruning(t *testing.T) { + rng := &LinearCongruentialGenerator{seed: 1} + paths, values := deduplicateWrites(sampleRandomRegisterWrites(rng, 300)) + + // Build both tries with all registers allocated. + m, pl := applyToBoth(t, + trie.NewEmptyMTrie(), payloadless.NewEmptyMTrie(), + paths, values, true, + ) + + // Unallocate the first quarter without pruning, keeping them as explicit default leaves. + numUnalloc := len(paths) / 4 + unallocValues := make([][]byte, numUnalloc) // all nil + m, pl = applyToBoth(t, m, pl, paths[:numUnalloc], unallocValues, false) + + require.Equal(t, uint64(len(paths)-numUnalloc), pl.AllocatedRegCount()) + require.Equal(t, m.AllocatedRegCount(), pl.AllocatedRegCount()) + + // Build three query categories: + // (a) still-allocated: paths[numUnalloc : len(paths)/2] + // (b) explicitly unallocated: paths[:numUnalloc] + // (c) never allocated: synthetic 0xfe.. paths + queryPaths := make([]ledger.Path, 0) + queryPaths = append(queryPaths, paths[numUnalloc:len(paths)/2]...) + queryPaths = append(queryPaths, paths[:numUnalloc]...) + for i := 0; i < 10; i++ { + var p ledger.Path + p[0] = byte(0xfe) + p[31] = byte(i) + queryPaths = append(queryPaths, p) + } + + mPaths := make([]ledger.Path, len(queryPaths)) + copy(mPaths, queryPaths) + plPaths := make([]ledger.Path, len(queryPaths)) + copy(plPaths, queryPaths) + + mBatch := m.UnsafeProofs(mPaths) + plBatch := pl.UnsafeProofs(plPaths) + require.Equal(t, mBatch.Size(), plBatch.Size()) + + mByPath := make(map[ledger.Path]*ledger.TrieProof, len(mPaths)) + for i, p := range mPaths { + mByPath[p] = mBatch.Proofs[i] + } + plByPath := make(map[ledger.Path]*ledger.PayloadlessTrieProof, len(plPaths)) + for i, p := range plPaths { + plByPath[p] = plBatch.Proofs[i] + } + + for _, p := range queryPaths { + mp := mByPath[p] + plp := plByPath[p] + + require.Equalf(t, mp.Inclusion, plp.Inclusion, "Inclusion mismatch for path %x", p[:]) + require.Equalf(t, mp.Steps, plp.Steps, "Steps mismatch for path %x", p[:]) + require.Equalf(t, mp.Flags, plp.Flags, "Flags mismatch for path %x", p[:]) + require.Equalf(t, mp.Interims, plp.Interims, "Interims mismatch for path %x", p[:]) + require.Equal(t, mp.Path, plp.Path) + + if mp.Inclusion { + if mp.Payload.IsEmpty() { + // Explicitly-unallocated register (default leaf kept by prune=false): + // payloadless reports Inclusion=true with LeafHash=nil. + require.Nilf(t, plp.LeafHash, + "payloadless should report nil LeafHash for explicitly-unallocated path %x", p[:]) + } else { + require.NotNilf(t, plp.LeafHash, + "payloadless should report non-nil LeafHash for allocated path %x", p[:]) + expected := hash.HashLeaf(hash.Hash(mp.Path), []byte(mp.Payload.Value())) + require.Equalf(t, expected, *plp.LeafHash, "leaf hash mismatch for path %x", p[:]) + } + } + } +} + +// TestEquivalence_RandomWalk runs random allocations, updates, and +// unallocations on both implementations and verifies the root hash, register +// count, and per-path reads (via UnsafeRead over the live set and recently +// unallocated paths) agree after every step. +func TestEquivalence_RandomWalk(t *testing.T) { + rand := unittest.GetPRG(t) + + const steps = 50 + const allocPerStep = 60 + const unallocPerStep = 30 + + m := trie.NewEmptyMTrie() + pl := payloadless.NewEmptyMTrie() + live := make(map[ledger.Path][]byte) + + for step := 0; step < steps; step++ { + updatePaths := make([]ledger.Path, 0, allocPerStep+unallocPerStep) + updateValues := make([][]byte, 0, allocPerStep+unallocPerStep) + + // Allocate / update some registers. + for i := 0; i < allocPerStep; i++ { + var p ledger.Path + _, err := rand.Read(p[:]) + require.NoError(t, err) + value := payloadValue(testutils.RandomPayload(1, 100)) + updatePaths = append(updatePaths, p) + updateValues = append(updateValues, value) + } + // Unallocate up to unallocPerStep existing registers. + count := 0 + for p := range live { + if count >= unallocPerStep { + break + } + updatePaths = append(updatePaths, p) + updateValues = append(updateValues, nil) + count++ + } + + m, pl = applyToBoth(t, m, pl, updatePaths, updateValues, true) + + // Track the expected live set. Re-establish from the post-update + // updatePaths/updateValues because applyToBoth used copies (the + // originals are untouched). + for i, p := range updatePaths { + if updateValues[i] == nil { + delete(live, p) + } else { + live[p] = updateValues[i] + } + } + + require.Equalf(t, m.RootHash(), pl.RootHash(), "root hashes diverged at step %d", step) + require.Equalf(t, uint64(len(live)), pl.AllocatedRegCount(), "reg count mismatch at step %d", step) + require.Equal(t, m.AllocatedRegCount(), pl.AllocatedRegCount()) + + // Per-path read: verify all live registers agree between implementations. + if len(live) > 0 { + livePaths := make([]ledger.Path, 0, len(live)) + for p := range live { + livePaths = append(livePaths, p) + } + mReadPaths := make([]ledger.Path, len(livePaths)) + copy(mReadPaths, livePaths) + plReadPaths := make([]ledger.Path, len(livePaths)) + copy(plReadPaths, livePaths) + + mPayloads := m.UnsafeRead(mReadPaths) + plHashes := pl.UnsafeRead(plReadPaths) + + // Index by path since UnsafeRead permutes the paths slice in place. + mByPath := make(map[ledger.Path]*ledger.Payload, len(livePaths)) + for i, p := range mReadPaths { + mByPath[p] = mPayloads[i] + } + plByPath := make(map[ledger.Path]*hash.Hash, len(livePaths)) + for i, p := range plReadPaths { + plByPath[p] = plHashes[i] + } + + for p, expectedValue := range live { + require.Falsef(t, mByPath[p].IsEmpty(), "mtrie should return non-empty payload for live path at step %d", step) + require.NotNilf(t, plByPath[p], "payloadless missing live path at step %d", step) + expectedLeafHash := hash.HashLeaf(hash.Hash(p), expectedValue) + require.Equalf(t, expectedLeafHash, *plByPath[p], "leaf hash mismatch for live path at step %d", step) + } + } + + // Per-path read: verify recently-unallocated paths return nil in both implementations. + deadPaths := make([]ledger.Path, 0) + for i, v := range updateValues { + if v == nil { + if _, stillLive := live[updatePaths[i]]; !stillLive { + deadPaths = append(deadPaths, updatePaths[i]) + } + } + } + if len(deadPaths) > 0 { + mDeadPaths := make([]ledger.Path, len(deadPaths)) + copy(mDeadPaths, deadPaths) + plDeadPaths := make([]ledger.Path, len(deadPaths)) + copy(plDeadPaths, deadPaths) + + mDeadPayloads := m.UnsafeRead(mDeadPaths) + plDeadHashes := pl.UnsafeRead(plDeadPaths) + + for i := range deadPaths { + // mtrie.UnsafeRead returns EmptyPayload() (non-nil) for missing paths; payloadless returns nil. + require.Truef(t, mDeadPayloads[i].IsEmpty(), "mtrie should return empty payload for unallocated path at step %d", step) + require.Nilf(t, plDeadHashes[i], "payloadless should return nil for unallocated path at step %d", step) + } + } + } +} + +func prefixForPrune(prune bool) string { + if prune { + return "with_pruning" + } + return "without_pruning" +} diff --git a/ledger/complete/payloadless/json_test.go b/ledger/complete/payloadless/json_test.go new file mode 100644 index 00000000000..6b6e4d88447 --- /dev/null +++ b/ledger/complete/payloadless/json_test.go @@ -0,0 +1,148 @@ +package payloadless_test + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/hash" + "github.com/onflow/flow-go/ledger/common/testutils" + "github.com/onflow/flow-go/ledger/complete/payloadless" +) + +func Test_DumpJSONEmpty(t *testing.T) { + + tr := payloadless.NewEmptyMTrie() + + var buffer bytes.Buffer + err := tr.DumpAsJSON(&buffer) + require.NoError(t, err) + + js := buffer.String() + assert.Empty(t, js) +} + +// Test_DumpJSON_WithDefaultLeaf verifies that DumpAsJSON emits an explicit +// {"path":...,"leafHash":null} entry for an unallocated register that is +// kept as a default leaf in an unpruned trie (prune=false). This pins the +// decision that default leaves are included in the dump rather than silently +// skipped (unlike AllLeafHashes, which skips them). +func Test_DumpJSON_WithDefaultLeaf(t *testing.T) { + path1 := testutils.PathByUint16(1) + path2 := testutils.PathByUint16(2) + + value1 := []byte{1} + + // Build an unpruned trie with one allocated and one explicitly-unallocated register. + tr, _, err := payloadless.NewTrieWithUpdatedRegisters(payloadless.NewEmptyMTrie(), + []ledger.Path{path1, path2}, [][]byte{value1, nil}, false) + require.NoError(t, err) + require.Equal(t, uint64(1), tr.AllocatedRegCount()) + + var buffer bytes.Buffer + err = tr.DumpAsJSON(&buffer) + require.NoError(t, err) + + js := buffer.String() + split := strings.Split(js, "\n") + rows := make([]string, 0) + for _, s := range split { + if len(s) > 0 { + rows = append(rows, s) + } + } + + // Expect two rows: one for path1 (allocated) and one for path2 (default leaf). + require.Len(t, rows, 2) + + type entry struct { + Path string `json:"path"` + LeafHash *string `json:"leafHash"` + } + + var foundAllocated, foundDefault bool + for _, row := range rows { + var e entry + require.NoError(t, json.Unmarshal([]byte(row), &e), "invalid JSON row: %s", row) + + pathHex := e.Path + path1Hex := hex.EncodeToString(path1[:]) + path2Hex := hex.EncodeToString(path2[:]) + + switch pathHex { + case path1Hex: + require.NotNil(t, e.LeafHash, "allocated register should have non-null leafHash") + expectedLeafHash := hash.HashLeaf(hash.Hash(path1), value1) + require.Equal(t, hex.EncodeToString(expectedLeafHash[:]), *e.LeafHash) + foundAllocated = true + case path2Hex: + require.Nil(t, e.LeafHash, "default leaf (explicitly-unallocated register) should have null leafHash") + foundDefault = true + } + } + require.True(t, foundAllocated, "row for allocated path not found") + require.True(t, foundDefault, "row for default (unallocated) path not found") +} + +func Test_DumpJSONNonEmpty(t *testing.T) { + path1 := testutils.PathByUint16(1) + path2 := testutils.PathByUint16(2) + path3 := testutils.PathByUint16(3) + + value1 := []byte{1} + value2 := []byte{2} + value3 := []byte{3} + + paths := []ledger.Path{path1, path2, path3} + values := [][]byte{value1, value2, value3} + + tr, _, err := payloadless.NewTrieWithUpdatedRegisters(payloadless.NewEmptyMTrie(), paths, values, true) + require.NoError(t, err) + + var buffer bytes.Buffer + err = tr.DumpAsJSON(&buffer) + require.NoError(t, err) + + js := buffer.String() + split := strings.Split(js, "\n") + + // filter out empty strings + rows := make([]string, 0) + for _, s := range split { + if len(s) > 0 { + rows = append(rows, s) + } + } + + require.Len(t, rows, 3) + + // Each row is a JSON object {"path":"","leafHash":""}. We assert each + // path is present together with the leaf hash HashLeaf(path, value). + type entry struct { + Path string `json:"path"` + LeafHash string `json:"leafHash"` + } + for i, p := range paths { + expectedLeafHash := hash.HashLeaf(hash.Hash(p), values[i]) + expectedPathHex := hex.EncodeToString(p[:]) + expectedHashHex := hex.EncodeToString(expectedLeafHash[:]) + + found := false + for _, row := range rows { + var e entry + require.NoError(t, json.Unmarshal([]byte(row), &e), "invalid JSON row: %s", row) + if e.Path == expectedPathHex { + require.Equal(t, expectedHashHex, e.LeafHash) + found = true + break + } + } + require.True(t, found, "row for path %s not found", expectedPathHex) + } +} diff --git a/ledger/complete/payloadless/node_test.go b/ledger/complete/payloadless/node_test.go index 87e904757c7..c0b7c955a5d 100644 --- a/ledger/complete/payloadless/node_test.go +++ b/ledger/complete/payloadless/node_test.go @@ -487,6 +487,129 @@ func Test_Compactify_BothChildrenPopulated(t *testing.T) { require.Equal(t, n5.Hash(), nn5.Hash()) } +// --------------------------------------------------------------------------------------------- +// NewInterimNode (hash pinning) and AllLeafHashes +// --------------------------------------------------------------------------------------------- +// +// These pin the hash of nodes built with the (non-compactifying) NewInterimNode constructor against +// values from the Python Merkle reference implementation, complementing the NewLeaf-focused anchors +// below. They also cover AllLeafHashes, the only Node method the white-box tests above do not touch. + +// Test_InterimNodeWithoutChildren verifies that the hash value of an interim node without children is +// computed correctly. We test the hash at the lowest-possible height (0), at an interim height (9) and (16). +func Test_InterimNodeWithoutChildren(t *testing.T) { + n := NewInterimNode(0, nil, nil) + expectedRootHashHex := "18373b4b038cbbf37456c33941a7e346e752acd8fafa896933d4859002b62619" + require.Equal(t, expectedRootHashHex, hashToString(n.Hash())) + require.Equal(t, ledger.GetDefaultHashForHeight(0), n.Hash()) + + n = NewInterimNode(9, nil, nil) + expectedRootHashHex = "a37f98dbac56e315fbd4b9f9bc85fbd1b138ed4ae453b128c22c99401495af6d" + require.Equal(t, expectedRootHashHex, hashToString(n.Hash())) + require.Equal(t, ledger.GetDefaultHashForHeight(9), n.Hash()) + + n = NewInterimNode(16, nil, nil) + expectedRootHashHex = "6e24e2397f130d9d17bef32b19a77b8f5bcf03fb7e9e75fd89b8a455675d574a" + require.Equal(t, expectedRootHashHex, hashToString(n.Hash())) + require.Equal(t, ledger.GetDefaultHashForHeight(16), n.Hash()) +} + +// Test_InterimNodeWithOneChild verifies that the hash value of an interim node with +// only one child (left or right) is computed correctly. The nil sibling contributes +// DefaultHashForHeight(h-1), which is why the left-only and right-only nodes below hash +// differently: +// +// NewInterimNode(1, c, nil) NewInterimNode(1, nil, c) +// (h=1) (h=1) +// / \ / \ +// c=Leaf(56809) nil=D(0) nil=D(0) c=Leaf(56809) +// (h=0) (h=0) +func Test_InterimNodeWithOneChild(t *testing.T) { + path := testutils.PathByUint16(56809) + v := []byte(testutils.LightPayload(56810, 59656).Value()) + c := NewLeaf(path, v, 0) + + n := NewInterimNode(1, c, nil) + expectedRootHashHex := "aa496f68adbbf43197f7e4b6ba1a63a47b9ce19b1587ca9ce587a7f29cad57d5" + require.Equal(t, expectedRootHashHex, hashToString(n.Hash())) + + n = NewInterimNode(1, nil, c) + expectedRootHashHex = "9845f2c9e9c067ec6efba06ffb7c1be387b2a893ae979b1f6cb091bda1b7e12d" + require.Equal(t, expectedRootHashHex, hashToString(n.Hash())) +} + +// Test_InterimNodeWithBothChildren verifies that the hash value of an interim node with +// both children (left and right) is computed correctly: +// +// NewInterimNode(1, leftChild, rightChild) +// (h=1) +// / \ +// leftChild=Leaf rightChild=Leaf +// (56809, h=0) (2, h=0) +// +// Note: the construction is synthetic — the child paths are not consistent with their +// branch positions (NewInterimNode does not validate this); only the resulting node hash +// (a function of the two child hashes) is under test. +func Test_InterimNodeWithBothChildren(t *testing.T) { + leftPath := testutils.PathByUint16(56809) + leftValue := []byte(testutils.LightPayload(56810, 59656).Value()) + leftChild := NewLeaf(leftPath, leftValue, 0) + + rightPath := testutils.PathByUint16(2) + rightValue := []byte(testutils.LightPayload(11, 22).Value()) + rightChild := NewLeaf(rightPath, rightValue, 0) + + n := NewInterimNode(1, leftChild, rightChild) + expectedRootHashHex := "1e4754fb35ec011b6192e205de403c1031d8ce64bd3d1ff8f534a20595af90c3" + require.Equal(t, expectedRootHashHex, hashToString(n.Hash())) +} + +// Test_AllLeafHashes verifies that AllLeafHashes collects the leaf hash of every allocated register in +// the subtree (here: three leaves across a small two-level interim structure), and that unallocated +// (default) leaves are skipped. Structure under test: +// +// n5 (h=2) +// / \ +// n4 (h=1) n6 (h=1) +// / \ / \ +// n1 (h=0) n2 (h=0) n3(h=0) n_default(h=0) +// +// n1, n2, n3 are distinct allocated registers; n_default is an unallocated register. +// AllLeafHashes must return exactly the three hashes for n1, n2, n3 and skip n_default. +func Test_AllLeafHashes(t *testing.T) { + path1 := testutils.PathByUint16(1) + path2 := testutils.PathByUint16(2) + path3 := testutils.PathByUint16(3) + path4 := testutils.PathByUint16(4) // unallocated + + v1 := []byte(testutils.LightPayload(2, 3).Value()) + v2 := []byte(testutils.LightPayload(4, 5).Value()) + v3 := []byte(testutils.LightPayload(6, 7).Value()) + + n1 := NewLeaf(path1, v1, 0) + n2 := NewLeaf(path2, v2, 0) + n3 := NewLeaf(path3, v3, 0) + nDefault := NewLeaf(path4, nil, 0) // default (unallocated) leaf; AllLeafHashes must skip this + + n4 := NewInterimNode(1, n1, n2) + n6 := NewInterimNode(1, n3, nDefault) + n5 := NewInterimNode(2, n4, n6) + + leafHashes := n5.AllLeafHashes() + require.Equal(t, 3, len(leafHashes)) + + // Verify the returned set equals the three expected leaf hashes. + expected := map[hash.Hash]bool{ + hash.HashLeaf(hash.Hash(path1), v1): true, + hash.HashLeaf(hash.Hash(path2), v2): true, + hash.HashLeaf(hash.Hash(path3), v3): true, + } + for _, lh := range leafHashes { + require.NotNil(t, lh) + require.Truef(t, expected[*lh], "unexpected leaf hash returned by AllLeafHashes: %x", *lh) + } +} + // --------------------------------------------------------------------------------------------- // Python reference-implementation anchors // --------------------------------------------------------------------------------------------- diff --git a/ledger/complete/payloadless/trie_test.go b/ledger/complete/payloadless/trie_test.go index c66d466d4bc..e4cf27734ac 100644 --- a/ledger/complete/payloadless/trie_test.go +++ b/ledger/complete/payloadless/trie_test.go @@ -1,63 +1,978 @@ package payloadless_test import ( + "bytes" + "encoding/binary" + "encoding/hex" + "math" + "sort" "testing" "github.com/stretchr/testify/require" + "gotest.tools/assert" "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/bitutils" + "github.com/onflow/flow-go/ledger/common/hash" "github.com/onflow/flow-go/ledger/common/testutils" "github.com/onflow/flow-go/ledger/complete/payloadless" + "github.com/onflow/flow-go/utils/unittest" ) -// Test_AllocatedRegCount verifies that AllocatedRegCount tracks all four allocated<->unallocated -// register transitions of an update: unallocated->allocated (delta +1), allocated->allocated with a -// different value (delta 0), allocated->unallocated (delta -1), and unallocated->unallocated (delta 0). +// TestEmptyTrie tests whether the root hash of an empty trie matches the formal specification. +func Test_EmptyTrie(t *testing.T) { + // Make new Trie (independently of MForest): + emptyTrie := payloadless.NewEmptyMTrie() + rootHash := emptyTrie.RootHash() + require.Equal(t, ledger.GetDefaultHashForHeight(ledger.NodeMaxHeight), hash.Hash(rootHash)) + + // verify root hash + expectedRootHashHex := "568f4ec740fe3b5de88034cb7b1fbddb41548b068f31aebc8ae9189e429c5749" + require.Equal(t, expectedRootHashHex, rootHashToString(rootHash)) + + // check String() method does not panic: + _ = emptyTrie.String() +} + +// Test_TrieWithLeftRegister tests whether the root hash of trie with only the left-most +// register populated matches the formal specification. +// The expected value is coming from a reference implementation in python and is hard-coded here. +func Test_TrieWithLeftRegister(t *testing.T) { + // Make new Trie (independently of MForest): + emptyTrie := payloadless.NewEmptyMTrie() + path := testutils.PathByUint16LeftPadded(0) + value := payloadValue(testutils.LightPayload(11, 12345)) + leftPopulatedTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(emptyTrie, []ledger.Path{path}, [][]byte{value}, true) + require.NoError(t, err) + require.Equal(t, uint16(0), maxDepthTouched) + require.Equal(t, uint64(1), leftPopulatedTrie.AllocatedRegCount()) + expectedRootHashHex := "b30c99cc3e027a6ff463876c638041b1c55316ed935f1b3699e52a2c3e3eaaab" + require.Equal(t, expectedRootHashHex, rootHashToString(leftPopulatedTrie.RootHash())) +} + +// Test_TrieWithRightRegister tests whether the root hash of trie with only the right-most +// register populated matches the formal specification. +// The expected value is coming from a reference implementation in python and is hard-coded here. +func Test_TrieWithRightRegister(t *testing.T) { + // Make new Trie (independently of MForest): + emptyTrie := payloadless.NewEmptyMTrie() + // build a path with all 1s + var path ledger.Path + for i := 0; i < len(path); i++ { + path[i] = uint8(255) + } + value := payloadValue(testutils.LightPayload(12346, 54321)) + rightPopulatedTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(emptyTrie, []ledger.Path{path}, [][]byte{value}, true) + require.NoError(t, err) + require.Equal(t, uint16(0), maxDepthTouched) + require.Equal(t, uint64(1), rightPopulatedTrie.AllocatedRegCount()) + expectedRootHashHex := "4313d22bcabbf21b1cfb833d38f1921f06a91e7198a6672bc68fa24eaaa1a961" + require.Equal(t, expectedRootHashHex, rootHashToString(rightPopulatedTrie.RootHash())) +} + +// Test_TrieWithMiddleRegister tests the root hash of trie holding only a single +// allocated register somewhere in the middle. +// The expected value is coming from a reference implementation in python and is hard-coded here. +func Test_TrieWithMiddleRegister(t *testing.T) { + // Make new Trie (independently of MForest): + emptyTrie := payloadless.NewEmptyMTrie() + + path := testutils.PathByUint16LeftPadded(56809) + value := payloadValue(testutils.LightPayload(12346, 59656)) + leftPopulatedTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(emptyTrie, []ledger.Path{path}, [][]byte{value}, true) + require.NoError(t, err) + require.Equal(t, uint16(0), maxDepthTouched) + require.Equal(t, uint64(1), leftPopulatedTrie.AllocatedRegCount()) + expectedRootHashHex := "4a29dad0b7ae091a1f035955e0c9aab0692b412f60ae83290b6290d4bf3eb296" + require.Equal(t, expectedRootHashHex, rootHashToString(leftPopulatedTrie.RootHash())) +} + +// Test_TrieWithManyRegisters tests whether the root hash of a trie storing 12001 randomly selected registers +// matches the formal specification. +// The expected value is coming from a reference implementation in python and is hard-coded here. +func Test_TrieWithManyRegisters(t *testing.T) { + // Make new Trie (independently of MForest): + emptyTrie := payloadless.NewEmptyMTrie() + // allocate single random register + rng := &LinearCongruentialGenerator{seed: 0} + paths, values := deduplicateWrites(sampleRandomRegisterWrites(rng, 12001)) + updatedTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(emptyTrie, paths, values, true) + require.NoError(t, err) + require.Equal(t, uint16(255), maxDepthTouched) + require.Equal(t, uint64(12001), updatedTrie.AllocatedRegCount()) + expectedRootHashHex := "74f748dbe563bb5819d6c09a34362a048531fd9647b4b2ea0b6ff43f200198aa" + require.Equal(t, expectedRootHashHex, rootHashToString(updatedTrie.RootHash())) +} + +// Test_FullTrie tests whether the root hash of a trie, +// whose left-most 65536 registers are populated, matches the formal specification. +// The expected value is coming from a reference implementation in python and is hard-coded here. +func Test_FullTrie(t *testing.T) { + // Make new Trie (independently of MForest): + emptyTrie := payloadless.NewEmptyMTrie() + + // allocate 65536 left-most registers + numberRegisters := 65536 + rng := &LinearCongruentialGenerator{seed: 0} + paths := make([]ledger.Path, 0, numberRegisters) + values := make([][]byte, 0, numberRegisters) + for i := 0; i < numberRegisters; i++ { + paths = append(paths, testutils.PathByUint16LeftPadded(uint16(i))) + temp := rng.next() + values = append(values, payloadValue(testutils.LightPayload(temp, temp))) + } + updatedTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(emptyTrie, paths, values, true) + require.NoError(t, err) + require.Equal(t, uint16(256), maxDepthTouched) + require.Equal(t, uint64(numberRegisters), updatedTrie.AllocatedRegCount()) + expectedRootHashHex := "6b3a48d672744f5586c571c47eae32d7a4a3549c1d4fa51a0acfd7b720471de9" + require.Equal(t, expectedRootHashHex, rootHashToString(updatedTrie.RootHash())) +} + +// TestUpdateTrie tests whether iteratively updating a Trie matches the formal specification. +// The expected root hashes are coming from a reference implementation in python and is hard-coded here. +func Test_UpdateTrie(t *testing.T) { + expectedRootHashes := []string{ + "08db9aeed2b9fcc66b63204a26a4c28652e44e3035bd87ba0ed632a227b3f6dd", + "2f4b0f490fa05e5b3bbd43176e367c3e9b64cdb710e45d4508fff11759d7a08e", + "668811792995cd960e7e343540a360682ac375f7ec5533f774c464cd6b34adc9", + "169c145eaeda2038a0e409068a12cb26bde5e890115ad1ef624f422007fb2d2a", + "8f87b503a706d9eaf50873030e0e627850c841cc0cf382187b81ba26cec57588", + "faacc057336e10e13ff6f5667aefc3ac9d9d390b34ee50391a6f7f305dfdf761", + "049e035735a13fee09a3c36a7f567daf05baee419ac90ade538108492d80b279", + "bb8340a9772ab6d6aa4862b23c8bb830da226cdf6f6c26f1e1e850077be600af", + "8b9b7eb5c489bf4aeffd86d3a215dc045856094d0abe5cf7b4cc3f835d499168", + "6514743e986f20fcf22a02e50ba352a5bfde50fe949b57b990aeb863cfcd81d1", + "33c3d386e1c7c707f727fdeb65c52117537d175da9ab3f60a0a576301d20756e", + "09df0bc6eee9d0f76df05d19b2ac550cde8c4294cd6eafaa1332718bd62e912f", + "8b1fccbf7d1eca093441305ebff72d3f12b8b7cce5b4f89d6f464fc5df83b0d3", + "0830e2d015742e284c56075050e94d3ff9618a46f28aa9066379f012e45c05fc", + "9d95255bb75dddc317deda4e45223aa4a5ac02eaa537dc9e602d6f03fa26d626", + "74f748dbe563bb5819d6c09a34362a048531fd9647b4b2ea0b6ff43f200198aa", + "c06903580432a27dee461e9022a6546cb4ddec2f8598c48429e9ba7a96a892da", + "a117f94e9cc6114e19b7639eaa630304788979cf92037736bbeb23ed1504638a", + "d382c97020371d8788d4c27971a89f1617f9bbf21c49c922f1b683cc36a4646c", + "ce633e9ca6329d6984c37a46e0a479bb1841674c2db00970dacfe035882d4aba", + } + + // Make new Trie (independently of MForest): + emptyTrie := payloadless.NewEmptyMTrie() + + // allocate single random register + rng := &LinearCongruentialGenerator{seed: 0} + path := testutils.PathByUint16LeftPadded(rng.next()) + temp := rng.next() + value := payloadValue(testutils.LightPayload(temp, temp)) + updatedTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(emptyTrie, []ledger.Path{path}, [][]byte{value}, true) + require.NoError(t, err) + require.Equal(t, uint16(0), maxDepthTouched) + require.Equal(t, uint64(1), updatedTrie.AllocatedRegCount()) + expectedRootHashHex := "08db9aeed2b9fcc66b63204a26a4c28652e44e3035bd87ba0ed632a227b3f6dd" + require.Equal(t, expectedRootHashHex, rootHashToString(updatedTrie.RootHash())) + + var paths []ledger.Path + var values [][]byte + parentTrieRegCount := updatedTrie.AllocatedRegCount() + for r := 0; r < 20; r++ { + paths, values = deduplicateWrites(sampleRandomRegisterWrites(rng, r*100)) + updatedTrie, maxDepthTouched, err = payloadless.NewTrieWithUpdatedRegisters(updatedTrie, paths, values, true) + require.NoError(t, err) + switch r { + case 0: + require.Equal(t, uint16(0), maxDepthTouched) + case 1: + require.Equal(t, uint16(254), maxDepthTouched) + default: + require.Equal(t, uint16(255), maxDepthTouched) + } + require.Equal(t, parentTrieRegCount+uint64(len(values)), updatedTrie.AllocatedRegCount()) + require.Equal(t, expectedRootHashes[r], rootHashToString(updatedTrie.RootHash())) + + parentTrieRegCount = updatedTrie.AllocatedRegCount() + } + // update with the same registers with the same values + newTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(updatedTrie, paths, values, true) + require.NoError(t, err) + require.Equal(t, uint16(255), maxDepthTouched) + require.Equal(t, updatedTrie.AllocatedRegCount(), newTrie.AllocatedRegCount()) + require.Equal(t, expectedRootHashes[19], rootHashToString(updatedTrie.RootHash())) + // check the root node pointers are equal + require.True(t, updatedTrie.RootNode() == newTrie.RootNode()) +} + +// Test_UnallocateRegisters tests whether unallocating registers matches the formal specification. +// Unallocating here means, to set the stored register value to an empty byte slice. +// The expected value is coming from a reference implementation in python and is hard-coded here. +func Test_UnallocateRegisters(t *testing.T) { + rng := &LinearCongruentialGenerator{seed: 0} + emptyTrie := payloadless.NewEmptyMTrie() + + // we first draw 99 random key-value pairs that will be first allocated and later unallocated: + paths1, values1 := deduplicateWrites(sampleRandomRegisterWrites(rng, 99)) + updatedTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(emptyTrie, paths1, values1, true) + require.NoError(t, err) + require.Equal(t, uint16(254), maxDepthTouched) + require.Equal(t, uint64(len(values1)), updatedTrie.AllocatedRegCount()) + + // we then write an additional 117 registers + paths2, values2 := deduplicateWrites(sampleRandomRegisterWrites(rng, 117)) + updatedTrie, maxDepthTouched, err = payloadless.NewTrieWithUpdatedRegisters(updatedTrie, paths2, values2, true) + require.NoError(t, err) + require.Equal(t, uint16(254), maxDepthTouched) + require.Equal(t, uint64(len(values1)+len(values2)), updatedTrie.AllocatedRegCount()) + + // and now we override the first 99 registers with default values, i.e. unallocate them. + // Mix nil and []byte{} values to verify both representations of "unallocated" are equivalent. + emptyValues0 := make([][]byte, len(values1)) + for i := range emptyValues0 { + if i%2 == 0 { + emptyValues0[i] = []byte{} + } + // odd indices remain nil + } + updatedTrie, maxDepthTouched, err = payloadless.NewTrieWithUpdatedRegisters(updatedTrie, paths1, emptyValues0, true) + require.NoError(t, err) + require.Equal(t, uint16(254), maxDepthTouched) + require.Equal(t, uint64(len(values2)), updatedTrie.AllocatedRegCount()) + + // this should be identical to the first 99 registers never been written + expectedRootHashHex := "d81e27a93f2bef058395f70e00fb5d3c8e426e22b3391d048b34017e1ecb483e" + comparisonTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(emptyTrie, paths2, values2, true) + require.NoError(t, err) + require.Equal(t, uint16(254), maxDepthTouched) + require.Equal(t, uint64(len(values2)), comparisonTrie.AllocatedRegCount()) + require.Equal(t, expectedRootHashHex, rootHashToString(comparisonTrie.RootHash())) + require.Equal(t, expectedRootHashHex, rootHashToString(updatedTrie.RootHash())) +} + +// simple Linear congruential RNG +// https://en.wikipedia.org/wiki/Linear_congruential_generator +// with configuration for 16bit output used by Microsoft Visual Basic 6 and earlier +type LinearCongruentialGenerator struct { + seed uint64 +} + +func (rng *LinearCongruentialGenerator) next() uint16 { + rng.seed = (rng.seed*1140671485 + 12820163) % 65536 + return uint16(rng.seed) +} + +// payloadValue extracts the raw value bytes from a *ledger.Payload, which is +// what the payloadless trie hashes. +func payloadValue(p *ledger.Payload) []byte { + return []byte(p.Value()) +} + +// sampleRandomRegisterWrites generates path-value tuples for `number` randomly selected registers; +// caution: registers might repeat +func sampleRandomRegisterWrites(rng *LinearCongruentialGenerator, number int) ([]ledger.Path, [][]byte) { + paths := make([]ledger.Path, 0, number) + values := make([][]byte, 0, number) + for i := 0; i < number; i++ { + path := testutils.PathByUint16LeftPadded(rng.next()) + paths = append(paths, path) + t := rng.next() + values = append(values, payloadValue(testutils.LightPayload(t, t))) + } + return paths, values +} + +// sampleRandomRegisterWritesWithPrefix generates path-value tuples for `number` randomly selected registers; +// each path is starting with the specified `prefix` and is filled to the full length with random bytes +// caution: register paths might repeat +func sampleRandomRegisterWritesWithPrefix(rng *LinearCongruentialGenerator, number int, prefix []byte) ([]ledger.Path, [][]byte) { + prefixLen := len(prefix) + if prefixLen >= hash.HashLen { + panic("prefix must be shorter than full path length, so there is some space left for random path segment") + } + + paths := make([]ledger.Path, 0, number) + values := make([][]byte, 0, number) + nextRandomBytes := make([]byte, 2) + nextRandomByteIndex := 2 // index of next unused byte in nextRandomBytes; if value is >= 2, we need to generate new random bytes + for i := 0; i < number; i++ { + var p ledger.Path + copy(p[:prefixLen], prefix) + for b := prefixLen; b < hash.HashLen; b++ { + if nextRandomByteIndex >= 2 { + // pre-generate next 2 bytes + binary.BigEndian.PutUint16(nextRandomBytes, rng.next()) + nextRandomByteIndex = 0 + } + p[b] = nextRandomBytes[nextRandomByteIndex] + nextRandomByteIndex++ + } + paths = append(paths, p) + + t := rng.next() + values = append(values, payloadValue(testutils.LightPayload(t, t))) + } + return paths, values +} + +// deduplicateWrites retains only the last register write +func deduplicateWrites(paths []ledger.Path, values [][]byte) ([]ledger.Path, [][]byte) { + mapping := make(map[ledger.Path]int) + if len(paths) != len(values) { + panic("size mismatch (paths and values)") + } + for i, path := range paths { + // we override the latest in the slice + mapping[path] = i + } + dedupedPaths := make([]ledger.Path, 0, len(mapping)) + dedupedValues := make([][]byte, 0, len(mapping)) + for path := range mapping { + dedupedPaths = append(dedupedPaths, path) + dedupedValues = append(dedupedValues, values[mapping[path]]) + } + return dedupedPaths, dedupedValues +} + +func TestSplitByPath(t *testing.T) { + rand := unittest.GetPRG(t) + + const pathsNumber = 100 + const redundantPaths = 10 + const pathsSize = 32 + randomIndex := rand.Intn(pathsSize * 8) + + // create path slice with redundant paths + paths := make([]ledger.Path, 0, pathsNumber) + for i := 0; i < pathsNumber-redundantPaths; i++ { + var p ledger.Path + _, err := rand.Read(p[:]) + require.NoError(t, err) + paths = append(paths, p) + } + for i := 0; i < redundantPaths; i++ { + paths = append(paths, paths[i]) + } + + // save a sorted paths copy for later check + sortedPaths := make([]ledger.Path, len(paths)) + copy(sortedPaths, paths) + sort.Slice(sortedPaths, func(i, j int) bool { + return bytes.Compare(sortedPaths[i][:], sortedPaths[j][:]) < 0 + }) + + // split paths + index := payloadless.SplitPaths(paths, randomIndex) + + // check correctness + for i := 0; i < index; i++ { + assert.Equal(t, bitutils.ReadBit(paths[i][:], randomIndex), 0) + } + for i := index; i < len(paths); i++ { + assert.Equal(t, bitutils.ReadBit(paths[i][:], randomIndex), 1) + } + + // check the multi-set didn't change + sort.Slice(paths, func(i, j int) bool { + return bytes.Compare(paths[i][:], paths[j][:]) < 0 + }) + for i := 0; i < len(paths); i++ { + assert.Equal(t, paths[i], sortedPaths[i]) + } +} + +// Test_DifferentiateEmptyVsLeaf tests correct behaviour for a very specific edge case for pruning: +// - By convention, a node in the trie is a leaf if both children are nil. +// - Therefore, we consider a completely unallocated subtrie also as a potential leaf. +// +// An edge case can now arise when unallocating a previously allocated leaf (see vertex '■' in the illustration below): +// +// - Before the update, both children of the leaf are nil (because it is a leaf) +// - After the update-algorithm updated the sub-Trie with root ■, both children of the updated vertex are +// also nil. But the sub-trie has now changed: the register previously represented by ■ is now gone. +// +// This case must be explicitly handled by the update algorithm: +// +// - (i) If the vertex is an interim node, i.e. it had at least one child, it is legal to re-use the vertex if neither +// of its child-subtries were affected by the update. +// - (ii) If the vertex is a leaf, only checking that neither child-subtries were affected by the update is insufficient. +// This is because the register the leaf represents might itself be affected by the update. // -// Superseded once mtrie's TestTrieAllocatedRegCountRegSize is ported to this package (it exercises the -// same four transitions at scale); this is an interim, focused guard. -func Test_AllocatedRegCount(t *testing.T) { - // Base trie: a minimal non-empty trie holding two allocated registers. With two registers the - // root is an interim node, so an update to an existing register descends to its leaf and reaches - // base case (1.a.i) there — the representative shape, rather than the empty-trie edge case. - pathA := testutils.PathByUint16LeftPadded(0) - pathB := testutils.PathByUint16LeftPadded(1) - pathC := testutils.PathByUint16LeftPadded(2) // not allocated in the base trie - base, _, err := payloadless.NewTrieWithUpdatedRegisters( - payloadless.NewEmptyMTrie(), - []ledger.Path{pathA, pathB}, [][]byte{{0x01}, {0x11}}, true) +// Condition (ii) is particularly subtle, if there are register updates in the subtrie of the leaf: +// +// - From an API perspective, it is a legal operation to set an unallocated register to nil (essentially a no-op). +// +// - Though, the Trie-update algorithm only realizes that the register is already unallocated, once it traverses +// into the respective sub-trie. When bubbling up from the recursion, nothing has changed in the children of ■ +// but the vertex ■ itself has changed from an allocated leaf register to an unallocated register. +func Test_DifferentiateEmptyVsLeaf(t *testing.T) { + // ⋮ commonPrefix29bytes 101 .... + // o + // / \ + // / \ + // / \ + // ■ o + // Left / \ + // SubTrie ⋮ ⋮ + // Right + // SubTrie + // Left Sub-Trie (■) is a single compactified leaf + // Right Sub-Trie contains multiple (18) allocated registers + + commonPrefix29bytes := "a0115ce6d49ffe0c9c3d8382826bbec896a9555e4c7720c45b558e7a9e" + leftSubTriePrefix, _ := hex.DecodeString(commonPrefix29bytes + "0") // in total 30 bytes + rightSubTriePrefix, _ := hex.DecodeString(commonPrefix29bytes + "1") // in total 30 bytes + + rng := &LinearCongruentialGenerator{seed: 0} + leftSubTriePath, leftSubTrieValue := sampleRandomRegisterWritesWithPrefix(rng, 1, leftSubTriePrefix) + rightSubTriePath, rightSubTrieValue := deduplicateWrites(sampleRandomRegisterWritesWithPrefix(rng, 18, rightSubTriePrefix)) + + // initialize Trie to the depicted state + paths := append(leftSubTriePath, rightSubTriePath...) + values := append(leftSubTrieValue, rightSubTrieValue...) + startTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(payloadless.NewEmptyMTrie(), paths, values, true) + require.NoError(t, err) + require.Equal(t, uint16(241), maxDepthTouched) + require.Equal(t, uint64(len(values)), startTrie.AllocatedRegCount()) + expectedRootHashHex := "8cf6659db0af7626ab0991e2a49019353d549aa4a8c4be1b33e8953d1a9b7fdd" + require.Equal(t, expectedRootHashHex, rootHashToString(startTrie.RootHash())) + + // Register update: + // * de-allocate the compactified leaf (■), i.e. set its value to nil. + // * also set a previously already unallocated register to nil as well + unallocatedRegister := leftSubTriePath[0] // copy path to leaf and modify it (next line) + unallocatedRegister[len(unallocatedRegister)-1] ^= 1 // path differs only in the last byte, i.e. register is also in the left Sub-Trie + updatedPaths := append(leftSubTriePath, unallocatedRegister) + updatedValues := [][]byte{nil, nil} + updatedTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(startTrie, updatedPaths, updatedValues, true) + require.NoError(t, err) + require.Equal(t, uint16(256), maxDepthTouched) + require.Equal(t, uint64(len(rightSubTrieValue)), updatedTrie.AllocatedRegCount()) + + // The updated trie should equal to a trie containing only the right sub-Trie + expectedUpdatedRootHashHex := "576e12a7ef5c760d5cc808ce50e9297919b21b87656b0cc0d9fe8a1a589cf42c" + require.Equal(t, expectedUpdatedRootHashHex, rootHashToString(updatedTrie.RootHash())) + referenceTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(payloadless.NewEmptyMTrie(), rightSubTriePath, rightSubTrieValue, true) require.NoError(t, err) - require.Equal(t, uint64(2), base.AllocatedRegCount()) + require.Equal(t, uint16(241), maxDepthTouched) + require.Equal(t, uint64(len(rightSubTrieValue)), referenceTrie.AllocatedRegCount()) + require.Equal(t, expectedUpdatedRootHashHex, rootHashToString(referenceTrie.RootHash())) +} + +func Test_Pruning(t *testing.T) { + rand := unittest.GetPRG(t) + emptyTrie := payloadless.NewEmptyMTrie() + + path1 := testutils.PathByUint16(1 << 12) // 000100... + path2 := testutils.PathByUint16(1 << 13) // 001000... + path4 := testutils.PathByUint16(1<<14 + 1<<13) // 01100... + path6 := testutils.PathByUint16(1 << 15) // 1000... + + value1 := payloadValue(testutils.LightPayload(2, 1)) + value2 := payloadValue(testutils.LightPayload(2, 2)) + value4 := payloadValue(testutils.LightPayload(2, 4)) + value6 := payloadValue(testutils.LightPayload(2, 6)) - t.Run("allocated -> allocated (different value): delta 0", func(t *testing.T) { - updated, _, err := payloadless.NewTrieWithUpdatedRegisters( - base, []ledger.Path{pathA}, [][]byte{{0x02}}, true) + paths := []ledger.Path{path1, path2, path4, path6} + values := [][]byte{value1, value2, value4, value6} + + // n7 + // / \ + // / \ + // n5 n6 (path6/value6) // 1000 + // / \ + // / \ + // / \ + // n3 n4 (path4/value4) // 01100... + // / \ + // / \ + // / \ + // n1 (path1, n2 (path2) + // value1) value2) + + baseTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(emptyTrie, paths, values, true) + require.NoError(t, err) + require.Equal(t, uint16(3), maxDepthTouched) + require.Equal(t, uint64(len(values)), baseTrie.AllocatedRegCount()) + + t.Run("leaf update with pruning test", func(t *testing.T) { + expectedRegCount := baseTrie.AllocatedRegCount() - 1 + + trie1, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(baseTrie, []ledger.Path{path1}, [][]byte{nil}, false) require.NoError(t, err) - require.Equal(t, uint64(2), updated.AllocatedRegCount()) // count unchanged - require.NotEqual(t, base.RootHash(), updated.RootHash()) // the value change took effect + require.Equal(t, uint16(3), maxDepthTouched) + require.Equal(t, expectedRegCount, trie1.AllocatedRegCount()) + + trie1withpruning, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(baseTrie, []ledger.Path{path1}, [][]byte{nil}, true) + require.NoError(t, err) + require.Equal(t, uint16(3), maxDepthTouched) + require.Equal(t, expectedRegCount, trie1withpruning.AllocatedRegCount()) + require.True(t, trie1withpruning.RootNode().VerifyCachedHash()) + + // after pruning + // n7 + // / \ + // / \ + // n5 n6 (path6/value6) // 1000 + // / \ + // / \ + // / \ + // n3 (path2 n4 (path4 + // /value2) /value4) // 01100... + require.Equal(t, trie1.RootHash(), trie1withpruning.RootHash()) }) - t.Run("allocated -> unallocated: delta -1", func(t *testing.T) { - updated, _, err := payloadless.NewTrieWithUpdatedRegisters( - base, []ledger.Path{pathA}, [][]byte{{}}, true) + t.Run("leaf update with two level pruning test", func(t *testing.T) { + expectedRegCount := baseTrie.AllocatedRegCount() - 1 + + // setting path4 to zero from baseTrie + trie2, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(baseTrie, []ledger.Path{path4}, [][]byte{nil}, false) + require.NoError(t, err) + require.Equal(t, uint16(2), maxDepthTouched) + require.Equal(t, expectedRegCount, trie2.AllocatedRegCount()) + + // pruning is not activated here because n3 is not a leaf node + trie2withpruning, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(baseTrie, []ledger.Path{path4}, [][]byte{nil}, true) + require.NoError(t, err) + require.Equal(t, uint16(2), maxDepthTouched) + require.Equal(t, expectedRegCount, trie2withpruning.AllocatedRegCount()) + require.True(t, trie2withpruning.RootNode().VerifyCachedHash()) + + require.Equal(t, trie2.RootHash(), trie2withpruning.RootHash()) + + // now setting path2 to zero should do the pruning for two levels + expectedRegCount -= 1 + + trie22, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(trie2, []ledger.Path{path2}, [][]byte{nil}, false) require.NoError(t, err) - require.Equal(t, uint64(1), updated.AllocatedRegCount()) // one register freed - require.NotEqual(t, base.RootHash(), updated.RootHash()) + require.Equal(t, uint16(3), maxDepthTouched) + require.Equal(t, expectedRegCount, trie22.AllocatedRegCount()) + + trie22withpruning, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(trie2withpruning, []ledger.Path{path2}, [][]byte{nil}, true) + require.NoError(t, err) + require.Equal(t, uint16(3), maxDepthTouched) + require.Equal(t, expectedRegCount, trie22withpruning.AllocatedRegCount()) + + // after pruning + // n7 + // / \ + // / \ + // n5 (path1, n6 (path6/value6) // 1000 + // /value1) + + require.Equal(t, trie22.RootHash(), trie22withpruning.RootHash()) + require.True(t, trie22withpruning.RootNode().VerifyCachedHash()) + }) - t.Run("unallocated -> allocated: delta +1", func(t *testing.T) { - updated, _, err := payloadless.NewTrieWithUpdatedRegisters( - base, []ledger.Path{pathC}, [][]byte{{0x22}}, true) + t.Run("several updates at the same time", func(t *testing.T) { + // setting path4 to zero from baseTrie + trie3, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(baseTrie, []ledger.Path{path2, path4, path6}, [][]byte{nil, nil, nil}, false) + require.NoError(t, err) + require.Equal(t, uint16(3), maxDepthTouched) + require.Equal(t, uint64(1), trie3.AllocatedRegCount()) + + // this should prune two levels + trie3withpruning, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(baseTrie, []ledger.Path{path2, path4, path6}, [][]byte{nil, nil, nil}, true) require.NoError(t, err) - require.Equal(t, uint64(3), updated.AllocatedRegCount()) // one register allocated - require.NotEqual(t, base.RootHash(), updated.RootHash()) + require.Equal(t, uint16(3), maxDepthTouched) + require.Equal(t, uint64(1), trie3withpruning.AllocatedRegCount()) + + // after pruning + // n7 (path1/value1) + require.Equal(t, trie3.RootHash(), trie3withpruning.RootHash()) + require.True(t, trie3withpruning.RootNode().VerifyCachedHash()) + }) + + t.Run("smoke testing trie pruning", func(t *testing.T) { + unittest.SkipUnless(t, unittest.TEST_LONG_RUNNING, "skipping trie pruning smoke testing as its not needed to always run") + + numberOfSteps := 1000 + numberOfUpdates := 750 + numberOfRemovals := 750 + + var err error + activeTrie := payloadless.NewEmptyMTrie() + activeTrieWithPruning := payloadless.NewEmptyMTrie() + allPaths := make(map[ledger.Path][]byte) + var maxDepthTouched, maxDepthTouchedWithPruning uint16 + var parentTrieRegCount uint64 + + for step := 0; step < numberOfSteps; step++ { + + updatePaths := make([]ledger.Path, 0) + updateValues := make([][]byte, 0) + + var expectedRegCountDelta int64 + + for i := 0; i < numberOfUpdates; { + var path ledger.Path + _, err := rand.Read(path[:]) + require.NoError(t, err) + // deduplicate + if _, found := allPaths[path]; !found { + value := payloadValue(testutils.RandomPayload(1, 100)) + updatePaths = append(updatePaths, path) + updateValues = append(updateValues, value) + expectedRegCountDelta++ + i++ + } + } + + i := 0 + samplesNeeded := int(math.Min(float64(numberOfRemovals), float64(len(allPaths)))) + for p := range allPaths { + updatePaths = append(updatePaths, p) + updateValues = append(updateValues, nil) + expectedRegCountDelta-- + delete(allPaths, p) + i++ + if i > samplesNeeded { + break + } + } + + // only set it for the updates + for i := 0; i < numberOfUpdates; i++ { + allPaths[updatePaths[i]] = updateValues[i] + } + + activeTrie, maxDepthTouched, err = payloadless.NewTrieWithUpdatedRegisters(activeTrie, updatePaths, updateValues, false) + require.NoError(t, err) + require.Equal(t, uint64(int64(parentTrieRegCount)+expectedRegCountDelta), activeTrie.AllocatedRegCount()) + + activeTrieWithPruning, maxDepthTouchedWithPruning, err = payloadless.NewTrieWithUpdatedRegisters(activeTrieWithPruning, updatePaths, updateValues, true) + require.NoError(t, err) + require.True(t, maxDepthTouched >= maxDepthTouchedWithPruning) + require.Equal(t, uint64(int64(parentTrieRegCount)+expectedRegCountDelta), activeTrieWithPruning.AllocatedRegCount()) + + require.Equal(t, activeTrie.RootHash(), activeTrieWithPruning.RootHash()) + + parentTrieRegCount = activeTrie.AllocatedRegCount() + + // fetch all leaf hashes and compare + queryPaths := make([]ledger.Path, 0) + for path := range allPaths { + queryPaths = append(queryPaths, path) + } + + leafHashes := activeTrie.UnsafeRead(queryPaths) + for i, lh := range leafHashes { + expectedValue := allPaths[queryPaths[i]] + expectedLeafHash := hash.HashLeaf(hash.Hash(queryPaths[i]), expectedValue) + require.NotNil(t, lh) + require.Equal(t, expectedLeafHash, *lh) + } + + leafHashes = activeTrieWithPruning.UnsafeRead(queryPaths) + for i, lh := range leafHashes { + expectedValue := allPaths[queryPaths[i]] + expectedLeafHash := hash.HashLeaf(hash.Hash(queryPaths[i]), expectedValue) + require.NotNil(t, lh) + require.Equal(t, expectedLeafHash, *lh) + } + + } + }) +} + +func rootHashToString(rh ledger.RootHash) string { + return hex.EncodeToString(rh[:]) +} + +// TestTrieAllocatedRegCount tests allocated register count for updated trie. +// It tests the following updates with prune flag set to true: +// - update empty trie with new paths and values +// - update trie with existing paths and updated values +// - update trie with new paths and empty values +// - update trie with existing path and empty value one by one until trie is empty +// +// It also tests the following updates with prune flag set to false: +// - update trie with existing path and empty value one by one until trie is empty +// - update trie with removed paths and empty values +// - update trie with removed paths and non-empty values +func TestTrieAllocatedRegCount(t *testing.T) { + + rng := &LinearCongruentialGenerator{seed: 0} + + // Allocate 255 registers + numberRegisters := 255 + paths := make([]ledger.Path, numberRegisters) + values := make([][]byte, numberRegisters) + for i := 0; i < numberRegisters; i++ { + var p ledger.Path + p[0] = byte(i) + + paths[i] = p + values[i] = payloadValue(testutils.LightPayload(rng.next(), rng.next())) + } + + // Update trie with registers to test reg count with new registers. + updatedTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(payloadless.NewEmptyMTrie(), paths, values, true) + require.NoError(t, err) + require.True(t, maxDepthTouched <= 256) + require.Equal(t, uint64(len(values)), updatedTrie.AllocatedRegCount()) + + // Update trie with existing paths and updated values + // to test reg count with updated registers + // (old value present and new value present). + for i := 0; i < len(values); i += 2 { + values[i] = payloadValue(testutils.LightPayload(rng.next(), rng.next())) + } + + updatedTrie, maxDepthTouched, err = payloadless.NewTrieWithUpdatedRegisters(updatedTrie, paths, values, true) + require.NoError(t, err) + require.True(t, maxDepthTouched <= 256) + require.Equal(t, uint64(len(values)), updatedTrie.AllocatedRegCount()) + + rootHash := updatedTrie.RootHash() + + // Update trie with new paths and empty values + // to test reg count with new empty registers. + newPaths := []ledger.Path{} + newValues := [][]byte{} + for i := 0; i < len(paths); i++ { + oldPath := paths[i] + + path1, _ := ledger.ToPath(oldPath[:]) + path1[1] = 1 + value1 := []byte(nil) + + path2, _ := ledger.ToPath(oldPath[:]) + path2[1] = 2 + value2 := []byte(nil) + + newPaths = append(newPaths, oldPath, path1, path2) + newValues = append(newValues, values[i], value1, value2) + } + + updatedTrie, maxDepthTouched, err = payloadless.NewTrieWithUpdatedRegisters(updatedTrie, newPaths, newValues, true) + require.NoError(t, err) + require.Equal(t, rootHash, updatedTrie.RootHash()) + require.True(t, maxDepthTouched <= 256) + require.Equal(t, uint64(len(values)), updatedTrie.AllocatedRegCount()) + + t.Run("pruning", func(t *testing.T) { + expectedRegCount := uint64(len(values)) + + updatedTrieWithPruning := updatedTrie + + // Remove register one by one to test reg count with empty registers + // (old value present and new value empty) + for i := 0; i < len(paths); i++ { + newPaths := []ledger.Path{paths[i]} + newValues := [][]byte{nil} + + expectedRegCount-- + + updatedTrieWithPruning, maxDepthTouched, err = payloadless.NewTrieWithUpdatedRegisters(updatedTrieWithPruning, newPaths, newValues, true) + require.NoError(t, err) + require.True(t, maxDepthTouched <= 256) + require.Equal(t, expectedRegCount, updatedTrieWithPruning.AllocatedRegCount()) + } + + // After all registered are removed, reg count should be 0. + require.Equal(t, payloadless.EmptyTrieRootHash(), updatedTrieWithPruning.RootHash()) + require.Equal(t, uint64(0), updatedTrieWithPruning.AllocatedRegCount()) }) - t.Run("unallocated -> unallocated: delta 0", func(t *testing.T) { - updated, _, err := payloadless.NewTrieWithUpdatedRegisters( - base, []ledger.Path{pathC}, [][]byte{{}}, true) + t.Run("no pruning", func(t *testing.T) { + expectedRegCount := uint64(len(values)) + + updatedTrieNoPruning := updatedTrie + + // Remove register one by one to test reg count with empty registers + // (old value present and new value empty) + for i := 0; i < len(paths); i++ { + newPaths := []ledger.Path{paths[i]} + newValues := [][]byte{nil} + + expectedRegCount-- + + updatedTrieNoPruning, maxDepthTouched, err = payloadless.NewTrieWithUpdatedRegisters(updatedTrieNoPruning, newPaths, newValues, false) + require.NoError(t, err) + require.True(t, maxDepthTouched <= 256) + require.Equal(t, expectedRegCount, updatedTrieNoPruning.AllocatedRegCount()) + } + + // After all registered are removed, reg count should be 0. + require.Equal(t, payloadless.EmptyTrieRootHash(), updatedTrieNoPruning.RootHash()) + require.Equal(t, uint64(0), updatedTrieNoPruning.AllocatedRegCount()) + + // Update with removed paths and empty values + // (old value empty and new value empty) + newValues := make([][]byte, len(paths)) + + updatedTrieNoPruning, maxDepthTouched, err = payloadless.NewTrieWithUpdatedRegisters(updatedTrieNoPruning, paths, newValues, false) require.NoError(t, err) - require.Equal(t, uint64(2), updated.AllocatedRegCount()) // count unchanged - require.Equal(t, base.RootHash(), updated.RootHash()) // writing empty to a fresh path is a no-op + require.True(t, maxDepthTouched <= 256) + require.Equal(t, payloadless.EmptyTrieRootHash(), updatedTrieNoPruning.RootHash()) + require.Equal(t, uint64(0), updatedTrieNoPruning.AllocatedRegCount()) + + // Update with removed paths and non-empty values + // (old value empty and new value present) + updatedTrieNoPruning, maxDepthTouched, err = payloadless.NewTrieWithUpdatedRegisters(updatedTrieNoPruning, paths, values, false) + require.NoError(t, err) + require.Equal(t, rootHash, updatedTrieNoPruning.RootHash()) + require.True(t, maxDepthTouched <= 256) + require.Equal(t, uint64(len(values)), updatedTrieNoPruning.AllocatedRegCount()) + }) +} + +// TestTrieAllocatedRegCountWithMixedPruneFlag tests allocated register count +// for updated trie with mixed pruning flag. +// It tests the following updates: +// - step 1 : update empty trie with new paths and values (255 allocated registers) +// - step 2 : remove a value without pruning (254 allocated registers) +// - step 3a: remove previously removed value with pruning (254 allocated registers) +// - step 3b: update trie from step 2 with a new value (sibling of removed value) +// with pruning (255 allocated registers) +func TestTrieAllocatedRegCountWithMixedPruneFlag(t *testing.T) { + rng := &LinearCongruentialGenerator{seed: 0} + + // Allocate 255 registers + numberRegisters := 255 + paths := make([]ledger.Path, numberRegisters) + values := make([][]byte, numberRegisters) + for i := 0; i < numberRegisters; i++ { + var p ledger.Path + p[0] = byte(i) + + paths[i] = p + values[i] = payloadValue(testutils.LightPayload(rng.next(), rng.next())) + } + + expectedAllocatedRegCount := uint64(len(values)) + + // Update trie with registers to test reg count with new registers. + baseTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(payloadless.NewEmptyMTrie(), paths, values, true) + require.NoError(t, err) + require.True(t, maxDepthTouched <= 256) + require.Equal(t, expectedAllocatedRegCount, baseTrie.AllocatedRegCount()) + + // Remove one value without pruning + expectedAllocatedRegCount-- + + removePaths := []ledger.Path{paths[0]} + removeValues := [][]byte{nil} + unprunedTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(baseTrie, removePaths, removeValues, false) + require.NoError(t, err) + require.True(t, maxDepthTouched <= 256) + require.Equal(t, expectedAllocatedRegCount, unprunedTrie.AllocatedRegCount()) + + // Remove the same value (no affect) from unprunedTrie with pruning + // expected reg count remains unchanged. + updatedTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(unprunedTrie, removePaths, removeValues, true) + require.NoError(t, err) + require.True(t, maxDepthTouched <= 256) + require.Equal(t, expectedAllocatedRegCount, updatedTrie.AllocatedRegCount()) + + // Add sibling of removed path from unprunedTrie with pruning + newPath := paths[0] + bitutils.SetBit(newPath[:], ledger.PathLen*8-1) + newPaths := []ledger.Path{newPath} + newValues := [][]byte{payloadValue(testutils.LightPayload(rng.next(), rng.next()))} + + // expected reg count is incremented. + expectedAllocatedRegCount++ + + updatedTrie, maxDepthTouched, err = payloadless.NewTrieWithUpdatedRegisters(unprunedTrie, newPaths, newValues, true) + require.NoError(t, err) + require.True(t, maxDepthTouched <= 256) + require.Equal(t, expectedAllocatedRegCount, updatedTrie.AllocatedRegCount()) +} + +// TestReadSingleLeafHash tests reading a single leaf hash of existent/non-existent path +// for trie of different layouts. +func TestReadSingleLeafHash(t *testing.T) { + + emptyTrie := payloadless.NewEmptyMTrie() + + // Test reading leaf hash in empty trie + t.Run("empty trie", func(t *testing.T) { + savedRootHash := emptyTrie.RootHash() + + path := testutils.PathByUint16LeftPadded(0) + leafHash := emptyTrie.ReadSingleLeafHash(path) + require.Nil(t, leafHash) + require.Equal(t, savedRootHash, emptyTrie.RootHash()) + }) + + // Test reading leaf hash for existent/non-existent path + // in trie with compact leaf as root node. + t.Run("compact leaf as root", func(t *testing.T) { + path1 := testutils.PathByUint16LeftPadded(0) + value1 := payloadValue(testutils.RandomPayload(1, 100)) + + paths := []ledger.Path{path1} + values := [][]byte{value1} + + newTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(emptyTrie, paths, values, true) + require.NoError(t, err) + require.Equal(t, uint16(0), maxDepthTouched) + + savedRootHash := newTrie.RootHash() + + // Get leaf hash for existent path + retLeafHash := newTrie.ReadSingleLeafHash(path1) + require.NotNil(t, retLeafHash) + expectedLeafHash := hash.HashLeaf(hash.Hash(path1), value1) + require.Equal(t, expectedLeafHash, *retLeafHash) + require.Equal(t, savedRootHash, newTrie.RootHash()) + + // Get leaf hash for non-existent path + path2 := testutils.PathByUint16LeftPadded(1) + retLeafHash = newTrie.ReadSingleLeafHash(path2) + require.Nil(t, retLeafHash) + require.Equal(t, savedRootHash, newTrie.RootHash()) + }) + + // Test reading leaf hash for existent/non-existent path in an unpruned trie. + t.Run("trie", func(t *testing.T) { + path1 := testutils.PathByUint16(1 << 12) // 000100... + path2 := testutils.PathByUint16(1 << 13) // 001000... + path3 := testutils.PathByUint16(1 << 14) // 010000... + + value1 := payloadValue(testutils.RandomPayload(1, 100)) + value2 := payloadValue(testutils.RandomPayload(1, 100)) + var value3 []byte // empty + + paths := []ledger.Path{path1, path2, path3} + values := [][]byte{value1, value2, value3} + + // Create an unpruned trie with 3 leaf nodes (n1, n2, n3). + newTrie, maxDepthTouched, err := payloadless.NewTrieWithUpdatedRegisters(emptyTrie, paths, values, false) + require.NoError(t, err) + require.Equal(t, uint16(3), maxDepthTouched) + + savedRootHash := newTrie.RootHash() + + // n5 + // / + // / + // n4 + // / \ + // / \ + // n3 n3 (path3/ + // / \ value3) + // / \ + // n1 (path1/ n2 (path2/ + // value1) value2) + // + + // Test reading leaf hash for all possible paths for the first 4 bits. + for i := 0; i < 16; i++ { + path := testutils.PathByUint16(uint16(i << 12)) + + retLeafHash := newTrie.ReadSingleLeafHash(path) + require.Equal(t, savedRootHash, newTrie.RootHash()) + switch path { + case path1: + require.NotNil(t, retLeafHash) + expected := hash.HashLeaf(hash.Hash(path1), value1) + require.Equal(t, expected, *retLeafHash) + case path2: + require.NotNil(t, retLeafHash) + expected := hash.HashLeaf(hash.Hash(path2), value2) + require.Equal(t, expected, *retLeafHash) + default: + require.Nil(t, retLeafHash) + } + } }) } diff --git a/ledger/payloadless_proof_test.go b/ledger/payloadless_proof_test.go new file mode 100644 index 00000000000..a9efa8444d9 --- /dev/null +++ b/ledger/payloadless_proof_test.go @@ -0,0 +1,275 @@ +package ledger + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/ledger/common/hash" +) + +// TestNewPayloadlessTrieProof verifies the initial state of a freshly constructed proof. +func TestNewPayloadlessTrieProof(t *testing.T) { + p := NewPayloadlessTrieProof() + + require.Nil(t, p.LeafHash) + require.NotNil(t, p.Interims) + require.Equal(t, 0, len(p.Interims)) + require.False(t, p.Inclusion) + require.Equal(t, PathLen, len(p.Flags)) + require.Equal(t, uint8(0), p.Steps) +} + +// TestPayloadlessTrieProofEquals_Identical verifies that two identical proofs compare equal. +func TestPayloadlessTrieProofEquals_Identical(t *testing.T) { + leafHash := hash.HashLeaf(hash.DummyHash, []byte("v")) + + p1 := NewPayloadlessTrieProof() + p1.Path = Path(hash.DummyHash) + p1.LeafHash = &leafHash + p1.Inclusion = true + p1.Steps = 5 + p1.Flags[0] = 0x01 + p1.Interims = []hash.Hash{hash.DummyHash} + + p2 := NewPayloadlessTrieProof() + p2.Path = Path(hash.DummyHash) + p2LeafHash := leafHash + p2.LeafHash = &p2LeafHash + p2.Inclusion = true + p2.Steps = 5 + p2.Flags[0] = 0x01 + p2.Interims = []hash.Hash{hash.DummyHash} + + require.True(t, p1.Equals(p2)) + require.True(t, p2.Equals(p1)) +} + +// TestPayloadlessTrieProofEquals_Nil verifies Equals returns false when compared to nil. +func TestPayloadlessTrieProofEquals_Nil(t *testing.T) { + p := NewPayloadlessTrieProof() + require.False(t, p.Equals(nil)) +} + +// TestPayloadlessTrieProofEquals_DifferentPath verifies Equals returns false for different paths. +func TestPayloadlessTrieProofEquals_DifferentPath(t *testing.T) { + p1 := NewPayloadlessTrieProof() + p1.Path = Path(hash.DummyHash) + + var otherPath Path + otherPath[0] = 1 + p2 := NewPayloadlessTrieProof() + p2.Path = otherPath + + require.False(t, p1.Equals(p2)) +} + +// TestPayloadlessTrieProofEquals_LeafHash covers all leaf hash nullability/value combinations. +func TestPayloadlessTrieProofEquals_LeafHash(t *testing.T) { + h1 := hash.HashLeaf(hash.DummyHash, []byte("a")) + h2 := hash.HashLeaf(hash.DummyHash, []byte("b")) + + t.Run("both nil", func(t *testing.T) { + p1 := NewPayloadlessTrieProof() + p2 := NewPayloadlessTrieProof() + require.True(t, p1.Equals(p2)) + }) + + t.Run("one nil one non-nil", func(t *testing.T) { + p1 := NewPayloadlessTrieProof() + p2 := NewPayloadlessTrieProof() + p2.LeafHash = &h1 + require.False(t, p1.Equals(p2)) + require.False(t, p2.Equals(p1)) + }) + + t.Run("both non-nil equal", func(t *testing.T) { + p1 := NewPayloadlessTrieProof() + p2 := NewPayloadlessTrieProof() + h1Copy := h1 + p1.LeafHash = &h1 + p2.LeafHash = &h1Copy + require.True(t, p1.Equals(p2)) + }) + + t.Run("both non-nil different", func(t *testing.T) { + p1 := NewPayloadlessTrieProof() + p2 := NewPayloadlessTrieProof() + p1.LeafHash = &h1 + p2.LeafHash = &h2 + require.False(t, p1.Equals(p2)) + }) +} + +// TestPayloadlessTrieProofEquals_Inclusion verifies Equals respects the Inclusion flag. +func TestPayloadlessTrieProofEquals_Inclusion(t *testing.T) { + p1 := NewPayloadlessTrieProof() + p1.Inclusion = true + p2 := NewPayloadlessTrieProof() + p2.Inclusion = false + require.False(t, p1.Equals(p2)) +} + +// TestPayloadlessTrieProofEquals_Steps verifies Equals respects Steps. +func TestPayloadlessTrieProofEquals_Steps(t *testing.T) { + p1 := NewPayloadlessTrieProof() + p1.Steps = 1 + p2 := NewPayloadlessTrieProof() + p2.Steps = 2 + require.False(t, p1.Equals(p2)) +} + +// TestPayloadlessTrieProofEquals_Flags verifies Equals respects Flags. +func TestPayloadlessTrieProofEquals_Flags(t *testing.T) { + p1 := NewPayloadlessTrieProof() + p1.Flags[0] = 0x01 + p2 := NewPayloadlessTrieProof() + p2.Flags[0] = 0x02 + require.False(t, p1.Equals(p2)) +} + +// TestPayloadlessTrieProofEquals_Interims verifies Equals respects Interims (length and content). +func TestPayloadlessTrieProofEquals_Interims(t *testing.T) { + t.Run("different length", func(t *testing.T) { + p1 := NewPayloadlessTrieProof() + p1.Interims = []hash.Hash{hash.DummyHash} + p2 := NewPayloadlessTrieProof() + p2.Interims = []hash.Hash{hash.DummyHash, hash.DummyHash} + require.False(t, p1.Equals(p2)) + }) + + t.Run("different content", func(t *testing.T) { + var h1, h2 hash.Hash + h1[0] = 1 + h2[0] = 2 + p1 := NewPayloadlessTrieProof() + p1.Interims = []hash.Hash{h1} + p2 := NewPayloadlessTrieProof() + p2.Interims = []hash.Hash{h2} + require.False(t, p1.Equals(p2)) + }) +} + +// TestPayloadlessTrieProofString_DoesNotPanic exercises the String method on +// representative shapes (inclusion vs non-inclusion, with and without LeafHash). +func TestPayloadlessTrieProofString_DoesNotPanic(t *testing.T) { + p := NewPayloadlessTrieProof() + require.NotEmpty(t, p.String()) // empty/non-inclusion proof + + p.Inclusion = true + leafHash := hash.HashLeaf(hash.DummyHash, []byte("x")) + p.LeafHash = &leafHash + p.Interims = []hash.Hash{hash.DummyHash} + p.Steps = 1 + p.Flags[0] = 0x80 + require.NotEmpty(t, p.String()) +} + +// TestNewPayloadlessTrieBatchProof verifies the empty batch proof. +func TestNewPayloadlessTrieBatchProof(t *testing.T) { + bp := NewPayloadlessTrieBatchProof() + require.NotNil(t, bp) + require.NotNil(t, bp.Proofs) + require.Equal(t, 0, bp.Size()) +} + +// TestNewPayloadlessTrieBatchProofWithEmptyProofs verifies the pre-filled batch proof. +func TestNewPayloadlessTrieBatchProofWithEmptyProofs(t *testing.T) { + bp := NewPayloadlessTrieBatchProofWithEmptyProofs(3) + require.Equal(t, 3, bp.Size()) + for _, p := range bp.Proofs { + require.NotNil(t, p) + require.False(t, p.Inclusion) + require.Nil(t, p.LeafHash) + require.Equal(t, PathLen, len(p.Flags)) + } +} + +// TestPayloadlessTrieBatchProof_PathsAndLeafHashes verifies that Paths and LeafHashes +// return slices aligned with the underlying Proofs. +func TestPayloadlessTrieBatchProof_PathsAndLeafHashes(t *testing.T) { + var p1Path, p2Path Path + p1Path[0] = 1 + p2Path[0] = 2 + h1 := hash.HashLeaf(hash.Hash(p1Path), []byte("a")) + + bp := NewPayloadlessTrieBatchProofWithEmptyProofs(2) + bp.Proofs[0].Path = p1Path + bp.Proofs[0].LeafHash = &h1 + bp.Proofs[0].Inclusion = true + bp.Proofs[1].Path = p2Path + // Proofs[1].LeafHash stays nil (non-inclusion) + + require.Equal(t, []Path{p1Path, p2Path}, bp.Paths()) + + leafHashes := bp.LeafHashes() + require.Equal(t, 2, len(leafHashes)) + require.NotNil(t, leafHashes[0]) + require.Equal(t, h1, *leafHashes[0]) + require.Nil(t, leafHashes[1]) +} + +// TestPayloadlessTrieBatchProof_AppendProof verifies AppendProof grows the batch. +func TestPayloadlessTrieBatchProof_AppendProof(t *testing.T) { + bp := NewPayloadlessTrieBatchProof() + require.Equal(t, 0, bp.Size()) + + bp.AppendProof(NewPayloadlessTrieProof()) + require.Equal(t, 1, bp.Size()) + + bp.AppendProof(NewPayloadlessTrieProof()) + require.Equal(t, 2, bp.Size()) +} + +// TestPayloadlessTrieBatchProof_MergeInto verifies MergeInto appends proofs to a destination. +func TestPayloadlessTrieBatchProof_MergeInto(t *testing.T) { + src := NewPayloadlessTrieBatchProofWithEmptyProofs(2) + dest := NewPayloadlessTrieBatchProofWithEmptyProofs(3) + + src.MergeInto(dest) + require.Equal(t, 5, dest.Size()) + require.Equal(t, 2, src.Size()) // source unchanged in size +} + +// TestPayloadlessTrieBatchProof_String_DoesNotPanic exercises the String method. +func TestPayloadlessTrieBatchProof_String_DoesNotPanic(t *testing.T) { + bp := NewPayloadlessTrieBatchProofWithEmptyProofs(2) + require.NotEmpty(t, bp.String()) +} + +// TestPayloadlessTrieBatchProofEquals verifies the equality semantics of a batch proof. +func TestPayloadlessTrieBatchProofEquals(t *testing.T) { + build := func() *PayloadlessTrieBatchProof { + var p1Path Path + p1Path[0] = 1 + h1 := hash.HashLeaf(hash.Hash(p1Path), []byte("a")) + + bp := NewPayloadlessTrieBatchProofWithEmptyProofs(1) + bp.Proofs[0].Path = p1Path + bp.Proofs[0].LeafHash = &h1 + bp.Proofs[0].Inclusion = true + return bp + } + + t.Run("identical", func(t *testing.T) { + require.True(t, build().Equals(build())) + }) + + t.Run("nil", func(t *testing.T) { + require.False(t, build().Equals(nil)) + }) + + t.Run("different size", func(t *testing.T) { + bp1 := build() + bp2 := build() + bp2.AppendProof(NewPayloadlessTrieProof()) + require.False(t, bp1.Equals(bp2)) + }) + + t.Run("different proof content", func(t *testing.T) { + bp1 := build() + bp2 := build() + bp2.Proofs[0].Inclusion = false + require.False(t, bp1.Equals(bp2)) + }) +}