From c51de07315362d47a777a93ecf6337935486f66d Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 29 May 2026 08:44:35 -0700 Subject: [PATCH 1/9] add original mtrie implementation as base for comparison --- ledger/complete/payloadless/node.go | 272 +++++++++ ledger/complete/payloadless/trie.go | 855 ++++++++++++++++++++++++++++ 2 files changed, 1127 insertions(+) create mode 100644 ledger/complete/payloadless/node.go create mode 100644 ledger/complete/payloadless/trie.go diff --git a/ledger/complete/payloadless/node.go b/ledger/complete/payloadless/node.go new file mode 100644 index 00000000000..bd1d6b08140 --- /dev/null +++ b/ledger/complete/payloadless/node.go @@ -0,0 +1,272 @@ +package node + +import ( + "encoding/hex" + "fmt" + + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/hash" +) + +// Node defines an Mtrie node +// +// DEFINITIONS: +// - HEIGHT of a node v in a tree is the number of edges on the longest +// downward path between v and a tree leaf. +// +// Conceptually, an MTrie is a sparse Merkle Trie, which has two node types: +// - INTERIM node: has at least one child (i.e. lChild or rChild is not +// nil). Interim nodes do not store a path and have no payload. +// - LEAF node: has _no_ children. +// +// Per convention, we also consider nil as a leaf. Formally, nil is the generic +// representative for any empty (sub)-trie (i.e. a trie without allocated +// registers). +// +// Nodes are supposed to be treated as _immutable_ data structures. +// TODO: optimized data structures might be able to reduce memory consumption +type Node struct { + // Implementation Comments: + // Formally, a tree can hold up to 2^maxDepth number of registers. However, + // the current implementation is designed to operate on a sparsely populated + // tree, holding much less than 2^64 registers. + + lChild *Node // Left Child + rChild *Node // Right Child + height int // height where the Node is at + path ledger.Path // the storage path (dummy value for interim nodes) + payload *ledger.Payload // the payload this node is storing (leaf nodes only) + hashValue hash.Hash // hash value of node (cached) +} + +// NewNode creates a new Node. +// UNCHECKED requirement: combination of values must conform to +// a valid node type (see documentation of `Node` for details) +func NewNode(height int, + lchild, + rchild *Node, + path ledger.Path, + payload *ledger.Payload, + hashValue hash.Hash, +) *Node { + n := &Node{ + lChild: lchild, + rChild: rchild, + height: height, + path: path, + hashValue: hashValue, + payload: payload, + } + return n +} + +// NewLeaf creates a compact leaf Node. +// UNCHECKED requirement: height must be non-negative +// UNCHECKED requirement: payload is non nil +// UNCHECKED requirement: payload should be deep copied if received from external sources +func NewLeaf(path ledger.Path, + payload *ledger.Payload, + height int, +) *Node { + n := &Node{ + lChild: nil, + rChild: nil, + height: height, + path: path, + payload: payload, + } + n.hashValue = n.computeHash() + return n +} + +// NewInterimNode creates a new interim Node. +// UNCHECKED requirement: +// - for any child `c` that is non-nil, its height must satisfy: height = c.height + 1 +func NewInterimNode(height int, lchild, rchild *Node) *Node { + n := &Node{ + lChild: lchild, + rChild: rchild, + height: height, + payload: nil, + } + n.hashValue = n.computeHash() + return n +} + +// NewInterimCompactifiedNode creates a new compactified interim Node. For compactification, +// we only consider the immediate children. When starting with a maximally pruned trie and +// creating only InterimCompactifiedNodes during an update, the resulting trie remains maximally +// pruned. Details on compactification: +// - If _both_ immediate children represent completely unallocated sub-tries, then the sub-trie +// with the new interim node is also completely empty. We return nil. +// - If either child is a leaf (i.e. representing a single allocated register) _and_ the other +// child represents a completely unallocated sub-trie, the new interim node also only holds +// a single allocated register. In this case, we return a compactified leaf. +// +// UNCHECKED requirement: +// - for any child `c` that is non-nil, its height must satisfy: height = c.height + 1 +func NewInterimCompactifiedNode(height int, lChild, rChild *Node) *Node { + if lChild.IsDefaultNode() { + lChild = nil + } + if rChild.IsDefaultNode() { + rChild = nil + } + + // CASE (a): _both_ children do _not_ contain any allocated registers: + if lChild == nil && rChild == nil { + return nil // return nil representing as completely empty sub-trie + } + + // CASE (b): one child is a compactified leaf (single allocated register) _and_ the other child represents + // an empty subtrie => in total we have one allocated register, which we represent as single leaf node + if rChild == nil && lChild.IsLeaf() { + h := hash.HashInterNode(lChild.hashValue, ledger.GetDefaultHashForHeight(lChild.height)) + return &Node{height: height, path: lChild.path, payload: lChild.payload, hashValue: h} + } + if lChild == nil && rChild.IsLeaf() { + h := hash.HashInterNode(ledger.GetDefaultHashForHeight(rChild.height), rChild.hashValue) + return &Node{height: height, path: rChild.path, payload: rChild.payload, hashValue: h} + } + + // CASE (b): both children contain some allocated registers => we can't compactify; return a full interim leaf + return NewInterimNode(height, lChild, rChild) +} + +// IsDefaultNode returns true iff the sub-trie represented by this root node contains +// only unallocated registers. This is the case, if the node is nil or the node's hash +// is equal to the default hash value at the respective height. +func (n *Node) IsDefaultNode() bool { + if n == nil { + return true + } + return n.hashValue == ledger.GetDefaultHashForHeight(n.height) +} + +// computeHash returns the hashValue of the node +func (n *Node) computeHash() hash.Hash { + // check for leaf node + if n.lChild == nil && n.rChild == nil { + // if payload is non-nil, compute the hash based on the payload content + if n.payload != nil { + return ledger.ComputeCompactValue(hash.Hash(n.path), n.payload.Value(), n.height) + } + // if payload is nil, return the default hash + return ledger.GetDefaultHashForHeight(n.height) + } + + // this is an interim node at least one of lChild or rChild is not nil. + var h1, h2 hash.Hash + if n.lChild != nil { + h1 = n.lChild.Hash() + } else { + h1 = ledger.GetDefaultHashForHeight(n.height - 1) + } + + if n.rChild != nil { + h2 = n.rChild.Hash() + } else { + h2 = ledger.GetDefaultHashForHeight(n.height - 1) + } + return hash.HashInterNode(h1, h2) +} + +// VerifyCachedHash verifies the hash of a node is valid +func verifyCachedHashRecursive(n *Node) bool { + if n == nil { + return true + } + if !verifyCachedHashRecursive(n.lChild) || !verifyCachedHashRecursive(n.rChild) { + return false + } + + computedHash := n.computeHash() + return n.hashValue == computedHash +} + +// VerifyCachedHash verifies the hash of a node is valid +func (n *Node) VerifyCachedHash() bool { + return verifyCachedHashRecursive(n) +} + +// Hash returns the Node's hash value. +// Do NOT MODIFY returned slice! +func (n *Node) Hash() hash.Hash { + return n.hashValue +} + +// Height returns the Node's height. +// Per definition, the height of a node v in a tree is the number +// of edges on the longest downward path between v and a tree leaf. +func (n *Node) Height() int { + return n.height +} + +// Path returns a pointer to the Node's register storage path. +// If the node is not a leaf, the function returns `nil`. +func (n *Node) Path() *ledger.Path { + if n.IsLeaf() { + return &n.path + } + return nil +} + +// Payload returns the Node's payload. +// Do NOT MODIFY returned slices! +func (n *Node) Payload() *ledger.Payload { + return n.payload +} + +// LeftChild returns the Node's left child. +// Only INTERIM nodes have children. +// Do NOT MODIFY returned Node! +func (n *Node) LeftChild() *Node { return n.lChild } + +// RightChild returns the Node's right child. +// Only INTERIM nodes have children. +// Do NOT MODIFY returned Node! +func (n *Node) RightChild() *Node { return n.rChild } + +// IsLeaf returns true if and only if Node is a LEAF. +func (n *Node) IsLeaf() bool { + // Per definition, a node is a leaf if and only it has no children + return n == nil || (n.lChild == nil && n.rChild == nil) +} + +// FmtStr provides formatted string representation of the Node and sub tree +func (n *Node) FmtStr(prefix string, subpath string) string { + right := "" + if n.rChild != nil { + right = fmt.Sprintf("\n%v", n.rChild.FmtStr(prefix+"\t", subpath+"1")) + } + left := "" + if n.lChild != nil { + left = fmt.Sprintf("\n%v", n.lChild.FmtStr(prefix+"\t", subpath+"0")) + } + payloadSize := 0 + if n.payload != nil { + payloadSize = n.payload.Size() + } + hashStr := hex.EncodeToString(n.hashValue[:]) + hashStr = hashStr[:3] + "..." + hashStr[len(hashStr)-3:] + return fmt.Sprintf("%v%v: (path:%v, payloadSize:%d hash:%v)[%s] (obj %p) %v %v ", prefix, n.height, n.path, payloadSize, hashStr, subpath, n, left, right) +} + +// AllPayloads returns the payload of this node and all payloads of the subtrie +func (n *Node) AllPayloads() []*ledger.Payload { + return n.appendSubtreePayloads([]*ledger.Payload{}) +} + +// appendSubtreePayloads appends the payloads of the subtree with this node as root +// to the provided Payload slice. Follows same pattern as Go's native append method. +func (n *Node) appendSubtreePayloads(result []*ledger.Payload) []*ledger.Payload { + if n == nil { + return result + } + if n.IsLeaf() { + return append(result, n.Payload()) + } + result = n.lChild.appendSubtreePayloads(result) + result = n.rChild.appendSubtreePayloads(result) + return result +} diff --git a/ledger/complete/payloadless/trie.go b/ledger/complete/payloadless/trie.go new file mode 100644 index 00000000000..2c65584045f --- /dev/null +++ b/ledger/complete/payloadless/trie.go @@ -0,0 +1,855 @@ +package trie + +import ( + "encoding/json" + "fmt" + "io" + "sync" + + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/bitutils" + "github.com/onflow/flow-go/ledger/complete/mtrie/node" +) + +// MTrie represents a perfect in-memory full binary Merkle tree with uniform height. +// For a detailed description of the storage model, please consult `mtrie/README.md` +// +// A MTrie is a thin wrapper around a the trie's root Node. An MTrie implements the +// logic for forming MTrie-graphs from the elementary nodes. Specifically: +// - how Nodes (graph vertices) form a Trie, +// - how register values are read from the trie, +// - how Merkle proofs are generated from a trie, and +// - how a new Trie with updated values is generated. +// +// `MTrie`s are _immutable_ data structures. Updating register values is implemented through +// copy-on-write, which creates a new `MTrie`. For minimal memory consumption, all sub-tries +// that where not affected by the write operation are shared between the original MTrie +// (before the register updates) and the updated MTrie (after the register writes). +// +// MTrie expects that for a specific path, the register's key never changes. +// +// DEFINITIONS and CONVENTIONS: +// - HEIGHT of a node v in a tree is the number of edges on the longest downward path +// between v and a tree leaf. The height of a tree is the height of its root. +// The height of a Trie is always the height of the fully-expanded tree. +type MTrie struct { + root *node.Node + regCount uint64 // number of registers allocated in the trie + regSize uint64 // size of registers allocated in the trie +} + +// NewEmptyMTrie returns an empty Mtrie (root is nil) +func NewEmptyMTrie() *MTrie { + return &MTrie{root: nil} +} + +// IsEmpty checks if a trie is empty. +// +// An empty try doesn't mean a trie with no allocated registers. +func (mt *MTrie) IsEmpty() bool { + return mt.root == nil +} + +// NewMTrie returns a Mtrie given the root +func NewMTrie(root *node.Node, regCount uint64, regSize uint64) (*MTrie, error) { + if root != nil && root.Height() != ledger.NodeMaxHeight { + return nil, fmt.Errorf("height of root node must be %d but is %d, hash: %s", ledger.NodeMaxHeight, root.Height(), root.Hash().String()) + } + return &MTrie{ + root: root, + regCount: regCount, + regSize: regSize, + }, nil +} + +// RootHash returns the trie's root hash. +// Concurrency safe (as Tries are immutable structures by convention) +func (mt *MTrie) RootHash() ledger.RootHash { + if mt.IsEmpty() { + // case of an empty trie + return EmptyTrieRootHash() + } + return ledger.RootHash(mt.root.Hash()) +} + +// AllocatedRegCount returns the number of allocated registers in the trie. +// Concurrency safe (as Tries are immutable structures by convention) +func (mt *MTrie) AllocatedRegCount() uint64 { + return mt.regCount +} + +// AllocatedRegSize returns the size (number of bytes) of allocated registers in the trie. +// Concurrency safe (as Tries are immutable structures by convention) +func (mt *MTrie) AllocatedRegSize() uint64 { + return mt.regSize +} + +// RootNode returns the Trie's root Node +// Concurrency safe (as Tries are immutable structures by convention) +func (mt *MTrie) RootNode() *node.Node { + return mt.root +} + +// String returns the trie's string representation. +// Concurrency safe (as Tries are immutable structures by convention) +func (mt *MTrie) String() string { + if mt.IsEmpty() { + return fmt.Sprintf("Empty Trie with default root hash: %v\n", mt.RootHash()) + } + trieStr := fmt.Sprintf("Trie root hash: %v\n", mt.RootHash()) + return trieStr + mt.root.FmtStr("", "") +} + +// UnsafeValueSizes returns payload value sizes for the given paths. +// UNSAFE: requires _all_ paths to have a length of mt.Height bits. +// CAUTION: while getting payload value sizes, `paths` is permuted IN-PLACE for optimized processing. +// Return: +// - `sizes` []int +// For each path, the corresponding payload value size is written into sizes. AFTER +// the size operation completes, the order of `path` and `sizes` are such that +// for `path[i]` the corresponding register value size is referenced by `sizes[i]`. +// +// TODO move consistency checks from Forest into Trie to obtain a safe, self-contained API +func (mt *MTrie) UnsafeValueSizes(paths []ledger.Path) []int { + sizes := make([]int, len(paths)) // pre-allocate slice for the result + valueSizes(sizes, paths, mt.root) + return sizes +} + +// valueSizes returns value sizes of all the registers in `paths“ in subtree with `head` as root node. +// For each `path[i]`, the corresponding value size is written into `sizes[i]` for the same index `i`. +// CAUTION: +// - while reading the payloads, `paths` is permuted IN-PLACE for optimized processing. +// - unchecked requirement: all paths must go through the `head` node +func valueSizes(sizes []int, paths []ledger.Path, head *node.Node) { + // check for empty paths + if len(paths) == 0 { + return + } + + // path not found + if head == nil { + return + } + + // reached a leaf node + if head.IsLeaf() { + for i, p := range paths { + if *head.Path() == p { + payload := head.Payload() + if payload != nil { + sizes[i] = payload.Value().Size() + } + // NOTE: break isn't used here because precondition + // doesn't require paths being deduplicated. + } + } + return + } + + // reached an interim node with only one path + if len(paths) == 1 { + path := paths[0][:] + + // traverse nodes following the path until a leaf node or nil node is reached. + // "for" loop helps to skip partition and recursive call when there's only one path to follow. + for { + depth := ledger.NodeMaxHeight - head.Height() // distance to the tree root + bit := bitutils.ReadBit(path, depth) + if bit == 0 { + head = head.LeftChild() + } else { + head = head.RightChild() + } + if head.IsLeaf() { + break + } + } + + valueSizes(sizes, paths, head) + return + } + + // reached an interim node with more than one paths + + // partition step to quick sort the paths: + // lpaths contains all paths that have `0` at the partitionIndex + // rpaths contains all paths that have `1` at the partitionIndex + depth := ledger.NodeMaxHeight - head.Height() // distance to the tree root + partitionIndex := SplitPaths(paths, depth) + lpaths, rpaths := paths[:partitionIndex], paths[partitionIndex:] + lsizes, rsizes := sizes[:partitionIndex], sizes[partitionIndex:] + + // read values from left and right subtrees in parallel + parallelRecursionThreshold := 32 // threshold to avoid the parallelization going too deep in the recursion + if len(lpaths) < parallelRecursionThreshold || len(rpaths) < parallelRecursionThreshold { + valueSizes(lsizes, lpaths, head.LeftChild()) + valueSizes(rsizes, rpaths, head.RightChild()) + } else { + // concurrent read of left and right subtree + wg := sync.WaitGroup{} + wg.Go(func() { + valueSizes(lsizes, lpaths, head.LeftChild()) + }) + valueSizes(rsizes, rpaths, head.RightChild()) + wg.Wait() // wait for all threads + } +} + +// ReadSinglePayload reads and returns a payload for a single path. +func (mt *MTrie) ReadSinglePayload(path ledger.Path) *ledger.Payload { + return readSinglePayload(path, mt.root) +} + +// readSinglePayload reads and returns a payload for a single path in subtree with `head` as root node. +func readSinglePayload(path ledger.Path, head *node.Node) *ledger.Payload { + pathBytes := path[:] + + if head == nil { + return ledger.EmptyPayload() + } + + depth := ledger.NodeMaxHeight - head.Height() // distance to the tree root + + // Traverse nodes following the path until a leaf node or nil node is reached. + for !head.IsLeaf() { + bit := bitutils.ReadBit(pathBytes, depth) + if bit == 0 { + head = head.LeftChild() + } else { + head = head.RightChild() + } + depth++ + } + + if head != nil && *head.Path() == path { + return head.Payload() + } + + return ledger.EmptyPayload() +} + +// UnsafeRead reads payloads for the given paths. +// UNSAFE: requires _all_ paths to have a length of mt.Height bits. +// CAUTION: while reading the payloads, `paths` is permuted IN-PLACE for optimized processing. +// Return: +// - `payloads` []*ledger.Payload +// For each path, the corresponding payload is written into payloads. AFTER +// the read operation completes, the order of `path` and `payloads` are such that +// for `path[i]` the corresponding register value is referenced by 0`payloads[i]`. +// +// TODO move consistency checks from Forest into Trie to obtain a safe, self-contained API +func (mt *MTrie) UnsafeRead(paths []ledger.Path) []*ledger.Payload { + payloads := make([]*ledger.Payload, len(paths)) // pre-allocate slice for the result + read(payloads, paths, mt.root) + return payloads +} + +// read reads all the registers in subtree with `head` as root node. For each +// `path[i]`, the corresponding payload is written into `payloads[i]` for the same index `i`. +// CAUTION: +// - while reading the payloads, `paths` is permuted IN-PLACE for optimized processing. +// - unchecked requirement: all paths must go through the `head` node +func read(payloads []*ledger.Payload, paths []ledger.Path, head *node.Node) { + // check for empty paths + if len(paths) == 0 { + return + } + + // path not found + if head == nil { + for i := range paths { + payloads[i] = ledger.EmptyPayload() + } + return + } + + // reached a leaf node + if head.IsLeaf() { + for i, p := range paths { + if *head.Path() == p { + payloads[i] = head.Payload() + } else { + payloads[i] = ledger.EmptyPayload() + } + } + return + } + + // reached an interim node + if len(paths) == 1 { + // call readSinglePayload to skip partition and recursive calls when there is only one path + payloads[0] = readSinglePayload(paths[0], head) + return + } + + // partition step to quick sort the paths: + // lpaths contains all paths that have `0` at the partitionIndex + // rpaths contains all paths that have `1` at the partitionIndex + depth := ledger.NodeMaxHeight - head.Height() // distance to the tree root + partitionIndex := SplitPaths(paths, depth) + lpaths, rpaths := paths[:partitionIndex], paths[partitionIndex:] + lpayloads, rpayloads := payloads[:partitionIndex], payloads[partitionIndex:] + + // read values from left and right subtrees in parallel + parallelRecursionThreshold := 32 // threshold to avoid the parallelization going too deep in the recursion + if len(lpaths) < parallelRecursionThreshold || len(rpaths) < parallelRecursionThreshold { + read(lpayloads, lpaths, head.LeftChild()) + read(rpayloads, rpaths, head.RightChild()) + } else { + // concurrent read of left and right subtree + wg := sync.WaitGroup{} + wg.Go(func() { + read(lpayloads, lpaths, head.LeftChild()) + }) + read(rpayloads, rpaths, head.RightChild()) + wg.Wait() // wait for all threads + } +} + +// NewTrieWithUpdatedRegisters constructs a new trie containing all registers from the parent trie, +// and returns: +// - updated trie +// - max depth touched during update (this isn't affected by prune flag) +// - error +// +// The key-value pairs specify the registers whose values are supposed to hold updated values +// compared to the parent trie. Constructing the new trie is done in a COPY-ON-WRITE manner: +// - The original trie remains unchanged. +// - subtries that remain unchanged are from the parent trie instead of copied. +// +// UNSAFE: method requires the following conditions to be satisfied: +// - keys are NOT duplicated +// - requires _all_ paths to have a length of mt.Height bits. +// +// CAUTION: `updatedPaths` and `updatedPayloads` are permuted IN-PLACE for optimized processing. +// CAUTION: MTrie expects that for a specific path, the payload's key never changes. +// TODO: move consistency checks from MForest to here, to make API safe and self-contained +func NewTrieWithUpdatedRegisters( + parentTrie *MTrie, + updatedPaths []ledger.Path, + updatedPayloads []ledger.Payload, + prune bool, +) (*MTrie, uint16, error) { + updatedRoot, regCountDelta, regSizeDelta, lowestHeightTouched := update( + ledger.NodeMaxHeight, + parentTrie.root, + updatedPaths, + updatedPayloads, + nil, + prune, + ) + + updatedTrieRegCount := int64(parentTrie.AllocatedRegCount()) + regCountDelta + updatedTrieRegSize := int64(parentTrie.AllocatedRegSize()) + regSizeDelta + maxDepthTouched := uint16(ledger.NodeMaxHeight - lowestHeightTouched) + + updatedTrie, err := NewMTrie(updatedRoot, uint64(updatedTrieRegCount), uint64(updatedTrieRegSize)) + if err != nil { + return nil, 0, fmt.Errorf("constructing updated trie failed: %w", err) + } + return updatedTrie, maxDepthTouched, nil +} + +// updateResult is a wrapper of return values from update(). +// It's used to communicate values from goroutine. +type updateResult struct { + child *node.Node + allocatedRegCountDelta int64 + allocatedRegSizeDelta int64 + lowestHeightTouched int +} + +// update traverses the subtree recursively and create new nodes with +// the updated payloads on the given paths +// +// it returns: +// - new updated node or original node if nothing was updated +// - allocated register count delta in subtrie (allocatedRegCountDelta) +// - allocated register size delta in subtrie (allocatedRegSizeDelta) +// - lowest height reached during recursive update in subtrie (lowestHeightTouched) +// +// update also compact a subtree into a single compact leaf node in the case where +// there is only 1 payload stored in the subtree. +// +// allocatedRegCountDelta and allocatedRegSizeDelta are used to compute updated +// trie's allocated register count and size. lowestHeightTouched is used to +// compute max depth touched during update. +// CAUTION: while updating, `paths` and `payloads` are permuted IN-PLACE for optimized processing. +// UNSAFE: method requires the following conditions to be satisfied: +// - paths all share the same common prefix [0 : mt.maxHeight-1 - nodeHeight) +// (excluding the bit at index headHeight) +// - paths are NOT duplicated +func update( + nodeHeight int, // the height of the node during traversing the subtree + currentNode *node.Node, // the current node on the travesing path, if it's nil it means the trie has no node on this path + paths []ledger.Path, // the paths to update the payloads + payloads []ledger.Payload, // the payloads to be updated at the given paths + compactLeaf *node.Node, // a compact leaf node from its ancester, it could be nil + prune bool, // prune is a flag for whether pruning nodes with empty payload. not pruning is useful for generating proof, expecially non-inclusion proof +) (n *node.Node, allocatedRegCountDelta int64, allocatedRegSizeDelta int64, lowestHeightTouched int) { + // No new path to update + if len(paths) == 0 { + if compactLeaf != nil { + // if a compactLeaf from a higher height is still left, + // then expand the compact leaf node to the current height by creating a new compact leaf + // node with the same path and payload. + // The old node shouldn't be recycled as it is still used by the tree copy before the update. + n = node.NewLeaf(*compactLeaf.Path(), compactLeaf.Payload(), nodeHeight) + return n, 0, 0, nodeHeight + } + // if no path to update and there is no compact leaf node on this path, we return + // the current node regardless it exists or not. + return currentNode, 0, 0, nodeHeight + } + + if len(paths) == 1 && currentNode == nil && compactLeaf == nil { + // if there is only 1 path to update, and the existing tree has no node on this path, also + // no compact leaf node from its ancester, it means we are storing a payload on a new path, + n = node.NewLeaf(paths[0], payloads[0].DeepCopy(), nodeHeight) + if payloads[0].IsEmpty() { + // if we are storing an empty node, then no register is allocated + // allocatedRegCountDelta and allocatedRegSizeDelta should both be 0 + return n, 0, 0, nodeHeight + } + // if we are storing a non-empty node, we are allocating a new register + return n, 1, int64(payloads[0].Size()), nodeHeight + } + + if currentNode != nil && currentNode.IsLeaf() { // if we're here then compactLeaf == nil + // check if the current node path is among the updated paths + found := false + currentPath := *currentNode.Path() + for i, p := range paths { + if p == currentPath { + // the case where the recursion stops: only one path to update + if len(paths) == 1 { + // check if the only path to update has the same payload. + // if payload is the same, we could skip the update to avoid creating duplicated node + if !currentNode.Payload().ValueEquals(&payloads[i]) { + n = node.NewLeaf(paths[i], payloads[i].DeepCopy(), nodeHeight) + + allocatedRegCountDelta, allocatedRegSizeDelta = + computeAllocatedRegDeltas(currentNode.Payload(), &payloads[i]) + + return n, allocatedRegCountDelta, allocatedRegSizeDelta, nodeHeight + } + // avoid creating a new node when the same payload is written + return currentNode, 0, 0, nodeHeight + } + // the case where the recursion carries on: len(paths)>1 + found = true + + allocatedRegCountDelta, allocatedRegSizeDelta = + computeAllocatedRegDeltasFromHigherHeight(currentNode.Payload()) + + break + } + } + if !found { + // if the current node carries a path not included in the input path, then the current node + // represents a compact leaf that needs to be carried down the recursion. + compactLeaf = currentNode + } + } + + // in the remaining code: + // - either len(paths) > 1 + // - or len(paths) == 1 and compactLeaf!= nil + // - or len(paths) == 1 and currentNode != nil && !currentNode.IsLeaf() + + // Split paths and payloads to recurse: + // lpaths contains all paths that have `0` at the partitionIndex + // rpaths contains all paths that have `1` at the partitionIndex + depth := ledger.NodeMaxHeight - nodeHeight // distance to the tree root + partitionIndex := splitByPath(paths, payloads, depth) + lpaths, rpaths := paths[:partitionIndex], paths[partitionIndex:] + lpayloads, rpayloads := payloads[:partitionIndex], payloads[partitionIndex:] + + // check if there is a compact leaf that needs to get deep to height 0 + var lcompactLeaf, rcompactLeaf *node.Node + if compactLeaf != nil { + // if yes, check which branch it will go to. + path := *compactLeaf.Path() + if bitutils.ReadBit(path[:], depth) == 0 { + lcompactLeaf = compactLeaf + } else { + rcompactLeaf = compactLeaf + } + } + + // set the node children + var oldLeftChild, oldRightChild *node.Node + if currentNode != nil { + oldLeftChild = currentNode.LeftChild() + oldRightChild = currentNode.RightChild() + } + + // recurse over each branch + var newLeftChild, newRightChild *node.Node + var lRegCountDelta, rRegCountDelta int64 + var lRegSizeDelta, rRegSizeDelta int64 + var lLowestHeightTouched, rLowestHeightTouched int + parallelRecursionThreshold := 16 + if len(lpaths) < parallelRecursionThreshold || len(rpaths) < parallelRecursionThreshold { + // runtime optimization: if there are _no_ updates for either left or right sub-tree, proceed single-threaded + newLeftChild, lRegCountDelta, lRegSizeDelta, lLowestHeightTouched = update(nodeHeight-1, oldLeftChild, lpaths, lpayloads, lcompactLeaf, prune) + newRightChild, rRegCountDelta, rRegSizeDelta, rLowestHeightTouched = update(nodeHeight-1, oldRightChild, rpaths, rpayloads, rcompactLeaf, prune) + } else { + // runtime optimization: process the left child in a separate thread + + // Since we're receiving 4 values from goroutine, use a + // struct and channel to reduce allocs/op. + // Although WaitGroup approach can be faster than channel (esp. with 2+ goroutines), + // we only use 1 goroutine here and need to communicate results from it. So using + // channel is faster and uses fewer allocs/op in this case. + results := make(chan updateResult, 1) + go func(retChan chan<- updateResult) { + child, regCountDelta, regSizeDelta, lowestHeightTouched := update(nodeHeight-1, oldLeftChild, lpaths, lpayloads, lcompactLeaf, prune) + retChan <- updateResult{child, regCountDelta, regSizeDelta, lowestHeightTouched} + }(results) + + newRightChild, rRegCountDelta, rRegSizeDelta, rLowestHeightTouched = update(nodeHeight-1, oldRightChild, rpaths, rpayloads, rcompactLeaf, prune) + + // Wait for results from goroutine. + ret := <-results + newLeftChild, lRegCountDelta, lRegSizeDelta, lLowestHeightTouched = ret.child, ret.allocatedRegCountDelta, ret.allocatedRegSizeDelta, ret.lowestHeightTouched + } + + allocatedRegCountDelta += lRegCountDelta + rRegCountDelta + allocatedRegSizeDelta += lRegSizeDelta + rRegSizeDelta + lowestHeightTouched = minInt(lLowestHeightTouched, rLowestHeightTouched) + + // mitigate storage exhaustion attack: avoids creating a new node when the exact same + // payload is re-written at a register. CAUTION: we only check that the children are + // unchanged. This is only sufficient for interim nodes (for leaf nodes, the children + // might be unchanged, i.e. both nil, but the payload could have changed). + // In case the current node was a leaf, we _cannot reuse_ it, because we potentially + // updated registers in the sub-trie + if !currentNode.IsLeaf() && newLeftChild == oldLeftChild && newRightChild == oldRightChild { + return currentNode, 0, 0, lowestHeightTouched + } + + // if prune is on, then will check and create a compact leaf node if one child is nil, and the + // other child is a leaf node + if prune { + n = node.NewInterimCompactifiedNode(nodeHeight, newLeftChild, newRightChild) + return n, allocatedRegCountDelta, allocatedRegSizeDelta, lowestHeightTouched + } + + n = node.NewInterimNode(nodeHeight, newLeftChild, newRightChild) + return n, allocatedRegCountDelta, allocatedRegSizeDelta, lowestHeightTouched +} + +// computeAllocatedRegDeltasFromHigherHeight returns the deltas +// needed to compute the allocated reg count and reg size when +// a payload is updated or unallocated at a lower height. +func computeAllocatedRegDeltasFromHigherHeight(oldPayload *ledger.Payload) (allocatedRegCountDelta, allocatedRegSizeDelta int64) { + if !oldPayload.IsEmpty() { + // Allocated register will be updated or unallocated at lower height. + allocatedRegCountDelta-- + } + oldPayloadSize := oldPayload.Size() + allocatedRegSizeDelta -= int64(oldPayloadSize) + return +} + +// computeAllocatedRegDeltas returns the allocated reg count +// and reg size deltas computed from old payload and new payload. +// PRECONDITION: !oldPayload.Equals(newPayload) +func computeAllocatedRegDeltas(oldPayload, newPayload *ledger.Payload) (allocatedRegCountDelta, allocatedRegSizeDelta int64) { + allocatedRegCountDelta = 0 + if newPayload.IsEmpty() { + // Old payload is not empty while new payload is empty. + // Allocated register will be unallocated. + allocatedRegCountDelta = -1 + } else if oldPayload.IsEmpty() { + // Old payload is empty while new payload is not empty. + // Unallocated register will be allocated. + allocatedRegCountDelta = 1 + } + + oldPayloadSize := oldPayload.Size() + newPayloadSize := newPayload.Size() + allocatedRegSizeDelta = int64(newPayloadSize - oldPayloadSize) + return +} + +// UnsafeProofs provides proofs for the given paths. +// +// CAUTION: while updating, `paths` and `proofs` are permuted IN-PLACE for optimized processing. +// UNSAFE: requires _all_ paths to have a length of mt.Height bits. +// Paths in the input query don't have to be deduplicated, though deduplication would +// result in allocating less dynamic memory to store the proofs. +func (mt *MTrie) UnsafeProofs(paths []ledger.Path) *ledger.TrieBatchProof { + batchProofs := ledger.NewTrieBatchProofWithEmptyProofs(len(paths)) + prove(mt.root, paths, batchProofs.Proofs) + return batchProofs +} + +// prove traverses the subtree and stores proofs for the given register paths in +// the provided `proofs` slice +// CAUTION: while updating, `paths` and `proofs` are permuted IN-PLACE for optimized processing. +// UNSAFE: method requires the following conditions to be satisfied: +// - paths all share the same common prefix [0 : mt.maxHeight-1 - nodeHeight) +// (excluding the bit at index headHeight) +func prove(head *node.Node, paths []ledger.Path, proofs []*ledger.TrieProof) { + // check for empty paths + if len(paths) == 0 { + return + } + + // we've reached the end of a trie + // and path is not found (noninclusion proof) + if head == nil { + // by default, proofs are non-inclusion proofs + return + } + + // we've reached a leaf + if head.IsLeaf() { + for i, path := range paths { + // value matches (inclusion proof) + if *head.Path() == path { + proofs[i].Path = *head.Path() + proofs[i].Payload = head.Payload() + proofs[i].Inclusion = true + } + } + // by default, proofs are non-inclusion proofs + return + } + + // increment steps for all the proofs + for _, p := range proofs { + p.Steps++ + } + + // partition step to quick sort the paths: + // lpaths contains all paths that have `0` at the partitionIndex + // rpaths contains all paths that have `1` at the partitionIndex + depth := ledger.NodeMaxHeight - head.Height() // distance to the tree root + partitionIndex := splitTrieProofsByPath(paths, proofs, depth) + lpaths, rpaths := paths[:partitionIndex], paths[partitionIndex:] + lproofs, rproofs := proofs[:partitionIndex], proofs[partitionIndex:] + + parallelRecursionThreshold := 64 // threshold to avoid the parallelization going too deep in the recursion + if len(lpaths) < parallelRecursionThreshold || len(rpaths) < parallelRecursionThreshold { + // runtime optimization: below the parallelRecursionThreshold, we proceed single-threaded + addSiblingTrieHashToProofs(head.RightChild(), depth, lproofs) + prove(head.LeftChild(), lpaths, lproofs) + + addSiblingTrieHashToProofs(head.LeftChild(), depth, rproofs) + prove(head.RightChild(), rpaths, rproofs) + } else { + wg := sync.WaitGroup{} + wg.Go(func() { + addSiblingTrieHashToProofs(head.RightChild(), depth, lproofs) + prove(head.LeftChild(), lpaths, lproofs) + }) + + addSiblingTrieHashToProofs(head.LeftChild(), depth, rproofs) + prove(head.RightChild(), rpaths, rproofs) + wg.Wait() + } +} + +// addSiblingTrieHashToProofs inspects the sibling Trie and adds its root hash +// to the proofs, if the trie contains non-empty registers (i.e. the +// siblingTrie has a non-default hash). +func addSiblingTrieHashToProofs(siblingTrie *node.Node, depth int, proofs []*ledger.TrieProof) { + if siblingTrie == nil || len(proofs) == 0 { + return + } + + // This code is necessary, because we do not remove nodes from the trie + // when a register is deleted. Instead, we just set the respective leaf's + // payload to empty. While this will cause the lead's hash to become the + // default hash, the node itself remains as part of the trie. + // However, a proof has the convention that the hash of the sibling trie + // should only be included, if it is _non-default_. Therefore, we can + // neither use `siblingTrie == nil` nor `siblingTrie.RegisterCount == 0`, + // as the sibling trie might contain leaves with default value (which are + // still counted as occupied registers) + // TODO: On update, prune subtries which only contain empty registers. + // Then, a child is nil if and only if the subtrie is empty. + + nodeHash := siblingTrie.Hash() + isDef := nodeHash == ledger.GetDefaultHashForHeight(siblingTrie.Height()) + if !isDef { // in proofs, we only provide non-default value hashes + for _, p := range proofs { + bitutils.SetBit(p.Flags, depth) + p.Interims = append(p.Interims, nodeHash) + } + } +} + +// Equals compares two tries for equality. +// Tries are equal iff they store the same data (i.e. root hash matches) +// and their number and height are identical +func (mt *MTrie) Equals(o *MTrie) bool { + if o == nil { + return false + } + return o.RootHash() == mt.RootHash() +} + +// DumpAsJSON dumps the trie key value pairs to a file having each key value pair as a json row +func (mt *MTrie) DumpAsJSON(w io.Writer) error { + + // Use encoder to prevent building entire trie in memory + enc := json.NewEncoder(w) + + err := dumpAsJSON(mt.root, enc) + if err != nil { + return err + } + + return nil +} + +// dumpAsJSON serializes the sub-trie with root n to json and feeds it into encoder +func dumpAsJSON(n *node.Node, encoder *json.Encoder) error { + if n.IsLeaf() { + if n != nil { + err := encoder.Encode(n.Payload()) + if err != nil { + return err + } + } + return nil + } + + if lChild := n.LeftChild(); lChild != nil { + err := dumpAsJSON(lChild, encoder) + if err != nil { + return err + } + } + + if rChild := n.RightChild(); rChild != nil { + err := dumpAsJSON(rChild, encoder) + if err != nil { + return err + } + } + return nil +} + +// EmptyTrieRootHash returns the rootHash of an empty Trie for the specified path size [bytes] +func EmptyTrieRootHash() ledger.RootHash { + return ledger.RootHash(ledger.GetDefaultHashForHeight(ledger.NodeMaxHeight)) +} + +// AllPayloads returns all payloads +func (mt *MTrie) AllPayloads() []*ledger.Payload { + return mt.root.AllPayloads() +} + +// IsAValidTrie verifies the content of the trie for potential issues +func (mt *MTrie) IsAValidTrie() bool { + // TODO add checks on the health of node max height ... + return mt.root.VerifyCachedHash() +} + +// splitByPath permutes the input paths to be partitioned into 2 parts. The first part contains paths with a zero bit +// at the input bitIndex, the second part contains paths with a one at the bitIndex. The index of partition +// is returned. The same permutation is applied to the payloads slice. +// +// This would be the partition step of an ascending quick sort of paths (lexicographic order) +// with the pivot being the path with all zeros and 1 at bitIndex. +// The comparison of paths is only based on the bit at bitIndex, the function therefore assumes all paths have +// equal bits from 0 to bitIndex-1 +// +// For instance, if `paths` contains the following 3 paths, and bitIndex is `1`: +// [[0,0,1,1], [0,1,0,1], [0,0,0,1]] +// then `splitByPath` returns 2 and updates `paths` into: +// [[0,0,1,1], [0,0,0,1], [0,1,0,1]] +func splitByPath(paths []ledger.Path, payloads []ledger.Payload, bitIndex int) int { + i := 0 + for j, path := range paths { + bit := bitutils.ReadBit(path[:], bitIndex) + if bit == 0 { + paths[i], paths[j] = paths[j], paths[i] + payloads[i], payloads[j] = payloads[j], payloads[i] + i++ + } + } + return i +} + +// SplitPaths permutes the input paths to be partitioned into 2 parts. The first part contains paths with a zero bit +// at the input bitIndex, the second part contains paths with a one at the bitIndex. The index of partition +// is returned. +// +// This would be the partition step of an ascending quick sort of paths (lexicographic order) +// with the pivot being the path with all zeros and 1 at bitIndex. +// The comparison of paths is only based on the bit at bitIndex, the function therefore assumes all paths have +// equal bits from 0 to bitIndex-1 +func SplitPaths(paths []ledger.Path, bitIndex int) int { + i := 0 + for j, path := range paths { + bit := bitutils.ReadBit(path[:], bitIndex) + if bit == 0 { + paths[i], paths[j] = paths[j], paths[i] + i++ + } + } + return i +} + +// splitTrieProofsByPath permutes the input paths to be partitioned into 2 parts. The first part contains paths +// with a zero bit at the input bitIndex, the second part contains paths with a one at the bitIndex. The index +// of partition is returned. The same permutation is applied to the proofs slice. +// +// This would be the partition step of an ascending quick sort of paths (lexicographic order) +// with the pivot being the path with all zeros and 1 at bitIndex. +// The comparison of paths is only based on the bit at bitIndex, the function therefore assumes all paths have +// equal bits from 0 to bitIndex-1 +func splitTrieProofsByPath(paths []ledger.Path, proofs []*ledger.TrieProof, bitIndex int) int { + i := 0 + for j, path := range paths { + bit := bitutils.ReadBit(path[:], bitIndex) + if bit == 0 { + paths[i], paths[j] = paths[j], paths[i] + proofs[i], proofs[j] = proofs[j], proofs[i] + i++ + } + } + return i +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} + +// TraverseNodes traverses all nodes of the trie in DFS order +func TraverseNodes(trie *MTrie, processNode func(*node.Node) error) error { + return traverseRecursive(trie.root, processNode) +} + +func traverseRecursive(n *node.Node, processNode func(*node.Node) error) error { + if n == nil { + return nil + } + + err := processNode(n) + if err != nil { + return err + } + + err = traverseRecursive(n.LeftChild(), processNode) + if err != nil { + return err + } + + err = traverseRecursive(n.RightChild(), processNode) + if err != nil { + return err + } + + return nil +} From a3a4d2a511f6348eb282e3dd6ae37ef87906dce3 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 29 May 2026 08:47:04 -0700 Subject: [PATCH 2/9] add payloadless node and trie implementation --- ledger/complete/payloadless/node.go | 142 ++++++---- ledger/complete/payloadless/trie.go | 408 +++++++++++----------------- ledger/payloadless_proof.go | 182 +++++++++++++ ledger/trie.go | 28 +- 4 files changed, 451 insertions(+), 309 deletions(-) create mode 100644 ledger/payloadless_proof.go diff --git a/ledger/complete/payloadless/node.go b/ledger/complete/payloadless/node.go index bd1d6b08140..90843089831 100644 --- a/ledger/complete/payloadless/node.go +++ b/ledger/complete/payloadless/node.go @@ -1,4 +1,4 @@ -package node +package payloadless import ( "encoding/hex" @@ -8,7 +8,11 @@ import ( "github.com/onflow/flow-go/ledger/common/hash" ) -// Node defines an Mtrie node +// Node defines a payloadless Mtrie node. +// +// Unlike the regular mtrie Node which stores full payloads, a payloadless Node +// stores only the leaf hash (HashLeaf(path, value)) for leaf nodes. This enables +// significant memory savings while preserving the same root hash as a full trie. // // DEFINITIONS: // - HEIGHT of a node v in a tree is the number of edges on the longest @@ -16,8 +20,8 @@ import ( // // Conceptually, an MTrie is a sparse Merkle Trie, which has two node types: // - INTERIM node: has at least one child (i.e. lChild or rChild is not -// nil). Interim nodes do not store a path and have no payload. -// - LEAF node: has _no_ children. +// nil). Interim nodes do not store a path and have no leafHash. +// - LEAF node: has _no_ children. Stores a path and (optionally) a leafHash. // // Per convention, we also consider nil as a leaf. Formally, nil is the generic // representative for any empty (sub)-trie (i.e. a trie without allocated @@ -31,12 +35,12 @@ type Node struct { // the current implementation is designed to operate on a sparsely populated // tree, holding much less than 2^64 registers. - lChild *Node // Left Child - rChild *Node // Right Child - height int // height where the Node is at - path ledger.Path // the storage path (dummy value for interim nodes) - payload *ledger.Payload // the payload this node is storing (leaf nodes only) - hashValue hash.Hash // hash value of node (cached) + lChild *Node // Left Child + rChild *Node // Right Child + height int // height where the Node is at + path ledger.Path // the storage path (dummy value for interim nodes) + leafHash *hash.Hash // HashLeaf(path, value) - the height-0 base hash (leaf nodes only; nil for unallocated registers) + hashValue hash.Hash // hash value of node (cached) } // NewNode creates a new Node. @@ -46,7 +50,7 @@ func NewNode(height int, lchild, rchild *Node, path ledger.Path, - payload *ledger.Payload, + leafHash *hash.Hash, hashValue hash.Hash, ) *Node { n := &Node{ @@ -54,40 +58,61 @@ func NewNode(height int, rChild: rchild, height: height, path: path, + leafHash: leafHash, hashValue: hashValue, - payload: payload, } return n } -// NewLeaf creates a compact leaf Node. +// NewLeaf creates a leaf Node from a path and the original payload value. +// The leafHash is computed as HashLeaf(path, value), and the node hash is +// computed using the original value to ensure the same root hash as a full trie. +// // UNCHECKED requirement: height must be non-negative -// UNCHECKED requirement: payload is non nil -// UNCHECKED requirement: payload should be deep copied if received from external sources -func NewLeaf(path ledger.Path, - payload *ledger.Payload, - height int, -) *Node { - n := &Node{ - lChild: nil, - rChild: nil, - height: height, - path: path, - payload: payload, +func NewLeaf(path ledger.Path, value []byte, height int) *Node { + // For empty values, create a default node + if len(value) == 0 { + return &Node{ + height: height, + path: path, + leafHash: nil, + hashValue: ledger.GetDefaultHashForHeight(height), + } + } + + // Compute the leaf hash (height-0 base hash) + leafHash := hash.HashLeaf(hash.Hash(path), value) + + return NewLeafWithHash(path, leafHash, height) +} + +// NewLeafWithHash creates a leaf Node from a pre-computed leaf hash. +// This is used when converting from a full trie or loading from a payloadless checkpoint. +// +// The nodeHash is computed by extending the leafHash (height-0) to the specified height. +// +// UNCHECKED requirement: height must be non-negative +// UNCHECKED requirement: leafHash must be HashLeaf(path, originalValue) +func NewLeafWithHash(path ledger.Path, leafHash hash.Hash, height int) *Node { + // Compute the node hash by extending the base hash to the target height + nodeHash := ledger.ComputeCompactValueFromBaseHash(hash.Hash(path), leafHash, height) + + return &Node{ + height: height, + path: path, + leafHash: &leafHash, + hashValue: nodeHash, } - n.hashValue = n.computeHash() - return n } // NewInterimNode creates a new interim Node. // UNCHECKED requirement: // - for any child `c` that is non-nil, its height must satisfy: height = c.height + 1 -func NewInterimNode(height int, lchild, rchild *Node) *Node { +func NewInterimNode(height int, lChild, rChild *Node) *Node { n := &Node{ - lChild: lchild, - rChild: rchild, - height: height, - payload: nil, + lChild: lChild, + rChild: rChild, + height: height, } n.hashValue = n.computeHash() return n @@ -122,11 +147,11 @@ func NewInterimCompactifiedNode(height int, lChild, rChild *Node) *Node { // an empty subtrie => in total we have one allocated register, which we represent as single leaf node if rChild == nil && lChild.IsLeaf() { h := hash.HashInterNode(lChild.hashValue, ledger.GetDefaultHashForHeight(lChild.height)) - return &Node{height: height, path: lChild.path, payload: lChild.payload, hashValue: h} + return &Node{height: height, path: lChild.path, leafHash: lChild.leafHash, hashValue: h} } if lChild == nil && rChild.IsLeaf() { h := hash.HashInterNode(ledger.GetDefaultHashForHeight(rChild.height), rChild.hashValue) - return &Node{height: height, path: rChild.path, payload: rChild.payload, hashValue: h} + return &Node{height: height, path: rChild.path, leafHash: rChild.leafHash, hashValue: h} } // CASE (b): both children contain some allocated registers => we can't compactify; return a full interim leaf @@ -147,11 +172,11 @@ func (n *Node) IsDefaultNode() bool { func (n *Node) computeHash() hash.Hash { // check for leaf node if n.lChild == nil && n.rChild == nil { - // if payload is non-nil, compute the hash based on the payload content - if n.payload != nil { - return ledger.ComputeCompactValue(hash.Hash(n.path), n.payload.Value(), n.height) + // if leafHash is non-nil, extend the height-0 base hash to the node's height + if n.leafHash != nil { + return ledger.ComputeCompactValueFromBaseHash(hash.Hash(n.path), *n.leafHash, n.height) } - // if payload is nil, return the default hash + // if leafHash is nil, return the default hash return ledger.GetDefaultHashForHeight(n.height) } @@ -211,10 +236,11 @@ func (n *Node) Path() *ledger.Path { return nil } -// Payload returns the Node's payload. -// Do NOT MODIFY returned slices! -func (n *Node) Payload() *ledger.Payload { - return n.payload +// LeafHash returns the Node's leaf hash HashLeaf(path, value). +// Returns nil for interim nodes and for leaves that represent unallocated registers. +// Do NOT MODIFY returned hash! +func (n *Node) LeafHash() *hash.Hash { + return n.leafHash } // LeftChild returns the Node's left child. @@ -243,30 +269,36 @@ func (n *Node) FmtStr(prefix string, subpath string) string { if n.lChild != nil { left = fmt.Sprintf("\n%v", n.lChild.FmtStr(prefix+"\t", subpath+"0")) } - payloadSize := 0 - if n.payload != nil { - payloadSize = n.payload.Size() + leafHashStr := "nil" + if n.leafHash != nil { + leafHashStr = hex.EncodeToString(n.leafHash[:])[:6] + "..." } hashStr := hex.EncodeToString(n.hashValue[:]) hashStr = hashStr[:3] + "..." + hashStr[len(hashStr)-3:] - return fmt.Sprintf("%v%v: (path:%v, payloadSize:%d hash:%v)[%s] (obj %p) %v %v ", prefix, n.height, n.path, payloadSize, hashStr, subpath, n, left, right) + return fmt.Sprintf("%v%v: (path:%v, leafHash:%s, hash:%v)[%s] (obj %p) %v %v", + prefix, n.height, n.path, leafHashStr, hashStr, subpath, n, left, right) } -// AllPayloads returns the payload of this node and all payloads of the subtrie -func (n *Node) AllPayloads() []*ledger.Payload { - return n.appendSubtreePayloads([]*ledger.Payload{}) +// AllLeafHashes returns the leaf hash of this node and all leaf hashes of the subtrie. +// Empty leaves (unallocated registers) are skipped. +func (n *Node) AllLeafHashes() []*hash.Hash { + return n.appendSubtreeLeafHashes([]*hash.Hash{}) } -// appendSubtreePayloads appends the payloads of the subtree with this node as root -// to the provided Payload slice. Follows same pattern as Go's native append method. -func (n *Node) appendSubtreePayloads(result []*ledger.Payload) []*ledger.Payload { +// appendSubtreeLeafHashes appends the leaf hashes of the subtree with this node as root +// to the provided slice. Follows same pattern as Go's native append method. +// Empty leaves (unallocated registers) are skipped. +func (n *Node) appendSubtreeLeafHashes(result []*hash.Hash) []*hash.Hash { if n == nil { return result } if n.IsLeaf() { - return append(result, n.Payload()) + if n.leafHash != nil { + return append(result, n.leafHash) + } + return result } - result = n.lChild.appendSubtreePayloads(result) - result = n.rChild.appendSubtreePayloads(result) + result = n.lChild.appendSubtreeLeafHashes(result) + result = n.rChild.appendSubtreeLeafHashes(result) return result } diff --git a/ledger/complete/payloadless/trie.go b/ledger/complete/payloadless/trie.go index 2c65584045f..6004dc8f478 100644 --- a/ledger/complete/payloadless/trie.go +++ b/ledger/complete/payloadless/trie.go @@ -1,4 +1,4 @@ -package trie +package payloadless import ( "encoding/json" @@ -8,7 +8,7 @@ import ( "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/bitutils" - "github.com/onflow/flow-go/ledger/complete/mtrie/node" + "github.com/onflow/flow-go/ledger/common/hash" ) // MTrie represents a perfect in-memory full binary Merkle tree with uniform height. @@ -33,9 +33,8 @@ import ( // between v and a tree leaf. The height of a tree is the height of its root. // The height of a Trie is always the height of the fully-expanded tree. type MTrie struct { - root *node.Node + root *Node regCount uint64 // number of registers allocated in the trie - regSize uint64 // size of registers allocated in the trie } // NewEmptyMTrie returns an empty Mtrie (root is nil) @@ -51,14 +50,13 @@ func (mt *MTrie) IsEmpty() bool { } // NewMTrie returns a Mtrie given the root -func NewMTrie(root *node.Node, regCount uint64, regSize uint64) (*MTrie, error) { +func NewMTrie(root *Node, regCount uint64) (*MTrie, error) { if root != nil && root.Height() != ledger.NodeMaxHeight { return nil, fmt.Errorf("height of root node must be %d but is %d, hash: %s", ledger.NodeMaxHeight, root.Height(), root.Hash().String()) } return &MTrie{ root: root, regCount: regCount, - regSize: regSize, }, nil } @@ -78,15 +76,9 @@ func (mt *MTrie) AllocatedRegCount() uint64 { return mt.regCount } -// AllocatedRegSize returns the size (number of bytes) of allocated registers in the trie. -// Concurrency safe (as Tries are immutable structures by convention) -func (mt *MTrie) AllocatedRegSize() uint64 { - return mt.regSize -} - // RootNode returns the Trie's root Node // Concurrency safe (as Tries are immutable structures by convention) -func (mt *MTrie) RootNode() *node.Node { +func (mt *MTrie) RootNode() *Node { return mt.root } @@ -100,113 +92,19 @@ func (mt *MTrie) String() string { return trieStr + mt.root.FmtStr("", "") } -// UnsafeValueSizes returns payload value sizes for the given paths. -// UNSAFE: requires _all_ paths to have a length of mt.Height bits. -// CAUTION: while getting payload value sizes, `paths` is permuted IN-PLACE for optimized processing. -// Return: -// - `sizes` []int -// For each path, the corresponding payload value size is written into sizes. AFTER -// the size operation completes, the order of `path` and `sizes` are such that -// for `path[i]` the corresponding register value size is referenced by `sizes[i]`. -// -// TODO move consistency checks from Forest into Trie to obtain a safe, self-contained API -func (mt *MTrie) UnsafeValueSizes(paths []ledger.Path) []int { - sizes := make([]int, len(paths)) // pre-allocate slice for the result - valueSizes(sizes, paths, mt.root) - return sizes -} - -// valueSizes returns value sizes of all the registers in `paths“ in subtree with `head` as root node. -// For each `path[i]`, the corresponding value size is written into `sizes[i]` for the same index `i`. -// CAUTION: -// - while reading the payloads, `paths` is permuted IN-PLACE for optimized processing. -// - unchecked requirement: all paths must go through the `head` node -func valueSizes(sizes []int, paths []ledger.Path, head *node.Node) { - // check for empty paths - if len(paths) == 0 { - return - } - - // path not found - if head == nil { - return - } - - // reached a leaf node - if head.IsLeaf() { - for i, p := range paths { - if *head.Path() == p { - payload := head.Payload() - if payload != nil { - sizes[i] = payload.Value().Size() - } - // NOTE: break isn't used here because precondition - // doesn't require paths being deduplicated. - } - } - return - } - - // reached an interim node with only one path - if len(paths) == 1 { - path := paths[0][:] - - // traverse nodes following the path until a leaf node or nil node is reached. - // "for" loop helps to skip partition and recursive call when there's only one path to follow. - for { - depth := ledger.NodeMaxHeight - head.Height() // distance to the tree root - bit := bitutils.ReadBit(path, depth) - if bit == 0 { - head = head.LeftChild() - } else { - head = head.RightChild() - } - if head.IsLeaf() { - break - } - } - - valueSizes(sizes, paths, head) - return - } - - // reached an interim node with more than one paths - - // partition step to quick sort the paths: - // lpaths contains all paths that have `0` at the partitionIndex - // rpaths contains all paths that have `1` at the partitionIndex - depth := ledger.NodeMaxHeight - head.Height() // distance to the tree root - partitionIndex := SplitPaths(paths, depth) - lpaths, rpaths := paths[:partitionIndex], paths[partitionIndex:] - lsizes, rsizes := sizes[:partitionIndex], sizes[partitionIndex:] - - // read values from left and right subtrees in parallel - parallelRecursionThreshold := 32 // threshold to avoid the parallelization going too deep in the recursion - if len(lpaths) < parallelRecursionThreshold || len(rpaths) < parallelRecursionThreshold { - valueSizes(lsizes, lpaths, head.LeftChild()) - valueSizes(rsizes, rpaths, head.RightChild()) - } else { - // concurrent read of left and right subtree - wg := sync.WaitGroup{} - wg.Go(func() { - valueSizes(lsizes, lpaths, head.LeftChild()) - }) - valueSizes(rsizes, rpaths, head.RightChild()) - wg.Wait() // wait for all threads - } -} - -// ReadSinglePayload reads and returns a payload for a single path. -func (mt *MTrie) ReadSinglePayload(path ledger.Path) *ledger.Payload { - return readSinglePayload(path, mt.root) +// ReadSingleLeafHash reads and returns the leaf hash for a single path. +// Returns nil if no leaf exists at the given path or if the leaf represents +// an unallocated register. +func (mt *MTrie) ReadSingleLeafHash(path ledger.Path) *hash.Hash { + return readSingleLeafHash(path, mt.root) } -// readSinglePayload reads and returns a payload for a single path in subtree with `head` as root node. -func readSinglePayload(path ledger.Path, head *node.Node) *ledger.Payload { +// readSingleLeafHash reads and returns the leaf hash for a single path in subtree with `head` as root node. +func readSingleLeafHash(path ledger.Path, head *Node) *hash.Hash { pathBytes := path[:] if head == nil { - return ledger.EmptyPayload() + return nil } depth := ledger.NodeMaxHeight - head.Height() // distance to the tree root @@ -223,34 +121,36 @@ func readSinglePayload(path ledger.Path, head *node.Node) *ledger.Payload { } if head != nil && *head.Path() == path { - return head.Payload() + return head.LeafHash() } - return ledger.EmptyPayload() + return nil } -// UnsafeRead reads payloads for the given paths. +// UnsafeRead reads leaf hashes for the given paths. // UNSAFE: requires _all_ paths to have a length of mt.Height bits. -// CAUTION: while reading the payloads, `paths` is permuted IN-PLACE for optimized processing. +// CAUTION: while reading the leaf hashes, `paths` is permuted IN-PLACE for optimized processing. // Return: -// - `payloads` []*ledger.Payload -// For each path, the corresponding payload is written into payloads. AFTER -// the read operation completes, the order of `path` and `payloads` are such that -// for `path[i]` the corresponding register value is referenced by 0`payloads[i]`. +// - `leafHashes` []*hash.Hash +// For each path, the corresponding leaf hash is written into leafHashes. AFTER +// the read operation completes, the order of `path` and `leafHashes` are such that +// for `path[i]` the corresponding leaf hash is referenced by `leafHashes[i]`. +// A nil entry indicates that no leaf exists at that path or the leaf represents +// an unallocated register. // // TODO move consistency checks from Forest into Trie to obtain a safe, self-contained API -func (mt *MTrie) UnsafeRead(paths []ledger.Path) []*ledger.Payload { - payloads := make([]*ledger.Payload, len(paths)) // pre-allocate slice for the result - read(payloads, paths, mt.root) - return payloads +func (mt *MTrie) UnsafeRead(paths []ledger.Path) []*hash.Hash { + leafHashes := make([]*hash.Hash, len(paths)) // pre-allocate slice for the result + read(leafHashes, paths, mt.root) + return leafHashes } // read reads all the registers in subtree with `head` as root node. For each -// `path[i]`, the corresponding payload is written into `payloads[i]` for the same index `i`. +// `path[i]`, the corresponding leaf hash is written into `leafHashes[i]` for the same index `i`. // CAUTION: -// - while reading the payloads, `paths` is permuted IN-PLACE for optimized processing. +// - while reading the leaf hashes, `paths` is permuted IN-PLACE for optimized processing. // - unchecked requirement: all paths must go through the `head` node -func read(payloads []*ledger.Payload, paths []ledger.Path, head *node.Node) { +func read(leafHashes []*hash.Hash, paths []ledger.Path, head *Node) { // check for empty paths if len(paths) == 0 { return @@ -258,9 +158,7 @@ func read(payloads []*ledger.Payload, paths []ledger.Path, head *node.Node) { // path not found if head == nil { - for i := range paths { - payloads[i] = ledger.EmptyPayload() - } + // leafHashes entries remain nil return } @@ -268,18 +166,17 @@ func read(payloads []*ledger.Payload, paths []ledger.Path, head *node.Node) { if head.IsLeaf() { for i, p := range paths { if *head.Path() == p { - payloads[i] = head.Payload() - } else { - payloads[i] = ledger.EmptyPayload() + leafHashes[i] = head.LeafHash() } + // else: leafHashes[i] remains nil } return } // reached an interim node if len(paths) == 1 { - // call readSinglePayload to skip partition and recursive calls when there is only one path - payloads[0] = readSinglePayload(paths[0], head) + // call readSingleLeafHash to skip partition and recursive calls when there is only one path + leafHashes[0] = readSingleLeafHash(paths[0], head) return } @@ -289,20 +186,20 @@ func read(payloads []*ledger.Payload, paths []ledger.Path, head *node.Node) { depth := ledger.NodeMaxHeight - head.Height() // distance to the tree root partitionIndex := SplitPaths(paths, depth) lpaths, rpaths := paths[:partitionIndex], paths[partitionIndex:] - lpayloads, rpayloads := payloads[:partitionIndex], payloads[partitionIndex:] + lLeafHashes, rLeafHashes := leafHashes[:partitionIndex], leafHashes[partitionIndex:] // read values from left and right subtrees in parallel parallelRecursionThreshold := 32 // threshold to avoid the parallelization going too deep in the recursion if len(lpaths) < parallelRecursionThreshold || len(rpaths) < parallelRecursionThreshold { - read(lpayloads, lpaths, head.LeftChild()) - read(rpayloads, rpaths, head.RightChild()) + read(lLeafHashes, lpaths, head.LeftChild()) + read(rLeafHashes, rpaths, head.RightChild()) } else { // concurrent read of left and right subtree wg := sync.WaitGroup{} wg.Go(func() { - read(lpayloads, lpaths, head.LeftChild()) + read(lLeafHashes, lpaths, head.LeftChild()) }) - read(rpayloads, rpaths, head.RightChild()) + read(rLeafHashes, rpaths, head.RightChild()) wg.Wait() // wait for all threads } } @@ -322,29 +219,28 @@ func read(payloads []*ledger.Payload, paths []ledger.Path, head *node.Node) { // - keys are NOT duplicated // - requires _all_ paths to have a length of mt.Height bits. // -// CAUTION: `updatedPaths` and `updatedPayloads` are permuted IN-PLACE for optimized processing. -// CAUTION: MTrie expects that for a specific path, the payload's key never changes. +// CAUTION: `updatedPaths` and `updatedValues` are permuted IN-PLACE for optimized processing. +// CAUTION: MTrie expects that for a specific path, the value's key never changes. // TODO: move consistency checks from MForest to here, to make API safe and self-contained func NewTrieWithUpdatedRegisters( parentTrie *MTrie, updatedPaths []ledger.Path, - updatedPayloads []ledger.Payload, + updatedValues [][]byte, prune bool, ) (*MTrie, uint16, error) { - updatedRoot, regCountDelta, regSizeDelta, lowestHeightTouched := update( + updatedRoot, allocatedRegCountDelta, lowestHeightTouched := update( ledger.NodeMaxHeight, parentTrie.root, updatedPaths, - updatedPayloads, + updatedValues, nil, prune, ) - updatedTrieRegCount := int64(parentTrie.AllocatedRegCount()) + regCountDelta - updatedTrieRegSize := int64(parentTrie.AllocatedRegSize()) + regSizeDelta + updatedTrieRegCount := int64(parentTrie.AllocatedRegCount()) + allocatedRegCountDelta maxDepthTouched := uint16(ledger.NodeMaxHeight - lowestHeightTouched) - updatedTrie, err := NewMTrie(updatedRoot, uint64(updatedTrieRegCount), uint64(updatedTrieRegSize)) + updatedTrie, err := NewMTrie(updatedRoot, uint64(updatedTrieRegCount)) if err != nil { return nil, 0, fmt.Errorf("constructing updated trie failed: %w", err) } @@ -354,66 +250,67 @@ func NewTrieWithUpdatedRegisters( // updateResult is a wrapper of return values from update(). // It's used to communicate values from goroutine. type updateResult struct { - child *node.Node + child *Node allocatedRegCountDelta int64 - allocatedRegSizeDelta int64 lowestHeightTouched int } // update traverses the subtree recursively and create new nodes with -// the updated payloads on the given paths +// the updated values on the given paths // // it returns: // - new updated node or original node if nothing was updated // - allocated register count delta in subtrie (allocatedRegCountDelta) -// - allocated register size delta in subtrie (allocatedRegSizeDelta) // - lowest height reached during recursive update in subtrie (lowestHeightTouched) // // update also compact a subtree into a single compact leaf node in the case where -// there is only 1 payload stored in the subtree. +// there is only 1 value stored in the subtree. // -// allocatedRegCountDelta and allocatedRegSizeDelta are used to compute updated -// trie's allocated register count and size. lowestHeightTouched is used to -// compute max depth touched during update. -// CAUTION: while updating, `paths` and `payloads` are permuted IN-PLACE for optimized processing. +// allocatedRegCountDelta is used to compute updated trie's allocated register count. +// lowestHeightTouched is used to compute max depth touched during update. +// CAUTION: while updating, `paths` and `values` are permuted IN-PLACE for optimized processing. // UNSAFE: method requires the following conditions to be satisfied: // - paths all share the same common prefix [0 : mt.maxHeight-1 - nodeHeight) // (excluding the bit at index headHeight) // - paths are NOT duplicated func update( nodeHeight int, // the height of the node during traversing the subtree - currentNode *node.Node, // the current node on the travesing path, if it's nil it means the trie has no node on this path - paths []ledger.Path, // the paths to update the payloads - payloads []ledger.Payload, // the payloads to be updated at the given paths - compactLeaf *node.Node, // a compact leaf node from its ancester, it could be nil - prune bool, // prune is a flag for whether pruning nodes with empty payload. not pruning is useful for generating proof, expecially non-inclusion proof -) (n *node.Node, allocatedRegCountDelta int64, allocatedRegSizeDelta int64, lowestHeightTouched int) { + currentNode *Node, // the current node on the travesing path, if it's nil it means the trie has no node on this path + paths []ledger.Path, // the paths to update the values + values [][]byte, // the values to be updated at the given paths + compactLeaf *Node, // a compact leaf node from its ancester, it could be nil + prune bool, // prune is a flag for whether pruning nodes with empty values. not pruning is useful for generating proof, expecially non-inclusion proof +) (n *Node, allocatedRegCountDelta int64, lowestHeightTouched int) { // No new path to update if len(paths) == 0 { if compactLeaf != nil { // if a compactLeaf from a higher height is still left, // then expand the compact leaf node to the current height by creating a new compact leaf - // node with the same path and payload. + // node with the same path and value. // The old node shouldn't be recycled as it is still used by the tree copy before the update. - n = node.NewLeaf(*compactLeaf.Path(), compactLeaf.Payload(), nodeHeight) - return n, 0, 0, nodeHeight + if compactLeaf.leafHash != nil { + n = NewLeafWithHash(compactLeaf.path, *compactLeaf.leafHash, nodeHeight) + } else { + n = NewLeaf(compactLeaf.path, nil, nodeHeight) + } + return n, 0, nodeHeight } // if no path to update and there is no compact leaf node on this path, we return // the current node regardless it exists or not. - return currentNode, 0, 0, nodeHeight + return currentNode, 0, nodeHeight } if len(paths) == 1 && currentNode == nil && compactLeaf == nil { // if there is only 1 path to update, and the existing tree has no node on this path, also - // no compact leaf node from its ancester, it means we are storing a payload on a new path, - n = node.NewLeaf(paths[0], payloads[0].DeepCopy(), nodeHeight) - if payloads[0].IsEmpty() { - // if we are storing an empty node, then no register is allocated - // allocatedRegCountDelta and allocatedRegSizeDelta should both be 0 - return n, 0, 0, nodeHeight + // no compact leaf node from its ancester, it means we are storing a value on a new path, + n = NewLeaf(paths[0], values[0], nodeHeight) + if len(values[0]) == 0 { + // if we are storing an empty value, then no register is allocated + // allocatedRegCountDelta should be 0 + return n, 0, nodeHeight } - // if we are storing a non-empty node, we are allocating a new register - return n, 1, int64(payloads[0].Size()), nodeHeight + // if we are storing a non-empty value, we are allocating a new register + return n, 1, nodeHeight } if currentNode != nil && currentNode.IsLeaf() { // if we're here then compactLeaf == nil @@ -424,25 +321,38 @@ func update( if p == currentPath { // the case where the recursion stops: only one path to update if len(paths) == 1 { - // check if the only path to update has the same payload. - // if payload is the same, we could skip the update to avoid creating duplicated node - if !currentNode.Payload().ValueEquals(&payloads[i]) { - n = node.NewLeaf(paths[i], payloads[i].DeepCopy(), nodeHeight) + // check if the only path to update has the same value. + // if value is the same, we could skip the update to avoid creating duplicated node + hadValue := currentNode.leafHash != nil + hasValue := len(values[i]) > 0 + var newLeafHash hash.Hash + if hasValue { + newLeafHash = hash.HashLeaf(hash.Hash(paths[i]), values[i]) + } - allocatedRegCountDelta, allocatedRegSizeDelta = - computeAllocatedRegDeltas(currentNode.Payload(), &payloads[i]) + if hadValue == hasValue { + // when value equals, if didn't have value before, then still no value after update; + // if had value before, then the leaf hash is still the same after update, + // so we can reuse the current node without creating a new one. + if !hasValue || *currentNode.leafHash == newLeafHash { + // avoid creating a new node when the same value is written + return currentNode, 0, nodeHeight + } + } - return n, allocatedRegCountDelta, allocatedRegSizeDelta, nodeHeight + // the value is updated, we need to create a new leaf node with the updated value. + // The old node shouldn't be recycled as it is still used by the trie before the update. + if hasValue { + n = NewLeafWithHash(paths[i], newLeafHash, nodeHeight) + } else { + n = NewLeaf(paths[i], nil, nodeHeight) } - // avoid creating a new node when the same payload is written - return currentNode, 0, 0, nodeHeight + allocatedRegCountDelta = computeAllocatedRegCountDelta(hadValue, hasValue) + return n, allocatedRegCountDelta, nodeHeight } // the case where the recursion carries on: len(paths)>1 found = true - - allocatedRegCountDelta, allocatedRegSizeDelta = - computeAllocatedRegDeltasFromHigherHeight(currentNode.Payload()) - + allocatedRegCountDelta = computeAllocatedRegCountDeltaFromHigherHeight(currentNode.leafHash != nil) break } } @@ -458,16 +368,16 @@ func update( // - or len(paths) == 1 and compactLeaf!= nil // - or len(paths) == 1 and currentNode != nil && !currentNode.IsLeaf() - // Split paths and payloads to recurse: + // Split paths and values to recurse: // lpaths contains all paths that have `0` at the partitionIndex // rpaths contains all paths that have `1` at the partitionIndex depth := ledger.NodeMaxHeight - nodeHeight // distance to the tree root - partitionIndex := splitByPath(paths, payloads, depth) + partitionIndex := splitByPath(paths, values, depth) lpaths, rpaths := paths[:partitionIndex], paths[partitionIndex:] - lpayloads, rpayloads := payloads[:partitionIndex], payloads[partitionIndex:] + lvalues, rvalues := values[:partitionIndex], values[partitionIndex:] // check if there is a compact leaf that needs to get deep to height 0 - var lcompactLeaf, rcompactLeaf *node.Node + var lcompactLeaf, rcompactLeaf *Node if compactLeaf != nil { // if yes, check which branch it will go to. path := *compactLeaf.Path() @@ -479,99 +389,91 @@ func update( } // set the node children - var oldLeftChild, oldRightChild *node.Node + var oldLeftChild, oldRightChild *Node if currentNode != nil { oldLeftChild = currentNode.LeftChild() oldRightChild = currentNode.RightChild() } // recurse over each branch - var newLeftChild, newRightChild *node.Node + var newLeftChild, newRightChild *Node var lRegCountDelta, rRegCountDelta int64 - var lRegSizeDelta, rRegSizeDelta int64 var lLowestHeightTouched, rLowestHeightTouched int parallelRecursionThreshold := 16 if len(lpaths) < parallelRecursionThreshold || len(rpaths) < parallelRecursionThreshold { // runtime optimization: if there are _no_ updates for either left or right sub-tree, proceed single-threaded - newLeftChild, lRegCountDelta, lRegSizeDelta, lLowestHeightTouched = update(nodeHeight-1, oldLeftChild, lpaths, lpayloads, lcompactLeaf, prune) - newRightChild, rRegCountDelta, rRegSizeDelta, rLowestHeightTouched = update(nodeHeight-1, oldRightChild, rpaths, rpayloads, rcompactLeaf, prune) + newLeftChild, lRegCountDelta, lLowestHeightTouched = update(nodeHeight-1, oldLeftChild, lpaths, lvalues, lcompactLeaf, prune) + newRightChild, rRegCountDelta, rLowestHeightTouched = update(nodeHeight-1, oldRightChild, rpaths, rvalues, rcompactLeaf, prune) } else { // runtime optimization: process the left child in a separate thread - // Since we're receiving 4 values from goroutine, use a + // Since we're receiving 3 values from goroutine, use a // struct and channel to reduce allocs/op. // Although WaitGroup approach can be faster than channel (esp. with 2+ goroutines), // we only use 1 goroutine here and need to communicate results from it. So using // channel is faster and uses fewer allocs/op in this case. results := make(chan updateResult, 1) go func(retChan chan<- updateResult) { - child, regCountDelta, regSizeDelta, lowestHeightTouched := update(nodeHeight-1, oldLeftChild, lpaths, lpayloads, lcompactLeaf, prune) - retChan <- updateResult{child, regCountDelta, regSizeDelta, lowestHeightTouched} + child, regCountDelta, lowestHeightTouched := update(nodeHeight-1, oldLeftChild, lpaths, lvalues, lcompactLeaf, prune) + retChan <- updateResult{child, regCountDelta, lowestHeightTouched} }(results) - newRightChild, rRegCountDelta, rRegSizeDelta, rLowestHeightTouched = update(nodeHeight-1, oldRightChild, rpaths, rpayloads, rcompactLeaf, prune) + newRightChild, rRegCountDelta, rLowestHeightTouched = update(nodeHeight-1, oldRightChild, rpaths, rvalues, rcompactLeaf, prune) // Wait for results from goroutine. ret := <-results - newLeftChild, lRegCountDelta, lRegSizeDelta, lLowestHeightTouched = ret.child, ret.allocatedRegCountDelta, ret.allocatedRegSizeDelta, ret.lowestHeightTouched + newLeftChild, lRegCountDelta, lLowestHeightTouched = ret.child, ret.allocatedRegCountDelta, ret.lowestHeightTouched } allocatedRegCountDelta += lRegCountDelta + rRegCountDelta - allocatedRegSizeDelta += lRegSizeDelta + rRegSizeDelta lowestHeightTouched = minInt(lLowestHeightTouched, rLowestHeightTouched) // mitigate storage exhaustion attack: avoids creating a new node when the exact same - // payload is re-written at a register. CAUTION: we only check that the children are + // value is re-written at a register. CAUTION: we only check that the children are // unchanged. This is only sufficient for interim nodes (for leaf nodes, the children - // might be unchanged, i.e. both nil, but the payload could have changed). + // might be unchanged, i.e. both nil, but the value could have changed). // In case the current node was a leaf, we _cannot reuse_ it, because we potentially // updated registers in the sub-trie if !currentNode.IsLeaf() && newLeftChild == oldLeftChild && newRightChild == oldRightChild { - return currentNode, 0, 0, lowestHeightTouched + return currentNode, 0, lowestHeightTouched } // if prune is on, then will check and create a compact leaf node if one child is nil, and the // other child is a leaf node if prune { - n = node.NewInterimCompactifiedNode(nodeHeight, newLeftChild, newRightChild) - return n, allocatedRegCountDelta, allocatedRegSizeDelta, lowestHeightTouched + n = NewInterimCompactifiedNode(nodeHeight, newLeftChild, newRightChild) + return n, allocatedRegCountDelta, lowestHeightTouched } - n = node.NewInterimNode(nodeHeight, newLeftChild, newRightChild) - return n, allocatedRegCountDelta, allocatedRegSizeDelta, lowestHeightTouched + n = NewInterimNode(nodeHeight, newLeftChild, newRightChild) + return n, allocatedRegCountDelta, lowestHeightTouched } -// computeAllocatedRegDeltasFromHigherHeight returns the deltas -// needed to compute the allocated reg count and reg size when -// a payload is updated or unallocated at a lower height. -func computeAllocatedRegDeltasFromHigherHeight(oldPayload *ledger.Payload) (allocatedRegCountDelta, allocatedRegSizeDelta int64) { - if !oldPayload.IsEmpty() { +// computeAllocatedRegCountDeltaFromHigherHeight returns the delta +// needed to compute the allocated reg count when +// a value is updated or unallocated at a lower height. +func computeAllocatedRegCountDeltaFromHigherHeight(hadValue bool) (allocatedRegCountDelta int64) { + if hadValue { // Allocated register will be updated or unallocated at lower height. allocatedRegCountDelta-- } - oldPayloadSize := oldPayload.Size() - allocatedRegSizeDelta -= int64(oldPayloadSize) return } -// computeAllocatedRegDeltas returns the allocated reg count -// and reg size deltas computed from old payload and new payload. -// PRECONDITION: !oldPayload.Equals(newPayload) -func computeAllocatedRegDeltas(oldPayload, newPayload *ledger.Payload) (allocatedRegCountDelta, allocatedRegSizeDelta int64) { +// computeAllocatedRegCountDelta returns the allocated reg count +// delta computed from the presence of the old and new value. +// PRECONDITION: hadValue != hasValue OR the stored value changed +func computeAllocatedRegCountDelta(hadValue, hasValue bool) (allocatedRegCountDelta int64) { allocatedRegCountDelta = 0 - if newPayload.IsEmpty() { - // Old payload is not empty while new payload is empty. + if !hasValue { + // Old value is present while new value is empty. // Allocated register will be unallocated. allocatedRegCountDelta = -1 - } else if oldPayload.IsEmpty() { - // Old payload is empty while new payload is not empty. + } else if !hadValue { + // Old value is empty while new value is present. // Unallocated register will be allocated. allocatedRegCountDelta = 1 } - - oldPayloadSize := oldPayload.Size() - newPayloadSize := newPayload.Size() - allocatedRegSizeDelta = int64(newPayloadSize - oldPayloadSize) return } @@ -581,8 +483,8 @@ func computeAllocatedRegDeltas(oldPayload, newPayload *ledger.Payload) (allocate // UNSAFE: requires _all_ paths to have a length of mt.Height bits. // Paths in the input query don't have to be deduplicated, though deduplication would // result in allocating less dynamic memory to store the proofs. -func (mt *MTrie) UnsafeProofs(paths []ledger.Path) *ledger.TrieBatchProof { - batchProofs := ledger.NewTrieBatchProofWithEmptyProofs(len(paths)) +func (mt *MTrie) UnsafeProofs(paths []ledger.Path) *ledger.PayloadlessTrieBatchProof { + batchProofs := ledger.NewPayloadlessTrieBatchProofWithEmptyProofs(len(paths)) prove(mt.root, paths, batchProofs.Proofs) return batchProofs } @@ -593,7 +495,7 @@ func (mt *MTrie) UnsafeProofs(paths []ledger.Path) *ledger.TrieBatchProof { // UNSAFE: method requires the following conditions to be satisfied: // - paths all share the same common prefix [0 : mt.maxHeight-1 - nodeHeight) // (excluding the bit at index headHeight) -func prove(head *node.Node, paths []ledger.Path, proofs []*ledger.TrieProof) { +func prove(head *Node, paths []ledger.Path, proofs []*ledger.PayloadlessTrieProof) { // check for empty paths if len(paths) == 0 { return @@ -612,7 +514,7 @@ func prove(head *node.Node, paths []ledger.Path, proofs []*ledger.TrieProof) { // value matches (inclusion proof) if *head.Path() == path { proofs[i].Path = *head.Path() - proofs[i].Payload = head.Payload() + proofs[i].LeafHash = head.LeafHash() proofs[i].Inclusion = true } } @@ -657,7 +559,7 @@ func prove(head *node.Node, paths []ledger.Path, proofs []*ledger.TrieProof) { // addSiblingTrieHashToProofs inspects the sibling Trie and adds its root hash // to the proofs, if the trie contains non-empty registers (i.e. the // siblingTrie has a non-default hash). -func addSiblingTrieHashToProofs(siblingTrie *node.Node, depth int, proofs []*ledger.TrieProof) { +func addSiblingTrieHashToProofs(siblingTrie *Node, depth int, proofs []*ledger.PayloadlessTrieProof) { if siblingTrie == nil || len(proofs) == 0 { return } @@ -694,7 +596,8 @@ func (mt *MTrie) Equals(o *MTrie) bool { return o.RootHash() == mt.RootHash() } -// DumpAsJSON dumps the trie key value pairs to a file having each key value pair as a json row +// DumpAsJSON dumps the trie leaf entries to a writer having each leaf as a json row. +// Each entry contains the leaf's path and its stored leaf hash. func (mt *MTrie) DumpAsJSON(w io.Writer) error { // Use encoder to prevent building entire trie in memory @@ -708,11 +611,17 @@ func (mt *MTrie) DumpAsJSON(w io.Writer) error { return nil } +// dumpLeafEntry is the JSON form of a leaf encoded by DumpAsJSON. +type dumpLeafEntry struct { + Path ledger.Path `json:"path"` + LeafHash *hash.Hash `json:"leafHash"` +} + // dumpAsJSON serializes the sub-trie with root n to json and feeds it into encoder -func dumpAsJSON(n *node.Node, encoder *json.Encoder) error { +func dumpAsJSON(n *Node, encoder *json.Encoder) error { if n.IsLeaf() { if n != nil { - err := encoder.Encode(n.Payload()) + err := encoder.Encode(dumpLeafEntry{Path: n.path, LeafHash: n.leafHash}) if err != nil { return err } @@ -741,9 +650,10 @@ func EmptyTrieRootHash() ledger.RootHash { return ledger.RootHash(ledger.GetDefaultHashForHeight(ledger.NodeMaxHeight)) } -// AllPayloads returns all payloads -func (mt *MTrie) AllPayloads() []*ledger.Payload { - return mt.root.AllPayloads() +// AllLeafHashes returns all leaf hashes stored in the trie. Empty leaves +// (unallocated registers) are skipped. +func (mt *MTrie) AllLeafHashes() []*hash.Hash { + return mt.root.AllLeafHashes() } // IsAValidTrie verifies the content of the trie for potential issues @@ -754,7 +664,7 @@ func (mt *MTrie) IsAValidTrie() bool { // splitByPath permutes the input paths to be partitioned into 2 parts. The first part contains paths with a zero bit // at the input bitIndex, the second part contains paths with a one at the bitIndex. The index of partition -// is returned. The same permutation is applied to the payloads slice. +// is returned. The same permutation is applied to the values slice. // // This would be the partition step of an ascending quick sort of paths (lexicographic order) // with the pivot being the path with all zeros and 1 at bitIndex. @@ -765,13 +675,13 @@ func (mt *MTrie) IsAValidTrie() bool { // [[0,0,1,1], [0,1,0,1], [0,0,0,1]] // then `splitByPath` returns 2 and updates `paths` into: // [[0,0,1,1], [0,0,0,1], [0,1,0,1]] -func splitByPath(paths []ledger.Path, payloads []ledger.Payload, bitIndex int) int { +func splitByPath(paths []ledger.Path, values [][]byte, bitIndex int) int { i := 0 for j, path := range paths { bit := bitutils.ReadBit(path[:], bitIndex) if bit == 0 { paths[i], paths[j] = paths[j], paths[i] - payloads[i], payloads[j] = payloads[j], payloads[i] + values[i], values[j] = values[j], values[i] i++ } } @@ -806,7 +716,7 @@ func SplitPaths(paths []ledger.Path, bitIndex int) int { // with the pivot being the path with all zeros and 1 at bitIndex. // The comparison of paths is only based on the bit at bitIndex, the function therefore assumes all paths have // equal bits from 0 to bitIndex-1 -func splitTrieProofsByPath(paths []ledger.Path, proofs []*ledger.TrieProof, bitIndex int) int { +func splitTrieProofsByPath(paths []ledger.Path, proofs []*ledger.PayloadlessTrieProof, bitIndex int) int { i := 0 for j, path := range paths { bit := bitutils.ReadBit(path[:], bitIndex) @@ -827,11 +737,11 @@ func minInt(a, b int) int { } // TraverseNodes traverses all nodes of the trie in DFS order -func TraverseNodes(trie *MTrie, processNode func(*node.Node) error) error { +func TraverseNodes(trie *MTrie, processNode func(*Node) error) error { return traverseRecursive(trie.root, processNode) } -func traverseRecursive(n *node.Node, processNode func(*node.Node) error) error { +func traverseRecursive(n *Node, processNode func(*Node) error) error { if n == nil { return nil } diff --git a/ledger/payloadless_proof.go b/ledger/payloadless_proof.go new file mode 100644 index 00000000000..e502042f82a --- /dev/null +++ b/ledger/payloadless_proof.go @@ -0,0 +1,182 @@ +package ledger + +import ( + "bytes" + "fmt" + + "github.com/onflow/flow-go/ledger/common/hash" +) + +// PayloadlessTrieProof includes all the information needed to walk +// through a payloadless trie branch from an specific leaf node (key) +// up to the root of the trie. +// +// Unlike [TrieProof], which stores the full Payload at the leaf, a +// PayloadlessTrieProof stores only the leaf hash (HashLeaf(path, value)), +// matching the storage of the payloadless trie. The caller is responsible +// for retrieving the original value separately if needed. +type PayloadlessTrieProof struct { + Path Path // path + LeafHash *hash.Hash // leaf hash HashLeaf(path, value); nil for empty leaves and non-inclusion proofs + Interims []hash.Hash // the non-default intermediate nodes in the proof + Inclusion bool // flag indicating if this is an inclusion or exclusion proof + Flags []byte // The flags of the proofs (is set if an intermediate node has a non-default) + Steps uint8 // number of steps for the proof (path len) // TODO: should this be a type allowing for larger values? +} + +// NewPayloadlessTrieProof creates a new instance of PayloadlessTrieProof +func NewPayloadlessTrieProof() *PayloadlessTrieProof { + return &PayloadlessTrieProof{ + LeafHash: nil, + Interims: make([]hash.Hash, 0), + Inclusion: false, + Flags: make([]byte, PathLen), + Steps: 0, + } +} + +func (p *PayloadlessTrieProof) String() string { + flagStr := "" + for _, f := range p.Flags { + flagStr += fmt.Sprintf("%08b", f) + } + proofStr := fmt.Sprintf("size: %d flags: %v\n", p.Steps, flagStr) + leafHashStr := "nil" + if p.LeafHash != nil { + leafHashStr = fmt.Sprintf("%x", *p.LeafHash) + } + proofStr += fmt.Sprintf("\t path: %v leafHash: %s\n", p.Path, leafHashStr) + + if p.Inclusion { + proofStr += "\t inclusion proof:\n" + } else { + proofStr += "\t noninclusion proof:\n" + } + interimIndex := 0 + for j := 0; j < int(p.Steps); j++ { + // if bit is set + if p.Flags[j/8]&(1<<(7-j%8)) != 0 { + proofStr += fmt.Sprintf("\t\t %d: [%x]\n", j, p.Interims[interimIndex]) + interimIndex++ + } + } + return proofStr +} + +// Equals compares this proof to another payloadless proof +func (p *PayloadlessTrieProof) Equals(o *PayloadlessTrieProof) bool { + if o == nil { + return false + } + if !p.Path.Equals(o.Path) { + return false + } + if (p.LeafHash == nil) != (o.LeafHash == nil) { + return false + } + if p.LeafHash != nil && *p.LeafHash != *o.LeafHash { + return false + } + if len(p.Interims) != len(o.Interims) { + return false + } + for i, inter := range p.Interims { + if inter != o.Interims[i] { + return false + } + } + if p.Inclusion != o.Inclusion { + return false + } + if !bytes.Equal(p.Flags, o.Flags) { + return false + } + if p.Steps != o.Steps { + return false + } + return true +} + +// PayloadlessTrieBatchProof is a struct that holds the payloadless proofs for several keys +// +// so there is no need for two calls (read, proofs) +type PayloadlessTrieBatchProof struct { + Proofs []*PayloadlessTrieProof +} + +// NewPayloadlessTrieBatchProof creates a new instance of PayloadlessTrieBatchProof +func NewPayloadlessTrieBatchProof() *PayloadlessTrieBatchProof { + bp := new(PayloadlessTrieBatchProof) + bp.Proofs = make([]*PayloadlessTrieProof, 0) + return bp +} + +// NewPayloadlessTrieBatchProofWithEmptyProofs creates an instance of PayloadlessTrieBatchProof +// filled with n newly created proofs (empty) +func NewPayloadlessTrieBatchProofWithEmptyProofs(numberOfProofs int) *PayloadlessTrieBatchProof { + bp := new(PayloadlessTrieBatchProof) + bp.Proofs = make([]*PayloadlessTrieProof, numberOfProofs) + for i := range numberOfProofs { + bp.Proofs[i] = NewPayloadlessTrieProof() + } + return bp +} + +// Size returns the number of proofs +func (bp *PayloadlessTrieBatchProof) Size() int { + return len(bp.Proofs) +} + +// Paths returns the slice of paths for this batch proof +func (bp *PayloadlessTrieBatchProof) Paths() []Path { + paths := make([]Path, len(bp.Proofs)) + for i, p := range bp.Proofs { + paths[i] = p.Path + } + return paths +} + +// LeafHashes returns the slice of leaf hashes for this batch proof +func (bp *PayloadlessTrieBatchProof) LeafHashes() []*hash.Hash { + leafHashes := make([]*hash.Hash, len(bp.Proofs)) + for i, p := range bp.Proofs { + leafHashes[i] = p.LeafHash + } + return leafHashes +} + +func (bp *PayloadlessTrieBatchProof) String() string { + res := fmt.Sprintf("payloadless trie batch proof includes %d proofs: \n", bp.Size()) + for _, proof := range bp.Proofs { + res = res + "\n" + proof.String() + } + return res +} + +// AppendProof adds a proof to the batch proof +func (bp *PayloadlessTrieBatchProof) AppendProof(p *PayloadlessTrieProof) { + bp.Proofs = append(bp.Proofs, p) +} + +// MergeInto adds all of its proofs into the dest batch proof +func (bp *PayloadlessTrieBatchProof) MergeInto(dest *PayloadlessTrieBatchProof) { + for _, p := range bp.Proofs { + dest.AppendProof(p) + } +} + +// Equals compares this batch proof to another batch proof +func (bp *PayloadlessTrieBatchProof) Equals(o *PayloadlessTrieBatchProof) bool { + if o == nil { + return false + } + if len(bp.Proofs) != len(o.Proofs) { + return false + } + for i, proof := range bp.Proofs { + if !proof.Equals(o.Proofs[i]) { + return false + } + } + return true +} diff --git a/ledger/trie.go b/ledger/trie.go index 386095a2923..f96c4b57449 100644 --- a/ledger/trie.go +++ b/ledger/trie.go @@ -73,11 +73,29 @@ func ComputeCompactValue(path hash.Hash, value []byte, nodeHeight int) hash.Hash return GetDefaultHashForHeight(nodeHeight) } - var out hash.Hash - out = hash.HashLeaf(path, value) // we first compute the hash of the fully-expanded leaf - for h := 1; h <= nodeHeight; h++ { // then, we hash our way upwards towards the root until we hit the specified nodeHeight - // h is the height of the node, whose hash we are computing in this iteration. - // The hash is computed from the node's children at height h-1. + baseHash := hash.HashLeaf(path, value) // compute the hash of the fully-expanded leaf (height 0) + return ComputeCompactValueFromBaseHash(path, baseHash, nodeHeight) +} + +// ComputeCompactValueFromBaseHash computes the node hash from a pre-computed base hash +// (the height-0 hash, i.e., HashLeaf(path, value)). This is useful for payloadless tries +// where the base hash is stored instead of the actual value. +// +// The function extends the base hash from height 0 to nodeHeight by hashing upward +// through the trie structure, combining with default hashes at each level. +func ComputeCompactValueFromBaseHash(path hash.Hash, baseHash hash.Hash, nodeHeight int) hash.Hash { + return ExtendHashToHeight(path, baseHash, 0, nodeHeight) +} + +// ExtendHashToHeight extends a hash from one height to another by continuing the hash chain +// along the path. This is used in payloadless mode to extend a leaf's hash when the leaf +// is moved to a higher position in the trie. +// UNCHECKED requirement: fromHeight < toHeight +// UNCHECKED requirement: fromHeight >= 0 +func ExtendHashToHeight(path hash.Hash, baseHash hash.Hash, fromHeight, toHeight int) hash.Hash { + out := baseHash + for h := fromHeight + 1; h <= toHeight; h++ { + // h is the height of the node whose hash we are computing bit := bitutils.ReadBit(path[:], NodeMaxHeight-h) if bit == 1 { // right branching out = hash.HashInterNode(GetDefaultHashForHeight(h-1), out) From f935c3441701e3174d47fb2792d5cd08a812a977 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 18 Jun 2026 16:58:08 -0700 Subject: [PATCH 3/9] remove ExtendHashToHeight and replace baseHash to leafHash in naming --- ledger/complete/payloadless/node.go | 12 ++++++------ ledger/trie.go | 25 ++++++++----------------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/ledger/complete/payloadless/node.go b/ledger/complete/payloadless/node.go index 90843089831..f84782000bb 100644 --- a/ledger/complete/payloadless/node.go +++ b/ledger/complete/payloadless/node.go @@ -39,7 +39,7 @@ type Node struct { rChild *Node // Right Child height int // height where the Node is at path ledger.Path // the storage path (dummy value for interim nodes) - leafHash *hash.Hash // HashLeaf(path, value) - the height-0 base hash (leaf nodes only; nil for unallocated registers) + leafHash *hash.Hash // HashLeaf(path, value) - the height-0 leaf hash (leaf nodes only; nil for unallocated registers) hashValue hash.Hash // hash value of node (cached) } @@ -80,7 +80,7 @@ func NewLeaf(path ledger.Path, value []byte, height int) *Node { } } - // Compute the leaf hash (height-0 base hash) + // Compute the leaf hash (height-0) leafHash := hash.HashLeaf(hash.Hash(path), value) return NewLeafWithHash(path, leafHash, height) @@ -94,8 +94,8 @@ func NewLeaf(path ledger.Path, value []byte, height int) *Node { // UNCHECKED requirement: height must be non-negative // UNCHECKED requirement: leafHash must be HashLeaf(path, originalValue) func NewLeafWithHash(path ledger.Path, leafHash hash.Hash, height int) *Node { - // Compute the node hash by extending the base hash to the target height - nodeHash := ledger.ComputeCompactValueFromBaseHash(hash.Hash(path), leafHash, height) + // Compute the node hash by extending the leaf hash to the target height + nodeHash := ledger.ComputeCompactValueFromLeafHash(hash.Hash(path), leafHash, height) return &Node{ height: height, @@ -172,9 +172,9 @@ func (n *Node) IsDefaultNode() bool { func (n *Node) computeHash() hash.Hash { // check for leaf node if n.lChild == nil && n.rChild == nil { - // if leafHash is non-nil, extend the height-0 base hash to the node's height + // if leafHash is non-nil, extend the height-0 leaf hash to the node's height if n.leafHash != nil { - return ledger.ComputeCompactValueFromBaseHash(hash.Hash(n.path), *n.leafHash, n.height) + return ledger.ComputeCompactValueFromLeafHash(hash.Hash(n.path), *n.leafHash, n.height) } // if leafHash is nil, return the default hash return ledger.GetDefaultHashForHeight(n.height) diff --git a/ledger/trie.go b/ledger/trie.go index f96c4b57449..12c6442b9d3 100644 --- a/ledger/trie.go +++ b/ledger/trie.go @@ -73,28 +73,19 @@ func ComputeCompactValue(path hash.Hash, value []byte, nodeHeight int) hash.Hash return GetDefaultHashForHeight(nodeHeight) } - baseHash := hash.HashLeaf(path, value) // compute the hash of the fully-expanded leaf (height 0) - return ComputeCompactValueFromBaseHash(path, baseHash, nodeHeight) + leafHash := hash.HashLeaf(path, value) // compute the hash of the fully-expanded leaf (height 0) + return ComputeCompactValueFromLeafHash(path, leafHash, nodeHeight) } -// ComputeCompactValueFromBaseHash computes the node hash from a pre-computed base hash +// ComputeCompactValueFromLeafHash computes the node hash from a pre-computed leaf hash // (the height-0 hash, i.e., HashLeaf(path, value)). This is useful for payloadless tries -// where the base hash is stored instead of the actual value. +// where the leaf hash is stored instead of the actual value. // -// The function extends the base hash from height 0 to nodeHeight by hashing upward +// The function extends the leaf hash from height 0 to nodeHeight by hashing upward // through the trie structure, combining with default hashes at each level. -func ComputeCompactValueFromBaseHash(path hash.Hash, baseHash hash.Hash, nodeHeight int) hash.Hash { - return ExtendHashToHeight(path, baseHash, 0, nodeHeight) -} - -// ExtendHashToHeight extends a hash from one height to another by continuing the hash chain -// along the path. This is used in payloadless mode to extend a leaf's hash when the leaf -// is moved to a higher position in the trie. -// UNCHECKED requirement: fromHeight < toHeight -// UNCHECKED requirement: fromHeight >= 0 -func ExtendHashToHeight(path hash.Hash, baseHash hash.Hash, fromHeight, toHeight int) hash.Hash { - out := baseHash - for h := fromHeight + 1; h <= toHeight; h++ { +func ComputeCompactValueFromLeafHash(path hash.Hash, leafHash hash.Hash, nodeHeight int) hash.Hash { + out := leafHash + for h := 1; h <= nodeHeight; h++ { // h is the height of the node whose hash we are computing bit := bitutils.ReadBit(path[:], NodeMaxHeight-h) if bit == 1 { // right branching From f42a6d8a17b617f7ed202d2135a0baa40ca6660c Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 18 Jun 2026 16:59:10 -0700 Subject: [PATCH 4/9] add comments for NewMTrie --- ledger/complete/payloadless/trie.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ledger/complete/payloadless/trie.go b/ledger/complete/payloadless/trie.go index 6004dc8f478..634cfbc1075 100644 --- a/ledger/complete/payloadless/trie.go +++ b/ledger/complete/payloadless/trie.go @@ -49,7 +49,9 @@ func (mt *MTrie) IsEmpty() bool { return mt.root == nil } -// NewMTrie returns a Mtrie given the root +// NewMTrie returns a Mtrie given the root. +// +// No error returns are expected during normal operation. func NewMTrie(root *Node, regCount uint64) (*MTrie, error) { if root != nil && root.Height() != ledger.NodeMaxHeight { return nil, fmt.Errorf("height of root node must be %d but is %d, hash: %s", ledger.NodeMaxHeight, root.Height(), root.Hash().String()) From 14e63968352d7638c2131b330e797d5b6561f69b Mon Sep 17 00:00:00 2001 From: Alex Hentschel Date: Tue, 7 Jul 2026 19:59:57 -0700 Subject: [PATCH 5/9] Suggested amendments for PR #8569: encapsulate leaf re-leveling, refine docs, add node/trie tests Encapsulate the "move a compact leaf to a new height" operation inside node.go (NewRelevelledLeaf) so trie.go business logic no longer branches on the allocated- vs-unallocated-register distinction; refine node/trie documentation and the mtrie README Update case taxonomy; add exhaustive node_test.go / trie_test.go coverage. Co-authored-by: Cursor --- ledger/common/hash/sha3.go | 2 +- ledger/common/hash/sha3_internal_test.go | 27 + ledger/complete/mtrie/README.md | 96 +-- ledger/complete/mtrie/trie/trie_test.go | 31 +- ledger/complete/payloadless/node.go | 241 +++++-- ledger/complete/payloadless/node_test.go | 820 +++++++++++++++++++++++ ledger/complete/payloadless/trie.go | 242 ++++--- ledger/complete/payloadless/trie_test.go | 63 ++ 8 files changed, 1300 insertions(+), 222 deletions(-) create mode 100644 ledger/common/hash/sha3_internal_test.go create mode 100644 ledger/complete/payloadless/node_test.go create mode 100644 ledger/complete/payloadless/trie_test.go diff --git a/ledger/common/hash/sha3.go b/ledger/common/hash/sha3.go index 4e67d2aa48d..65a6f8cd403 100644 --- a/ledger/common/hash/sha3.go +++ b/ledger/common/hash/sha3.go @@ -112,7 +112,7 @@ func (d *state) hash256Plus(p1 Hash, p2 []byte) Hash { } // hash256plus256 absorbs two 256 bits slices of data into the hash's state -// applies the permutation, and outpute the result in out +// applies the permutation, and outputs the result in out func (d *state) hash256plus256(p1, p2 Hash) Hash { copyIn512(d, p1, p2) // permute diff --git a/ledger/common/hash/sha3_internal_test.go b/ledger/common/hash/sha3_internal_test.go new file mode 100644 index 00000000000..4761e154af9 --- /dev/null +++ b/ledger/common/hash/sha3_internal_test.go @@ -0,0 +1,27 @@ +package hash + +import ( + "crypto/rand" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Test_hash256Plus_matches_hash256plus256 checks that for a 32-byte p2 the two +// absorption routines produce the same digest. hash256Plus is the variable-length +// path (p1 is 256 bits, p2 is an arbitrary-length slice) while hash256plus256 is the +// fixed 256+256-bit fast path. When p2 is exactly 32 bytes both absorb the identical +// 512-bit input under the same SHA3-256 padding, so their outputs must be equal. +func Test_hash256Plus_matches_hash256plus256(t *testing.T) { + var p1, p2 Hash + _, err := rand.Read(p1[:]) + require.NoError(t, err) + _, err = rand.Read(p2[:]) + require.NoError(t, err) + + h1 := (&state{}).hash256Plus(p1, p2[:]) + h2 := (&state{}).hash256plus256(p1, p2) + + assert.Equal(t, h1, h2) +} diff --git a/ledger/complete/mtrie/README.md b/ledger/complete/mtrie/README.md index f53aa282bb4..f87d3f35a0a 100644 --- a/ledger/complete/mtrie/README.md +++ b/ledger/complete/mtrie/README.md @@ -29,7 +29,7 @@ derived from the key, called path. While all register paths have the same fixed (measured in bits), the keys and values are variable-length byte slices. A register holds both the key and value, which forms a payload. A path is derived deterministically from the key part of the payload. We define an **unallocated register** as holding no value, i.e. a nil payload or an empty value byte slice. -By default, each register is unallocated. In contrast, an **allocated_ register** +By default, each register is unallocated. In contrast, an **allocated register** holds a non-nil payload and a value with positive storage size, i.e. a byte slice with length larger than zero. Note that we do not introduce the concept of registers with `nil` values. @@ -42,14 +42,14 @@ Therefore, we have two different node types in the tree: - following established graph-theoretic conventions, the `height` of a leaf is zero. - the `hash` value is defined as: - For an _unallocated_ register, the `hash` is just the hash of a global constant. - Therefore, the leafs for all unallocated registers have the same hash. + Therefore, the leaves for all unallocated registers have the same hash. We refer to the hash of an unallocated register as `default hash at height 0`. - For _allocated_ registers, the `hash` value is `H(path, value)` for `H` the hash function. * An **INTERIM** node is a vertex in the tree: - it has exactly two children, called `LeftChild` and `RightChild`, which are both of the same height; - the children can either be leafs or interim nodes. + the children can either be leaves or interim nodes. - the `height` of an interim node `n` is `n.height = LeftChild.height + 1 = RightChild.height + 1`; - (Hence, an interim node `n` can only have a `n.height > 0`, as only leafs have height zero). + (Hence, an interim node `n` can only have a `n.height > 0`, as only leaves have height zero). - the `hash` value is defined as `H(LeftChild, RightChild)` #### Convention for mapping a register `key` to a path in the tree @@ -170,11 +170,11 @@ We define the function `Update` to implement the recursive algorithm to apply th During the recursion, we can encounter the following cases: -* **Case 0: `node` is an interim node.** (generic recursion step) As described, we further descend down the left or right child, depending on the bit value `path[i]`. -* **Case 1: `node` is a leaf.** A leaf can be either fully expanded (height zero) or compactified (height larger than zero). - - **case 1.a: `node.path == path`**, i.e. the leaf's represents the register that we are looking to update. +* **Case 1: `node` is an interim node.** (generic recursion step) As described, we further descend down the left or right child, depending on the bit value `path[i]`. +* **Case 2: `node` is a leaf.** A leaf can be either fully expanded (height zero) or compactified (height larger than zero). + - **case 2.a: `node.path == path`**, i.e. the leaf's represents the register that we are looking to update. The tree update is done by creating a new node with the new input `payload`. - - **case 1.b: `node.path ≠ path`**, i.e. the leaf represents a _different_ register than the one we want to update. + - **case 2.b: `node.path ≠ path`**, i.e. the leaf represents a _different_ register than the one we want to update. While the register with `path`, falls in the same sub-trie as the allocated register, it is still unallocated. This implies that `node` must be a compactified leaf. Therefore, in the updated trie, the previously compactified leaf has to be replaced by sub-trie containing @@ -182,15 +182,16 @@ During the recursion, we can encounter the following cases: write the contents of the previously existing register as well as the new register `(path, payload)` to the interim-node's children. We set `compactLeaf := node` and continue the recursive construction of the new the sub-tree. -* **Case 2: `node == nil`**: A `nil` sub-trie means that the sub-trie is empty and at least a new leaf has to be created. - - **case 2.a: there is only one leaf to create**. If there is only one leaf to create (either the one representing the input `(path, payload)`, -or the one representing a compactified leaf carried over from a higher height), then a new leaf is created. -The new leaf can be either fully expanded or compactified. - - **case 2.b: there are 2 leafs to create**. If there are 2 leafs to create (both the input `(path, payload)` and the compactified leaf carried over), +* **Case 3: `node == nil`**: the sub-trie is empty, so at least one new leaf must be created. + - **case 3.a: exactly one leaf is created** (fully expanded or compactified). It represents one of: + - **(i)** the input register `(path, payload)`: the input path falls into this empty sub-trie and no compactified leaf was carried over; + - **(ii)** the compactified leaf carried over from a higher height: no input path falls into this sub-trie, so that carried leaf is re-created at the current (lower) height. + + - **case 3.b: there are 2 leaves to create**. If there are 2 leaves to create (both the input `(path, payload)` and the compactified leaf carried over), then we are still at an interim-node height. Hence, we create a new interim-node with `nil` children, check the path index of both the input `path` -and the compactified node `path` and continue the recursion over the children. Eventually the recursion calls will fall into 2.a +and the compactified node `path` and continue the recursion over the children. Eventually the recursion calls will fall into 3.a as we reach the first different bit index between the 2 paths. This case can be seen as a special case of the -generic case 0 above, but just called with a `node = nil`. +generic case 1 above, but just called with a `node = nil`. #### General algorithm @@ -201,23 +202,24 @@ We now generalize this algorithm to an arbitrary number of `K` register updates: The first partition has all updates for registers with `paths[k][i] = 0` and goes into the left child recursion, while the second partition has the updates pertaining to registers with `paths[k][i] = 1` and goes into the right child recursion. This results in sorting the overall input `paths` using an implicit quick sort. - - if `len(paths) == 0` and there is no compact leaf carried over (`compactLeaf == nil`), no update will be done -and the original sub-trie can be re-used in the new trie. + - **Case 0 (re-use):** if `len(paths) == 0` and there is no compact leaf carried over (`compactLeaf == nil`), no update will be done and the original sub-trie can be re-used in the new trie. -* **Case 0: `node` is an interim node.** An interim-node is created, the paths are split into left and right. -* **Case 1: `node` is a leaf.** Instead of comparing the path of the leaf with the unique input path, the leaf path is linearly searched within -all the input paths. Case 1.a is when the leaf path is found among the inputs, Case 1.b is when the leaf path is not found. +* **Case 1: `node` is an interim node.** An interim-node is created, the paths are split into left and right. +* **Case 2: `node` is a leaf.** Instead of comparing the path of the leaf with the unique input path, the leaf path is linearly searched within +all the input paths. Case 2.a is when the leaf path is found among the inputs, Case 2.b is when the leaf path is not found. The linear search in the recursive step has an overall complexity `O(K)` (for all recursion steps combined). - Case 1.a is now split into two subcases: - - **case 1.a.i: `node.path ∈ path` and `len(paths) == 1`**. A new node is created with the new updated payload. This would be a leaf in the new trie. - - **case 1.a.ii: `node.path ∈ path` and `len(paths) > 1`**. We are necessarily on a compactified leaf and we don't care about its own payload as it will get + Case 2.a is now split into two subcases: + - **case 2.a.i: `node.path ∈ paths` and `len(paths) == 1`**. A new node is created with the new updated payload. This would be a leaf in the new trie. + - **case 2.a.ii: `node.path ∈ paths` and `len(paths) > 1`**. We are necessarily on a compactified leaf and we don't care about its own payload as it will get updated by the new input payload. We therefore continue the recursion with `compactLeaf = nil` and the same input paths and payloads. - - **case 1.b: `node.path ∉ path`**. If the leaf path is not found among the inputs, `node` must be a compactified leaf + - **case 2.b: `node.path ∉ paths`**. If the leaf path is not found among the inputs, `node` must be a compactified leaf (as multiple different registers fall in its respective sub-trie). We call the recursion with the same inputs but with `compactLeaf` being set to the current node. -* **Case 2: `node == nil`** : The sub-trie is empty - - **Case 2a: `node == nil` and there is only one leaf to create**, i.e. `len(paths) == 1 && compactLeaf == nil` or `len(paths) == 0 && compactLeaf ≠ nil`. - - **Case 2b: there are 2 or more leafs to create**. An interim-node is created, the paths are split into left and right, and `compactLeaf` is carried over into the left or right child. We note that this case is very similar to Case 0 where the current node is `nil`. The pseudo-code below will treat case 0 and 2.b in the same code section. +* **Case 3: `node == nil`** : The sub-trie is empty + - **Case 3a: `node == nil` and there is only one leaf to create**, i.e. (i) `len(paths) == 1 && compactLeaf == nil` or (ii) `len(paths) == 0 && compactLeaf ≠ nil`. +The sub-trie is empty before the update and will hold exactly one register after the update. So we stop the recursion and create a single compactified leaf at the current `height`: +in case (i) from the sole input `(paths[0], payloads[0])`, in case (ii) from the carried-over compactified leaf `(compactLeaf.path, compactLeaf.payload)`. + - **Case 3b: there are 2 or more leaves to create**. An interim-node is created, the paths are split into left and right, and `compactLeaf` is carried over into the left or right child. We note that this case is very similar to Case 1 where the current node is `nil`. The pseudo-code below will treat case 1 and 3.b in the same code section. **Lemma**: _Consider a trie `m` before the update. The following condition holds for the `Update` algorithm: If `compactLeaf ≠ nil` then `node == nil`._ @@ -231,13 +233,13 @@ Initially, the `Update` algorithm starts with: The initial condition satisfies the lemma. Let's consider the first recursion step where `compactLeaf` may switch from `nil` (initial value) -to a non-`nil` value. This switch happens only in Case 1.b where we replace a compactified leaf by a trie holding multiple -registers. In this case (1.b), a new interim-node with `nil` children is created, and the recursion is carried forward with `node` being set to the `nil` children. The following steps will necessary fall under case 2 since `node` is `nil`. Subcases of case 2 would always keep `node` set to `nil`. +to a non-`nil` value. This switch happens only in Case 2.b where we replace a compactified leaf by a trie holding multiple +registers. In this case (2.b), a new interim-node with `nil` children is created, and the recursion is carried forward with `node` being set to the `nil` children. The following steps will necessary fall under case 3 since `node` is `nil`. Subcases of case 3 would always keep `node` set to `nil`. Q.E.D. #### Further optimization and resource-exhaustion attack: -In order to counter a resource-exhaustion attack where an existing allocated register is being updated with the same payload, resulting in creating new unnecessary nodes, we slightly adjust step 1.a. When `len(paths)==1` and the input path is equal to the current leaf path, we only create a new leaf if the input payload is different than the one stored initially in the leaf. If the two payloads are equal, we just re-cycle the initial leaf. +In order to counter a resource-exhaustion attack where an existing allocated register is being updated with the same payload, resulting in creating new unnecessary nodes, we slightly adjust step 2.a. When `len(paths)==1` and the input path is equal to the current leaf path, we only create a new leaf if the input payload is different than the one stored initially in the leaf. If the two payloads are equal, we just re-cycle the initial leaf. Morever, we create a new interim-node from the left and right children only if the returned children are different than the original node children. If the children are equal, we just re-cycle the same interim-node. #### Putting everything together: @@ -248,7 +250,7 @@ This results in the following `Update` algorithm. When applying the updates `(pa ```golang FUNCTION Update(height Int, node Node, paths []Path, payloads []Payload, compactLeaf Node, prune bool) Node { if len(paths) == 0 { - // If a compactLeaf from a higher height is carried over, then we are necessarily in case 2.a + // If a compactLeaf from a higher height is carried over, then we are necessarily in case 3.a // (node == nil and only one register to create) if compactLeaf != nil { return NewLeaf(compactLeaf.path, compactLeaf.payload, height) @@ -257,38 +259,38 @@ FUNCTION Update(height Int, node Node, paths []Path, payloads []Payload, compact return node } - // The remaining sub-case of 2.a (node == nil and only one register to create): + // The remaining sub-case of 3.a (node == nil and only one register to create): // the register payload is the input and no compactified leaf is to be carried over. if len(paths) == 1 && node == nil && compactLeaf == nil { return NewLeaf(paths[0], payloads[0], height) } - // case 1: we reach a non-nil leaf. Per Lemma, compactLeaf is necessarily nil + // case 2: we reach a non-nil leaf. Per Lemma, compactLeaf is necessarily nil if node != nil && node.IsLeaf() { if node.path ∈ paths { - if len(paths) == 1 { // case 1.a.i + if len(paths) == 1 { // case 2.a.i // the resource-exhaustion counter-measure if !node.payload == payloads[i] { return NewLeaf(paths[i], payloads[i], height) } return node // re-cycle the same node } - // case 1.a.ii: len(paths)>1 + // case 2.a.ii: len(paths)>1 // Value of compactified leaf will be overwritten. Hence, we don't have to carry it forward. - // Case 1.a.ii is the call: Update(height, nil, paths, payload, nil), but we can optimize the extra call and just continue the function to case 2.b with the same parameters. + // Case 2.a.ii is the call: Update(height, nil, paths, payload, nil), but we can optimize the extra call and just continue the function to case 3.b with the same parameters. } else { - // case 1.b: node's path was not found among the inputs and we should carry the node to lower heights as a compactLeaf parameter. - // Case 1.b is the call: Update(height, nil, paths, payload, node), but we can optimize the extra call and just continue the function to case 2.b with + // case 2.b: node's path was not found among the inputs and we should carry the node to lower heights as a compactLeaf parameter. + // Case 2.b is the call: Update(height, nil, paths, payload, node), but we can optimize the extra call and just continue the function to case 3.b with // compactLeaf set as node. compactLeaf = node } } // The remaining logic below handles the remaining recursion step which is common for the - // case 0: node ≠ nil and there are many paths to update (len(paths)>1) - // case 1.a.ii: node ≠ nil and node.path ∈ path and len(paths) > 1 - // case 1.b: node ≠ nil and node.path ∉ path - // case 2.b: node == nil and there is more than one register to update: + // case 1: node ≠ nil and node is an interim node + // case 2.a.ii: node ≠ nil and node.path ∈ paths and len(paths) > 1 + // case 2.b: node ≠ nil and node.path ∉ paths + // case 3.b: node == nil and there is more than one register to update: // - len(paths) == 1 and compactLeaf != nil // - or alternatively len(paths) > 1 @@ -297,7 +299,7 @@ FUNCTION Update(height Int, node Node, paths []Path, payloads []Payload, compact // rpaths contains all paths that have `1` at the bit index lpaths, rpaths, lpayloads, rpayloads = Split(paths, payloads, 256 - height) - // As part of cases 1.b and 2.b, we have to determine whether compactLeaf falls into the left or right sub-trie: + // As part of cases 2.b and 3.b, we have to determine whether compactLeaf falls into the left or right sub-trie: if compactLeaf != nil { // if yes, check which branch it will go to. if Bit(compactLeaf.path, 256 - height) == 0 { @@ -307,16 +309,16 @@ FUNCTION Update(height Int, node Node, paths []Path, payloads []Payload, compact lcompactLeaf = nil rcompactLeaf = compactLeaf } - } else { // for cases 0 and 1.a.ii, we don't have a compactified leaf to carry forward + } else { // for cases 1 and 2.a.ii, we don't have a compactified leaf to carry forward lcompactLeaf = nil rcompactLeaf = nil } // the difference between cases with node ≠ nil vs the case with node == nil - if node != nil { // cases 0, 1.a.ii, and 1.b + if node != nil { // cases 1, 2.a.ii, and 2.b lchild = node.leftChild rchild = node.rightChild - } else { // case 2.b + } else { // case 3.b lchild = nil rchild = nil } @@ -341,4 +343,4 @@ nodeToBeReturned := NewInterimNode(height, newlChild, newrChild) return nodeToBeReturned } -``` \ No newline at end of file +``` diff --git a/ledger/complete/mtrie/trie/trie_test.go b/ledger/complete/mtrie/trie/trie_test.go index ca62da06de2..dcc80e02473 100644 --- a/ledger/complete/mtrie/trie/trie_test.go +++ b/ledger/complete/mtrie/trie/trie_test.go @@ -20,6 +20,9 @@ import ( ) // TestEmptyTrie tests whether the root hash of an empty trie matches the formal specification. +// The expected value originates from the standalone Merkle reference implementation (the source of +// truth): github.com/onflow/flow-internal → reference_implementations/merkle_tree.py. Run it to +// regenerate; MTrie is the implementation under test, checked against it. func Test_EmptyTrie(t *testing.T) { // Make new Trie (independently of MForest): emptyTrie := trie.NewEmptyMTrie() @@ -36,7 +39,9 @@ func Test_EmptyTrie(t *testing.T) { // Test_TrieWithLeftRegister tests whether the root hash of trie with only the left-most // register populated matches the formal specification. -// The expected value is coming from a reference implementation in python and is hard-coded here. +// The expected value originates from the standalone Merkle reference implementation (the source of +// truth): github.com/onflow/flow-internal → reference_implementations/merkle_tree.py. Run it to +// regenerate; MTrie is the implementation under test, checked against it. func Test_TrieWithLeftRegister(t *testing.T) { // Make new Trie (independently of MForest): emptyTrie := trie.NewEmptyMTrie() @@ -53,7 +58,9 @@ func Test_TrieWithLeftRegister(t *testing.T) { // Test_TrieWithRightRegister tests whether the root hash of trie with only the right-most // register populated matches the formal specification. -// The expected value is coming from a reference implementation in python and is hard-coded here. +// The expected value originates from the standalone Merkle reference implementation (the source of +// truth): github.com/onflow/flow-internal → reference_implementations/merkle_tree.py. Run it to +// regenerate; MTrie is the implementation under test, checked against it. func Test_TrieWithRightRegister(t *testing.T) { // Make new Trie (independently of MForest): emptyTrie := trie.NewEmptyMTrie() @@ -74,7 +81,9 @@ func Test_TrieWithRightRegister(t *testing.T) { // Test_TrieWithMiddleRegister tests the root hash of trie holding only a single // allocated register somewhere in the middle. -// The expected value is coming from a reference implementation in python and is hard-coded here. +// The expected value originates from the standalone Merkle reference implementation (the source of +// truth): github.com/onflow/flow-internal → reference_implementations/merkle_tree.py. Run it to +// regenerate; MTrie is the implementation under test, checked against it. func Test_TrieWithMiddleRegister(t *testing.T) { // Make new Trie (independently of MForest): emptyTrie := trie.NewEmptyMTrie() @@ -92,7 +101,9 @@ func Test_TrieWithMiddleRegister(t *testing.T) { // Test_TrieWithManyRegisters tests whether the root hash of a trie storing 12001 randomly selected registers // matches the formal specification. -// The expected value is coming from a reference implementation in python and is hard-coded here. +// The expected value originates from the standalone Merkle reference implementation (the source of +// truth): github.com/onflow/flow-internal → reference_implementations/merkle_tree.py. Run it to +// regenerate; MTrie is the implementation under test, checked against it. func Test_TrieWithManyRegisters(t *testing.T) { // Make new Trie (independently of MForest): emptyTrie := trie.NewEmptyMTrie() @@ -114,7 +125,9 @@ func Test_TrieWithManyRegisters(t *testing.T) { // Test_FullTrie tests whether the root hash of a trie, // whose left-most 65536 registers are populated, matches the formal specification. -// The expected value is coming from a reference implementation in python and is hard-coded here. +// The expected value originates from the standalone Merkle reference implementation (the source of +// truth): github.com/onflow/flow-internal → reference_implementations/merkle_tree.py. Run it to +// regenerate; MTrie is the implementation under test, checked against it. func Test_FullTrie(t *testing.T) { // Make new Trie (independently of MForest): emptyTrie := trie.NewEmptyMTrie() @@ -142,7 +155,9 @@ func Test_FullTrie(t *testing.T) { } // TestUpdateTrie tests whether iteratively updating a Trie matches the formal specification. -// The expected root hashes are coming from a reference implementation in python and is hard-coded here. +// The expected root hashes originate from the standalone Merkle reference implementation (the source +// of truth): github.com/onflow/flow-internal → reference_implementations/merkle_tree.py. Run it to +// regenerate; MTrie is the implementation under test, checked against it. func Test_UpdateTrie(t *testing.T) { expectedRootHashes := []string{ "08db9aeed2b9fcc66b63204a26a4c28652e44e3035bd87ba0ed632a227b3f6dd", @@ -223,7 +238,9 @@ func Test_UpdateTrie(t *testing.T) { // Test_UnallocateRegisters tests whether unallocating registers matches the formal specification. // Unallocating here means, to set the stored register value to an empty byte slice. -// The expected value is coming from a reference implementation in python and is hard-coded here. +// The expected value originates from the standalone Merkle reference implementation (the source of +// truth): github.com/onflow/flow-internal → reference_implementations/merkle_tree.py. Run it to +// regenerate; MTrie is the implementation under test, checked against it. func Test_UnallocateRegisters(t *testing.T) { rng := &LinearCongruentialGenerator{seed: 0} emptyTrie := trie.NewEmptyMTrie() diff --git a/ledger/complete/payloadless/node.go b/ledger/complete/payloadless/node.go index f84782000bb..7052cab442c 100644 --- a/ledger/complete/payloadless/node.go +++ b/ledger/complete/payloadless/node.go @@ -11,17 +11,25 @@ import ( // Node defines a payloadless Mtrie node. // // Unlike the regular mtrie Node which stores full payloads, a payloadless Node -// stores only the leaf hash (HashLeaf(path, value)) for leaf nodes. This enables -// significant memory savings while preserving the same root hash as a full trie. +// stores only the leaf hash for leaf nodes. This enables significant memory savings +// while preserving the same root hash as a full trie. // // DEFINITIONS: -// - HEIGHT of a node v in a tree is the number of edges on the longest -// downward path between v and a tree leaf. +// - HEIGHT of a node v in a tree is the number of edges on the longest downward +// path between v and a tree leaf (in a hypothetical, fully expanded (perfect) +// tree, i.e. without compactification or nil-pruning). // // Conceptually, an MTrie is a sparse Merkle Trie, which has two node types: // - INTERIM node: has at least one child (i.e. lChild or rChild is not // nil). Interim nodes do not store a path and have no leafHash. -// - LEAF node: has _no_ children. Stores a path and (optionally) a leafHash. +// - LEAF node: has _no_ children. It represents a single register, storing a path and +// the ability to provide the hash commitment of the register's content. Nodes at height +// 𝒽 = 0 are always leaves. Nodes with 𝒽 > 0 are either leaves or interim nodes. A leaf at +// height 𝒽 represents a height-𝒽 subtree that holds a single allocated register. Because +// such a subtree contains no branches, it is collapsed into one node — a COMPACTIFIED LEAF +// (per mtrie/README.md the term spans all heights 𝒽 ≥ 0). Compactification is optional and +// always root-hash-preserving. See documentation of `Node.leafHash` and `Node.hashValue` +// for storage and hashing details. // // Per convention, we also consider nil as a leaf. Formally, nil is the generic // representative for any empty (sub)-trie (i.e. a trie without allocated @@ -31,21 +39,60 @@ import ( // TODO: optimized data structures might be able to reduce memory consumption type Node struct { // Implementation Comments: - // Formally, a tree can hold up to 2^maxDepth number of registers. However, + // Formally, a tree of height 𝒽 can hold up to 2^𝒽 number of registers. However, // the current implementation is designed to operate on a sparsely populated // tree, holding much less than 2^64 registers. - lChild *Node // Left Child - rChild *Node // Right Child - height int // height where the Node is at - path ledger.Path // the storage path (dummy value for interim nodes) - leafHash *hash.Hash // HashLeaf(path, value) - the height-0 leaf hash (leaf nodes only; nil for unallocated registers) - hashValue hash.Hash // hash value of node (cached) + height int // height where the Node is at (root node has largest height of 256) + lChild *Node // Left Child (nil for all leaves, including compactified leaves of height > 0) + rChild *Node // Right Child (nil for all leaves, including compactified leaves of height > 0) + path ledger.Path // the storage path (dummy value for interim nodes) + + // leafHash is nil for all non-leaf nodes. For leaves (node with both children nil), leafHash + // is nil if and only if the leaf represents an unallocated register (value is nil or empty). + // Formally: + // ╭ hash(path, value) if len(value) > 0 + // leafHash = ┥ + // ╰ nil otherwise + leafHash *hash.Hash + + // hashValue is the cached hash of this node (always set for every node). By construction, + // a node's hashValue equals the hash that the equivalent node would have in the fully-expanded + // (perfect) tree; compactification and nil-pruning are chosen precisely to preserve this + // value (see mtrie/README.md for details), which is why they leave the root hash unchanged. + // + // We specify hashValue per node *type* (independent of compactification or nil-pruning). Here + // DefaultHashForHeight(𝒽) denotes the hash of a subtree with height 𝒽 holding only unallocated + // registers (defined recursively in mtrie/README.md). DefaultHashForHeight is independent of + // the sub-trie's location in the overall trie (identical for all empty subtrees of height 𝒽). + // + // • LEAF node (both children nil) at height 𝒽, for register (path, value): + // ╭ subtree-root hash for {(path, value)} if len(value) > 0 (allocated) + // hashValue = ┥ + // ╰ DefaultHashForHeight(𝒽) if len(value) == 0 (unallocated) + // where the `subtree-root hash for {(path, value)}` is the hash of the height-𝒽 subtree that + // holds only this single register: start from the height-0 leaf with hash(path, value) and + // hash upward 𝒽 levels, at each level combining with the default hash of the (empty) sibling + // subtree. At 𝒽 == 0 the upward climb is empty, so this is just hash(path, value). + // + // • INTERIM node (at least one non-nil child) at height 𝒽 > 0: + // hashValue = hash( lChild.hashValue , rChild.hashValue ) + // where a nil child contributes DefaultHashForHeight(𝒽-1) (the hash of the empty + // subtree of height 𝒽-1 it represents). + // + // Notes: + // • For an allocated register, the height-0 leaf's hashValue equals its leafHash by design. + // • hashValue == DefaultHashForHeight(𝒽) + // ⟺ the sub-tree rooted at this node holds only unallocated registers + // ⟺ every node in that sub-tree has leafHash == nil + // (the forward direction relies on our requirement of a collision-resistant hash function). + hashValue hash.Hash } // NewNode creates a new Node. -// UNCHECKED requirement: combination of values must conform to -// a valid node type (see documentation of `Node` for details) +// CAUTION: INSECURE! Only intended to reconstruct Nodes from their serialization! +// UNCHECKED requirement: combination of values must conform to a valid node type (see +// documentation of `Node` for details) func NewNode(height int, lchild, rchild *Node, @@ -64,36 +111,114 @@ func NewNode(height int, return n } -// NewLeaf creates a leaf Node from a path and the original payload value. -// The leafHash is computed as HashLeaf(path, value), and the node hash is -// computed using the original value to ensure the same root hash as a full trie. +// NewLeaf constructs the leaf Node 𝓃 representing the single register (path, value) at the +// given height 𝒽. By construction, 𝓃.Hash() equals the hash that the height-𝒽 subtree +// containing the register would have in the fully-expanded trie. In other words, tries built +// from these constructors share the same root hash as a fully-expanded trie. That subtree hash +// is computed by compactification: the recursive application of hashing (see mtrie/README.md for +// details). It starts from the register's height-0 leaf hash and hashes upward 𝒽 levels, +// combining at each level with the default hash of the empty sibling subtree. // -// UNCHECKED requirement: height must be non-negative +// A `nil` or empty `value` denotes an unallocated register; the result is the default node for the +// given height (hash is DefaultHashForHeight(height) independent of path; see newDefaultLeaf for details). +// +// It is safe for `path` or `value` to be mutated after the call to `NewLeaf` returns. +// +// UNCHECKED requirement: height must be non-negative. func NewLeaf(path ledger.Path, value []byte, height int) *Node { - // For empty values, create a default node - if len(value) == 0 { - return &Node{ - height: height, - path: path, - leafHash: nil, - hashValue: ledger.GetDefaultHashForHeight(height), - } + if len(value) == 0 { // For empty values, create a default node + return newDefaultLeaf(path, height) } - // Compute the leaf hash (height-0) - leafHash := hash.HashLeaf(hash.Hash(path), value) + // Leaf represent an allocated register: + leafHash := hash.HashLeaf(hash.Hash(path), value) // we pre-compute leaf hash at height-0 here + return newLeafWithHash(path, leafHash, height) // handles compactification up to given height if necessary +} - return NewLeafWithHash(path, leafHash, height) +// newDefaultLeaf constructs the default node, which represents an unallocated register (`nil` or empty value) +// compactified to a trie node at the given height. Its leafHash is nil and its hash is DefaultHashForHeight(height). +// +// Note that the hash of an empty subtree (at any height) is technically independent of path. However, subtries +// containing only unallocated registers should be replaced by nil in a compactified trie. Explicitly creating a +// default node is only useful if the caller wants to deliberately represent a specific register that is +// not yet allocated. Explicitly representing specific unallocated registers is an interim shortcut until we have +// specialized (more efficient) non-inclusion proofs implemented (at the moment, non-inclusion proofs fall back +// on inclusion proofs of explicitly represented default nodes). +// +// It is safe for `path` to be mutated after this function returns. +// +// UNCHECKED requirement: height must be non-negative. +func newDefaultLeaf(path ledger.Path, height int) *Node { + return &Node{ + height: height, + path: path, + leafHash: nil, + hashValue: ledger.GetDefaultHashForHeight(height), + } } -// NewLeafWithHash creates a leaf Node from a pre-computed leaf hash. -// This is used when converting from a full trie or loading from a payloadless checkpoint. +// NewRelevelledLeaf creates a new compactified leaf 𝓁' for the same register (path, value) as the input +// leaf 𝓁, re-levelled to height `relevellingHeight`. This is needed when a register r is allocated or +// removed in the neighbourhood of 𝓁, changing the height at which 𝓁 can be compactified. Example: +// +// trie without r trie with r // +// parent parent +// ╱ ╲ ╱ ╲ +// 𝓁 △ ◀──▶ ◯ △ height 𝒽 +// ┊ ╱ ╲ +// ┊ 𝓁' 𝓃 height 𝒽-1 +// ┊ ┊ +// • • height 0 (fully-expanded perfect trie) +// +// parent : genuine branch at height 𝒽+1; its other child △ is a non-empty sibling subtree +// (the reason 𝓁 sits at height 𝒽, not higher). Unchanged by allocating r. +// ◯ : interim node materialized at 𝓁's former position ( height-𝒽 ). +// 𝓁, 𝓁' : the SAME register (path, value); 𝓁' is 𝓁 re-levelled to a lower height 𝒽' (𝒽-1 shown). +// 𝓃 : compactified leaf representing register r. +// • : the register's actual leaf at height 0 in the fully-expanded (perfect) trie. +// dotted : single-child perfect-trie path (┊) that compactification collapses into one node. +// +// Implementation correctly handles leaves that represent either allocated or unallocated registers. For an +// unallocated register represented by an explicit default leaf (which carries the register's path), a new +// default leaf at `relevellingHeight` is created; this is useful for our shortcut for non-inclusion proofs +// utilizing explicitly represented default leaf nodes. A `nil` input yields a `nil` result. +// +// UNCHECKED requirement: `leaf.IsLeaf()` must be true +func NewRelevelledLeaf(leaf *Node, relevellingHeight int) *Node { + // If the leaf represents an unallocated register, return the default node at the relevelling height. + // Implementation details: + // - A nil input represents an empty sub-trie that carries no register path. By convention a default leaf + // must carry the path of the register it represents, which a nil input cannot supply, so we return nil + // (an empty sub-trie stays empty). + // - A default leaf (leafHash == nil, i.e. an unallocated register) is relevelled to the default node at height `relevellingHeight`. + // The resulting node's hash is `DefaultHashForHeight(relevellingHeight)`; it is path-independent, but height-dependent. + // - For an allocated register (leafHash ≠ nil), the `leafHash` by convention always contains the hash of the fully-expanded leaf at + // height 0. The hash of any compactified leaf is computed by iteratively hashing the `leafHash` together with the + // default hash at the corresponding height (sibling subtree representing only empty registers). + if leaf == nil { // empty sub-trie carries no register path => cannot form a path-bearing default leaf; stays empty + return nil + } + if leaf.leafHash == nil { // leaf.leafHash is nil ⟺ unallocated register ⟺ node is a default leaf + return newDefaultLeaf(leaf.path, relevellingHeight) + } + if relevellingHeight == leaf.height { // same height => no relevelling needed + return leaf + } + + // Leaf represent an allocated register: + return newLeafWithHash(leaf.path, *leaf.leafHash, relevellingHeight) // handles compactification up to given relevellingHeight if necessary +} + +// newLeafWithHash creates a leaf Node from a pre-computed leaf hash. +// This is used when converting from a full trie or loading from a payloadless checkpoint. // The nodeHash is computed by extending the leafHash (height-0) to the specified height. // +// It is safe for `path` to be mutated after this function returns. +// // UNCHECKED requirement: height must be non-negative // UNCHECKED requirement: leafHash must be HashLeaf(path, originalValue) -func NewLeafWithHash(path ledger.Path, leafHash hash.Hash, height int) *Node { +func newLeafWithHash(path ledger.Path, leafHash hash.Hash, height int) *Node { // Compute the node hash by extending the leaf hash to the target height nodeHash := ledger.ComputeCompactValueFromLeafHash(hash.Hash(path), leafHash, height) @@ -118,7 +243,7 @@ func NewInterimNode(height int, lChild, rChild *Node) *Node { return n } -// NewInterimCompactifiedNode creates a new compactified interim Node. For compactification, +// NewInterimCompactifiedNode creates a new interim Node - compactified if possible. For compactification, // we only consider the immediate children. When starting with a maximally pruned trie and // creating only InterimCompactifiedNodes during an update, the resulting trie remains maximally // pruned. Details on compactification: @@ -140,7 +265,7 @@ func NewInterimCompactifiedNode(height int, lChild, rChild *Node) *Node { // CASE (a): _both_ children do _not_ contain any allocated registers: if lChild == nil && rChild == nil { - return nil // return nil representing as completely empty sub-trie + return nil // return nil representing a completely empty sub-trie } // CASE (b): one child is a compactified leaf (single allocated register) _and_ the other child represents @@ -154,13 +279,16 @@ func NewInterimCompactifiedNode(height int, lChild, rChild *Node) *Node { return &Node{height: height, path: rChild.path, leafHash: rChild.leafHash, hashValue: h} } - // CASE (b): both children contain some allocated registers => we can't compactify; return a full interim leaf + // CASE (c): both children contain some allocated registers => we can't compactify; return a full interim node return NewInterimNode(height, lChild, rChild) } // IsDefaultNode returns true iff the sub-trie represented by this root node contains -// only unallocated registers. This is the case, if the node is nil or the node's hash -// is equal to the default hash value at the respective height. +// only unallocated registers. This is the case, if and only if the node is nil or the +// node's hash is equal to the default hash value at the respective height. +// +// This function is universally applicable, irrespective of compactification, nil-pruning, +// or whether the node is a leaf or an interim node. func (n *Node) IsDefaultNode() bool { if n == nil { return true @@ -168,6 +296,29 @@ func (n *Node) IsDefaultNode() bool { return n.hashValue == ledger.GetDefaultHashForHeight(n.height) } +// IsAllocatedRegisterLeaf reports whether this node is a leaf representing an allocated register +// (equivalently, a non-default compactified leaf, at any height 𝒽 ≥ 0). It is a computationally very lightweight +// check intended to be run on leaf nodes, optimized for minimal computational cost rather than universal applicability. +// +// On leaves it coincides with the negation of `IsDefaultNode`: +// +// 𝓃.IsAllocatedRegisterLeaf() = ¬ 𝓃.IsDefaultNode() for any node 𝓃 with 𝓃.IsLeaf() == true +// +// CAUTION: +// - For non-leaf nodes, this function always returns false. We do *not* check whether the node could be +// compactified into a non-default leaf. This is a deliberate trade-off for minimal computational cost. +// - A false result therefore does NOT imply the sub-trie is empty/default: any interim node returns +// false regardless of the subtree beneath it. To test whether a sub-trie is empty/default, use the +// universally-applicable `IsDefaultNode`. +func (n *Node) IsAllocatedRegisterLeaf() bool { + if n == nil { + return false + } + return n.leafHash != nil +} + +// a height-0 allocated leaf is a degenerate compactified leaf, so returning true + // computeHash returns the hashValue of the node func (n *Node) computeHash() hash.Hash { // check for leaf node @@ -180,7 +331,7 @@ func (n *Node) computeHash() hash.Hash { return ledger.GetDefaultHashForHeight(n.height) } - // this is an interim node at least one of lChild or rChild is not nil. + // this is an interim node; at least one of lChild or rChild is non-nil. var h1, h2 hash.Hash if n.lChild != nil { h1 = n.lChild.Hash() @@ -196,7 +347,8 @@ func (n *Node) computeHash() hash.Hash { return hash.HashInterNode(h1, h2) } -// VerifyCachedHash verifies the hash of a node is valid +// verifyCachedHashRecursive recursively verifies that every node in the subtree +// rooted at n has a cached hashValue matching its recomputed hash. func verifyCachedHashRecursive(n *Node) bool { if n == nil { return true @@ -209,13 +361,16 @@ func verifyCachedHashRecursive(n *Node) bool { return n.hashValue == computedHash } -// VerifyCachedHash verifies the hash of a node is valid +// VerifyCachedHash verifies that every node in the subtree rooted at this node has +// a cached `hashValue` matching its freshly recomputed hash. +// CAUTION: this recomputes the hash of every node in the subtree and is therefore +// very expensive on large tries. func (n *Node) VerifyCachedHash() bool { return verifyCachedHashRecursive(n) } -// Hash returns the Node's hash value. -// Do NOT MODIFY returned slice! +// Hash returns the Node's cached hash value, which is a fixed-size array, so the +// returned value is a copy; mutating it does not affect the Node. func (n *Node) Hash() hash.Hash { return n.hashValue } @@ -255,7 +410,7 @@ func (n *Node) RightChild() *Node { return n.rChild } // IsLeaf returns true if and only if Node is a LEAF. func (n *Node) IsLeaf() bool { - // Per definition, a node is a leaf if and only it has no children + // Per definition, a node is a leaf if and only if it has no children return n == nil || (n.lChild == nil && n.rChild == nil) } diff --git a/ledger/complete/payloadless/node_test.go b/ledger/complete/payloadless/node_test.go new file mode 100644 index 00000000000..f0e0230169d --- /dev/null +++ b/ledger/complete/payloadless/node_test.go @@ -0,0 +1,820 @@ +package payloadless + +// White-box tests for the payloadless Node constructors. They live in `package payloadless` +// (not `payloadless_test`) so they can exercise the un-exported constructors `newDefaultLeaf` +// and `newLeafWithHash` and inspect internal fields (`leafHash`, `path`, `height`, `hashValue`, +// `lChild`, `rChild`) directly. +// +// Hash-correctness is verified three ways: +// - Unrolled independent composition for small heights (0,1,2): we spell out the expected hash by +// hand — nesting `HashInterNode` level by level with empty siblings — rather than reusing the +// production loop in `ledger.ComputeCompactValueFromLeafHash`. Using left-, right-, and +// mixed-branching paths, this is an independent cross-check that the constructor computes the +// same hash. +// - Cross-construction equivalences: relevelling a leaf must equal freshly constructing it at the +// target height; a compactified leaf must equal its fully-expanded `NewInterimNode` reference. +// - Python reference-implementation anchors: node hashes for fixed (path, value, height) inputs, +// independently computed by the standalone Python Merkle reference implementation (the source of +// truth) and hard-coded here as an oracle. A compactified leaf at height h holding one register +// has the same hash as the root of a height-h subtree containing only it, so the reference +// computes these node hashes from first principles (its node-level scenario section prints them). +// Reference: github.com/onflow/flow-internal → reference_implementations/merkle_tree.py; run it to +// regenerate. This is the same convention the sibling mtrie trie tests use (mtrie/trie/trie_test.go, +// e.g. Test_TrieWithLeftRegister). A mismatch means this Go implementation and the reference +// disagree (e.g. a change in the hashing primitives or default-hash table). + +import ( + "encoding/hex" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/hash" + "github.com/onflow/flow-go/ledger/common/testutils" +) + +// Shared path fixtures covering the three branching regimes a compactified leaf's fold can take. A +// leaf at height h folds up h levels; at each level the path bit selects which side the (default) +// empty sibling sits on. Testing only uniform paths (all-left or all-right) would miss a wrong-bit- +// index bug, since both boundaries are symmetric under such a bug; a mixed path is the discriminator. +// - `pathLeft` = all bits 0 → left branch at every height (empty sibling always on the RIGHT); +// - `pathRight` = all bits 1 → right branch at every height (empty sibling always on the LEFT); +// - `pathMixed` = 0x…0135 → interleaved: right,left,right,left,right,right,left,left,right over +// heights 1..9 (bit(255)=1, bit(254)=0, …). +var ( + pathLeft = testutils.PathByUint16LeftPadded(0) // 32 bytes 0x00 + // pathRight is the all-ones path (32 bytes 0xFF): branches right at every height. + pathRight = ledger.Path{ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + } + pathMixed = testutils.PathByUint16LeftPadded(0x135) // ...0000_0001_0011_0101 + value = []byte{0x9a, 0xbc, 0xde} +) + +// pathFixture names one of the three branching-regime path fixtures above. +type pathFixture struct { + name string + path ledger.Path +} + +// branchRegimePaths is the left-only, right-only, and mixed path fixtures, for sweeping any +// branch-sensitive scenario across all three regimes. +var branchRegimePaths = []pathFixture{ + {"pathLeft", pathLeft}, + {"pathRight", pathRight}, + {"pathMixed", pathMixed}, +} + +// --------------------------------------------------------------------------------------------- +// NewLeaf +// --------------------------------------------------------------------------------------------- + +// Test_NewLeaf_Allocated verifies NewLeaf for a non-empty value (an allocated register). At height 0 +// the node hash equals the height-0 leaf hash; at height > 0 it equals the fully-expanded reference. +func Test_NewLeaf_Allocated(t *testing.T) { + leafHashL := hash.HashLeaf(hash.Hash(pathLeft), value) // height-0 leaf hash for pathLeft + leafHashR := hash.HashLeaf(hash.Hash(pathRight), value) // ... for pathRight + leafHashM := hash.HashLeaf(hash.Hash(pathMixed), value) // ... for pathMixed + + t.Run("height 0 is an uncompactified leaf", func(t *testing.T) { + n := NewLeaf(pathLeft, value, 0) + require.True(t, n.IsLeaf()) + require.Nil(t, n.lChild) + require.Nil(t, n.rChild) + require.Equal(t, 0, n.height) + require.Equal(t, pathLeft, *n.Path()) + require.NotNil(t, n.leafHash) + require.Equal(t, leafHashL, *n.leafHash) // leafHash == HashLeaf(path, value) + require.Equal(t, leafHashL, n.Hash()) // at height 0, node hash == leaf hash + require.False(t, n.IsDefaultNode()) + require.True(t, n.VerifyCachedHash()) + }) + + // Unrolled independent references for small heights: the empty-sibling placement is spelled out by + // hand for each branching regime — left (sibling right), right (sibling left), and mixed. + t.Run("height 1, left branch (pathLeft): sibling on the right", func(t *testing.T) { + n := NewLeaf(pathLeft, value, 1) + want := hash.HashInterNode(leafHashL, ledger.GetDefaultHashForHeight(0)) + require.Equal(t, want, n.Hash()) + require.Equal(t, leafHashL, *n.leafHash) // leafHash unchanged by compactification + require.True(t, n.VerifyCachedHash()) + }) + + t.Run("height 1, right branch (pathRight): sibling on the left", func(t *testing.T) { + n := NewLeaf(pathRight, value, 1) + want := hash.HashInterNode(ledger.GetDefaultHashForHeight(0), leafHashR) + require.Equal(t, want, n.Hash()) + require.True(t, n.VerifyCachedHash()) + }) + + t.Run("height 2, both right (pathRight)", func(t *testing.T) { + n := NewLeaf(pathRight, value, 2) + h1 := hash.HashInterNode(ledger.GetDefaultHashForHeight(0), leafHashR) // height 1: right + want := hash.HashInterNode(ledger.GetDefaultHashForHeight(1), h1) // height 2: right + require.Equal(t, want, n.Hash()) + require.True(t, n.VerifyCachedHash()) + }) + + t.Run("height 2, mixed right-then-left (pathMixed)", func(t *testing.T) { + n := NewLeaf(pathMixed, value, 2) + h1 := hash.HashInterNode(ledger.GetDefaultHashForHeight(0), leafHashM) // height 1: right (bit 255 = 1) + want := hash.HashInterNode(h1, ledger.GetDefaultHashForHeight(1)) // height 2: left (bit 254 = 0) + require.Equal(t, want, n.Hash()) + require.True(t, n.VerifyCachedHash()) + }) + + t.Run("compactified leaf equals fully-expanded interim reference (all branch regimes)", func(t *testing.T) { + for _, p := range branchRegimePaths { + for _, height := range []int{1, 9, 256} { + ref := fullyExpandedAllocatedRef(t, p.path, value, height) + n := NewLeaf(p.path, value, height) + require.Equal(t, ref, n.Hash(), "compactified leaf must equal fully-expanded reference: %s @ height %d", p.name, height) + require.True(t, n.VerifyCachedHash()) + } + } + }) +} + +// Test_NewLeaf_Empty verifies NewLeaf for empty and nil values (an unallocated register); the result +// is the default node whose hash is DefaultHashForHeight(height), independent of path. +func Test_NewLeaf_Empty(t *testing.T) { + testCases := []struct { + name string + value []byte + }{ + {"nil value", nil}, + {"empty value", []byte{}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for _, height := range []int{0, 9, 256} { + n := NewLeaf(pathLeft, tc.value, height) + require.True(t, n.IsLeaf()) + require.Nil(t, n.leafHash) + require.Equal(t, ledger.GetDefaultHashForHeight(height), n.Hash()) + require.True(t, n.IsDefaultNode()) + require.Equal(t, pathLeft, *n.Path()) + require.True(t, n.VerifyCachedHash()) + } + }) + } +} + +// Test_NewLeaf_InputImmutability verifies the docstring guarantee that `path` and `value` may be +// mutated by the caller after NewLeaf returns without affecting the constructed node. +func Test_NewLeaf_InputImmutability(t *testing.T) { + t.Run("allocated leaf", func(t *testing.T) { + p := testutils.PathByUint16LeftPadded(7) + v := []byte{0x01, 0x02, 0x03, 0x04} + n := NewLeaf(p, v, 5) + hBefore := n.Hash() + pathBefore := *n.Path() + + p[0] ^= 0xFF // mutate the caller's path array + p[31] ^= 0xFF + v[0] ^= 0xFF // mutate the caller's value slice + + require.Equal(t, hBefore, n.Hash(), "node hash must not change when caller mutates inputs") + require.Equal(t, pathBefore, *n.Path(), "node path must not change when caller mutates inputs") + }) + + t.Run("default leaf", func(t *testing.T) { + p := testutils.PathByUint16LeftPadded(7) + n := NewLeaf(p, nil, 5) + pathBefore := *n.Path() + p[0] ^= 0xFF + require.Equal(t, pathBefore, *n.Path()) + require.True(t, n.IsDefaultNode()) + }) +} + +// --------------------------------------------------------------------------------------------- +// newDefaultLeaf +// --------------------------------------------------------------------------------------------- + +// Test_newDefaultLeaf verifies the default-node constructor: nil leafHash, hash == DefaultHashForHeight, +// and path-independence of the hash. +func Test_newDefaultLeaf(t *testing.T) { + for _, height := range []int{0, 9, 256} { + n := newDefaultLeaf(pathLeft, height) + require.True(t, n.IsLeaf()) + require.Nil(t, n.lChild) + require.Nil(t, n.rChild) + require.Nil(t, n.leafHash) + require.Equal(t, height, n.height) + require.Equal(t, pathLeft, *n.Path()) + require.Equal(t, ledger.GetDefaultHashForHeight(height), n.Hash()) + require.True(t, n.IsDefaultNode()) + require.True(t, n.VerifyCachedHash()) + } + + t.Run("hash is path-independent", func(t *testing.T) { + a := newDefaultLeaf(pathLeft, 9) + b := newDefaultLeaf(pathRight, 9) + require.Equal(t, a.Hash(), b.Hash(), "default-node hash must not depend on path") + }) +} + +// --------------------------------------------------------------------------------------------- +// newLeafWithHash +// --------------------------------------------------------------------------------------------- + +// Test_newLeafWithHash verifies constructing a leaf from a pre-computed height-0 leaf hash, and that +// it is consistent with NewLeaf (which derives the leaf hash from (path, value) internally). +func Test_newLeafWithHash(t *testing.T) { + leafHash := hash.HashLeaf(hash.Hash(pathLeft), value) + + t.Run("stores leaf hash and computes node hash", func(t *testing.T) { + for _, height := range []int{0, 1, 9} { + n := newLeafWithHash(pathLeft, leafHash, height) + require.NotNil(t, n.leafHash) + require.Equal(t, leafHash, *n.leafHash) + require.Equal(t, ledger.ComputeCompactValueFromLeafHash(hash.Hash(pathLeft), leafHash, height), n.Hash()) + require.Equal(t, pathLeft, *n.Path()) + require.True(t, n.VerifyCachedHash()) + } + }) + + t.Run("height 0 node hash equals the leaf hash", func(t *testing.T) { + n := newLeafWithHash(pathLeft, leafHash, 0) + require.Equal(t, leafHash, n.Hash()) + }) + + t.Run("consistent with NewLeaf (all branch regimes)", func(t *testing.T) { + for _, p := range branchRegimePaths { + lh := hash.HashLeaf(hash.Hash(p.path), value) + for _, height := range []int{0, 1, 9, 256} { + viaHash := newLeafWithHash(p.path, lh, height) + viaValue := NewLeaf(p.path, value, height) + require.Equal(t, viaValue.Hash(), viaHash.Hash(), "%s @ height %d", p.name, height) + require.Equal(t, *viaValue.leafHash, *viaHash.leafHash) + } + } + }) +} + +// --------------------------------------------------------------------------------------------- +// NewRelevelledLeaf +// --------------------------------------------------------------------------------------------- + +// Test_NewRelevelledLeaf_Allocated verifies re-levelling an allocated leaf: the height-0 leaf hash is +// preserved, and the re-levelled node equals a leaf freshly constructed at the target height. +// +// Re-levelling depends only on the preserved height-0 leaf hash and the target height, so the result +// must equal a fresh construction at the target REGARDLESS of whether the target is LOWER than, EQUAL +// to, or HIGHER than the origin leaf's own height. We therefore sweep origins at several heights and +// relevel each to targets below, at, and above its own height — across all three branching regimes +// (the target-height fold places empty siblings on path-dependent sides). +func Test_NewRelevelledLeaf_Allocated(t *testing.T) { + heights := []int{0, 1, 9, 256} + for _, p := range branchRegimePaths { + for _, originHeight := range heights { + origin := NewLeaf(p.path, value, originHeight) + for _, target := range heights { + direction := "equal" + if target < originHeight { + direction = "down" + } else if target > originHeight { + direction = "up" + } + t.Run(fmt.Sprintf("%s origin h%d -> target h%d (%s)", p.name, originHeight, target, direction), func(t *testing.T) { + relevelled := NewRelevelledLeaf(origin, target) + fresh := NewLeaf(p.path, value, target) + require.NotNil(t, relevelled.leafHash) + require.Equal(t, *origin.leafHash, *relevelled.leafHash, "height-0 leaf hash must be preserved") + require.Equal(t, fresh.Hash(), relevelled.Hash(), "relevel(origin, h) must equal NewLeaf(path,value,h)") + require.Equal(t, p.path, *relevelled.Path()) + require.True(t, relevelled.VerifyCachedHash()) + }) + } + } + } +} + +// Test_NewRelevelledLeaf_Default verifies re-levelling an unallocated (default) leaf produces the +// default node at the target height, carrying the SAME register path as the input. +// +// We sweep origins at several heights and relevel each to targets below, at, and above its own height. +// We also sweep all three branching regimes (left/right/mixed): treating the function as a black box, the +// path is an input we must check is preserved, not assume is passed through. This is not redundant — note +// pathLeft is the all-zero path, which aliases ledger.DummyPath, so a bug that dropped the register path +// and substituted DummyPath would go undetected if we only tested pathLeft. The non-zero regimes catch it. +func Test_NewRelevelledLeaf_Default(t *testing.T) { + heights := []int{0, 1, 9, 256} + for _, p := range branchRegimePaths { + for _, originHeight := range heights { + origin := newDefaultLeaf(p.path, originHeight) + for _, target := range heights { + direction := "equal" + if target < originHeight { + direction = "down" + } else if target > originHeight { + direction = "up" + } + t.Run(fmt.Sprintf("%s origin h%d -> target h%d (%s)", p.name, originHeight, target, direction), func(t *testing.T) { + relevelled := NewRelevelledLeaf(origin, target) + require.Nil(t, relevelled.leafHash) + require.Equal(t, ledger.GetDefaultHashForHeight(target), relevelled.Hash()) + require.True(t, relevelled.IsDefaultNode()) + require.Equal(t, p.path, *relevelled.Path(), "register path must be preserved") + require.True(t, relevelled.VerifyCachedHash()) + }) + } + } + } +} + +// Test_NewRelevelledLeaf_Nil verifies re-levelling a nil input. A nil represents an empty sub-trie that +// carries no register path. By convention a default leaf must carry the path of the register it represents, +// which a nil input cannot supply, so NewRelevelledLeaf returns nil (an empty sub-trie stays empty), +// regardless of the target height. The "same vs different height" axis does not apply: a nil input carries +// no height of its own. +func Test_NewRelevelledLeaf_Nil(t *testing.T) { + for _, target := range []int{0, 1, 9, 256} { + t.Run(fmt.Sprintf("target h%d", target), func(t *testing.T) { + require.Nil(t, NewRelevelledLeaf(nil, target), "nil input represents an empty sub-trie => nil output") + }) + } +} + +// --------------------------------------------------------------------------------------------- +// NewInterimCompactifiedNode +// --------------------------------------------------------------------------------------------- + +// Test_Compactify_EmptySubtrie: both children represent empty sub-tries => the compactified interim +// node is nil (a completely empty sub-trie). +func Test_Compactify_EmptySubtrie(t *testing.T) { + // n3 + // / \ + // n1(-) n2(-) + n1 := NewLeaf(testutils.PathByUint16LeftPadded(0), nil, 4) // path: ...0000 0000 + n2 := NewLeaf(testutils.PathByUint16LeftPadded(1<<4), nil, 4) // path: ...0001 0000 + + t.Run("both children empty", func(t *testing.T) { + require.Nil(t, NewInterimCompactifiedNode(5, n1, n2)) + }) + t.Run("one child nil and one child empty", func(t *testing.T) { + require.Nil(t, NewInterimCompactifiedNode(5, nil, n2)) + require.Nil(t, NewInterimCompactifiedNode(5, n1, nil)) + }) + t.Run("both children nil", func(t *testing.T) { + require.Nil(t, NewInterimCompactifiedNode(5, nil, nil)) + }) +} + +// Test_Compactify_ToLeaf: one child empty and the other a single allocated register => the +// compactified interim node is a leaf, hash-invariant against the fully-expanded reference. +func Test_Compactify_ToLeaf(t *testing.T) { + path1 := testutils.PathByUint16LeftPadded(0) // ...0000 0000 + path2 := testutils.PathByUint16LeftPadded(1 << 4) // ...0001 0000 + valueA := []byte{0x02, 0x02} + + t.Run("left child empty", func(t *testing.T) { + // n3 + // / \ + // n1(-) n2(A) + n1 := NewLeaf(path1, nil, 4) + n2 := NewLeaf(path2, valueA, 4) + n3 := NewInterimNode(5, n1, n2) + + nn3 := NewInterimCompactifiedNode(5, n1, n2) + requireIsLeafWithHash(t, nn3, n3.Hash()) + require.Equal(t, n2.leafHash, nn3.leafHash) // allocated register's leaf hash carried over + + nn3 = NewInterimCompactifiedNode(5, nil, n2) + requireIsLeafWithHash(t, nn3, n3.Hash()) + }) + + t.Run("right child empty", func(t *testing.T) { + // n3 + // / \ + // n1(A) n2(-) + n1 := NewLeaf(path1, valueA, 4) + n2 := NewLeaf(path2, nil, 4) + n3 := NewInterimNode(5, n1, n2) + + nn3 := NewInterimCompactifiedNode(5, n1, n2) + requireIsLeafWithHash(t, nn3, n3.Hash()) + require.Equal(t, n1.leafHash, nn3.leafHash) + + nn3 = NewInterimCompactifiedNode(5, n1, nil) + requireIsLeafWithHash(t, nn3, n3.Hash()) + }) +} + +// Test_Compactify_EmptyChild: one child empty and the other holds multiple allocated registers => +// the empty sub-trie is replaced by nil while the root hash is preserved. +func Test_Compactify_EmptyChild(t *testing.T) { + valueA := []byte{0x02, 0x02} + valueB := []byte{0x04, 0x04} + + t.Run("right child empty", func(t *testing.T) { + // n5 + // / \ + // n3 n4(-) + // / \ + // n1(A) n2(B) + n1 := NewLeaf(testutils.PathByUint16LeftPadded(0), valueA, 4) // path: ...0000 0000 + n2 := NewLeaf(testutils.PathByUint16LeftPadded(1<<4), valueB, 4) // path: ...0001 0000 + n3 := NewInterimNode(5, n1, n2) + n4 := NewLeaf(testutils.PathByUint16LeftPadded(3<<4), nil, 5) // path: ...0011 0000 + n5 := NewInterimNode(6, n3, n4) + + nn5 := NewInterimCompactifiedNode(6, n3, n4) + require.Equal(t, n3, nn5.LeftChild()) + require.Nil(t, nn5.RightChild()) + require.True(t, nn5.VerifyCachedHash()) + require.Equal(t, n5.Hash(), nn5.Hash()) + }) + + t.Run("left child empty", func(t *testing.T) { + // n5 + // / \ + // n3(-) n4 + // / \ + // n1(A) n2(B) + n1 := NewLeaf(testutils.PathByUint16LeftPadded(2<<4), valueA, 4) // path: ...0010 0000 + n2 := NewLeaf(testutils.PathByUint16LeftPadded(3<<4), valueB, 4) // path: ...0011 0000 + n3 := NewLeaf(testutils.PathByUint16LeftPadded(0), nil, 5) // path: ...0000 0000 + n4 := NewInterimNode(5, n1, n2) + n5 := NewInterimNode(6, n3, n4) + + nn5 := NewInterimCompactifiedNode(6, n3, n4) + require.Nil(t, nn5.LeftChild()) + require.Equal(t, n4, nn5.RightChild()) + require.True(t, nn5.VerifyCachedHash()) + require.Equal(t, n5.Hash(), nn5.Hash()) + }) +} + +// Test_Compactify_BothChildrenPopulated: both children hold allocated registers => no compactification +// is possible; the result reproduces the full interim node. +func Test_Compactify_BothChildrenPopulated(t *testing.T) { + // n5 + // / \ + // n3 n4(C) + // / \ + // n1(A) n2(B) + path1 := testutils.PathByUint16LeftPadded(0) // ...0000 0000 + path2 := testutils.PathByUint16LeftPadded(1 << 4) // ...0001 0000 + path4 := testutils.PathByUint16LeftPadded(3 << 4) // ...0011 0000 + valueA := []byte{0x02, 0x02} + valueB := []byte{0x03, 0x03} + valueC := []byte{0x04, 0x04} + + n1 := NewLeaf(path1, valueA, 4) + n2 := NewLeaf(path2, valueB, 4) + n3 := NewInterimNode(5, n1, n2) + n4 := NewLeaf(path4, valueC, 5) + n5 := NewInterimNode(6, n3, n4) + + nn3 := NewInterimCompactifiedNode(5, n1, n2) + require.Equal(t, n1, nn3.LeftChild()) + require.Equal(t, n2, nn3.RightChild()) + require.True(t, nn3.VerifyCachedHash()) + require.Equal(t, n3.Hash(), nn3.Hash()) + + nn5 := NewInterimCompactifiedNode(6, nn3, n4) + require.Equal(t, nn3, nn5.LeftChild()) + require.Equal(t, n4, nn5.RightChild()) + require.True(t, nn5.VerifyCachedHash()) + require.Equal(t, n5.Hash(), nn5.Hash()) +} + +// --------------------------------------------------------------------------------------------- +// Python reference-implementation anchors +// --------------------------------------------------------------------------------------------- + +// Test_ReferenceImplementationHashes pins the node hash for fixed (path, value, height) inputs against +// values independently computed by the standalone Python Merkle reference implementation (the source +// of truth): github.com/onflow/flow-internal → reference_implementations/merkle_tree.py. Run it to +// regenerate (its node-level scenario section prints these). A compactified leaf at height h holding a +// single register has the same hash as the root of a height-h subtree containing only it, which is how +// the reference derives these from first principles. A mismatch means this Go implementation and the +// reference disagree (e.g. a systematic change in the hashing primitives or default-hash table). The +// allocated-leaf fixtures are additionally validated by the unrolled / cross-construction tests above; +// refDefaultHeight9 also matches the default-hash-at-height-9 value pinned by the sibling mtrie test +// node.Test_InterimNodeWithoutChildren. +func Test_ReferenceImplementationHashes(t *testing.T) { + // Values printed by merkle_tree.py's node-level scenario section, one map per branching regime. + // The reference feeds each path's full 32 bytes into the leaf hash, so pathLeft/pathRight/pathMixed + // have distinct anchors — independently checking left, right, and mixed folds at the node level. + refHashes := map[string]map[int]string{ + "pathLeft": { + 0: "a584b083863e72398b959acd39e39d691f4dee619c28a68f7023a20a69f534cb", + 1: "357c6c27f26aedc8beaf61b0dd808a509140439eaf5e45a707a5cbf3a891b9ef", + 9: "0696044ab99fe5afa4c349df6c654db0e40ebdad61f76437d6fb1aca2c2ae89f", + 256: "df9bb7f23235c734d77068f8e90a616a1ea219e0bc9f8910be4a551b158ba222", + }, + "pathRight": { + 0: "d7dca2e61b984c07206df415456fdb6be2af390fd526e23ed06b59e877130af9", + 1: "9cc143de8f84d07bad66f160536a2a9185cc84985a38d43478313d863cb3a48d", + 9: "40539cfa47c57ca8d82832d0a8472b814ea3c47f5932466e5855abd183ea6bdf", + 256: "ca228d496d4910f0826c12916a564d14a2ed74b74025a88b127cb40641fbdabc", + }, + "pathMixed": { + 0: "1b4a471b379d1ab1e1512308fe264dbe44b1a274628654432c1f74828a3c5c0c", + 1: "e5d47968dafd3c2fac0f88c059854c21d8a8c07dbe369eb581760c31d6fdb154", + 9: "b6acb1296d222e9a13910ba8bc0c8b7d2caf6b74479e7e94564a6ec66c5fb81a", + 256: "9d3f716355f0f45890a4e62f7fed65b9df32b17446d00b27c8b39b323c82cd8e", + }, + } + for _, p := range branchRegimePaths { + for _, height := range []int{0, 1, 9, 256} { + n := NewLeaf(p.path, value, height) + require.Equal(t, refHashes[p.name][height], hashToString(n.Hash()), "reference-hash mismatch for %s at height %d", p.name, height) + } + } + + // default node (path-independent); matches node.Test_InterimNodeWithoutChildren at height 9. + const refDefaultHeight9 = "a37f98dbac56e315fbd4b9f9bc85fbd1b138ed4ae453b128c22c99401495af6d" + require.Equal(t, refDefaultHeight9, hashToString(newDefaultLeaf(pathLeft, 9).Hash())) +} + +// ============================================================================================= +// Predicate methods: IsDefaultNode / IsAllocatedRegisterLeaf +// ============================================================================================= +// +// INTENT (from node.go): +// • IsDefaultNode(𝓃) ≡ "the subtree rooted at 𝓃 holds ZERO allocated registers" (an all-empty / +// default subtree). Universally applicable: correct for nil, leaves, AND interim nodes, +// irrespective of compactification or nil-pruning. Mechanism: nil ⇒ true; else +// hashValue == D(𝒽) where D(𝒽) := GetDefaultHashForHeight(𝒽). +// • IsAllocatedRegisterLeaf(𝓃): a CHEAP, leaf-only check (reads only leafHash). +// – for a LEAF 𝓃 (incl. nil): ≡ ¬IsDefaultNode(𝓃) ≡ "𝓃 is a leaf holding one allocated register". +// – for an INTERIM 𝓃: ALWAYS false — deliberately, even when the interim subtree holds +// allocated registers. This is the documented divergence from ¬IsDefaultNode on interim nodes +// (the cheap check under-approximates: it never claims an interim node is a non-default leaf). +// +// CORRECTNESS (why the implementations match their intent — two standing assumptions): +// (H) collision-resistance of the hash: an all-empty height-𝒽 subtree hashes to D(𝒽) unconditionally, +// and — only under collision-resistance — the converse holds (hashValue==D(𝒽) ⟹ all-empty). This +// is the equivalence the struct doc pins and the compaction-model lemma proves by induction. +// (I) node invariant: hashValue is the correct cached hash, and leafHash==nil for every non-leaf node. +// Maintained by every constructor except the INSECURE NewNode. (These tests never call NewNode, +// so both predicates are exercised only on invariant-respecting nodes.) +// +// INDUCTIVE COVERAGE ARGUMENT (why the case set below is exhaustive): +// The reachable Node space is generated by the constructors; every node is, by structural induction +// on height 𝒽: +// BASE (leaves): nil │ default leaf (leafHash==nil) │ allocated leaf (leafHash≠nil), 𝒽∈{0, >0}. +// STEP (interim): NewInterimNode(𝒽, L, R) with L,R each nil or a node at height 𝒽-1. +// IsDefaultNode reads hashValue, and an interim's hashValue is a function of ONLY its two children's +// hashValues (a nil child contributes D(𝒽-1)). Hence, given the predicate on every height-<𝒽 node, +// the value at 𝒽 is fixed by whether BOTH children are default. It therefore suffices to cover: +// – every leaf base case (L0–L3), and +// – the interim step for each child-emptiness combination: both-empty (I0), one-non-empty (I1), +// both-non-empty (I2); plus a ≥2-level NESTED case (D/D') to witness that the step composes +// across depth AND across mixed empty-representations (nil / default-leaf / all-empty-interim) — +// the "any recursive mix" corollary of the all-empty hash-equivalence lemma. +// That set exhausts the structural forms the induction ranges over. +// +// Interim test nodes are built with NewInterimNode (NOT NewInterimCompactifiedNode) precisely so the +// node stays an interim node: NewInterimCompactifiedNode would prune/compactify away the very interim +// structure whose predicate behaviour we are verifying. + +// Test_IsDefaultNode is the EXHAUSTIVE inductive sweep. Per node it asserts the full node-class +// invariant bundle (via requireDefaultSubtree / requireAllocatedLeaf / requireNonDefaultInterim), +// which checks BOTH predicates together — so this test also fully exercises +// IsAllocatedRegisterLeaf across the case set. Test_IsAllocatedRegisterLeaf below re-covers +// the leaf/interim partition with emphasis on that predicate's deliberate interim under-approximation. +func Test_IsDefaultNode(t *testing.T) { + t.Run("L0: nil node is default", func(t *testing.T) { + var n *Node + requireDefaultSubtree(t, n) + }) + + t.Run("L1: default leaf (unallocated register) is default at any height, path-independent", func(t *testing.T) { + for _, h := range []int{0, 8, 256} { + requireDefaultSubtree(t, newDefaultLeaf(pathLeft, h)) + requireDefaultSubtree(t, NewLeaf(pathLeft, nil, h)) + requireDefaultSubtree(t, NewLeaf(pathLeft, []byte{}, h)) + requireDefaultSubtree(t, newDefaultLeaf(pathRight, h)) // path-independence + } + }) + + t.Run("L2/L3: allocated leaf is NOT default (uncompactified h=0 and compactified h>0)", func(t *testing.T) { + for _, h := range []int{0, 1, 8, 256} { + requireAllocatedLeaf(t, NewLeaf(pathLeft, value, h)) + } + }) + + t.Run("I0: all-empty interim node is default (unpruned; interim universality)", func(t *testing.T) { + // 𝓃 (interim, height 𝒽) hashValue = H(D(𝒽-1), D(𝒽-1)) = D(𝒽) + // ╱ ╲ + // cL cR cL, cR ∈ {nil, default-leaf} (all-empty) + for _, h := range []int{1, 2, 256} { + require.False(t, emptyInterimNode(h).IsLeaf(), "@h%d must be an interim node", h) + requireDefaultSubtree(t, emptyInterimNode(h)) // both children default-leaf + requireDefaultSubtree(t, NewInterimNode(h, nil, newDefaultLeaf(pathRight, h-1))) // nil + default-leaf + requireDefaultSubtree(t, NewInterimNode(h, newDefaultLeaf(pathLeft, h-1), nil)) // default-leaf + nil + } + }) + + t.Run("I1: interim with exactly one allocated register is NOT default", func(t *testing.T) { + // 𝓃 (interim, height 𝒽) + // ╱ ╲ + // a cR a = allocated leaf; cR ∈ {nil, default-leaf} + for _, h := range []int{1, 2, 256} { + requireNonDefaultInterim(t, oneRegisterInterimNode(h)) // a + nil + requireNonDefaultInterim(t, NewInterimNode(h, nil, NewLeaf(pathRight, value, h-1))) // nil + a + requireNonDefaultInterim(t, NewInterimNode(h, NewLeaf(pathLeft, value, h-1), newDefaultLeaf(pathRight, h-1))) // a + default-leaf + } + }) + + t.Run("I2: interim with allocated registers in BOTH children is NOT default", func(t *testing.T) { + for _, h := range []int{1, 2, 256} { + requireNonDefaultInterim(t, twoRegisterInterimNode(h)) + } + }) + + t.Run("D: nested all-empty subtree (recursive mix nil / default-leaf / all-empty-interim) is default", func(t *testing.T) { + // r (interim, h=2) all-empty ⇒ D(2) + // ╱ ╲ + // m (h=1) d (h=1) m = all-empty interim, d = default leaf + // ╱ ╲ + // nil e (h=0) e = default leaf + m := NewInterimNode(1, nil, newDefaultLeaf(pathLeft, 0)) + d := newDefaultLeaf(pathRight, 1) + r := NewInterimNode(2, m, d) + require.False(t, r.IsLeaf()) + requireDefaultSubtree(t, r) + }) + + t.Run("D': nested subtree with a single buried allocated register is NOT default", func(t *testing.T) { + // r (interim, h=3) 1 allocated register ⇒ NOT default + // ╱ ╲ + // m3 (h=2) nil + // ╱ ╲ + // nil m2 (h=1) + // ╱ ╲ + // a (h=0) nil a = allocated leaf + m2 := NewInterimNode(1, NewLeaf(pathLeft, value, 0), nil) + m3 := NewInterimNode(2, nil, m2) + r := NewInterimNode(3, m3, nil) + requireNonDefaultInterim(t, r) + }) +} + +// Test_IsAllocatedRegisterLeaf re-covers the case space partitioned by leaf-vs-interim, with the +// spotlight on this predicate's peculiarities: it recognises allocated leaves at ANY height (compactified +// or not), and it ALWAYS returns false on interim nodes — coinciding with ¬IsDefaultNode on all-empty +// interims but deliberately DIVERGING from it on interims that hold allocated registers. +func Test_IsAllocatedRegisterLeaf(t *testing.T) { + t.Run("nil ⇒ false", func(t *testing.T) { + var n *Node + requireDefaultSubtree(t, n) // bundle asserts IsAllocatedRegisterLeaf()==false + }) + + t.Run("default leaf ⇒ false (any height)", func(t *testing.T) { + for _, h := range []int{0, 8, 256} { + requireDefaultSubtree(t, newDefaultLeaf(pathLeft, h)) + requireDefaultSubtree(t, NewLeaf(pathLeft, nil, h)) + } + }) + + t.Run("allocated leaf ⇒ true (uncompactified h=0 and compactified h>0)", func(t *testing.T) { + requireAllocatedLeaf(t, NewLeaf(pathLeft, value, 0)) // uncompactified + for _, h := range []int{1, 8, 256} { // compactified + requireAllocatedLeaf(t, NewLeaf(pathLeft, value, h)) + } + }) + + t.Run("interim node ⇒ ALWAYS false, regardless of allocated content", func(t *testing.T) { + // all-empty interim (also default): false COINCIDES with ¬IsDefaultNode. + require.False(t, emptyInterimNode(1).IsLeaf()) + requireDefaultSubtree(t, emptyInterimNode(1)) + + // interim WITH allocated registers (NOT default): false DIVERGES from ¬IsDefaultNode (=true). + requireNonDefaultInterim(t, oneRegisterInterimNode(1)) + requireNonDefaultInterim(t, twoRegisterInterimNode(1)) + }) +} + +// Test_LeafPredicateGuarantee verifies the guarantee stated in node.go: for every LEAF node 𝓃 +// (including nil), IsAllocatedRegisterLeaf() == ¬IsDefaultNode(). +func Test_LeafPredicateGuarantee(t *testing.T) { + var nilNode *Node + leaves := []*Node{ + nilNode, + newDefaultLeaf(pathLeft, 0), + newDefaultLeaf(pathLeft, 8), + newDefaultLeaf(pathLeft, 256), + NewLeaf(pathLeft, value, 0), + NewLeaf(pathLeft, value, 8), + NewLeaf(pathRight, value, 256), + } + for i, n := range leaves { + require.True(t, n.IsLeaf(), "case %d must be a leaf", i) + require.Equal(t, !n.IsDefaultNode(), n.IsAllocatedRegisterLeaf(), "leaf guarantee violated at case %d", i) + } +} + +// --------------------------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------------------------- + +func hashToString(h hash.Hash) string { + return hex.EncodeToString(h[:]) +} + +// --- node builders (reuse the pathLeft / pathRight / value fixtures) ------------------------------ +// These construct INTERIM nodes of a given emptiness class via NewInterimNode (NOT +// NewInterimCompactifiedNode), so the node stays interim — the shape whose predicate behaviour the +// predicate tests target. + +// emptyInterimNode returns an interim node (both children non-nil default leaves) whose subtree is +// entirely unallocated; its hash is D(height). +func emptyInterimNode(height int) *Node { + return NewInterimNode(height, newDefaultLeaf(pathLeft, height-1), newDefaultLeaf(pathRight, height-1)) +} + +// oneRegisterInterimNode returns an interim node holding exactly one allocated register (allocated leaf +// on the left, empty right child). +func oneRegisterInterimNode(height int) *Node { + return NewInterimNode(height, NewLeaf(pathLeft, value, height-1), nil) +} + +// twoRegisterInterimNode returns an interim node holding two allocated registers (one per child). +func twoRegisterInterimNode(height int) *Node { + return NewInterimNode(height, NewLeaf(pathLeft, value, height-1), NewLeaf(pathRight, value, height-1)) +} + +// --- node-class invariant assertions -------------------------------------------------------------- +// Each names one of the three node classes the predicate analysis revolves around and asserts the +// FULL invariant bundle for that class from BOTH predicates plus structure. Extracted because the +// bundle recurs across every predicate case. + +// requireDefaultSubtree asserts that 𝓃 represents an all-empty (default) subtree, for ANY +// representation (nil, default leaf, or all-empty interim node): IsDefaultNode is true and the cheap +// leaf-check is false; a non-nil node's cached hash is the height default and internally consistent. +func requireDefaultSubtree(t *testing.T, n *Node) { + t.Helper() + require.True(t, n.IsDefaultNode(), "expected an all-empty (default) subtree") + require.False(t, n.IsAllocatedRegisterLeaf(), "a default subtree is never a non-default leaf") + if n != nil { + require.True(t, n.VerifyCachedHash()) + require.Equal(t, ledger.GetDefaultHashForHeight(n.height), n.Hash()) + } +} + +// requireAllocatedLeaf asserts that 𝓃 is a leaf holding exactly one allocated register: it is a leaf, +// not default, and the cheap leaf-check recognises it. +func requireAllocatedLeaf(t *testing.T, n *Node) { + t.Helper() + require.True(t, n.IsLeaf(), "expected a leaf") + require.False(t, n.IsDefaultNode(), "an allocated leaf is not default") + require.True(t, n.IsAllocatedRegisterLeaf(), "cheap leaf-check must recognise an allocated leaf") + require.True(t, n.VerifyCachedHash()) +} + +// requireNonDefaultInterim asserts that 𝓃 is an interim node holding ≥1 allocated register: it is NOT a +// leaf, not default, and the cheap leaf-check deliberately returns false (under-approximation on interim +// nodes — the documented divergence from ¬IsDefaultNode). +func requireNonDefaultInterim(t *testing.T, n *Node) { + t.Helper() + require.False(t, n.IsLeaf(), "expected an interim node") + require.False(t, n.IsDefaultNode(), "expected a non-default (allocated) subtree") + require.False(t, n.IsAllocatedRegisterLeaf(), "cheap leaf-check must return false on interim nodes") + require.True(t, n.VerifyCachedHash()) +} + +// fullyExpandedAllocatedRef builds the hash of the height-`height` subtree holding the single +// allocated register (path, value) by explicitly composing interim nodes with default (empty) +// siblings, using NewInterimNode. This is the fully-expanded (perfect-trie) reference against which +// a compactified leaf's hash must be invariant. The register's leaf sits on the side selected by the +// path bit at each level, matching the fully-expanded trie geometry. +func fullyExpandedAllocatedRef(t *testing.T, path ledger.Path, value []byte, height int) hash.Hash { + t.Helper() + node := NewLeaf(path, value, 0) // height-0 (uncompactified) leaf + for h := 1; h <= height; h++ { + bit := readPathBit(path, ledger.NodeMaxHeight-h) + if bit == 1 { // register branches right => empty sibling on the left + node = NewInterimNode(h, nil, node) + } else { // register branches left => empty sibling on the right + node = NewInterimNode(h, node, nil) + } + } + return node.Hash() +} + +// readPathBit returns the bit of `path` at position `index` (0 == MSB of byte 0). +func readPathBit(path ledger.Path, index int) int { + return int((path[index>>3] >> (7 - (index & 7))) & 1) +} + +// requireIsLeafWithHash verifies that `n` is a leaf node whose hash equals `expectedHash`. +func requireIsLeafWithHash(t *testing.T, n *Node, expectedHash hash.Hash) { + t.Helper() + require.Nil(t, n.LeftChild()) + require.Nil(t, n.RightChild()) + require.Equal(t, expectedHash, n.Hash()) + require.True(t, n.VerifyCachedHash()) + require.True(t, n.IsLeaf()) +} diff --git a/ledger/complete/payloadless/trie.go b/ledger/complete/payloadless/trie.go index 634cfbc1075..a254fe80124 100644 --- a/ledger/complete/payloadless/trie.go +++ b/ledger/complete/payloadless/trie.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + "slices" "sync" "github.com/onflow/flow-go/ledger" @@ -14,7 +15,7 @@ import ( // MTrie represents a perfect in-memory full binary Merkle tree with uniform height. // For a detailed description of the storage model, please consult `mtrie/README.md` // -// A MTrie is a thin wrapper around a the trie's root Node. An MTrie implements the +// A MTrie is a thin wrapper around a trie's root Node. An MTrie implements the // logic for forming MTrie-graphs from the elementary nodes. Specifically: // - how Nodes (graph vertices) form a Trie, // - how register values are read from the trie, @@ -23,7 +24,7 @@ import ( // // `MTrie`s are _immutable_ data structures. Updating register values is implemented through // copy-on-write, which creates a new `MTrie`. For minimal memory consumption, all sub-tries -// that where not affected by the write operation are shared between the original MTrie +// that were not affected by the write operation are shared between the original MTrie // (before the register updates) and the updated MTrie (after the register writes). // // MTrie expects that for a specific path, the register's key never changes. @@ -44,7 +45,7 @@ func NewEmptyMTrie() *MTrie { // IsEmpty checks if a trie is empty. // -// An empty try doesn't mean a trie with no allocated registers. +// An empty trie (root is nil) is not the same as a trie whose registers are all unallocated. func (mt *MTrie) IsEmpty() bool { return mt.root == nil } @@ -215,7 +216,7 @@ func read(leafHashes []*hash.Hash, paths []ledger.Path, head *Node) { // The key-value pairs specify the registers whose values are supposed to hold updated values // compared to the parent trie. Constructing the new trie is done in a COPY-ON-WRITE manner: // - The original trie remains unchanged. -// - subtries that remain unchanged are from the parent trie instead of copied. +// - subtries that remain unchanged are referenced from the parent trie instead of copied. // // UNSAFE: method requires the following conditions to be satisfied: // - keys are NOT duplicated @@ -257,7 +258,7 @@ type updateResult struct { lowestHeightTouched int } -// update traverses the subtree recursively and create new nodes with +// update traverses the subtree recursively and creates new nodes with // the updated values on the given paths // // it returns: @@ -265,7 +266,7 @@ type updateResult struct { // - allocated register count delta in subtrie (allocatedRegCountDelta) // - lowest height reached during recursive update in subtrie (lowestHeightTouched) // -// update also compact a subtree into a single compact leaf node in the case where +// update also compacts a subtree into a single compact leaf node in the case where // there is only 1 value stored in the subtree. // // allocatedRegCountDelta is used to compute updated trie's allocated register count. @@ -277,98 +278,110 @@ type updateResult struct { // - paths are NOT duplicated func update( nodeHeight int, // the height of the node during traversing the subtree - currentNode *Node, // the current node on the travesing path, if it's nil it means the trie has no node on this path + currentNode *Node, // the current node on the traversing path, if it's nil it means the trie has no node on this path paths []ledger.Path, // the paths to update the values values [][]byte, // the values to be updated at the given paths - compactLeaf *Node, // a compact leaf node from its ancester, it could be nil - prune bool, // prune is a flag for whether pruning nodes with empty values. not pruning is useful for generating proof, expecially non-inclusion proof + compactLeaf *Node, // a compact leaf node from its ancestor, it could be nil + prune bool, // prune flag specifies whether the update should prune nodes with empty values; not pruning is useful for generating proof, especially non-inclusion proof ) (n *Node, allocatedRegCountDelta int64, lowestHeightTouched int) { - // No new path to update + // IMPLEMENTATION Notes: + // - This method proceeds recursively, essentially partitioning the remaining set of `paths` and corresponding `values` according to each bit of + // the path we are descending down. In essence, the base case of the recursion is when there is only a single path-value pair left to be updated + // in the trie. We *always create a leaf* in the recursion base case, no matter whether the leaf represents an unallocated or allocated register. + // Explicitly representing specific unallocated registers is an interim shortcut until we have specialized (more efficient) non-inclusion proofs + // implemented (at the moment, non-inclusion proofs fall back on inclusion proofs of explicitly represented default leaf nodes). + // - When descending upwards again from the recursion, `NewInterimCompactifiedNode` takes care of the compaction if `prune` is true. When `prune` + // is enabled (unchanged for the entire update), it follows by induction that the trie always produces maximally compactified leaves for all + // registers written during the update (untouched registers may remain in the state before the update, potentially uncompactified). + // - Important: we only track the number of *allocated* registers (the change thereof to be precise). Therefore, whenever a new leaf is created, + // we need to check if it represents an unallocated register and return the appropriate change of allocated register count. + + // [Recursion Base Case] empty update (len(paths) == 0), i.e. no register to write in this sub-trie. if len(paths) == 0 { - if compactLeaf != nil { - // if a compactLeaf from a higher height is still left, - // then expand the compact leaf node to the current height by creating a new compact leaf - // node with the same path and value. - // The old node shouldn't be recycled as it is still used by the tree copy before the update. - if compactLeaf.leafHash != nil { - n = NewLeafWithHash(compactLeaf.path, *compactLeaf.leafHash, nodeHeight) - } else { - n = NewLeaf(compactLeaf.path, nil, nodeHeight) - } + if compactLeaf != nil { // this implies currentNode == nil per Lemma in mtrie/README.md + // README case 3.a.ii: the sole leaf to create is the compactified leaf carried over from a higher height. + // We re-level it to the current height by creating a new compact leaf node with the same path and value. + // The old node isn't modified, as it is still used by the trie before the update. No matter whether + // `compactLeaf` represents an unallocated or allocated register, the register count remains unchanged. + n = NewRelevelledLeaf(compactLeaf, nodeHeight) return n, 0, nodeHeight } - // if no path to update and there is no compact leaf node on this path, we return - // the current node regardless it exists or not. + // README Case 0 (re-use): no path to update and no compact leaf carried over ⇒ no update at all, so we + // re-use the existing sub-trie (mtrie/README.md § Update: "no update will be done and the original + // sub-trie can be re-used"). We return `currentNode` regardless of whether it exists. return currentNode, 0, nodeHeight } + // [Recursion Base Case] README case 3.a.i: currentNode == nil: A single register is written into a previously empty (e.g. pruned) sub-trie. if len(paths) == 1 && currentNode == nil && compactLeaf == nil { - // if there is only 1 path to update, and the existing tree has no node on this path, also - // no compact leaf node from its ancester, it means we are storing a value on a new path, n = NewLeaf(paths[0], values[0], nodeHeight) - if len(values[0]) == 0 { - // if we are storing an empty value, then no register is allocated - // allocatedRegCountDelta should be 0 - return n, 0, nodeHeight - } - // if we are storing a non-empty value, we are allocating a new register - return n, 1, nodeHeight + allocatedRegCountDelta = computeAllocatedRegCountDelta(false, n.IsAllocatedRegisterLeaf()) + return n, allocatedRegCountDelta, nodeHeight } - if currentNode != nil && currentNode.IsLeaf() { // if we're here then compactLeaf == nil - // check if the current node path is among the updated paths - found := false + // Every remaining configuration has len(paths) ≥ 1. The code branches on currentNode next. By the Lemma (mtrie/README.md § Update), the + // configuration currentNode ≠ nil AND compactLeaf ≠ nil cannot occur. So the configurations that remain, classified by currentNode, are: + // • currentNode is a leaf (⟹ compactLeaf == nil, by README's Lemma): README Case 2 + // • currentNode is an interim node (⟹ compactLeaf == nil, by README's Lemma): README Case 1 + // • currentNode == nil (⟹ ≥2 leaves to create; as single-leaf case 3.a was covered above): only README case 3.b remains + + // README Case 2: currentNode is a leaf (⟹ compactLeaf == nil per Lemma in mtrie/README.md). + if currentNode != nil && currentNode.IsLeaf() { currentPath := *currentNode.Path() - for i, p := range paths { - if p == currentPath { - // the case where the recursion stops: only one path to update - if len(paths) == 1 { - // check if the only path to update has the same value. - // if value is the same, we could skip the update to avoid creating duplicated node - hadValue := currentNode.leafHash != nil - hasValue := len(values[i]) > 0 - var newLeafHash hash.Hash - if hasValue { - newLeafHash = hash.HashLeaf(hash.Hash(paths[i]), values[i]) - } - - if hadValue == hasValue { - // when value equals, if didn't have value before, then still no value after update; - // if had value before, then the leaf hash is still the same after update, - // so we can reuse the current node without creating a new one. - if !hasValue || *currentNode.leafHash == newLeafHash { - // avoid creating a new node when the same value is written - return currentNode, 0, nodeHeight - } - } - - // the value is updated, we need to create a new leaf node with the updated value. - // The old node shouldn't be recycled as it is still used by the trie before the update. - if hasValue { - n = NewLeafWithHash(paths[i], newLeafHash, nodeHeight) - } else { - n = NewLeaf(paths[i], nil, nodeHeight) - } - allocatedRegCountDelta = computeAllocatedRegCountDelta(hadValue, hasValue) - return n, allocatedRegCountDelta, nodeHeight - } - // the case where the recursion carries on: len(paths)>1 - found = true - allocatedRegCountDelta = computeAllocatedRegCountDeltaFromHigherHeight(currentNode.leafHash != nil) - break + + // [Recursion Base Case] README case 2.a.i: the single updated path coincides with `currentNode`'s path, + // so we overwrite the register represented by the existing leaf in place. + if len(paths) == 1 && (paths[0] == currentPath) { + // In most cases, the new register value will be different from the old value, in which case we need to instantiate a new leaf + // anyway. So we optimistically create the new leaf first. But if a posterior check reveals that the new leaf's hash is identical + // to the old `currentNode`, we just return `currentNode` to avoid duplication (optimistically created leaf is garbage collected). + n := NewLeaf(paths[0], values[0], nodeHeight) + if n.hashValue == currentNode.hashValue { + return currentNode, 0, nodeHeight } + allocatedRegCountDelta = computeAllocatedRegCountDelta(currentNode.IsAllocatedRegisterLeaf(), n.IsAllocatedRegisterLeaf()) + return n, allocatedRegCountDelta, nodeHeight } - if !found { - // if the current node carries a path not included in the input path, then the current node - // represents a compact leaf that needs to be carried down the recursion. + + // -- from here on, until the end of the method, we are handling the recursive cases -- + + // [Recursive Case] README case 2.a.ii or 2.b, both FALL THROUGH to the SPLIT-AND-RECURSE section below + if slices.Contains(paths, currentPath) { // `currentNode.path ∈ paths` and `len(paths) > 1`: README case 2.a.ii + // The register at `currentNode`'s path is among the updated `paths`, so its value will be overwritten. Here we + // only account for removing `currentNode`'s own contribution to the count; the new value is counted separately, + // deeper in the recursion, when its leaf is (re)created. Dropping `currentNode` yields -1 if it held an + // allocated register and 0 if it was a default (unallocated) leaf. + allocatedRegCountDelta = computeAllocatedRegCountDelta(currentNode.IsAllocatedRegisterLeaf(), false) // drop `currentNode` + } else { // `currentNode.path ∉ paths`: README case 2.b + // `currentNode` carries a path that is not among the updated `paths`. Hence it represents a compact leaf + // that must be carried down the recursion. compactLeaf = currentNode + // INSIGHT [not implemented]: formally `currentNode` could be set to nil here. This proof (presumably relatively short) + // should be written up in the readme, to keep the complexity out of the code. } } - - // in the remaining code: - // - either len(paths) > 1 - // - or len(paths) == 1 and compactLeaf!= nil - // - or len(paths) == 1 and currentNode != nil && !currentNode.IsLeaf() + // CAUTION: in the prior code block implementing README case 2.b, we set compactLeaf = currentNode while + // currentNode ≠ nil, so the Readme's Lemma no longer holds from here on. This is safe because currentNode + // is a *leaf* in case 2.b, so its LeftChild()/RightChild() are nil (fetched below): the split descends into + // empty children while compactLeaf carries the preserved register down. + + // Every remaining configuration has len(paths) ≥ 1. The configurations remaining to be handled are: + // case 1: node ≠ nil and currentNode is an interim node, with at least one register to write into its sub-trie (len(paths) ≥ 1). + // We descend into currentNode's existing children to apply the update while preserving the rest. + // case 2.a.ii: currentNode is a leaf with currentNode.path ∈ paths and len(paths) > 1. + // currentNode's own register is among those updated, and at least one *other* register (different path) is + // written into the same sub-trie. So currentNode's leaf must be replaced by a sub-trie holding two or more + // distinct registers (built by the recursion). + // case 2.b: currentNode is a leaf with currentNode.path ∉ paths. + // At least one register is written here (len(paths) ≥ 1), but currentNode's own register is not among them + // (no updated path equals currentNode.path). So currentNode's leaf must be replaced by a sub-trie holding two + // or more distinct registers: currentNode's carried register plus the updated one(s) (built by the recursion). + // case 3.b: currentNode == nil with more than one register to create, since either + // - len(paths) == 1 and compactLeaf ≠ nil, or + // - len(paths) > 1. + // Common to all these cases: the update continues by recursion. Either currentNode (a leaf or nil) is expanded into a + // new sub-trie holding two or more distinct registers (cases 2.a.ii, 2.b, 3.b), or currentNode is already an interim + // node and we recurse into its existing children (case 1). // Split paths and values to recurse: // lpaths contains all paths that have `0` at the partitionIndex @@ -403,18 +416,15 @@ func update( var lLowestHeightTouched, rLowestHeightTouched int parallelRecursionThreshold := 16 if len(lpaths) < parallelRecursionThreshold || len(rpaths) < parallelRecursionThreshold { - // runtime optimization: if there are _no_ updates for either left or right sub-tree, proceed single-threaded + // runtime optimization: if there are only few updates for either left or right sub-tree, proceed single-threaded newLeftChild, lRegCountDelta, lLowestHeightTouched = update(nodeHeight-1, oldLeftChild, lpaths, lvalues, lcompactLeaf, prune) newRightChild, rRegCountDelta, rLowestHeightTouched = update(nodeHeight-1, oldRightChild, rpaths, rvalues, rcompactLeaf, prune) } else { - // runtime optimization: process the left child in a separate thread - - // Since we're receiving 3 values from goroutine, use a - // struct and channel to reduce allocs/op. - // Although WaitGroup approach can be faster than channel (esp. with 2+ goroutines), - // we only use 1 goroutine here and need to communicate results from it. So using + // Runtime optimization: process the left child in a separate thread. This Recursive `update` call returns + // 3 values, which we have to wait for. Although WaitGroup approach can be faster than channel (esp. with + // 2+ goroutines), we only use 1 goroutine here and need to communicate results from it. So using // channel is faster and uses fewer allocs/op in this case. - results := make(chan updateResult, 1) + results := make(chan updateResult, 1) // channel capacity 1, so goroutine can push into channel and finish without being blocked go func(retChan chan<- updateResult) { child, regCountDelta, lowestHeightTouched := update(nodeHeight-1, oldLeftChild, lpaths, lvalues, lcompactLeaf, prune) retChan <- updateResult{child, regCountDelta, lowestHeightTouched} @@ -422,13 +432,13 @@ func update( newRightChild, rRegCountDelta, rLowestHeightTouched = update(nodeHeight-1, oldRightChild, rpaths, rvalues, rcompactLeaf, prune) - // Wait for results from goroutine. + // Wait for results from goroutine processing left child ret := <-results newLeftChild, lRegCountDelta, lLowestHeightTouched = ret.child, ret.allocatedRegCountDelta, ret.lowestHeightTouched } allocatedRegCountDelta += lRegCountDelta + rRegCountDelta - lowestHeightTouched = minInt(lLowestHeightTouched, rLowestHeightTouched) + lowestHeightTouched = min(lLowestHeightTouched, rLowestHeightTouched) // mitigate storage exhaustion attack: avoids creating a new node when the exact same // value is re-written at a register. CAUTION: we only check that the children are @@ -440,8 +450,15 @@ func update( return currentNode, 0, lowestHeightTouched } - // if prune is on, then will check and create a compact leaf node if one child is nil, and the - // other child is a leaf node + // -- from here on, until the end of the method, we are constructing the updated trie bottom up (increasing height) from the previously (recursively) updated children -- + + // Instantiate the node replacing `currentNode` from the recursively-updated children. Each of newLeftChild / newRightChild is itself an update result: + // nil, a compact leaf, or an interim node. If prune is on, `NewInterimCompactifiedNode` creates a compact leaf when one child is nil and the other is + // a leaf node. When starting from a maximally compactified trie, by induction this process yields an updated trie that is also maximally compactified + // (all leaves compactified). + // However, if we operate on a sub-trie where some leaves are expanded or unallocated registers are explicitly represented, the resulting updated trie + // may still contain leaves that could be pruned or compactified even when `prune == true`: a register deeper in an untouched, reused child stays as-is, + // since `NewInterimCompactifiedNode` inspects only the immediate children (for efficiency) and does not traverse sub-tries unless there are register updates. if prune { n = NewInterimCompactifiedNode(nodeHeight, newLeftChild, newRightChild) return n, allocatedRegCountDelta, lowestHeightTouched @@ -451,32 +468,16 @@ func update( return n, allocatedRegCountDelta, lowestHeightTouched } -// computeAllocatedRegCountDeltaFromHigherHeight returns the delta -// needed to compute the allocated reg count when -// a value is updated or unallocated at a lower height. -func computeAllocatedRegCountDeltaFromHigherHeight(hadValue bool) (allocatedRegCountDelta int64) { - if hadValue { - // Allocated register will be updated or unallocated at lower height. - allocatedRegCountDelta-- +// computeAllocatedRegCountDelta returns *change* in allocated reg count when updating a +// leaf to-and-from allocated register vs default leaf (representing an unallocated register). +func computeAllocatedRegCountDelta(wasAllocatedRegister, updatedToAllocatedRegister bool) (allocatedRegCountDelta int64) { + if wasAllocatedRegister == updatedToAllocatedRegister { + return 0 } - return -} - -// computeAllocatedRegCountDelta returns the allocated reg count -// delta computed from the presence of the old and new value. -// PRECONDITION: hadValue != hasValue OR the stored value changed -func computeAllocatedRegCountDelta(hadValue, hasValue bool) (allocatedRegCountDelta int64) { - allocatedRegCountDelta = 0 - if !hasValue { - // Old value is present while new value is empty. - // Allocated register will be unallocated. - allocatedRegCountDelta = -1 - } else if !hadValue { - // Old value is empty while new value is present. - // Unallocated register will be allocated. - allocatedRegCountDelta = 1 - } - return + if updatedToAllocatedRegister { + return 1 // previously unallocated register will be allocated. + } + return -1 // previously allocated register will be unallocated. } // UnsafeProofs provides proofs for the given paths. @@ -568,7 +569,7 @@ func addSiblingTrieHashToProofs(siblingTrie *Node, depth int, proofs []*ledger.P // This code is necessary, because we do not remove nodes from the trie // when a register is deleted. Instead, we just set the respective leaf's - // payload to empty. While this will cause the lead's hash to become the + // payload to empty. While this will cause the leaf's hash to become the // default hash, the node itself remains as part of the trie. // However, a proof has the convention that the hash of the sibling trie // should only be included, if it is _non-default_. Therefore, we can @@ -731,13 +732,6 @@ func splitTrieProofsByPath(paths []ledger.Path, proofs []*ledger.PayloadlessTrie return i } -func minInt(a, b int) int { - if a < b { - return a - } - return b -} - // TraverseNodes traverses all nodes of the trie in DFS order func TraverseNodes(trie *MTrie, processNode func(*Node) error) error { return traverseRecursive(trie.root, processNode) diff --git a/ledger/complete/payloadless/trie_test.go b/ledger/complete/payloadless/trie_test.go new file mode 100644 index 00000000000..c66d466d4bc --- /dev/null +++ b/ledger/complete/payloadless/trie_test.go @@ -0,0 +1,63 @@ +package payloadless_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/testutils" + "github.com/onflow/flow-go/ledger/complete/payloadless" +) + +// Test_AllocatedRegCount verifies that AllocatedRegCount tracks all four allocated<->unallocated +// register transitions of an update: unallocated->allocated (delta +1), allocated->allocated with a +// different value (delta 0), allocated->unallocated (delta -1), and unallocated->unallocated (delta 0). +// +// Superseded once mtrie's TestTrieAllocatedRegCountRegSize is ported to this package (it exercises the +// same four transitions at scale); this is an interim, focused guard. +func Test_AllocatedRegCount(t *testing.T) { + // Base trie: a minimal non-empty trie holding two allocated registers. With two registers the + // root is an interim node, so an update to an existing register descends to its leaf and reaches + // base case (1.a.i) there — the representative shape, rather than the empty-trie edge case. + pathA := testutils.PathByUint16LeftPadded(0) + pathB := testutils.PathByUint16LeftPadded(1) + pathC := testutils.PathByUint16LeftPadded(2) // not allocated in the base trie + base, _, err := payloadless.NewTrieWithUpdatedRegisters( + payloadless.NewEmptyMTrie(), + []ledger.Path{pathA, pathB}, [][]byte{{0x01}, {0x11}}, true) + require.NoError(t, err) + require.Equal(t, uint64(2), base.AllocatedRegCount()) + + t.Run("allocated -> allocated (different value): delta 0", func(t *testing.T) { + updated, _, err := payloadless.NewTrieWithUpdatedRegisters( + base, []ledger.Path{pathA}, [][]byte{{0x02}}, true) + require.NoError(t, err) + require.Equal(t, uint64(2), updated.AllocatedRegCount()) // count unchanged + require.NotEqual(t, base.RootHash(), updated.RootHash()) // the value change took effect + }) + + t.Run("allocated -> unallocated: delta -1", func(t *testing.T) { + updated, _, err := payloadless.NewTrieWithUpdatedRegisters( + base, []ledger.Path{pathA}, [][]byte{{}}, true) + require.NoError(t, err) + require.Equal(t, uint64(1), updated.AllocatedRegCount()) // one register freed + require.NotEqual(t, base.RootHash(), updated.RootHash()) + }) + + t.Run("unallocated -> allocated: delta +1", func(t *testing.T) { + updated, _, err := payloadless.NewTrieWithUpdatedRegisters( + base, []ledger.Path{pathC}, [][]byte{{0x22}}, true) + require.NoError(t, err) + require.Equal(t, uint64(3), updated.AllocatedRegCount()) // one register allocated + require.NotEqual(t, base.RootHash(), updated.RootHash()) + }) + + t.Run("unallocated -> unallocated: delta 0", func(t *testing.T) { + updated, _, err := payloadless.NewTrieWithUpdatedRegisters( + base, []ledger.Path{pathC}, [][]byte{{}}, true) + require.NoError(t, err) + require.Equal(t, uint64(2), updated.AllocatedRegCount()) // count unchanged + require.Equal(t, base.RootHash(), updated.RootHash()) // writing empty to a fresh path is a no-op + }) +} From 0077b9e0cdfcc1c3c70cfa8439860feab343f06a Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Wed, 8 Jul 2026 22:55:33 -0700 Subject: [PATCH 6/9] Update ledger/complete/payloadless/trie.go Co-authored-by: Leo Zhang --- ledger/complete/payloadless/trie.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/complete/payloadless/trie.go b/ledger/complete/payloadless/trie.go index a254fe80124..cae7b777337 100644 --- a/ledger/complete/payloadless/trie.go +++ b/ledger/complete/payloadless/trie.go @@ -335,7 +335,7 @@ func update( // In most cases, the new register value will be different from the old value, in which case we need to instantiate a new leaf // anyway. So we optimistically create the new leaf first. But if a posterior check reveals that the new leaf's hash is identical // to the old `currentNode`, we just return `currentNode` to avoid duplication (optimistically created leaf is garbage collected). - n := NewLeaf(paths[0], values[0], nodeHeight) + n = NewLeaf(paths[0], values[0], nodeHeight) if n.hashValue == currentNode.hashValue { return currentNode, 0, nodeHeight } From 1e8e7a7e5c4a49d79c8addd103b55cd6dfcc102d Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Wed, 8 Jul 2026 22:57:36 -0700 Subject: [PATCH 7/9] Update ledger/complete/payloadless/node.go Co-authored-by: Leo Zhang --- ledger/complete/payloadless/node.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ledger/complete/payloadless/node.go b/ledger/complete/payloadless/node.go index 7052cab442c..484ebe872ce 100644 --- a/ledger/complete/payloadless/node.go +++ b/ledger/complete/payloadless/node.go @@ -317,7 +317,6 @@ func (n *Node) IsAllocatedRegisterLeaf() bool { return n.leafHash != nil } -// a height-0 allocated leaf is a degenerate compactified leaf, so returning true // computeHash returns the hashValue of the node func (n *Node) computeHash() hash.Hash { From d6211503535fa49728283ecdd732e6edbccd18d7 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 9 Jul 2026 12:25:46 -0700 Subject: [PATCH 8/9] add comments to trie --- ledger/complete/payloadless/trie.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ledger/complete/payloadless/trie.go b/ledger/complete/payloadless/trie.go index cae7b777337..dfaddc5b937 100644 --- a/ledger/complete/payloadless/trie.go +++ b/ledger/complete/payloadless/trie.go @@ -98,6 +98,11 @@ func (mt *MTrie) String() string { // ReadSingleLeafHash reads and returns the leaf hash for a single path. // Returns nil if no leaf exists at the given path or if the leaf represents // an unallocated register. +// +// CAUTION: the returned pointer aliases the trie node's internal leaf hash. Tries are immutable and +// shared copy-on-write across trie versions, so the caller MUST NOT modify the pointee; doing so would +// corrupt the node's cached hash. Callers needing a mutable hash must copy it (see the Forest layer, +// which returns a defensive copy). Do NOT MODIFY the returned hash! func (mt *MTrie) ReadSingleLeafHash(path ledger.Path) *hash.Hash { return readSingleLeafHash(path, mt.root) } @@ -655,6 +660,11 @@ func EmptyTrieRootHash() ledger.RootHash { // AllLeafHashes returns all leaf hashes stored in the trie. Empty leaves // (unallocated registers) are skipped. +// +// CAUTION: each returned pointer aliases the corresponding trie node's internal leaf hash. Tries are +// immutable and shared copy-on-write across trie versions, so the caller MUST NOT modify any pointee; +// doing so would corrupt that node's cached hash. Callers needing mutable hashes must copy them. Do +// NOT MODIFY the returned hashes! func (mt *MTrie) AllLeafHashes() []*hash.Hash { return mt.root.AllLeafHashes() } From 9b0aa2aaf477f5bf99fda0da38ce7efc6ffbbef4 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 10 Jul 2026 09:52:34 -0700 Subject: [PATCH 9/9] fix lint --- ledger/complete/payloadless/node.go | 1 - ledger/complete/payloadless/node_test.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ledger/complete/payloadless/node.go b/ledger/complete/payloadless/node.go index 484ebe872ce..31f6fed77a2 100644 --- a/ledger/complete/payloadless/node.go +++ b/ledger/complete/payloadless/node.go @@ -317,7 +317,6 @@ func (n *Node) IsAllocatedRegisterLeaf() bool { return n.leafHash != nil } - // computeHash returns the hashValue of the node func (n *Node) computeHash() hash.Hash { // check for leaf node diff --git a/ledger/complete/payloadless/node_test.go b/ledger/complete/payloadless/node_test.go index f0e0230169d..87e904757c7 100644 --- a/ledger/complete/payloadless/node_test.go +++ b/ledger/complete/payloadless/node_test.go @@ -42,7 +42,7 @@ import ( // - `pathLeft` = all bits 0 → left branch at every height (empty sibling always on the RIGHT); // - `pathRight` = all bits 1 → right branch at every height (empty sibling always on the LEFT); // - `pathMixed` = 0x…0135 → interleaved: right,left,right,left,right,right,left,left,right over -// heights 1..9 (bit(255)=1, bit(254)=0, …). +// heights 1..9 (bit(255)=1, bit(254)=0, …). var ( pathLeft = testutils.PathByUint16LeftPadded(0) // 32 bytes 0x00 // pathRight is the all-ones path (32 bytes 0xFF): branches right at every height.