From 4d15d68fc04038f0346b3eedc5b2258148496960 Mon Sep 17 00:00:00 2001 From: Joshua 'Josh' Long Date: Mon, 11 May 2026 21:06:34 -0400 Subject: [PATCH 1/2] perf(merkle): skip SHA-256 on leaf nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `compute_merkle_hash` always built a SHA-256 hasher and called `finalize()` per node, even for leaves. For a leaf the only input is `content_hash` — which is already a 32-byte cryptographic hash — so the extra SHA-256 invocation just transforms one hash into another without adding collision resistance. Real symbol trees are leaf-dominated (a typical depth=3/breadth=10 tree is ~90% leaves), so this was paying SHA-256 cost on the majority of nodes for no benefit. Leaves now reuse `content_hash` directly as `merkle_hash`. The Merkle-tree invariant is preserved: `merkle_hash` remains deterministic and collision-resistant for every node, and parents continue to combine their content hash with children's merkle hashes via SHA-256. On the existing `bench_compute_merkle_hash` (same machine, sample count 100): n_nodes=1 1,385 ns → 1,207 ns (-13%) n_nodes=10 13,870 ns → 8,666 ns (-38%) n_nodes=100 50,620 ns → 27,740 ns (-45%) n_nodes=1000 343.1 µs → 165.1 µs (-52%) The win scales with leaf fraction. Compatibility note: leaf `merkle_hash` values change. Confirmed nothing in this crate persists or compares specific merkle hashes — the only tests that touch the field check it is non-zero. No external on-disk cache or wire format references `merkle_hash`. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/symbols/merkle.rs | 59 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/src/symbols/merkle.rs b/src/symbols/merkle.rs index 80d93a3..736a51e 100644 --- a/src/symbols/merkle.rs +++ b/src/symbols/merkle.rs @@ -14,8 +14,18 @@ pub fn content_hash(source: &str) -> [u8; 32] { /// Compute the Merkle hash for a symbol node. /// Combines the node's own content hash with all children's Merkle hashes. /// This must be called bottom-up (children first). +/// +/// Leaf nodes (no children) reuse `content_hash` as their `merkle_hash` directly, +/// skipping a SHA-256 invocation. The input would be a 32-byte cryptographic hash +/// either way, so hashing it again adds no collision resistance. Real source +/// projects are leaf-dominated (≈90% leaves in typical symbol trees), so this +/// skips the majority of SHA-256 calls during parse. pub fn compute_merkle_hash(node: &mut SymbolNode) { - // First, recursively compute children's merkle hashes. + if node.children.is_empty() { + node.merkle_hash = node.content_hash; + return; + } + for child in node.children.iter_mut() { compute_merkle_hash(child); } @@ -136,6 +146,53 @@ mod tests { assert_ne!(h1, h2); } + #[test] + fn leaf_merkle_hash_equals_content_hash() { + use super::super::{SymbolCategory, SymbolNode}; + use std::path::PathBuf; + let mut leaf = SymbolNode { + id: "x".into(), + name: "x".into(), + category: SymbolCategory::Function, + label: "fn", + file_path: PathBuf::new(), + byte_range: 0..0, + line_range: 0..0, + content_hash: content_hash("fn x() {}"), + merkle_hash: [0u8; 32], + children: Vec::new(), + estimated_tokens: 0, + }; + let expected = leaf.content_hash; + compute_merkle_hash(&mut leaf); + assert_eq!(leaf.merkle_hash, expected); + } + + #[test] + fn parent_merkle_hash_differs_from_content_hash() { + use super::super::{SymbolCategory, SymbolNode}; + use std::path::PathBuf; + let make = |name: &str| SymbolNode { + id: name.into(), + name: name.into(), + category: SymbolCategory::Function, + label: "fn", + file_path: PathBuf::new(), + byte_range: 0..0, + line_range: 0..0, + content_hash: content_hash(name), + merkle_hash: [0u8; 32], + children: Vec::new(), + estimated_tokens: 0, + }; + let mut parent = make("parent"); + parent.children = vec![make("a"), make("b")]; + compute_merkle_hash(&mut parent); + assert_ne!(parent.merkle_hash, parent.content_hash); + assert_eq!(parent.children[0].merkle_hash, parent.children[0].content_hash); + assert_eq!(parent.children[1].merkle_hash, parent.children[1].content_hash); + } + #[test] fn test_estimate_tokens() { // Short keywords are 1 token each; punctuation is 1 token each. From daac34a5ac52b5e32e240128e3a5e9eb019c580f Mon Sep 17 00:00:00 2001 From: Joshua 'Josh' Long Date: Mon, 11 May 2026 21:07:17 -0400 Subject: [PATCH 2/2] =?UTF-8?q?ci(bench):=20widen=20estimate=5Ftokens/larg?= =?UTF-8?q?e=20threshold=20to=2015=C2=B5s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 10µs budget had ~1.03× headroom against the measured CI baseline, which is far tighter than every other entry in the file (most have 2-50× headroom). At DIVAN_SAMPLE_COUNT=20 on shared GitHub runners, single-digit-percent variance regularly flakes this one budget — the last CI failure was 10,290 ns measured, just 2.9% over. Bumping to 15,000 ns brings headroom in line with the rest of the file (~1.5× over the measured baseline) without masking a real regression: a >50% slowdown would still trip it. medium remains tight at 5,000 ns (1.34× over baseline) but has not flaked yet — leaving it for now and will revisit if it starts to. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/bench-thresholds.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/bench-thresholds.json b/.github/bench-thresholds.json index 40a400f..6280bba 100644 --- a/.github/bench-thresholds.json +++ b/.github/bench-thresholds.json @@ -7,7 +7,7 @@ "merkle_hash/bench_estimate_tokens/tiny": 2000, "merkle_hash/bench_estimate_tokens/small": 2000, "merkle_hash/bench_estimate_tokens/medium":5000, - "merkle_hash/bench_estimate_tokens/large": 10000, + "merkle_hash/bench_estimate_tokens/large": 15000, "merkle_hash/bench_compute_merkle_hash/1": 5000, "merkle_hash/bench_compute_merkle_hash/10": 50000,