From a1661f812b7dfc4871479ec861d19f9cc8d41234 Mon Sep 17 00:00:00 2001 From: Joshua 'Josh' Long Date: Mon, 11 May 2026 20:02:25 -0400 Subject: [PATCH 1/2] perf(merkle): ASCII fast path in estimate_tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `estimate_tokens` walked `source.chars()` and called the Unicode-aware `char::is_alphanumeric()` and `char::is_whitespace()` on every iteration. Both do general-category property lookups across ~150 Unicode classes. Source code is virtually 100% ASCII, so we were paying full Unicode cost on every parsed symbol for zero benefit. The CI bench `merkle_hash/bench_estimate_tokens/large` had crept up to 10,290 ns against a 10,000 ns budget; medium was also tight at 1.34× of budget. Split into: - `estimate_tokens_ascii(&[u8])` — byte-level loop with `is_ascii_alphanumeric` / `is_unicode_ws_ascii` range checks. - `estimate_tokens_unicode(&str)` — the original chars-based body, kept intact as the fallback for any non-ASCII source. `str::is_ascii()` is a single SIMD-vectorized pass — typically cheaper than one full chars() iteration on its own — so the dispatch overhead is negligible. `is_unicode_ws_ascii` matches `char::is_whitespace()` exactly on ASCII bytes (including U+000B which `u8::is_ascii_whitespace()` excludes), keeping the two paths bit-for-bit equivalent. A new test asserts agreement across representative samples including the VT edge case. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/symbols/merkle.rs | 60 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/symbols/merkle.rs b/src/symbols/merkle.rs index 80d93a3..ea6dd29 100644 --- a/src/symbols/merkle.rs +++ b/src/symbols/merkle.rs @@ -82,6 +82,36 @@ fn normalize_source(source: &str) -> String { /// vocabulary. It will be most accurate for English-identifier code (Rust, Python, /// TypeScript) and less accurate for heavily symbolic code (e.g. APL, dense regex). pub fn estimate_tokens(source: &str) -> usize { + if source.is_ascii() { + return estimate_tokens_ascii(source.as_bytes()); + } + estimate_tokens_unicode(source) +} + +#[inline] +fn estimate_tokens_ascii(bytes: &[u8]) -> usize { + let mut count = 0usize; + let mut word_len = 0usize; + for &b in bytes { + if b.is_ascii_alphanumeric() || b == b'_' { + word_len += 1; + } else { + if word_len > 0 { + count += word_len.div_ceil(4); + word_len = 0; + } + if !is_unicode_ws_ascii(b) { + count += 1; + } + } + } + if word_len > 0 { + count += word_len.div_ceil(4); + } + count +} + +fn estimate_tokens_unicode(source: &str) -> usize { let mut count = 0usize; let mut word_len = 0usize; @@ -104,6 +134,15 @@ pub fn estimate_tokens(source: &str) -> usize { count } +/// ASCII subset of Unicode's `White_Space` property: matches `char::is_whitespace()` +/// for any ASCII byte. `u8::is_ascii_whitespace()` excludes U+000B (vertical tab), +/// which Unicode treats as whitespace — we include it so the fast path is bit-for-bit +/// equivalent to the Unicode path for ASCII input. +#[inline] +fn is_unicode_ws_ascii(b: u8) -> bool { + matches!(b, b'\t' | b'\n' | 0x0B | 0x0C | b'\r' | b' ') +} + #[cfg(test)] mod tests { use super::*; @@ -136,6 +175,27 @@ mod tests { assert_ne!(h1, h2); } + #[test] + fn ascii_and_unicode_paths_agree_on_ascii_input() { + // The fast path takes the ASCII branch; we manually compare against the + // Unicode fallback for the same input. + let samples = [ + "", + "fn foo() {}", + "calculate_result", + "let x = (a + b) * c;", + " \n\t ", + "a\x0Bb", // includes vertical tab — Unicode whitespace but not ASCII whitespace + ]; + for s in samples { + assert_eq!( + estimate_tokens_ascii(s.as_bytes()), + estimate_tokens_unicode(s), + "estimate_tokens disagreement on {s:?}", + ); + } + } + #[test] fn test_estimate_tokens() { // Short keywords are 1 token each; punctuation is 1 token each. From 267ffae5a2d3639ce37b4e579aadfab16619663b Mon Sep 17 00:00:00 2001 From: Joshua 'Josh' Long Date: Mon, 11 May 2026 20:34:56 -0400 Subject: [PATCH 2/2] perf(merkle): stream content_hash directly into SHA-256, drop allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `content_hash` called `normalize_source`, which allocated a `String` and filled it via `chars()` + `char::is_whitespace()` only to immediately feed it to the SHA-256 hasher and drop it. For source code (≈100% ASCII) this paid for a heap allocation, UTF-8 decoding, and Unicode-aware whitespace classification with nothing to show for it. The new ASCII fast path streams the same normalized byte sequence into the hasher through a 256-byte stack buffer: - No allocation. - No `chars()` decode — raw byte iteration over `&[u8]`. - `is_unicode_ws_ascii` (ASCII subset of `char::is_whitespace`) for classification. The Unicode path is preserved verbatim for any non-ASCII input. A new test hashes a representative set of ASCII inputs — including this very source file as a non-trivial payload — through both paths and asserts bit-for-bit hash equality, so persisted/cached `content_hash` values are unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/symbols/merkle.rs | 74 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/src/symbols/merkle.rs b/src/symbols/merkle.rs index ea6dd29..83140d6 100644 --- a/src/symbols/merkle.rs +++ b/src/symbols/merkle.rs @@ -4,13 +4,53 @@ use super::SymbolNode; /// Compute content hash from the raw source text of a symbol. /// Normalizes whitespace to make hashing resilient to formatting changes. +/// +/// For ASCII input — virtually all source code — we stream normalized bytes +/// directly into the SHA-256 hasher through a stack buffer, skipping the +/// intermediate `String` allocation and Unicode-aware classification used by +/// the general fallback. Both paths feed bit-for-bit identical byte sequences +/// to the hasher, so hash values are unchanged. pub fn content_hash(source: &str) -> [u8; 32] { - let normalized = normalize_source(source); let mut hasher = Sha256::new(); - hasher.update(normalized.as_bytes()); + if source.is_ascii() { + hash_normalized_ascii(source.trim().as_bytes(), &mut hasher); + } else { + let normalized = normalize_source(source); + hasher.update(normalized.as_bytes()); + } hasher.finalize().into() } +/// Stream `bytes` into `hasher` after collapsing runs of ASCII whitespace into a +/// single space, matching `normalize_source` byte-for-byte. Caller must pre-trim. +#[inline] +fn hash_normalized_ascii(bytes: &[u8], hasher: &mut Sha256) { + let mut buf = [0u8; 256]; + let mut len = 0; + let mut prev_was_space = false; + for &b in bytes { + let push = if is_unicode_ws_ascii(b) { + if prev_was_space { + continue; + } + prev_was_space = true; + b' ' + } else { + prev_was_space = false; + b + }; + buf[len] = push; + len += 1; + if len == buf.len() { + hasher.update(buf); + len = 0; + } + } + if len > 0 { + hasher.update(&buf[..len]); + } +} + /// Compute the Merkle hash for a symbol node. /// Combines the node's own content hash with all children's Merkle hashes. /// This must be called bottom-up (children first). @@ -175,6 +215,36 @@ mod tests { assert_ne!(h1, h2); } + /// Reference implementation: hash the output of the existing Unicode + /// `normalize_source` path. Used to assert the ASCII fast path is + /// bit-for-bit equivalent. + fn content_hash_via_unicode(source: &str) -> [u8; 32] { + let normalized = normalize_source(source); + let mut hasher = Sha256::new(); + hasher.update(normalized.as_bytes()); + hasher.finalize().into() + } + + #[test] + fn content_hash_ascii_path_matches_unicode_path() { + let samples = [ + "", + "fn foo() {}", + "fn foo() { }", + " \n\t fn bar() {}\n\n", + "let x = (a + b) * c;\nlet y = x * 2;", + "a\x0Bb", // vertical tab — Unicode whitespace, not ASCII whitespace + include_str!("merkle.rs"), // larger ASCII payload: this very file + ]; + for s in samples { + assert_eq!( + content_hash(s), + content_hash_via_unicode(s), + "content_hash disagreement on {s:?}", + ); + } + } + #[test] fn ascii_and_unicode_paths_agree_on_ascii_input() { // The fast path takes the ASCII branch; we manually compare against the