Perf/merkle ascii fastpath

src/symbols/merkle.rs

@@ -4,13 +4,53 @@ use super::SymbolNode;
 
 /// Compute content hash from the raw source text of a symbol.
 /// Normalizes whitespace to make hashing resilient to formatting changes.
+///
+/// For ASCII input — virtually all source code — we stream normalized bytes
+/// directly into the SHA-256 hasher through a stack buffer, skipping the
+/// intermediate `String` allocation and Unicode-aware classification used by
+/// the general fallback. Both paths feed bit-for-bit identical byte sequences
+/// to the hasher, so hash values are unchanged.
 pub fn content_hash(source: &str) -> [u8; 32] {
-    let normalized = normalize_source(source);
     let mut hasher = Sha256::new();
-    hasher.update(normalized.as_bytes());
+    if source.is_ascii() {
+        hash_normalized_ascii(source.trim().as_bytes(), &mut hasher);
+    } else {
+        let normalized = normalize_source(source);
+        hasher.update(normalized.as_bytes());
+    }
     hasher.finalize().into()
 }
 
+/// Stream `bytes` into `hasher` after collapsing runs of ASCII whitespace into a
+/// single space, matching `normalize_source` byte-for-byte. Caller must pre-trim.
+#[inline]
+fn hash_normalized_ascii(bytes: &[u8], hasher: &mut Sha256) {
+    let mut buf = [0u8; 256];
+    let mut len = 0;
+    let mut prev_was_space = false;
+    for &b in bytes {
+        let push = if is_unicode_ws_ascii(b) {
+            if prev_was_space {
+                continue;
+            }
+            prev_was_space = true;
+            b' '
+        } else {
+            prev_was_space = false;
+            b
+        };
+        buf[len] = push;
+        len += 1;
+        if len == buf.len() {
+            hasher.update(buf);
+            len = 0;
+        }
+    }
+    if len > 0 {
+        hasher.update(&buf[..len]);
+    }
+}
+
 /// Compute the Merkle hash for a symbol node.
 /// Combines the node's own content hash with all children's Merkle hashes.
 /// This must be called bottom-up (children first).
@@ -82,6 +122,36 @@ fn normalize_source(source: &str) -> String {
 /// vocabulary. It will be most accurate for English-identifier code (Rust, Python,
 /// TypeScript) and less accurate for heavily symbolic code (e.g. APL, dense regex).
 pub fn estimate_tokens(source: &str) -> usize {
+    if source.is_ascii() {
+        return estimate_tokens_ascii(source.as_bytes());
+    }
+    estimate_tokens_unicode(source)
+}
+
+#[inline]
+fn estimate_tokens_ascii(bytes: &[u8]) -> usize {
+    let mut count = 0usize;
+    let mut word_len = 0usize;
+    for &b in bytes {
+        if b.is_ascii_alphanumeric() || b == b'_' {
+            word_len += 1;
+        } else {
+            if word_len > 0 {
+                count += word_len.div_ceil(4);
+                word_len = 0;
+            }
+            if !is_unicode_ws_ascii(b) {
+                count += 1;
+            }
+        }
+    }
+    if word_len > 0 {
+        count += word_len.div_ceil(4);
+    }
+    count
+}
+
+fn estimate_tokens_unicode(source: &str) -> usize {
     let mut count = 0usize;
     let mut word_len = 0usize;
 
@@ -104,6 +174,15 @@ pub fn estimate_tokens(source: &str) -> usize {
     count
 }
 
+/// ASCII subset of Unicode's `White_Space` property: matches `char::is_whitespace()`
+/// for any ASCII byte. `u8::is_ascii_whitespace()` excludes U+000B (vertical tab),
+/// which Unicode treats as whitespace — we include it so the fast path is bit-for-bit
+/// equivalent to the Unicode path for ASCII input.
+#[inline]
+fn is_unicode_ws_ascii(b: u8) -> bool {
+    matches!(b, b'\t' | b'\n' | 0x0B | 0x0C | b'\r' | b' ')
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -136,6 +215,57 @@ mod tests {
         assert_ne!(h1, h2);
     }
 
+    /// Reference implementation: hash the output of the existing Unicode
+    /// `normalize_source` path. Used to assert the ASCII fast path is
+    /// bit-for-bit equivalent.
+    fn content_hash_via_unicode(source: &str) -> [u8; 32] {
+        let normalized = normalize_source(source);
+        let mut hasher = Sha256::new();
+        hasher.update(normalized.as_bytes());
+        hasher.finalize().into()
+    }
+
+    #[test]
+    fn content_hash_ascii_path_matches_unicode_path() {
+        let samples = [
+            "",
+            "fn foo() {}",
+            "fn  foo()  {  }",
+            "   \n\t  fn bar() {}\n\n",
+            "let x = (a + b) * c;\nlet y = x * 2;",
+            "a\x0Bb",  // vertical tab — Unicode whitespace, not ASCII whitespace
+            include_str!("merkle.rs"),  // larger ASCII payload: this very file
+        ];
+        for s in samples {
+            assert_eq!(
+                content_hash(s),
+                content_hash_via_unicode(s),
+                "content_hash disagreement on {s:?}",
+            );
+        }
+    }
+
+    #[test]
+    fn ascii_and_unicode_paths_agree_on_ascii_input() {
+        // The fast path takes the ASCII branch; we manually compare against the
+        // Unicode fallback for the same input.
+        let samples = [
+            "",
+            "fn foo() {}",
+            "calculate_result",
+            "let x = (a + b) * c;",
+            "  \n\t  ",
+            "a\x0Bb",  // includes vertical tab — Unicode whitespace but not ASCII whitespace
+        ];
+        for s in samples {
+            assert_eq!(
+                estimate_tokens_ascii(s.as_bytes()),
+                estimate_tokens_unicode(s),
+                "estimate_tokens disagreement on {s:?}",
+            );
+        }
+    }
+
     #[test]
     fn test_estimate_tokens() {
         // Short keywords are 1 token each; punctuation is 1 token each.