diff --git a/src/symbols/merkle.rs b/src/symbols/merkle.rs index 80d93a3..83140d6 100644 --- a/src/symbols/merkle.rs +++ b/src/symbols/merkle.rs @@ -4,13 +4,53 @@ use super::SymbolNode; /// Compute content hash from the raw source text of a symbol. /// Normalizes whitespace to make hashing resilient to formatting changes. +/// +/// For ASCII input — virtually all source code — we stream normalized bytes +/// directly into the SHA-256 hasher through a stack buffer, skipping the +/// intermediate `String` allocation and Unicode-aware classification used by +/// the general fallback. Both paths feed bit-for-bit identical byte sequences +/// to the hasher, so hash values are unchanged. pub fn content_hash(source: &str) -> [u8; 32] { - let normalized = normalize_source(source); let mut hasher = Sha256::new(); - hasher.update(normalized.as_bytes()); + if source.is_ascii() { + hash_normalized_ascii(source.trim().as_bytes(), &mut hasher); + } else { + let normalized = normalize_source(source); + hasher.update(normalized.as_bytes()); + } hasher.finalize().into() } +/// Stream `bytes` into `hasher` after collapsing runs of ASCII whitespace into a +/// single space, matching `normalize_source` byte-for-byte. Caller must pre-trim. +#[inline] +fn hash_normalized_ascii(bytes: &[u8], hasher: &mut Sha256) { + let mut buf = [0u8; 256]; + let mut len = 0; + let mut prev_was_space = false; + for &b in bytes { + let push = if is_unicode_ws_ascii(b) { + if prev_was_space { + continue; + } + prev_was_space = true; + b' ' + } else { + prev_was_space = false; + b + }; + buf[len] = push; + len += 1; + if len == buf.len() { + hasher.update(buf); + len = 0; + } + } + if len > 0 { + hasher.update(&buf[..len]); + } +} + /// Compute the Merkle hash for a symbol node. /// Combines the node's own content hash with all children's Merkle hashes. /// This must be called bottom-up (children first). @@ -82,6 +122,36 @@ fn normalize_source(source: &str) -> String { /// vocabulary. It will be most accurate for English-identifier code (Rust, Python, /// TypeScript) and less accurate for heavily symbolic code (e.g. APL, dense regex). pub fn estimate_tokens(source: &str) -> usize { + if source.is_ascii() { + return estimate_tokens_ascii(source.as_bytes()); + } + estimate_tokens_unicode(source) +} + +#[inline] +fn estimate_tokens_ascii(bytes: &[u8]) -> usize { + let mut count = 0usize; + let mut word_len = 0usize; + for &b in bytes { + if b.is_ascii_alphanumeric() || b == b'_' { + word_len += 1; + } else { + if word_len > 0 { + count += word_len.div_ceil(4); + word_len = 0; + } + if !is_unicode_ws_ascii(b) { + count += 1; + } + } + } + if word_len > 0 { + count += word_len.div_ceil(4); + } + count +} + +fn estimate_tokens_unicode(source: &str) -> usize { let mut count = 0usize; let mut word_len = 0usize; @@ -104,6 +174,15 @@ pub fn estimate_tokens(source: &str) -> usize { count } +/// ASCII subset of Unicode's `White_Space` property: matches `char::is_whitespace()` +/// for any ASCII byte. `u8::is_ascii_whitespace()` excludes U+000B (vertical tab), +/// which Unicode treats as whitespace — we include it so the fast path is bit-for-bit +/// equivalent to the Unicode path for ASCII input. +#[inline] +fn is_unicode_ws_ascii(b: u8) -> bool { + matches!(b, b'\t' | b'\n' | 0x0B | 0x0C | b'\r' | b' ') +} + #[cfg(test)] mod tests { use super::*; @@ -136,6 +215,57 @@ mod tests { assert_ne!(h1, h2); } + /// Reference implementation: hash the output of the existing Unicode + /// `normalize_source` path. Used to assert the ASCII fast path is + /// bit-for-bit equivalent. + fn content_hash_via_unicode(source: &str) -> [u8; 32] { + let normalized = normalize_source(source); + let mut hasher = Sha256::new(); + hasher.update(normalized.as_bytes()); + hasher.finalize().into() + } + + #[test] + fn content_hash_ascii_path_matches_unicode_path() { + let samples = [ + "", + "fn foo() {}", + "fn foo() { }", + " \n\t fn bar() {}\n\n", + "let x = (a + b) * c;\nlet y = x * 2;", + "a\x0Bb", // vertical tab — Unicode whitespace, not ASCII whitespace + include_str!("merkle.rs"), // larger ASCII payload: this very file + ]; + for s in samples { + assert_eq!( + content_hash(s), + content_hash_via_unicode(s), + "content_hash disagreement on {s:?}", + ); + } + } + + #[test] + fn ascii_and_unicode_paths_agree_on_ascii_input() { + // The fast path takes the ASCII branch; we manually compare against the + // Unicode fallback for the same input. + let samples = [ + "", + "fn foo() {}", + "calculate_result", + "let x = (a + b) * c;", + " \n\t ", + "a\x0Bb", // includes vertical tab — Unicode whitespace but not ASCII whitespace + ]; + for s in samples { + assert_eq!( + estimate_tokens_ascii(s.as_bytes()), + estimate_tokens_unicode(s), + "estimate_tokens disagreement on {s:?}", + ); + } + } + #[test] fn test_estimate_tokens() { // Short keywords are 1 token each; punctuation is 1 token each.