From 1fdd290b4c5ed0a841c9f2221e1f27a292ff9cb8 Mon Sep 17 00:00:00 2001 From: Joshua 'Josh' Long Date: Tue, 12 May 2026 08:40:07 -0400 Subject: [PATCH 1/5] perf(round-0): add monorepo-scale benches + heap probe harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 0 of the perf hardening effort: pure instrumentation, no behavior or hot-path changes. Establishes a reproducible yardstick for measuring symbol-pipeline cost at monorepo scale before any algorithmic work. Additions: - `benches/support/fixtures.rs::make_monorepo_tree(n_files, n_per_file)` builds a synthetic `ProjectTree` with ~80% leaves and ~20% one-level type-with-methods groups, distributed across nested `src/pkg_*/` paths and mixed `.rs`/`.ts`/`.py` extensions. Pure in-memory, no tree-sitter. - `benches/symbol_pipeline.rs` covers project-scale operations on the above fixture: `compute_merkle_hash_project`, `coverage_full_project`, `ledger_populate_full_project`. Tiers at 100/1,000/5,000 files (10k / 100k / 500k symbols). - `src/bin/heap_probe.rs` (gated on `dhat-heap` feature) drives the same fixture through dhat to produce per-site allocation profiles. Build: `cargo build --release --features dhat-heap --bin heap_probe`. - `dhat` added as an optional dependency via the `dhat-heap` feature so release builds and library consumers are unaffected. This commit deliberately does not change any production code path. It exists to let us measure → review → optimize in cleanly separated rounds. Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 4 ++ Cargo.lock | 35 ++++++++++ Cargo.toml | 13 ++++ benches/support/fixtures.rs | 62 ++++++++++++++++++ benches/symbol_pipeline.rs | 95 +++++++++++++++++++++++++++ src/bin/heap_probe.rs | 127 ++++++++++++++++++++++++++++++++++++ 6 files changed, 336 insertions(+) create mode 100644 benches/symbol_pipeline.rs create mode 100644 src/bin/heap_probe.rs diff --git a/.gitignore b/.gitignore index ae8dea9..7072cdc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ node_modules/ # Dolt database files (added by bd init) .dolt/ *.db + +# Heap probe / perf measurement artifacts — machine-specific. +.perf-baseline/ +dhat-heap.json diff --git a/Cargo.lock b/Cargo.lock index 843e5c9..d5d736c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,7 @@ dependencies = [ "clap", "color-eyre", "crossterm", + "dhat", "divan", "flume", "ignore", @@ -398,6 +399,22 @@ dependencies = [ "syn", ] +[[package]] +name = "dhat" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98cd11d84628e233de0ce467de10b8633f4ddaecafadefc86e13b84b8739b827" +dependencies = [ + "backtrace", + "lazy_static", + "mintex", + "parking_lot", + "rustc-hash", + "serde", + "serde_json", + "thousands", +] + [[package]] name = "digest" version = "0.10.7" @@ -827,6 +844,12 @@ dependencies = [ "adler2", ] +[[package]] +name = "mintex" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c505b3e17ed6b70a7ed2e67fbb2c560ee327353556120d6e72f5232b6880d536" + [[package]] name = "mio" version = "1.1.1" @@ -1061,6 +1084,12 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustix" version = "0.38.44" @@ -1325,6 +1354,12 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "thousands" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" + [[package]] name = "thread_local" version = "1.1.9" diff --git a/Cargo.toml b/Cargo.toml index 0c773ef..aca1675 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,10 @@ color-eyre = "0.6" serde-pickle = "1.2" flume = "0.12" toml = { version = "0.8", default-features = false, features = ["parse"] } +dhat = { version = "0.3", optional = true } + +[features] +dhat-heap = ["dep:dhat"] [dev-dependencies] tempfile = "3" @@ -53,6 +57,15 @@ harness = false name = "tracking" harness = false +[[bench]] +name = "symbol_pipeline" +harness = false + +[[bin]] +name = "heap_probe" +path = "src/bin/heap_probe.rs" +required-features = ["dhat-heap"] + [lib] name = "ambits" path = "src/lib.rs" diff --git a/benches/support/fixtures.rs b/benches/support/fixtures.rs index 55e8739..35351a3 100644 --- a/benches/support/fixtures.rs +++ b/benches/support/fixtures.rs @@ -1011,6 +1011,68 @@ pub fn make_project(n_files: usize, symbols_per_file: usize) -> ProjectTree { ProjectTree { root: PathBuf::from("/bench/project"), files } } +/// Build a synthetic monorepo `ProjectTree` for project-scale benchmarks. +/// +/// Distributes `n_files` across nested package directories (mimicking realistic +/// path depth) and varies file extensions across `rs`/`ts`/`py`. Within each +/// file the symbol mix is ~80% flat leaves and ~20% one-level-nested groups +/// (a type with four methods), so `compute_merkle_hash`'s leaf-vs-parent code +/// paths both get exercised in proportions roughly matching real source. +/// +/// Pure in-memory construction — no disk I/O, no tree-sitter. Use this for +/// benchmarking the code paths *downstream* of parsing. +pub fn make_monorepo_tree(n_files: usize, symbols_per_file: usize) -> ProjectTree { + const EXTENSIONS: [&str; 3] = ["rs", "ts", "py"]; + let files: Vec = (0..n_files) + .map(|file_idx| { + let ext = EXTENSIONS[file_idx % EXTENSIONS.len()]; + let pkg_idx = file_idx / 20; + let path_str = format!("src/pkg_{pkg_idx:04}/module_{file_idx:06}.{ext}"); + let symbols = build_file_symbols(&path_str, symbols_per_file); + FileSymbols { + file_path: PathBuf::from(&path_str), + symbols, + total_lines: symbols_per_file * 5, + } + }) + .collect(); + ProjectTree { + root: PathBuf::from("/bench/monorepo"), + files, + } +} + +fn build_file_symbols(file_path: &str, n: usize) -> Vec { + let groups = n / 5; + let remainder = n - groups * 5; + let mut out = Vec::with_capacity(groups + remainder); + let file_path_buf = PathBuf::from(file_path); + for g in 0..groups { + let parent_id = format!("{file_path}::Type_{g:04}"); + let mut parent = make_sym(&parent_id, &format!("Type_{g:04}")); + parent.category = SymbolCategory::Type; + parent.label = "struct"; + parent.file_path = file_path_buf.clone(); + parent.children = (0..4) + .map(|i| { + let child_id = format!("{parent_id}::method_{i}"); + let mut sym = make_sym(&child_id, &format!("method_{i}")); + sym.file_path = file_path_buf.clone(); + sym + }) + .collect(); + ambits::symbols::merkle::compute_merkle_hash(&mut parent); + out.push(parent); + } + for i in 0..remainder { + let leaf_id = format!("{file_path}::fn_{i:04}"); + let mut sym = make_sym(&leaf_id, &format!("fn_{i:04}")); + sym.file_path = file_path_buf.clone(); + out.push(sym); + } + out +} + /// Build a ContextLedger pre-populated with entries for all symbols in `project`. pub fn make_populated_ledger(project: &ProjectTree) -> ContextLedger { let mut ledger = ContextLedger::new(); diff --git a/benches/symbol_pipeline.rs b/benches/symbol_pipeline.rs new file mode 100644 index 0000000..4a5b1ac --- /dev/null +++ b/benches/symbol_pipeline.rs @@ -0,0 +1,95 @@ +//! Project-scale benches: simulate operating on a synthetic monorepo's +//! symbol tree (up to ~500k symbols). These do NOT invoke tree-sitter — +//! they target the code paths downstream of parsing: merkle hashing, +//! coverage report construction, and ledger population. +//! +//! Tier sizing (`SYMBOLS_PER_FILE = 100`): +//! 100 files → 10,000 symbols +//! 1,000 files → 100,000 symbols +//! 5,000 files → 500,000 symbols (top tier for monorepo target) + +#[path = "support/fixtures.rs"] +mod fixtures; + +use ambits::coverage::CoverageReport; +use ambits::symbols::ProjectTree; +use ambits::symbols::merkle::compute_merkle_hash; +use ambits::tracking::{ContextLedger, ReadDepth}; +use fixtures::make_monorepo_tree; + +fn main() { + divan::main(); +} + +const SYMBOLS_PER_FILE: usize = 100; + +/// Time `compute_merkle_hash` over every top-level symbol in a project-scale tree. +/// Captures merkle pass cost on monorepo loads — this is the operation parsers +/// perform once per file after symbol extraction. +#[divan::bench(args = [100, 1_000, 5_000], sample_count = 20)] +fn compute_merkle_hash_project(bencher: divan::Bencher, n_files: &usize) { + bencher + .with_inputs(|| make_monorepo_tree(*n_files, SYMBOLS_PER_FILE)) + .bench_local_values(|mut tree| { + for file in tree.files.iter_mut() { + for sym in file.symbols.iter_mut() { + compute_merkle_hash(divan::black_box(sym)); + } + } + divan::black_box(tree) + }); +} + +/// Time `CoverageReport::from_project` on a fully-populated ledger. Models +/// the end-of-session coverage report at monorepo scale. +#[divan::bench(args = [100, 1_000], sample_count = 20)] +fn coverage_full_project(bencher: divan::Bencher, n_files: &usize) { + let project = make_monorepo_tree(*n_files, SYMBOLS_PER_FILE); + let ledger = populate_full(&project); + bencher.bench(|| { + divan::black_box(CoverageReport::from_project( + divan::black_box(&project), + divan::black_box(&ledger), + None, + )) + }); +} + +/// Time recording every symbol in the project tree into a fresh ledger. Models +/// the cost of ingesting a session that read the entire repo once. +#[divan::bench(args = [100, 1_000], sample_count = 20)] +fn ledger_populate_full_project(bencher: divan::Bencher, n_files: &usize) { + let project = make_monorepo_tree(*n_files, SYMBOLS_PER_FILE); + bencher + .with_inputs(ContextLedger::new) + .bench_local_values(|mut ledger| { + for file in &project.files { + for sym in &file.symbols { + ledger.record( + sym.id.clone(), + ReadDepth::Overview, + sym.content_hash, + "agent-0".to_string(), + sym.estimated_tokens, + ); + } + } + divan::black_box(ledger) + }); +} + +fn populate_full(project: &ProjectTree) -> ContextLedger { + let mut ledger = ContextLedger::new(); + for file in &project.files { + for sym in &file.symbols { + ledger.record( + sym.id.clone(), + ReadDepth::FullBody, + sym.content_hash, + "agent-0".to_string(), + sym.estimated_tokens, + ); + } + } + ledger +} diff --git a/src/bin/heap_probe.rs b/src/bin/heap_probe.rs new file mode 100644 index 0000000..b9809c7 --- /dev/null +++ b/src/bin/heap_probe.rs @@ -0,0 +1,127 @@ +//! Project-scale heap profile harness. +//! +//! Build: cargo build --release --features dhat-heap --bin heap_probe +//! Run: ./target/release/heap_probe +//! +//! Writes `dhat-heap.json` to CWD. View with `dh_view.html` or `dhat` from +//! https://valgrind.org/docs/manual/dh-manual.html — drop the JSON onto the +//! viewer to get an interactive breakdown of allocation sites and bytes. +//! +//! Default tier (omit arg) is 1000 files ≈ 100k symbols. + +use std::path::PathBuf; + +use ambits::symbols::merkle::{compute_merkle_hash, content_hash}; +use ambits::symbols::{FileSymbols, ProjectTree, SymbolCategory, SymbolNode}; + +#[global_allocator] +static ALLOC: dhat::Alloc = dhat::Alloc; + +const SYMBOLS_PER_FILE: usize = 100; + +fn main() { + let _profiler = dhat::Profiler::new_heap(); + + let n_files: usize = std::env::args() + .nth(1) + .and_then(|s| s.parse().ok()) + .unwrap_or(1_000); + + eprintln!( + "heap_probe: building tree with {n_files} files × {SYMBOLS_PER_FILE} symbols/file", + ); + + let project = make_monorepo_tree(n_files, SYMBOLS_PER_FILE); + + // Drive a few representative operations against the tree so dhat sees + // the steady-state allocation pattern of real consumers, not just the + // initial construction. + let mut total_symbols = 0usize; + for file in &project.files { + for sym in &file.symbols { + total_symbols += count_subtree(sym); + } + } + eprintln!("heap_probe: total symbols (including nested children) = {total_symbols}"); + + // Keep the tree live until the profiler captures the heap. + std::hint::black_box(&project); +} + +fn count_subtree(node: &SymbolNode) -> usize { + 1 + node.children.iter().map(count_subtree).sum::() +} + +// ─── Inlined monorepo fixture (mirrors benches/support/fixtures.rs) ────────── +// Duplicated here so the binary is self-contained; this lives outside any +// bench harness and is short enough that duplication beats the path-include +// gymnastics needed to share with the bench fixture file. + +fn make_monorepo_tree(n_files: usize, symbols_per_file: usize) -> ProjectTree { + const EXTENSIONS: [&str; 3] = ["rs", "ts", "py"]; + let files: Vec = (0..n_files) + .map(|file_idx| { + let ext = EXTENSIONS[file_idx % EXTENSIONS.len()]; + let pkg_idx = file_idx / 20; + let path_str = format!("src/pkg_{pkg_idx:04}/module_{file_idx:06}.{ext}"); + let symbols = build_file_symbols(&path_str, symbols_per_file); + FileSymbols { + file_path: PathBuf::from(&path_str), + symbols, + total_lines: symbols_per_file * 5, + } + }) + .collect(); + ProjectTree { + root: PathBuf::from("/bench/monorepo"), + files, + } +} + +fn build_file_symbols(file_path: &str, n: usize) -> Vec { + let groups = n / 5; + let remainder = n - groups * 5; + let mut out = Vec::with_capacity(groups + remainder); + let file_path_buf = PathBuf::from(file_path); + for g in 0..groups { + let parent_id = format!("{file_path}::Type_{g:04}"); + let mut parent = make_sym(&parent_id, &format!("Type_{g:04}")); + parent.category = SymbolCategory::Type; + parent.label = "struct"; + parent.file_path = file_path_buf.clone(); + parent.children = (0..4) + .map(|i| { + let child_id = format!("{parent_id}::method_{i}"); + let mut sym = make_sym(&child_id, &format!("method_{i}")); + sym.file_path = file_path_buf.clone(); + sym + }) + .collect(); + compute_merkle_hash(&mut parent); + out.push(parent); + } + for i in 0..remainder { + let leaf_id = format!("{file_path}::fn_{i:04}"); + let mut sym = make_sym(&leaf_id, &format!("fn_{i:04}")); + sym.file_path = file_path_buf.clone(); + out.push(sym); + } + out +} + +fn make_sym(id: &str, name: &str) -> SymbolNode { + let hash = content_hash(name); + SymbolNode { + id: id.to_string(), + name: name.to_string(), + category: SymbolCategory::Function, + label: "fn", + file_path: PathBuf::from("src/bench.rs"), + byte_range: 0..100, + line_range: 1..10, + content_hash: hash, + merkle_hash: hash, + children: Vec::new(), + estimated_tokens: 30, + } +} From ea68e897e61590452b86e80621b0eba2649ee1e0 Mon Sep 17 00:00:00 2001 From: Joshua 'Josh' Long Date: Tue, 12 May 2026 09:09:13 -0400 Subject: [PATCH 2/5] perf(round-1a): share file_path across symbols in a file via Arc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 0 heap profile showed `SymbolNode.file_path` was the third-largest allocation site after the children Vec and the file-level symbols Vec. Every symbol in a file held its own `PathBuf` clone of the file's path — for a 100-symbol file, the same path was duplicated 100 times. Switch `SymbolNode.file_path` from `PathBuf` to `Arc`. Each parser (rust, python, typescript, and the serena pickle loader) now constructs one `Arc` per file and clones the Arc (a refcount bump) into each child symbol. The actual `PathBuf` allocation happens once per file instead of once per symbol. Public API impact is minimal because most consumer code already goes through deref methods (`.as_path()`, `.display()`, `.to_string_lossy()`). The one in-tree case that did `assert_eq!(sym.file_path, Path::new(..))` now uses `sym.file_path.as_path()`. Measured impact (heap_probe, dhat, same machine as Round 0 baseline): tier peak bytes (before → after) peak blocks (before → after) 10k 3,100,748 → 2,657,673 (-14.3%) 32,206 → 22,407 (-30%) 100k 31,005,248 → 26,574,273 (-14.3%) 322,006 → 224,007 (-30%) 500k 155,025,248 → 132,870,273 (-14.3%) 1.61M → 1.12M (-30%) → 22.2 MB saved at the 500k-symbol tier → 490k fewer live heap blocks at peak (less allocator fragmentation pressure on long-running monorepo loads) → per-symbol cost: 310 B → 266 B Time (cargo bench --bench symbol_pipeline): compute_merkle_hash_project / 5000 57.13ms → 57.34ms (noise) coverage_full_project / 1000 4.90ms → 4.01ms (-18%) ledger_populate_full_project / 1000 5.18ms → 5.15ms (noise) Coverage report builder gets a real ~18% win because the path-related work it does on every symbol now operates on an 8-byte pointer instead of a 24-byte struct + heap deref. Co-Authored-By: Claude Opus 4.7 (1M context) --- benches/support/fixtures.rs | 14 ++++++++------ src/bin/heap_probe.rs | 11 ++++++----- src/parser/python.rs | 18 ++++++++++-------- src/parser/rust.rs | 14 ++++++++------ src/parser/typescript.rs | 28 +++++++++++++++------------- src/serena/mod.rs | 10 ++++++---- src/symbols/merkle.rs | 6 ++++-- src/symbols/mod.rs | 10 +++++++++- src/ui/stats.rs | 2 +- src/ui/tree_view.rs | 2 +- tests/e2e.rs | 6 ++++-- tests/helpers/mod.rs | 6 ++++-- 12 files changed, 76 insertions(+), 51 deletions(-) diff --git a/benches/support/fixtures.rs b/benches/support/fixtures.rs index 35351a3..2169133 100644 --- a/benches/support/fixtures.rs +++ b/benches/support/fixtures.rs @@ -1,6 +1,7 @@ #![allow(dead_code)] use std::path::PathBuf; +use std::sync::Arc; use ambits::symbols::merkle::content_hash; use ambits::symbols::{FileSymbols, ProjectTree, SymbolCategory, SymbolNode}; @@ -962,7 +963,7 @@ pub fn make_sym(id: &str, name: &str) -> SymbolNode { name: name.to_string(), category: SymbolCategory::Function, label: "fn", - file_path: PathBuf::from("src/bench.rs"), + file_path: Arc::new(PathBuf::from("src/bench.rs")), byte_range: 0..100, line_range: 1..10, content_hash: hash, @@ -996,9 +997,10 @@ pub fn make_flat_symbols(n: usize) -> Vec { /// Build a FileSymbols containing `n_symbols` flat symbols. pub fn make_file(path: &str, n_symbols: usize) -> FileSymbols { let file_path = PathBuf::from(path); + let file_path_arc = Arc::new(file_path.clone()); let symbols = make_flat_symbols(n_symbols) .into_iter() - .map(|mut s| { s.file_path = file_path.clone(); s }) + .map(|mut s| { s.file_path = Arc::clone(&file_path_arc); s }) .collect(); FileSymbols { file_path, symbols, total_lines: n_symbols * 5 } } @@ -1046,18 +1048,18 @@ fn build_file_symbols(file_path: &str, n: usize) -> Vec { let groups = n / 5; let remainder = n - groups * 5; let mut out = Vec::with_capacity(groups + remainder); - let file_path_buf = PathBuf::from(file_path); + let file_path_arc = Arc::new(PathBuf::from(file_path)); for g in 0..groups { let parent_id = format!("{file_path}::Type_{g:04}"); let mut parent = make_sym(&parent_id, &format!("Type_{g:04}")); parent.category = SymbolCategory::Type; parent.label = "struct"; - parent.file_path = file_path_buf.clone(); + parent.file_path = Arc::clone(&file_path_arc); parent.children = (0..4) .map(|i| { let child_id = format!("{parent_id}::method_{i}"); let mut sym = make_sym(&child_id, &format!("method_{i}")); - sym.file_path = file_path_buf.clone(); + sym.file_path = Arc::clone(&file_path_arc); sym }) .collect(); @@ -1067,7 +1069,7 @@ fn build_file_symbols(file_path: &str, n: usize) -> Vec { for i in 0..remainder { let leaf_id = format!("{file_path}::fn_{i:04}"); let mut sym = make_sym(&leaf_id, &format!("fn_{i:04}")); - sym.file_path = file_path_buf.clone(); + sym.file_path = Arc::clone(&file_path_arc); out.push(sym); } out diff --git a/src/bin/heap_probe.rs b/src/bin/heap_probe.rs index b9809c7..e9d898b 100644 --- a/src/bin/heap_probe.rs +++ b/src/bin/heap_probe.rs @@ -10,6 +10,7 @@ //! Default tier (omit arg) is 1000 files ≈ 100k symbols. use std::path::PathBuf; +use std::sync::Arc; use ambits::symbols::merkle::{compute_merkle_hash, content_hash}; use ambits::symbols::{FileSymbols, ProjectTree, SymbolCategory, SymbolNode}; @@ -82,18 +83,18 @@ fn build_file_symbols(file_path: &str, n: usize) -> Vec { let groups = n / 5; let remainder = n - groups * 5; let mut out = Vec::with_capacity(groups + remainder); - let file_path_buf = PathBuf::from(file_path); + let file_path_arc = Arc::new(PathBuf::from(file_path)); for g in 0..groups { let parent_id = format!("{file_path}::Type_{g:04}"); let mut parent = make_sym(&parent_id, &format!("Type_{g:04}")); parent.category = SymbolCategory::Type; parent.label = "struct"; - parent.file_path = file_path_buf.clone(); + parent.file_path = Arc::clone(&file_path_arc); parent.children = (0..4) .map(|i| { let child_id = format!("{parent_id}::method_{i}"); let mut sym = make_sym(&child_id, &format!("method_{i}")); - sym.file_path = file_path_buf.clone(); + sym.file_path = Arc::clone(&file_path_arc); sym }) .collect(); @@ -103,7 +104,7 @@ fn build_file_symbols(file_path: &str, n: usize) -> Vec { for i in 0..remainder { let leaf_id = format!("{file_path}::fn_{i:04}"); let mut sym = make_sym(&leaf_id, &format!("fn_{i:04}")); - sym.file_path = file_path_buf.clone(); + sym.file_path = Arc::clone(&file_path_arc); out.push(sym); } out @@ -116,7 +117,7 @@ fn make_sym(id: &str, name: &str) -> SymbolNode { name: name.to_string(), category: SymbolCategory::Function, label: "fn", - file_path: PathBuf::from("src/bench.rs"), + file_path: Arc::new(PathBuf::from("src/bench.rs")), byte_range: 0..100, line_range: 1..10, content_hash: hash, diff --git a/src/parser/python.rs b/src/parser/python.rs index f991928..3700550 100644 --- a/src/parser/python.rs +++ b/src/parser/python.rs @@ -29,7 +29,8 @@ //! | `MAX_SIZE = ...` (UPPER_SNAKE_CASE) | Variable | `"var"` | //! | class methods | Function | `"def"` | -use std::path::Path; +use std::path::{Path, PathBuf}; +use std::sync::Arc; use color_eyre::eyre::eyre; use tree_sitter::{Node, Parser}; @@ -73,8 +74,9 @@ impl LanguageParser for PythonParser { let path_prefix = path.to_string_lossy(); let src = source.as_bytes(); let mut symbols = Vec::new(); + let file_path_arc = Arc::new(path.to_path_buf()); - extract_symbols(root, src, path, &path_prefix, "", &mut symbols); + extract_symbols(root, src, &file_path_arc, &path_prefix, "", &mut symbols); for sym in symbols.iter_mut() { compute_merkle_hash(sym); @@ -136,7 +138,7 @@ const TYPE_ALIAS: SymbolMeta = SymbolMeta { category: SymbolCategory::Type, labe fn extract_symbols( node: Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -186,7 +188,7 @@ fn extract_symbols( name: name.clone(), category: meta.category, label: meta.label, - file_path: file_path.to_path_buf(), + file_path: Arc::clone(file_path), byte_range, line_range: start_line..end_line, content_hash: content_hash(text), @@ -224,7 +226,7 @@ fn extract_symbols( fn extract_decorated( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -261,7 +263,7 @@ fn extract_decorated( name: name.clone(), category: meta.category, label: meta.label, - file_path: file_path.to_path_buf(), + file_path: Arc::clone(file_path), byte_range, line_range: start_line..end_line, content_hash: content_hash(text), @@ -396,7 +398,7 @@ fn extract_type_alias(node: &Node, src: &[u8]) -> Option<(String, SymbolMeta)> { fn extract_from_compound_bodies( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -656,7 +658,7 @@ mod tests { .parse_file(Path::new("src/main.py"), "def foo():\n pass\n") .unwrap(); assert_eq!(file.file_path, Path::new("src/main.py")); - assert_eq!(file.symbols[0].file_path, Path::new("src/main.py")); + assert_eq!(file.symbols[0].file_path.as_path(), Path::new("src/main.py")); assert_eq!(file.symbols[0].id, "src/main.py::foo"); } diff --git a/src/parser/rust.rs b/src/parser/rust.rs index 7ebe8dd..8f0ac8b 100644 --- a/src/parser/rust.rs +++ b/src/parser/rust.rs @@ -1,4 +1,5 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; +use std::sync::Arc; use color_eyre::eyre::eyre; use tree_sitter::{Node, Parser}; @@ -38,8 +39,9 @@ impl LanguageParser for RustParser { let path_prefix = path.to_string_lossy(); let src = source.as_bytes(); let mut symbols = Vec::new(); + let file_path_arc = Arc::new(path.to_path_buf()); - extract_symbols(root, src, path, &path_prefix, "", &mut symbols); + extract_symbols(root, src, &file_path_arc, &path_prefix, "", &mut symbols); for sym in symbols.iter_mut() { compute_merkle_hash(sym); @@ -75,7 +77,7 @@ const MACRO: SymbolMeta = SymbolMeta { category: SymbolCategory::Macro, label: " fn extract_symbols( node: Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -114,7 +116,7 @@ fn extract_symbols( name: name.clone(), category: meta.category, label: meta.label, - file_path: file_path.to_path_buf(), + file_path: Arc::clone(file_path), byte_range, line_range: start_line..end_line, content_hash: content_hash(text), @@ -140,7 +142,7 @@ fn extract_symbols( fn extract_body_children( body: Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -168,7 +170,7 @@ fn extract_body_children( name, category: meta.category, label: meta.label, - file_path: file_path.to_path_buf(), + file_path: Arc::clone(file_path), byte_range, line_range: start_line..end_line, content_hash: content_hash(text), diff --git a/src/parser/typescript.rs b/src/parser/typescript.rs index 7f9f73b..5834202 100644 --- a/src/parser/typescript.rs +++ b/src/parser/typescript.rs @@ -32,7 +32,8 @@ //! | class properties / interface props | Variable | `"property"` | //! | `declare ...` | Variable | `"declare"` | -use std::path::Path; +use std::path::{Path, PathBuf}; +use std::sync::Arc; use color_eyre::eyre::eyre; use tree_sitter::{Node, Parser}; @@ -76,8 +77,9 @@ impl LanguageParser for TypescriptParser { let path_prefix = path.to_string_lossy(); let src = source.as_bytes(); let mut symbols = Vec::new(); + let file_path_arc = Arc::new(path.to_path_buf()); - extract_symbols(root, src, path, &path_prefix, "", &mut symbols); + extract_symbols(root, src, &file_path_arc, &path_prefix, "", &mut symbols); for sym in symbols.iter_mut() { compute_merkle_hash(sym); @@ -146,7 +148,7 @@ const DECLARE: SymbolMeta = SymbolMeta { category: SymbolCategory::Variable, lab fn extract_symbols( node: Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -250,7 +252,7 @@ fn extract_symbols( fn extract_arrow_fns( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -297,7 +299,7 @@ fn extract_arrow_fns( fn extract_ambient( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -329,7 +331,7 @@ fn extract_ambient( fn extract_ambient_vars( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, ambient_range: &std::ops::Range, @@ -364,7 +366,7 @@ fn extract_ambient_vars( fn emit_class( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, meta: &SymbolMeta, @@ -402,7 +404,7 @@ fn emit_class( fn extract_members( body: Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -451,7 +453,7 @@ fn extract_members( fn emit_interface( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, byte_range: std::ops::Range, @@ -484,7 +486,7 @@ fn emit_interface( fn emit_namespace( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -530,7 +532,7 @@ fn make_symbol( line_node: &Node, byte_range: std::ops::Range, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, children: Vec, @@ -550,7 +552,7 @@ fn make_symbol( name, category: meta.category, label: meta.label, - file_path: file_path.to_path_buf(), + file_path: Arc::clone(file_path), byte_range, line_range: start_line..end_line, content_hash: content_hash(text), @@ -566,7 +568,7 @@ fn make_symbol( fn build_named_symbol( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, path_prefix: &str, parent_name_path: &str, meta: &SymbolMeta, diff --git a/src/serena/mod.rs b/src/serena/mod.rs index 11b7509..bac37e3 100644 --- a/src/serena/mod.rs +++ b/src/serena/mod.rs @@ -94,9 +94,10 @@ fn parse_raw_pickle(value: &Value) -> Result> { as_list(&items[1]).ok_or_else(|| eyre!("Symbol list not an array for {file_path_str}"))?; let path_prefix = file_path.to_string_lossy(); + let file_path_arc = std::sync::Arc::new(file_path.clone()); let mut symbols = Vec::new(); for sym_val in symbol_list { - if let Ok(node) = convert_symbol(sym_val, &file_path, &path_prefix, "") { + if let Ok(node) = convert_symbol(sym_val, &file_path_arc, &path_prefix, "") { symbols.push(node); } } @@ -136,9 +137,10 @@ fn parse_document_pickle(value: &Value) -> Result> { .ok_or_else(|| eyre!("Cannot find symbols for {file_path_str}"))?; let path_prefix = file_path.to_string_lossy(); + let file_path_arc = std::sync::Arc::new(file_path.clone()); let mut symbols = Vec::new(); for sym_val in symbol_list { - if let Ok(node) = convert_symbol(sym_val, &file_path, &path_prefix, "") { + if let Ok(node) = convert_symbol(sym_val, &file_path_arc, &path_prefix, "") { symbols.push(node); } } @@ -156,7 +158,7 @@ fn parse_document_pickle(value: &Value) -> Result> { /// Convert a pickle Value dict into a SymbolNode. fn convert_symbol( val: &Value, - file_path: &Path, + file_path: &std::sync::Arc, path_prefix: &str, parent_id: &str, ) -> Result { @@ -209,7 +211,7 @@ fn convert_symbol( name, category, label, - file_path: file_path.to_path_buf(), + file_path: std::sync::Arc::clone(file_path), byte_range: (start_line * 40 + start_char)..(end_line * 40 + end_char), line_range: (start_line + 1)..(end_line + 1), // 1-indexed like tree-sitter content_hash, diff --git a/src/symbols/merkle.rs b/src/symbols/merkle.rs index 736a51e..de7880d 100644 --- a/src/symbols/merkle.rs +++ b/src/symbols/merkle.rs @@ -150,12 +150,13 @@ mod tests { fn leaf_merkle_hash_equals_content_hash() { use super::super::{SymbolCategory, SymbolNode}; use std::path::PathBuf; + use std::sync::Arc; let mut leaf = SymbolNode { id: "x".into(), name: "x".into(), category: SymbolCategory::Function, label: "fn", - file_path: PathBuf::new(), + file_path: Arc::new(PathBuf::new()), byte_range: 0..0, line_range: 0..0, content_hash: content_hash("fn x() {}"), @@ -172,12 +173,13 @@ mod tests { fn parent_merkle_hash_differs_from_content_hash() { use super::super::{SymbolCategory, SymbolNode}; use std::path::PathBuf; + use std::sync::Arc; let make = |name: &str| SymbolNode { id: name.into(), name: name.into(), category: SymbolCategory::Function, label: "fn", - file_path: PathBuf::new(), + file_path: Arc::new(PathBuf::new()), byte_range: 0..0, line_range: 0..0, content_hash: content_hash(name), diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index f6d35cc..1c9a2a3 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -1,6 +1,7 @@ use std::fmt; use std::ops::Range; use std::path::PathBuf; +use std::sync::Arc; pub mod merkle; @@ -46,7 +47,14 @@ pub struct SymbolNode { pub name: String, pub category: SymbolCategory, pub label: &'static str, // Language-specific label (e.g., "class", "struct", "def") - pub file_path: PathBuf, + /// Path to the source file that produced this symbol. Wrapped in `Arc` so + /// symbols within the same file share one backing `PathBuf` instead of + /// each carrying their own clone — saves real heap at monorepo scale where + /// a typical file has dozens of symbols. Reads go through deref: most + /// consumers don't notice the change (`.as_path()`, `.display()`, + /// `.to_string_lossy()` all work). Equality checks against `Path` / + /// `PathBuf` need `&**arc` or `arc.as_path()` to unwrap. + pub file_path: Arc, pub byte_range: Range, pub line_range: Range, pub content_hash: [u8; 32], diff --git a/src/ui/stats.rs b/src/ui/stats.rs index 611756c..f5a96f5 100644 --- a/src/ui/stats.rs +++ b/src/ui/stats.rs @@ -295,7 +295,7 @@ mod tests { let hash = ambits::symbols::merkle::content_hash(name); SymbolNode { id: id.into(), name: name.into(), category: SymbolCategory::Function, - label: "fn", file_path: PathBuf::new(), + label: "fn", file_path: std::sync::Arc::new(PathBuf::new()), byte_range: 0..100, line_range: 1..10, content_hash: hash, merkle_hash: hash, children: Vec::new(), estimated_tokens: 30, } diff --git a/src/ui/tree_view.rs b/src/ui/tree_view.rs index 83c2705..5db8588 100644 --- a/src/ui/tree_view.rs +++ b/src/ui/tree_view.rs @@ -120,7 +120,7 @@ mod tests { let hash = ambits::symbols::merkle::content_hash(name); SymbolNode { id: id.into(), name: name.into(), category: SymbolCategory::Function, - label: "fn", file_path: PathBuf::new(), + label: "fn", file_path: std::sync::Arc::new(PathBuf::new()), byte_range: 0..100, line_range: 1..10, content_hash: hash, merkle_hash: hash, children: Vec::new(), estimated_tokens: 30, } diff --git a/tests/e2e.rs b/tests/e2e.rs index 8746edd..c7c745c 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -4,6 +4,7 @@ use std::io::Write; use std::path::PathBuf; +use std::sync::Arc; use ambits::app::App; use ambits::coverage::{CoverageFormatter, CoverageReport, JsonFormatter, TextFormatter}; @@ -42,7 +43,7 @@ fn sym(id: &str, name: &str) -> SymbolNode { name: name.to_string(), category: SymbolCategory::Function, label: "fn", - file_path: PathBuf::new(), + file_path: Arc::new(PathBuf::new()), byte_range: 0..100, line_range: 1..10, content_hash: hash, @@ -54,10 +55,11 @@ fn sym(id: &str, name: &str) -> SymbolNode { fn file(path: &str, symbols: Vec) -> FileSymbols { let file_path = PathBuf::from(path); + let file_path_arc = Arc::new(file_path.clone()); let symbols = symbols .into_iter() .map(|mut s| { - s.file_path = file_path.clone(); + s.file_path = Arc::clone(&file_path_arc); s }) .collect(); diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs index 50f8689..6a4febf 100644 --- a/tests/helpers/mod.rs +++ b/tests/helpers/mod.rs @@ -1,4 +1,5 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::ingest::AgentToolCall; use crate::symbols::{FileSymbols, ProjectTree, SymbolCategory, SymbolNode}; @@ -13,7 +14,7 @@ pub fn sym(id: &str, name: &str) -> SymbolNode { name: name.to_string(), category: SymbolCategory::Function, label: "fn", - file_path: PathBuf::new(), + file_path: Arc::new(PathBuf::new()), byte_range: 0..100, line_range: 1..10, content_hash: hash, @@ -54,10 +55,11 @@ pub fn sym_with_children_and_lines( /// Create a FileSymbols entry, setting each symbol's `file_path` to match. pub fn file(path: &str, symbols: Vec) -> FileSymbols { let file_path = PathBuf::from(path); + let file_path_arc = Arc::new(file_path.clone()); let symbols = symbols .into_iter() .map(|mut s| { - s.file_path = file_path.clone(); + s.file_path = Arc::clone(&file_path_arc); s }) .collect(); From f3adefe4ff537c20e621b292771800a006722091 Mon Sep 17 00:00:00 2001 From: Joshua 'Josh' Long Date: Tue, 12 May 2026 09:37:31 -0400 Subject: [PATCH 3/5] perf(round-1b): intern symbol names via Arc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SymbolNode.name` was a `String`, allocated per symbol via tree-sitter text extraction. Real source has heavy name repetition within a file — methods named `new`, `default`, `build`, `init`; properties named `id`, `name`, `value`; in our synthetic fixture, `method_0`..`method_3` repeating across every `Type_NNNN` group. Switch to `Arc` plus a per-file `NameInterner` (a small `RefCell>>` in `src/symbols/mod.rs`). Each parser constructs one interner in `parse_file` and threads `&NameInterner` alongside the existing `&Arc`; recurrent names share a single heap allocation, unique names cost no more than a fresh `Arc::from`. Scope is intentionally per-file. A global interner would save more on truly common cross-file names (`new`, `id`) but adds thread-safety and lifetime complexity for a marginal additional win; we can revisit if a later round shows it matters. Comparisons against `&str` need `.as_ref()` because `Arc` doesn't implement `PartialEq<&str>` directly — touched every `assert_eq!(sym.name, "...")` test (~94 sites across the parser test modules) and the one in-tree string equality in `app::symbol_name_matches`. Measured impact (heap_probe, dhat, same machine): tier peak bytes (1a → 1b) peak blocks (1a → 1b) 10k 2,657,673 → 2,570,839 (-3.3%) 22,407 → 14,834 (-34%) 100k 26,574,273 → 25,691,839 (-3.3%) 224,007 → 148,034 (-34%) 500k 132,870,273 → 128,451,839 (-3.3%) 1.12M → 740,034 (-34%) Round 1a + 1b combined vs Round 0 baseline: 500k peak: 155.0 MB → 128.5 MB (-17.1%, -26.5 MB) 500k blocks: 1,610,006 → 740,034 (-54%) The block-count drop is the bigger structural win: half a million fewer live heap allocations at peak means less allocator fragmentation pressure in long-running monorepo sessions. Time (cargo bench --bench symbol_pipeline) unchanged across all three operations — interner lookup is HashMap-cheap and only happens during construction, not during the workload the bench measures. Co-Authored-By: Claude Opus 4.7 (1M context) --- benches/support/fixtures.rs | 18 +++-- src/app.rs | 4 +- src/bin/heap_probe.rs | 18 +++-- src/parser/python.rs | 130 ++++++++++++++++---------------- src/parser/rust.rs | 25 ++++--- src/parser/typescript.rs | 145 +++++++++++++++++++----------------- src/serena/mod.rs | 2 +- src/symbols/mod.rs | 44 ++++++++++- tests/e2e.rs | 2 +- tests/helpers/mod.rs | 2 +- 10 files changed, 233 insertions(+), 157 deletions(-) diff --git a/benches/support/fixtures.rs b/benches/support/fixtures.rs index 2169133..357166c 100644 --- a/benches/support/fixtures.rs +++ b/benches/support/fixtures.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use std::sync::Arc; use ambits::symbols::merkle::content_hash; -use ambits::symbols::{FileSymbols, ProjectTree, SymbolCategory, SymbolNode}; +use ambits::symbols::{FileSymbols, NameInterner, ProjectTree, SymbolCategory, SymbolNode}; use ambits::tracking::{ContextLedger, ReadDepth}; // ─── Rust source fixtures ───────────────────────────────────────────────────── @@ -960,7 +960,7 @@ pub fn make_sym(id: &str, name: &str) -> SymbolNode { let hash = content_hash(name); SymbolNode { id: id.to_string(), - name: name.to_string(), + name: Arc::from(name), category: SymbolCategory::Function, label: "fn", file_path: Arc::new(PathBuf::from("src/bench.rs")), @@ -1049,16 +1049,22 @@ fn build_file_symbols(file_path: &str, n: usize) -> Vec { let remainder = n - groups * 5; let mut out = Vec::with_capacity(groups + remainder); let file_path_arc = Arc::new(PathBuf::from(file_path)); + // One interner per file — same scope as production parsers. + let names = NameInterner::new(); for g in 0..groups { let parent_id = format!("{file_path}::Type_{g:04}"); - let mut parent = make_sym(&parent_id, &format!("Type_{g:04}")); + let parent_name = format!("Type_{g:04}"); + let mut parent = make_sym(&parent_id, &parent_name); + parent.name = names.intern(&parent_name); parent.category = SymbolCategory::Type; parent.label = "struct"; parent.file_path = Arc::clone(&file_path_arc); parent.children = (0..4) .map(|i| { let child_id = format!("{parent_id}::method_{i}"); - let mut sym = make_sym(&child_id, &format!("method_{i}")); + let child_name = format!("method_{i}"); + let mut sym = make_sym(&child_id, &child_name); + sym.name = names.intern(&child_name); sym.file_path = Arc::clone(&file_path_arc); sym }) @@ -1068,7 +1074,9 @@ fn build_file_symbols(file_path: &str, n: usize) -> Vec { } for i in 0..remainder { let leaf_id = format!("{file_path}::fn_{i:04}"); - let mut sym = make_sym(&leaf_id, &format!("fn_{i:04}")); + let leaf_name = format!("fn_{i:04}"); + let mut sym = make_sym(&leaf_id, &leaf_name); + sym.name = names.intern(&leaf_name); sym.file_path = Arc::clone(&file_path_arc); out.push(sym); } diff --git a/src/app.rs b/src/app.rs index 4139c54..60f75d2 100644 --- a/src/app.rs +++ b/src/app.rs @@ -668,7 +668,7 @@ fn flatten_symbol( rows.push(TreeRow { symbol_id: sym.id.clone(), - display_name: sym.name.clone(), + display_name: sym.name.to_string(), label: sym.label, depth, is_file: false, @@ -825,7 +825,7 @@ pub fn symbol_name_matches(sym: &SymbolNode, event: &AgentToolCall) -> bool { } // Plain name match (e.g. target = "handle_key", sym.name = "handle_key"). let norm_sym_name = normalize_name_path(&sym.name); - sym.name == *target_name || norm_sym_name == norm_target + sym.name.as_ref() == target_name.as_str() || norm_sym_name == norm_target } /// Check if a symbol's line range overlaps with the tool call's `target_lines`. diff --git a/src/bin/heap_probe.rs b/src/bin/heap_probe.rs index e9d898b..67d14d8 100644 --- a/src/bin/heap_probe.rs +++ b/src/bin/heap_probe.rs @@ -13,7 +13,7 @@ use std::path::PathBuf; use std::sync::Arc; use ambits::symbols::merkle::{compute_merkle_hash, content_hash}; -use ambits::symbols::{FileSymbols, ProjectTree, SymbolCategory, SymbolNode}; +use ambits::symbols::{FileSymbols, NameInterner, ProjectTree, SymbolCategory, SymbolNode}; #[global_allocator] static ALLOC: dhat::Alloc = dhat::Alloc; @@ -84,16 +84,22 @@ fn build_file_symbols(file_path: &str, n: usize) -> Vec { let remainder = n - groups * 5; let mut out = Vec::with_capacity(groups + remainder); let file_path_arc = Arc::new(PathBuf::from(file_path)); + // One interner per file — same scope as production parsers. + let names = NameInterner::new(); for g in 0..groups { let parent_id = format!("{file_path}::Type_{g:04}"); - let mut parent = make_sym(&parent_id, &format!("Type_{g:04}")); + let parent_name = format!("Type_{g:04}"); + let mut parent = make_sym(&parent_id, &parent_name); + parent.name = names.intern(&parent_name); parent.category = SymbolCategory::Type; parent.label = "struct"; parent.file_path = Arc::clone(&file_path_arc); parent.children = (0..4) .map(|i| { let child_id = format!("{parent_id}::method_{i}"); - let mut sym = make_sym(&child_id, &format!("method_{i}")); + let child_name = format!("method_{i}"); + let mut sym = make_sym(&child_id, &child_name); + sym.name = names.intern(&child_name); sym.file_path = Arc::clone(&file_path_arc); sym }) @@ -103,7 +109,9 @@ fn build_file_symbols(file_path: &str, n: usize) -> Vec { } for i in 0..remainder { let leaf_id = format!("{file_path}::fn_{i:04}"); - let mut sym = make_sym(&leaf_id, &format!("fn_{i:04}")); + let leaf_name = format!("fn_{i:04}"); + let mut sym = make_sym(&leaf_id, &leaf_name); + sym.name = names.intern(&leaf_name); sym.file_path = Arc::clone(&file_path_arc); out.push(sym); } @@ -114,7 +122,7 @@ fn make_sym(id: &str, name: &str) -> SymbolNode { let hash = content_hash(name); SymbolNode { id: id.to_string(), - name: name.to_string(), + name: Arc::from(name), category: SymbolCategory::Function, label: "fn", file_path: Arc::new(PathBuf::from("src/bench.rs")), diff --git a/src/parser/python.rs b/src/parser/python.rs index 3700550..ae8d6f3 100644 --- a/src/parser/python.rs +++ b/src/parser/python.rs @@ -36,7 +36,7 @@ use color_eyre::eyre::eyre; use tree_sitter::{Node, Parser}; use crate::symbols::merkle::{compute_merkle_hash, content_hash, estimate_tokens}; -use crate::symbols::{FileSymbols, SymbolCategory, SymbolNode}; +use crate::symbols::{FileSymbols, NameInterner, SymbolCategory, SymbolNode}; use super::LanguageParser; @@ -75,8 +75,9 @@ impl LanguageParser for PythonParser { let src = source.as_bytes(); let mut symbols = Vec::new(); let file_path_arc = Arc::new(path.to_path_buf()); + let names = NameInterner::new(); - extract_symbols(root, src, &file_path_arc, &path_prefix, "", &mut symbols); + extract_symbols(root, src, &file_path_arc, &names, &path_prefix, "", &mut symbols); for sym in symbols.iter_mut() { compute_merkle_hash(sym); @@ -139,6 +140,7 @@ fn extract_symbols( node: Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -152,7 +154,7 @@ fn extract_symbols( "class_definition" => child_name(&child, src).map(|n| (n, CLASS)), // Decorated definitions: unwrap the decorator to find the inner def/class. "decorated_definition" => { - extract_decorated(&child, src, file_path, path_prefix, parent_name_path, out); + extract_decorated(&child, src, file_path, names, path_prefix, parent_name_path, out); None } // Module-level assignments: annotated or UPPER_SNAKE_CASE only. @@ -163,7 +165,7 @@ fn extract_symbols( "if_statement" | "for_statement" | "while_statement" | "try_statement" | "with_statement" | "match_statement" => { extract_from_compound_bodies( - &child, src, file_path, path_prefix, parent_name_path, out, + &child, src, file_path, names, path_prefix, parent_name_path, out, ); None } @@ -185,7 +187,7 @@ fn extract_symbols( let mut sym = SymbolNode { id, - name: name.clone(), + name: names.intern(&name), category: meta.category, label: meta.label, file_path: Arc::clone(file_path), @@ -200,7 +202,7 @@ fn extract_symbols( // For classes, recurse into the body block to find methods. if meta.category == SymbolCategory::Type { if let Some(body) = child.child_by_field_name("body") { - extract_symbols(body, src, file_path, path_prefix, &name_path, &mut sym.children); + extract_symbols(body, src, file_path, names, path_prefix, &name_path, &mut sym.children); } } @@ -227,6 +229,7 @@ fn extract_decorated( node: &Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -260,7 +263,7 @@ fn extract_decorated( let mut sym = SymbolNode { id, - name: name.clone(), + name: names.intern(&name), category: meta.category, label: meta.label, file_path: Arc::clone(file_path), @@ -274,7 +277,7 @@ fn extract_decorated( if meta.category == SymbolCategory::Type { if let Some(body) = child.child_by_field_name("body") { - extract_symbols(body, src, file_path, path_prefix, &name_path, &mut sym.children); + extract_symbols(body, src, file_path, names, path_prefix, &name_path, &mut sym.children); } } @@ -399,6 +402,7 @@ fn extract_from_compound_bodies( node: &Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -406,12 +410,12 @@ fn extract_from_compound_bodies( match node.kind() { "for_statement" | "while_statement" | "with_statement" => { if let Some(body) = node.child_by_field_name("body") { - extract_symbols(body, src, file_path, path_prefix, parent_name_path, out); + extract_symbols(body, src, file_path, names, path_prefix, parent_name_path, out); } } "if_statement" => { if let Some(body) = node.child_by_field_name("consequence") { - extract_symbols(body, src, file_path, path_prefix, parent_name_path, out); + extract_symbols(body, src, file_path, names, path_prefix, parent_name_path, out); } let mut cursor = node.walk(); for child in node.children(&mut cursor) { @@ -419,14 +423,14 @@ fn extract_from_compound_bodies( "elif_clause" => { if let Some(body) = child.child_by_field_name("consequence") { extract_symbols( - body, src, file_path, path_prefix, parent_name_path, out, + body, src, file_path, names, path_prefix, parent_name_path, out, ); } } "else_clause" => { if let Some(body) = child.child_by_field_name("body") { extract_symbols( - body, src, file_path, path_prefix, parent_name_path, out, + body, src, file_path, names, path_prefix, parent_name_path, out, ); } } @@ -436,7 +440,7 @@ fn extract_from_compound_bodies( } "try_statement" => { if let Some(body) = node.child_by_field_name("body") { - extract_symbols(body, src, file_path, path_prefix, parent_name_path, out); + extract_symbols(body, src, file_path, names, path_prefix, parent_name_path, out); } let mut cursor = node.walk(); for child in node.children(&mut cursor) { @@ -446,7 +450,7 @@ fn extract_from_compound_bodies( for inner in child.children(&mut inner_cursor) { if inner.kind() == "block" { extract_symbols( - inner, src, file_path, path_prefix, parent_name_path, out, + inner, src, file_path, names, path_prefix, parent_name_path, out, ); } } @@ -454,7 +458,7 @@ fn extract_from_compound_bodies( "else_clause" => { if let Some(body) = child.child_by_field_name("body") { extract_symbols( - body, src, file_path, path_prefix, parent_name_path, out, + body, src, file_path, names, path_prefix, parent_name_path, out, ); } } @@ -469,7 +473,7 @@ fn extract_from_compound_bodies( if child.kind() == "case_clause" { if let Some(consequence) = child.child_by_field_name("consequence") { extract_symbols( - consequence, src, file_path, path_prefix, parent_name_path, out, + consequence, src, file_path, names, path_prefix, parent_name_path, out, ); } } @@ -508,7 +512,7 @@ mod tests { fn parse_simple_function() { let syms = parse("def foo():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].category, SymbolCategory::Function); assert_eq!(syms[0].label, "def"); assert_eq!(syms[0].id, "test.py::foo"); @@ -519,15 +523,15 @@ mod tests { fn parse_multiple_functions() { let syms = parse("def foo():\n pass\n\ndef bar():\n pass\n"); assert_eq!(syms.len(), 2); - assert_eq!(syms[0].name, "foo"); - assert_eq!(syms[1].name, "bar"); + assert_eq!(syms[0].name.as_ref(), "foo"); + assert_eq!(syms[1].name.as_ref(), "bar"); } #[test] fn parse_simple_class() { let syms = parse("class Foo:\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].label, "class"); assert_eq!(syms[0].id, "test.py::Foo"); @@ -539,13 +543,13 @@ mod tests { "class Foo:\n def __init__(self):\n pass\n def bar(self):\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].children.len(), 2); - assert_eq!(syms[0].children[0].name, "__init__"); + assert_eq!(syms[0].children[0].name.as_ref(), "__init__"); assert_eq!(syms[0].children[0].category, SymbolCategory::Function); assert_eq!(syms[0].children[0].id, "test.py::Foo/__init__"); - assert_eq!(syms[0].children[1].name, "bar"); + assert_eq!(syms[0].children[1].name.as_ref(), "bar"); assert_eq!(syms[0].children[1].id, "test.py::Foo/bar"); } @@ -553,7 +557,7 @@ mod tests { fn parse_decorated_function() { let syms = parse("@staticmethod\ndef foo():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].category, SymbolCategory::Function); assert_eq!(syms[0].label, "def"); // Decorated definitions include the decorator in the byte range @@ -564,7 +568,7 @@ mod tests { fn parse_decorated_class() { let syms = parse("@dataclass\nclass Point:\n x: int\n y: int\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Point"); + assert_eq!(syms[0].name.as_ref(), "Point"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].label, "class"); // Decorated class range includes the decorator @@ -577,11 +581,11 @@ mod tests { "@dataclass\nclass Point:\n def __init__(self):\n pass\n def distance(self):\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Point"); + assert_eq!(syms[0].name.as_ref(), "Point"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].children.len(), 2); - assert_eq!(syms[0].children[0].name, "__init__"); - assert_eq!(syms[0].children[1].name, "distance"); + assert_eq!(syms[0].children[0].name.as_ref(), "__init__"); + assert_eq!(syms[0].children[1].name.as_ref(), "distance"); } #[test] @@ -590,12 +594,12 @@ mod tests { "def helper():\n pass\n\nclass Foo:\n def method(self):\n pass\n\ndef another():\n pass\n", ); assert_eq!(syms.len(), 3); - assert_eq!(syms[0].name, "helper"); + assert_eq!(syms[0].name.as_ref(), "helper"); assert_eq!(syms[0].category, SymbolCategory::Function); - assert_eq!(syms[1].name, "Foo"); + assert_eq!(syms[1].name.as_ref(), "Foo"); assert_eq!(syms[1].category, SymbolCategory::Type); assert_eq!(syms[1].children.len(), 1); - assert_eq!(syms[2].name, "another"); + assert_eq!(syms[2].name.as_ref(), "another"); assert_eq!(syms[2].category, SymbolCategory::Function); } @@ -605,14 +609,14 @@ mod tests { "class Outer:\n class Inner:\n def method(self):\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Outer"); + assert_eq!(syms[0].name.as_ref(), "Outer"); assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "Inner"); + assert_eq!(syms[0].children[0].name.as_ref(), "Inner"); assert_eq!(syms[0].children[0].category, SymbolCategory::Type); assert_eq!(syms[0].children[0].id, "test.py::Outer/Inner"); // Inner class should have its own children assert_eq!(syms[0].children[0].children.len(), 1); - assert_eq!(syms[0].children[0].children[0].name, "method"); + assert_eq!(syms[0].children[0].children[0].name.as_ref(), "method"); assert_eq!(syms[0].children[0].children[0].id, "test.py::Outer/Inner/method"); } @@ -675,9 +679,9 @@ mod tests { "class Foo:\n @staticmethod\n def bar():\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "bar"); + assert_eq!(syms[0].children[0].name.as_ref(), "bar"); assert_eq!(syms[0].children[0].category, SymbolCategory::Function); } @@ -685,7 +689,7 @@ mod tests { fn parse_multiple_decorators() { let syms = parse("@decorator1\n@decorator2\ndef foo():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].line_range.start, 1); } @@ -697,7 +701,7 @@ mod tests { // async def is not a direct `function_definition` – it is an expression_statement // or may be wrapped. Verify actual behavior either way. assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -716,7 +720,7 @@ mod tests { fn parse_annotated_variable() { let syms = parse("x: int = 42\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "x"); + assert_eq!(syms[0].name.as_ref(), "x"); assert_eq!(syms[0].category, SymbolCategory::Variable); assert_eq!(syms[0].label, "var"); assert_eq!(syms[0].id, "test.py::x"); @@ -727,7 +731,7 @@ mod tests { fn parse_annotated_variable_no_value() { let syms = parse("x: int\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "x"); + assert_eq!(syms[0].name.as_ref(), "x"); assert_eq!(syms[0].category, SymbolCategory::Variable); } @@ -735,7 +739,7 @@ mod tests { fn parse_upper_case_constant() { let syms = parse("MAX_SIZE = 100\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "MAX_SIZE"); + assert_eq!(syms[0].name.as_ref(), "MAX_SIZE"); assert_eq!(syms[0].category, SymbolCategory::Variable); assert_eq!(syms[0].label, "var"); } @@ -744,7 +748,7 @@ mod tests { fn parse_upper_case_no_underscore() { let syms = parse("DEBUG = True\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "DEBUG"); + assert_eq!(syms[0].name.as_ref(), "DEBUG"); assert_eq!(syms[0].category, SymbolCategory::Variable); } @@ -776,11 +780,11 @@ mod tests { fn parse_variables_mixed_with_functions() { let syms = parse("MAX: int = 100\n\ndef foo():\n pass\n\nDEBUG = True\n"); assert_eq!(syms.len(), 3); - assert_eq!(syms[0].name, "MAX"); + assert_eq!(syms[0].name.as_ref(), "MAX"); assert_eq!(syms[0].category, SymbolCategory::Variable); - assert_eq!(syms[1].name, "foo"); + assert_eq!(syms[1].name.as_ref(), "foo"); assert_eq!(syms[1].category, SymbolCategory::Function); - assert_eq!(syms[2].name, "DEBUG"); + assert_eq!(syms[2].name.as_ref(), "DEBUG"); assert_eq!(syms[2].category, SymbolCategory::Variable); } @@ -790,7 +794,7 @@ mod tests { fn parse_type_alias() { let syms = parse("type Vector = list[float]\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Vector"); + assert_eq!(syms[0].name.as_ref(), "Vector"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].label, "type"); assert_eq!(syms[0].id, "test.py::Vector"); @@ -800,9 +804,9 @@ mod tests { fn parse_type_alias_with_functions() { let syms = parse("type ID = int\n\ndef process(x: ID) -> None:\n pass\n"); assert_eq!(syms.len(), 2); - assert_eq!(syms[0].name, "ID"); + assert_eq!(syms[0].name.as_ref(), "ID"); assert_eq!(syms[0].category, SymbolCategory::Type); - assert_eq!(syms[1].name, "process"); + assert_eq!(syms[1].name.as_ref(), "process"); assert_eq!(syms[1].category, SymbolCategory::Function); } @@ -812,7 +816,7 @@ mod tests { fn parse_function_inside_if() { let syms = parse("if True:\n def foo():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -822,8 +826,8 @@ mod tests { "if sys.platform == 'win32':\n def init():\n pass\nelse:\n def init():\n pass\n", ); assert_eq!(syms.len(), 2); - assert_eq!(syms[0].name, "init"); - assert_eq!(syms[1].name, "init"); + assert_eq!(syms[0].name.as_ref(), "init"); + assert_eq!(syms[1].name.as_ref(), "init"); } #[test] @@ -832,7 +836,7 @@ mod tests { "if False:\n pass\nelif True:\n def handler():\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "handler"); + assert_eq!(syms[0].name.as_ref(), "handler"); } #[test] @@ -841,9 +845,9 @@ mod tests { "try:\n class Foo:\n pass\nexcept Exception:\n class FallbackFoo:\n pass\n", ); assert_eq!(syms.len(), 2); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].category, SymbolCategory::Type); - assert_eq!(syms[1].name, "FallbackFoo"); + assert_eq!(syms[1].name.as_ref(), "FallbackFoo"); assert_eq!(syms[1].category, SymbolCategory::Type); } @@ -853,36 +857,36 @@ mod tests { "try:\n def setup():\n pass\nfinally:\n def cleanup():\n pass\n", ); assert_eq!(syms.len(), 2); - assert_eq!(syms[0].name, "setup"); - assert_eq!(syms[1].name, "cleanup"); + assert_eq!(syms[0].name.as_ref(), "setup"); + assert_eq!(syms[1].name.as_ref(), "cleanup"); } #[test] fn parse_function_inside_for() { let syms = parse("for i in range(1):\n def worker():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "worker"); + assert_eq!(syms[0].name.as_ref(), "worker"); } #[test] fn parse_function_inside_while() { let syms = parse("while True:\n def loop_body():\n pass\n break\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "loop_body"); + assert_eq!(syms[0].name.as_ref(), "loop_body"); } #[test] fn parse_function_inside_with() { let syms = parse("with open('f') as f:\n def process():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "process"); + assert_eq!(syms[0].name.as_ref(), "process"); } #[test] fn parse_decorated_function_inside_if() { let syms = parse("if True:\n @decorator\n def foo():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -892,7 +896,7 @@ mod tests { "try:\n if True:\n def deeply_nested():\n pass\nexcept:\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "deeply_nested"); + assert_eq!(syms[0].name.as_ref(), "deeply_nested"); } #[test] @@ -901,10 +905,10 @@ mod tests { "if True:\n class Foo:\n def method(self):\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "method"); + assert_eq!(syms[0].children[0].name.as_ref(), "method"); } // --- is_upper_snake_case unit tests --- diff --git a/src/parser/rust.rs b/src/parser/rust.rs index 8f0ac8b..3232054 100644 --- a/src/parser/rust.rs +++ b/src/parser/rust.rs @@ -5,7 +5,7 @@ use color_eyre::eyre::eyre; use tree_sitter::{Node, Parser}; use crate::symbols::merkle::{compute_merkle_hash, content_hash, estimate_tokens}; -use crate::symbols::{FileSymbols, SymbolCategory, SymbolNode}; +use crate::symbols::{FileSymbols, NameInterner, SymbolCategory, SymbolNode}; use super::LanguageParser; @@ -40,8 +40,9 @@ impl LanguageParser for RustParser { let src = source.as_bytes(); let mut symbols = Vec::new(); let file_path_arc = Arc::new(path.to_path_buf()); + let names = NameInterner::new(); - extract_symbols(root, src, &file_path_arc, &path_prefix, "", &mut symbols); + extract_symbols(root, src, &file_path_arc, &names, &path_prefix, "", &mut symbols); for sym in symbols.iter_mut() { compute_merkle_hash(sym); @@ -78,6 +79,7 @@ fn extract_symbols( node: Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -113,7 +115,7 @@ fn extract_symbols( let mut sym = SymbolNode { id, - name: name.clone(), + name: names.intern(&name), category: meta.category, label: meta.label, file_path: Arc::clone(file_path), @@ -130,7 +132,7 @@ fn extract_symbols( || meta.label == "trait" { if let Some(body) = child_by_kind(&child, "declaration_list") { - extract_body_children(body, src, file_path, path_prefix, &name_path, &mut sym.children); + extract_body_children(body, src, file_path, names, path_prefix, &name_path, &mut sym.children); } } @@ -143,6 +145,7 @@ fn extract_body_children( body: Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -167,7 +170,7 @@ fn extract_body_children( out.push(SymbolNode { id, - name, + name: names.intern(&name), category: meta.category, label: meta.label, file_path: Arc::clone(file_path), @@ -250,7 +253,7 @@ mod tests { fn parse_function() { let syms = parse("fn foo() {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -260,22 +263,22 @@ mod tests { "struct Point { x: i32 }\nimpl Point {\n fn new() -> Self { Self { x: 0 } }\n}", ); assert_eq!(syms.len(), 2); - assert_eq!(syms[0].name, "Point"); + assert_eq!(syms[0].name.as_ref(), "Point"); assert_eq!(syms[0].category, SymbolCategory::Type); - assert_eq!(syms[1].name, "Point"); + assert_eq!(syms[1].name.as_ref(), "Point"); assert_eq!(syms[1].category, SymbolCategory::Implementation); assert_eq!(syms[1].children.len(), 1); - assert_eq!(syms[1].children[0].name, "new"); + assert_eq!(syms[1].children[0].name.as_ref(), "new"); } #[test] fn parse_nested_module() { let syms = parse("mod inner {\n fn bar() {}\n}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "inner"); + assert_eq!(syms[0].name.as_ref(), "inner"); assert_eq!(syms[0].category, SymbolCategory::Module); assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "bar"); + assert_eq!(syms[0].children[0].name.as_ref(), "bar"); } #[test] diff --git a/src/parser/typescript.rs b/src/parser/typescript.rs index 5834202..1902c76 100644 --- a/src/parser/typescript.rs +++ b/src/parser/typescript.rs @@ -39,7 +39,7 @@ use color_eyre::eyre::eyre; use tree_sitter::{Node, Parser}; use crate::symbols::merkle::{compute_merkle_hash, content_hash, estimate_tokens}; -use crate::symbols::{FileSymbols, SymbolCategory, SymbolNode}; +use crate::symbols::{FileSymbols, NameInterner, SymbolCategory, SymbolNode}; use super::LanguageParser; @@ -78,8 +78,9 @@ impl LanguageParser for TypescriptParser { let src = source.as_bytes(); let mut symbols = Vec::new(); let file_path_arc = Arc::new(path.to_path_buf()); + let names = NameInterner::new(); - extract_symbols(root, src, &file_path_arc, &path_prefix, "", &mut symbols); + extract_symbols(root, src, &file_path_arc, &names, &path_prefix, "", &mut symbols); for sym in symbols.iter_mut() { compute_merkle_hash(sym); @@ -149,6 +150,7 @@ fn extract_symbols( node: Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -175,7 +177,7 @@ fn extract_symbols( let mut inner_cursor = child.walk(); for inner in child.children(&mut inner_cursor) { if inner.kind() == "internal_module" { - emit_namespace(&inner, src, file_path, path_prefix, parent_name_path, out); + emit_namespace(&inner, src, file_path, names, path_prefix, parent_name_path, out); } } continue; @@ -188,45 +190,45 @@ fn extract_symbols( match target.kind() { // `function foo()` or `function* gen()` - leaf symbol, no children. "function_declaration" | "generator_function_declaration" => { - if let Some(sym) = build_named_symbol(&target, src, file_path, path_prefix, parent_name_path, &FN, byte_range) { + if let Some(sym) = build_named_symbol(&target, src, file_path, names, path_prefix, parent_name_path, &FN, byte_range) { out.push(sym); } } // `class Foo { ... }` - container, recurse into class_body for members. "class_declaration" => { - emit_class(&target, src, file_path, path_prefix, parent_name_path, &CLASS, byte_range, out); + emit_class(&target, src, file_path, names, path_prefix, parent_name_path, &CLASS, byte_range, out); } // `abstract class Base { ... }` - same as class but different label. "abstract_class_declaration" => { - emit_class(&target, src, file_path, path_prefix, parent_name_path, &ABSTRACT_CLASS, byte_range, out); + emit_class(&target, src, file_path, names, path_prefix, parent_name_path, &ABSTRACT_CLASS, byte_range, out); } // `interface Config { ... }` - container, recurse into interface_body. "interface_declaration" => { - emit_interface(&target, src, file_path, path_prefix, parent_name_path, byte_range, out); + emit_interface(&target, src, file_path, names, path_prefix, parent_name_path, byte_range, out); } // `type Alias = ...` - leaf symbol. "type_alias_declaration" => { - if let Some(sym) = build_named_symbol(&target, src, file_path, path_prefix, parent_name_path, &TYPE, byte_range) { + if let Some(sym) = build_named_symbol(&target, src, file_path, names, path_prefix, parent_name_path, &TYPE, byte_range) { out.push(sym); } } // `enum Status { ... }` - leaf (we don't extract enum members). "enum_declaration" => { - if let Some(sym) = build_named_symbol(&target, src, file_path, path_prefix, parent_name_path, &ENUM, byte_range) { + if let Some(sym) = build_named_symbol(&target, src, file_path, names, path_prefix, parent_name_path, &ENUM, byte_range) { out.push(sym); } } // `namespace N { ... }` / `module M { ... }` - container, recurse. "internal_module" | "module" => { - emit_namespace(&target, src, file_path, path_prefix, parent_name_path, out); + emit_namespace(&target, src, file_path, names, path_prefix, parent_name_path, out); } // `const foo = () => {}` or `let bar = function() {}` - detect arrow/fn expressions. "lexical_declaration" | "variable_declaration" => { - extract_arrow_fns(&target, src, file_path, path_prefix, parent_name_path, out); + extract_arrow_fns(&target, src, file_path, names, path_prefix, parent_name_path, out); } // `declare function ...`, `declare class ...`, `declare const ...`, etc. "ambient_declaration" => { - extract_ambient(&target, src, file_path, path_prefix, parent_name_path, out); + extract_ambient(&target, src, file_path, names, path_prefix, parent_name_path, out); } // Imports, comments, expression statements, etc. - ignored. _ => {} @@ -253,6 +255,7 @@ fn extract_arrow_fns( node: &Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -283,7 +286,7 @@ fn extract_arrow_fns( // Use the full declaration range (includes const/let keyword). let byte_range = node.byte_range(); - out.push(make_symbol(name, &FN, node, byte_range, src, file_path, path_prefix, parent_name_path, Vec::new())); + out.push(make_symbol(name, &FN, node, byte_range, src, file_path, names, path_prefix, parent_name_path, Vec::new())); } } @@ -300,6 +303,7 @@ fn extract_ambient( node: &Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -314,14 +318,14 @@ fn extract_ambient( | "internal_module" | "module" => child_name(&child, src), "lexical_declaration" | "variable_declaration" => { // Extract variable names from declare const/let/var. - extract_ambient_vars(&child, src, file_path, path_prefix, parent_name_path, &ambient_range, out); + extract_ambient_vars(&child, src, file_path, names, path_prefix, parent_name_path, &ambient_range, out); None } _ => None, }; if let Some(name) = name { - out.push(make_symbol(name, &DECLARE, node, ambient_range, src, file_path, path_prefix, parent_name_path, Vec::new())); + out.push(make_symbol(name, &DECLARE, node, ambient_range, src, file_path, names, path_prefix, parent_name_path, Vec::new())); return; // One symbol per ambient_declaration. } } @@ -332,6 +336,7 @@ fn extract_ambient_vars( node: &Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, ambient_range: &std::ops::Range, @@ -351,7 +356,7 @@ fn extract_ambient_vars( None => continue, }; - out.push(make_symbol(name, &DECLARE, &child, ambient_range.clone(), src, file_path, path_prefix, parent_name_path, Vec::new())); + out.push(make_symbol(name, &DECLARE, &child, ambient_range.clone(), src, file_path, names, path_prefix, parent_name_path, Vec::new())); } } @@ -367,6 +372,7 @@ fn emit_class( node: &Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, meta: &SymbolMeta, @@ -386,10 +392,10 @@ fn emit_class( let mut children = Vec::new(); if let Some(body) = child_by_kind(node, "class_body") { - extract_members(body, src, file_path, path_prefix, &name_path, &mut children); + extract_members(body, src, file_path, names, path_prefix, &name_path, &mut children); } - out.push(make_symbol(name, meta, node, byte_range, src, file_path, path_prefix, parent_name_path, children)); + out.push(make_symbol(name, meta, node, byte_range, src, file_path, names, path_prefix, parent_name_path, children)); } /// Extract members from a `class_body` or `interface_body` node. @@ -405,6 +411,7 @@ fn extract_members( body: Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -442,7 +449,7 @@ fn extract_members( }; let byte_range = child.byte_range(); - out.push(make_symbol(name, meta, &child, byte_range, src, file_path, path_prefix, parent_name_path, Vec::new())); + out.push(make_symbol(name, meta, &child, byte_range, src, file_path, names, path_prefix, parent_name_path, Vec::new())); } } @@ -454,6 +461,7 @@ fn emit_interface( node: &Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, byte_range: std::ops::Range, @@ -472,10 +480,10 @@ fn emit_interface( let mut children = Vec::new(); if let Some(body) = child_by_kind(node, "interface_body") { - extract_members(body, src, file_path, path_prefix, &name_path, &mut children); + extract_members(body, src, file_path, names, path_prefix, &name_path, &mut children); } - out.push(make_symbol(name, &IFACE, node, byte_range, src, file_path, path_prefix, parent_name_path, children)); + out.push(make_symbol(name, &IFACE, node, byte_range, src, file_path, names, path_prefix, parent_name_path, children)); } /// Emit a `namespace`/`module` symbol and recurse into `statement_block` for nested declarations. @@ -487,6 +495,7 @@ fn emit_namespace( node: &Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -504,11 +513,11 @@ fn emit_namespace( let mut children = Vec::new(); if let Some(body) = child_by_kind(node, "statement_block") { - extract_symbols(body, src, file_path, path_prefix, &name_path, &mut children); + extract_symbols(body, src, file_path, names, path_prefix, &name_path, &mut children); } let byte_range = node.byte_range(); - out.push(make_symbol(name, &NS, node, byte_range, src, file_path, path_prefix, parent_name_path, children)); + out.push(make_symbol(name, &NS, node, byte_range, src, file_path, names, path_prefix, parent_name_path, children)); } // --------------------------------------------------------------------------- @@ -533,6 +542,7 @@ fn make_symbol( byte_range: std::ops::Range, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, children: Vec, @@ -549,7 +559,7 @@ fn make_symbol( SymbolNode { id, - name, + name: names.intern(&name), category: meta.category, label: meta.label, file_path: Arc::clone(file_path), @@ -569,13 +579,14 @@ fn build_named_symbol( node: &Node, src: &[u8], file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, meta: &SymbolMeta, byte_range: std::ops::Range, ) -> Option { let name = child_name(node, src)?; - Some(make_symbol(name, meta, node, byte_range, src, file_path, path_prefix, parent_name_path, Vec::new())) + Some(make_symbol(name, meta, node, byte_range, src, file_path, names, path_prefix, parent_name_path, Vec::new())) } // --------------------------------------------------------------------------- @@ -640,7 +651,7 @@ mod tests { fn find<'a>(symbols: &'a [SymbolNode], name: &str) -> &'a SymbolNode { symbols .iter() - .find(|s| s.name == name) + .find(|s| s.name.as_ref() == name) .unwrap_or_else(|| panic!("symbol '{}' not found", name)) } @@ -652,7 +663,7 @@ mod tests { fn function_declaration() { let syms = parse("function greet(name: string): string { return name; }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "greet"); + assert_eq!(syms[0].name.as_ref(), "greet"); assert_eq!(syms[0].label, "function"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -661,7 +672,7 @@ mod tests { fn generator_function() { let syms = parse("function* gen() { yield 1; }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "gen"); + assert_eq!(syms[0].name.as_ref(), "gen"); assert_eq!(syms[0].label, "function"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -670,7 +681,7 @@ mod tests { fn async_function() { let syms = parse("async function fetchData(): Promise {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "fetchData"); + assert_eq!(syms[0].name.as_ref(), "fetchData"); assert_eq!(syms[0].label, "function"); } @@ -682,7 +693,7 @@ mod tests { fn arrow_function_const() { let syms = parse("const handler = () => {};"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "handler"); + assert_eq!(syms[0].name.as_ref(), "handler"); assert_eq!(syms[0].label, "function"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -691,7 +702,7 @@ mod tests { fn arrow_function_with_body() { let syms = parse("const process = (data: string) => {\n return data;\n};"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "process"); + assert_eq!(syms[0].name.as_ref(), "process"); assert_eq!(syms[0].label, "function"); } @@ -699,7 +710,7 @@ mod tests { fn function_expression() { let syms = parse("const handler = function() {};"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "handler"); + assert_eq!(syms[0].name.as_ref(), "handler"); assert_eq!(syms[0].label, "function"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -708,7 +719,7 @@ mod tests { fn arrow_function_let() { let syms = parse("let handler = () => {};"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "handler"); + assert_eq!(syms[0].name.as_ref(), "handler"); assert_eq!(syms[0].label, "function"); } @@ -738,7 +749,7 @@ mod tests { fn class_declaration() { let syms = parse("class Foo {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].label, "class"); assert_eq!(syms[0].category, SymbolCategory::Type); } @@ -753,11 +764,11 @@ mod tests { ); assert_eq!(syms.len(), 1); let cls = &syms[0]; - assert_eq!(cls.name, "Service"); + assert_eq!(cls.name.as_ref(), "Service"); assert_eq!(cls.children.len(), 2); - assert_eq!(cls.children[0].name, "doWork"); + assert_eq!(cls.children[0].name.as_ref(), "doWork"); assert_eq!(cls.children[0].label, "method"); - assert_eq!(cls.children[1].name, "process"); + assert_eq!(cls.children[1].name.as_ref(), "process"); assert_eq!(cls.children[1].label, "method"); } @@ -772,10 +783,10 @@ mod tests { assert_eq!(syms.len(), 1); let cls = &syms[0]; assert_eq!(cls.children.len(), 2); - assert_eq!(cls.children[0].name, "host"); + assert_eq!(cls.children[0].name.as_ref(), "host"); assert_eq!(cls.children[0].label, "property"); assert_eq!(cls.children[0].category, SymbolCategory::Variable); - assert_eq!(cls.children[1].name, "port"); + assert_eq!(cls.children[1].name.as_ref(), "port"); assert_eq!(cls.children[1].label, "property"); } @@ -805,7 +816,7 @@ mod tests { ); assert_eq!(syms.len(), 1); assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "constructor"); + assert_eq!(syms[0].children[0].name.as_ref(), "constructor"); assert_eq!(syms[0].children[0].label, "method"); } @@ -822,13 +833,13 @@ mod tests { }", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Base"); + assert_eq!(syms[0].name.as_ref(), "Base"); assert_eq!(syms[0].label, "abstract class"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].children.len(), 2); - assert_eq!(syms[0].children[0].name, "doWork"); + assert_eq!(syms[0].children[0].name.as_ref(), "doWork"); assert_eq!(syms[0].children[0].label, "method"); - assert_eq!(syms[0].children[1].name, "concrete"); + assert_eq!(syms[0].children[1].name.as_ref(), "concrete"); assert_eq!(syms[0].children[1].label, "method"); } @@ -840,7 +851,7 @@ mod tests { fn interface_declaration() { let syms = parse("interface Foo {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].label, "interface"); assert_eq!(syms[0].category, SymbolCategory::Type); } @@ -858,15 +869,15 @@ mod tests { assert_eq!(syms.len(), 1); let iface = &syms[0]; assert_eq!(iface.children.len(), 4); - assert_eq!(iface.children[0].name, "host"); + assert_eq!(iface.children[0].name.as_ref(), "host"); assert_eq!(iface.children[0].label, "property"); assert_eq!(iface.children[0].category, SymbolCategory::Variable); - assert_eq!(iface.children[1].name, "port"); + assert_eq!(iface.children[1].name.as_ref(), "port"); assert_eq!(iface.children[1].label, "property"); - assert_eq!(iface.children[2].name, "connect"); + assert_eq!(iface.children[2].name.as_ref(), "connect"); assert_eq!(iface.children[2].label, "method"); assert_eq!(iface.children[2].category, SymbolCategory::Function); - assert_eq!(iface.children[3].name, "disconnect"); + assert_eq!(iface.children[3].name.as_ref(), "disconnect"); assert_eq!(iface.children[3].label, "method"); } @@ -878,7 +889,7 @@ mod tests { fn type_alias() { let syms = parse("type UserId = string;"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "UserId"); + assert_eq!(syms[0].name.as_ref(), "UserId"); assert_eq!(syms[0].label, "type"); assert_eq!(syms[0].category, SymbolCategory::Type); } @@ -887,7 +898,7 @@ mod tests { fn generic_type_alias() { let syms = parse("type Result = { ok: true; value: T } | { ok: false; error: Error };"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Result"); + assert_eq!(syms[0].name.as_ref(), "Result"); assert_eq!(syms[0].label, "type"); } @@ -899,7 +910,7 @@ mod tests { fn enum_declaration() { let syms = parse("enum Status { Active, Inactive, Pending }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Status"); + assert_eq!(syms[0].name.as_ref(), "Status"); assert_eq!(syms[0].label, "enum"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].children.len(), 0, "enum should not recurse into members"); @@ -909,7 +920,7 @@ mod tests { fn const_enum() { let syms = parse("const enum Direction { Up, Down, Left, Right }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Direction"); + assert_eq!(syms[0].name.as_ref(), "Direction"); assert_eq!(syms[0].label, "enum"); } @@ -925,12 +936,12 @@ mod tests { }", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Validation"); + assert_eq!(syms[0].name.as_ref(), "Validation"); assert_eq!(syms[0].label, "namespace"); assert_eq!(syms[0].category, SymbolCategory::Module); // Namespace should recurse into children. assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "isValid"); + assert_eq!(syms[0].children[0].name.as_ref(), "isValid"); assert_eq!(syms[0].children[0].label, "function"); } @@ -942,10 +953,10 @@ mod tests { }", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "MyModule"); + assert_eq!(syms[0].name.as_ref(), "MyModule"); assert_eq!(syms[0].label, "namespace"); assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "Inner"); + assert_eq!(syms[0].children[0].name.as_ref(), "Inner"); assert_eq!(syms[0].children[0].label, "class"); } @@ -957,7 +968,7 @@ mod tests { fn declare_function() { let syms = parse("declare function require(id: string): any;"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "require"); + assert_eq!(syms[0].name.as_ref(), "require"); assert_eq!(syms[0].label, "declare"); assert_eq!(syms[0].category, SymbolCategory::Variable); } @@ -966,7 +977,7 @@ mod tests { fn declare_const() { let syms = parse("declare const __dirname: string;"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "__dirname"); + assert_eq!(syms[0].name.as_ref(), "__dirname"); assert_eq!(syms[0].label, "declare"); } @@ -981,7 +992,7 @@ mod tests { fn declare_class() { let syms = parse("declare class Buffer { constructor(str: string); }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Buffer"); + assert_eq!(syms[0].name.as_ref(), "Buffer"); assert_eq!(syms[0].label, "declare"); } @@ -993,7 +1004,7 @@ mod tests { fn export_function() { let syms = parse("export function greet(): void {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "greet"); + assert_eq!(syms[0].name.as_ref(), "greet"); assert_eq!(syms[0].label, "function"); } @@ -1001,7 +1012,7 @@ mod tests { fn export_class() { let syms = parse("export class Service {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Service"); + assert_eq!(syms[0].name.as_ref(), "Service"); assert_eq!(syms[0].label, "class"); } @@ -1009,7 +1020,7 @@ mod tests { fn export_interface() { let syms = parse("export interface Config { host: string; }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Config"); + assert_eq!(syms[0].name.as_ref(), "Config"); assert_eq!(syms[0].label, "interface"); assert_eq!(syms[0].children.len(), 1); } @@ -1018,7 +1029,7 @@ mod tests { fn export_type_alias() { let syms = parse("export type Id = string;"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Id"); + assert_eq!(syms[0].name.as_ref(), "Id"); assert_eq!(syms[0].label, "type"); } @@ -1026,7 +1037,7 @@ mod tests { fn export_enum() { let syms = parse("export enum Color { Red, Green, Blue }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Color"); + assert_eq!(syms[0].name.as_ref(), "Color"); assert_eq!(syms[0].label, "enum"); } @@ -1034,7 +1045,7 @@ mod tests { fn export_arrow_function() { let syms = parse("export const handler = () => {};"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "handler"); + assert_eq!(syms[0].name.as_ref(), "handler"); assert_eq!(syms[0].label, "function"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -1043,7 +1054,7 @@ mod tests { fn export_default_function() { let syms = parse("export default function main() {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "main"); + assert_eq!(syms[0].name.as_ref(), "main"); assert_eq!(syms[0].label, "function"); } @@ -1051,7 +1062,7 @@ mod tests { fn export_default_class() { let syms = parse("export default class App {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "App"); + assert_eq!(syms[0].name.as_ref(), "App"); assert_eq!(syms[0].label, "class"); } diff --git a/src/serena/mod.rs b/src/serena/mod.rs index bac37e3..3f5f038 100644 --- a/src/serena/mod.rs +++ b/src/serena/mod.rs @@ -208,7 +208,7 @@ fn convert_symbol( let mut node = SymbolNode { id, - name, + name: std::sync::Arc::from(name.as_str()), category, label, file_path: std::sync::Arc::clone(file_path), diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index 1c9a2a3..175e61c 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -1,3 +1,5 @@ +use std::cell::RefCell; +use std::collections::HashMap; use std::fmt; use std::ops::Range; use std::path::PathBuf; @@ -44,7 +46,14 @@ impl fmt::Display for SymbolCategory { #[derive(Debug, Clone)] pub struct SymbolNode { pub id: SymbolId, - pub name: String, + /// Symbol's short name (`foo`, `Bar`, `method_0`, etc). Stored as + /// `Arc` so common names (`new`, `default`, `get`, `build`, + /// `method_0` repeating across types in a file) share one heap + /// allocation. Construct via [`NameInterner::intern`] from a parser, or + /// directly with `Arc::from(name)` from a test/fixture. Display, Eq, and + /// `as_ref()` work via deref to `&str`; explicit comparisons against + /// `String` need `&*sym.name == &**other` or `sym.name.as_ref() == other`. + pub name: Arc, pub category: SymbolCategory, pub label: &'static str, // Language-specific label (e.g., "class", "struct", "def") /// Path to the source file that produced this symbol. Wrapped in `Arc` so @@ -63,6 +72,39 @@ pub struct SymbolNode { pub estimated_tokens: usize, } +/// Per-file interner for symbol names. Real code has heavy name repetition +/// within a file (think methods named `new`, `default`, `build` across every +/// type in a module), so a small hash cache here lets us share one `Arc` +/// across every reuse instead of allocating one heap buffer per occurrence. +/// +/// Uses interior mutability so it can be threaded as `&NameInterner` alongside +/// other immutable parser state without requiring `&mut` propagation through +/// every recursive helper. +/// +/// Scope is intentionally per-file rather than global: simpler borrow story +/// (no thread safety needed), and the wins from within-file repetition +/// dominate. +#[derive(Debug, Default)] +pub struct NameInterner { + cache: RefCell>>, +} + +impl NameInterner { + pub fn new() -> Self { + Self::default() + } + + pub fn intern(&self, s: &str) -> Arc { + let mut cache = self.cache.borrow_mut(); + if let Some(arc) = cache.get(s) { + return Arc::clone(arc); + } + let arc: Arc = Arc::from(s); + cache.insert(s.to_string(), Arc::clone(&arc)); + arc + } +} + impl SymbolNode { pub fn total_symbols(&self) -> usize { 1 + self.children.iter().map(|c| c.total_symbols()).sum::() diff --git a/tests/e2e.rs b/tests/e2e.rs index c7c745c..68ed49d 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -40,7 +40,7 @@ fn sym(id: &str, name: &str) -> SymbolNode { let hash = content_hash(name); SymbolNode { id: id.to_string(), - name: name.to_string(), + name: Arc::from(name), category: SymbolCategory::Function, label: "fn", file_path: Arc::new(PathBuf::new()), diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs index 6a4febf..cf221c3 100644 --- a/tests/helpers/mod.rs +++ b/tests/helpers/mod.rs @@ -11,7 +11,7 @@ pub fn sym(id: &str, name: &str) -> SymbolNode { let hash = crate::symbols::merkle::content_hash(name); SymbolNode { id: id.to_string(), - name: name.to_string(), + name: Arc::from(name), category: SymbolCategory::Function, label: "fn", file_path: Arc::new(PathBuf::new()), From 9bfadeb0b80b404acd3617776cdc813ace712b45 Mon Sep 17 00:00:00 2001 From: Joshua 'Josh' Long Date: Tue, 12 May 2026 09:56:48 -0400 Subject: [PATCH 4/5] perf(round-2): swap SHA-256 for BLAKE3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `content_hash` and `compute_merkle_hash` use the symbol hash for change detection — no adversary threat model, no need for SHA-256's cryptographic guarantees. BLAKE3 with its SIMD-vectorised tree mode is substantially faster on the input sizes we hash (8 bytes – several KB) while keeping a 32-byte output that drops in to the existing `[u8; 32]` field. Swapped both callers: - `src/symbols/merkle.rs::content_hash` and `compute_merkle_hash` - `src/serena/mod.rs::convert_symbol` (the pickle loader's identity hash) `sha2` removed from `Cargo.toml` — no remaining users. `blake3` added as a normal dep (pulls only `arrayref`, `arrayvec`, `constant_time_eq`, all small). Hash values change for every symbol on this run vs prior runs; confirmed nothing in this crate persists or compares specific hash bytes — the only tests that touch the field assert it is non-zero, and no on-disk or wire format references the hash. Measured (cargo bench, same machine, sample_count 100 / 20): `merkle_hash/bench_content_hash` (unit-level): tiny 243 → 152 ns (-37%) small 854 → 573 ns (-33%) medium 4.99 → 3.42 µs (-32%) large 13.29 → 9.04 µs (-32%) `merkle_hash/bench_compute_merkle_hash` (unit-level tree fixture): n=1 1.39 → 0.46 µs (-67%) n=10 13.87 → 4.29 µs (-69%) n=100 50.62 → 15.47 µs (-69%) n=1000 343.1 → 113.4 µs (-67%) `symbol_pipeline/compute_merkle_hash_project` (monorepo scale): 100 files (10k symbols) 1.14 → 0.52 ms (-54%) 1000 files (100k symbols) 11.29 → 5.24 ms (-54%) 5000 files (500k symbols) 57.02 → 26.36 ms (-54%) Coverage and ledger ops drop ~0–7% (small, since merkle is a tiny fraction of their work). Heap at 500k unchanged at 128.5 MB / 740k blocks — BLAKE3's larger hasher state (~1.9 KB vs SHA-256's ~120 B) lives only on the stack during the call, no persistent footprint. Combined effect of Round 0 → Round 2 on a 500k-symbol monorepo: peak heap: 155.0 MB → 128.5 MB (-17.1%) blocks at peak: 1.61M → 740k (-54%) merkle pass: n/a (was on main with SHA-256, was 57.13 ms) baseline 57.13 ms → 26.36 ms (-54%) Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 90 +++++++++++++++---------------------------- Cargo.toml | 2 +- src/serena/mod.rs | 14 +++---- src/symbols/merkle.rs | 22 ++++++----- 4 files changed, 51 insertions(+), 77 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5d736c..1809d36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,6 +36,7 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" name = "ambits" version = "0.14.0" dependencies = [ + "blake3", "clap", "color-eyre", "crossterm", @@ -48,7 +49,6 @@ dependencies = [ "serde", "serde-pickle", "serde_json", - "sha2", "tempfile", "toml", "tree-sitter", @@ -107,6 +107,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "autocfg" version = "1.5.0" @@ -141,12 +153,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] -name = "block-buffer" -version = "0.10.4" +name = "blake3" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ - "generic-array", + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", ] [[package]] @@ -296,11 +313,17 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] @@ -355,16 +378,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "darling" version = "0.23.0" @@ -415,16 +428,6 @@ dependencies = [ "thousands", ] -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - [[package]] name = "divan" version = "0.1.21" @@ -547,16 +550,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -1208,17 +1201,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -1493,12 +1475,6 @@ dependencies = [ "tree-sitter-language", ] -[[package]] -name = "typenum" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" - [[package]] name = "unicode-ident" version = "1.0.22" @@ -1546,12 +1522,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index aca1675..324932e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,11 +23,11 @@ notify = "7" clap = { version = "4", features = ["derive"] } serde = { version = "1", features = ["derive"] } serde_json = "1" -sha2 = "0.10" color-eyre = "0.6" serde-pickle = "1.2" flume = "0.12" toml = { version = "0.8", default-features = false, features = ["parse"] } +blake3 = "1" dhat = { version = "0.3", optional = true } [features] diff --git a/src/serena/mod.rs b/src/serena/mod.rs index 3f5f038..5dbd180 100644 --- a/src/serena/mod.rs +++ b/src/serena/mod.rs @@ -4,7 +4,6 @@ use std::path::{Path, PathBuf}; use color_eyre::eyre::{bail, eyre, Result}; use serde_pickle::value::{HashableValue, Value}; -use sha2::{Digest, Sha256}; use ambits::symbols::merkle::compute_merkle_hash; use ambits::symbols::{FileSymbols, ProjectTree, SymbolCategory, SymbolNode}; @@ -186,13 +185,14 @@ fn convert_symbol( 1 }; - // Content hash from identity (no source text available in raw format) - let content_hash = { - let mut hasher = Sha256::new(); + // Content hash from identity (no source text available in raw format). + // BLAKE3 matches `symbols::merkle::content_hash` algorithm choice. + let content_hash: [u8; 32] = { + let mut hasher = blake3::Hasher::new(); hasher.update(name.as_bytes()); - hasher.update(kind_int.to_le_bytes()); - hasher.update(start_line.to_le_bytes()); - hasher.update(end_line.to_le_bytes()); + hasher.update(&kind_int.to_le_bytes()); + hasher.update(&start_line.to_le_bytes()); + hasher.update(&end_line.to_le_bytes()); hasher.finalize().into() }; diff --git a/src/symbols/merkle.rs b/src/symbols/merkle.rs index de7880d..69c825a 100644 --- a/src/symbols/merkle.rs +++ b/src/symbols/merkle.rs @@ -1,12 +1,16 @@ -use sha2::{Digest, Sha256}; - use super::SymbolNode; /// Compute content hash from the raw source text of a symbol. /// Normalizes whitespace to make hashing resilient to formatting changes. +/// +/// Uses BLAKE3 (256-bit output) — fast, SIMD-accelerated, no adversary threat +/// model required for our change-detection use case. Output width matches the +/// `[u8; 32]` field on `SymbolNode`. Hash values differ from previous SHA-256 +/// runs; no on-disk format references these hashes, so the change is internal +/// only. pub fn content_hash(source: &str) -> [u8; 32] { let normalized = normalize_source(source); - let mut hasher = Sha256::new(); + let mut hasher = blake3::Hasher::new(); hasher.update(normalized.as_bytes()); hasher.finalize().into() } @@ -16,10 +20,10 @@ pub fn content_hash(source: &str) -> [u8; 32] { /// This must be called bottom-up (children first). /// /// Leaf nodes (no children) reuse `content_hash` as their `merkle_hash` directly, -/// skipping a SHA-256 invocation. The input would be a 32-byte cryptographic hash -/// either way, so hashing it again adds no collision resistance. Real source +/// skipping a hash invocation. The input would be a 32-byte cryptographic-quality +/// hash either way, so hashing it again adds no collision resistance. Real source /// projects are leaf-dominated (≈90% leaves in typical symbol trees), so this -/// skips the majority of SHA-256 calls during parse. +/// skips the majority of hash calls during parse. pub fn compute_merkle_hash(node: &mut SymbolNode) { if node.children.is_empty() { node.merkle_hash = node.content_hash; @@ -30,10 +34,10 @@ pub fn compute_merkle_hash(node: &mut SymbolNode) { compute_merkle_hash(child); } - let mut hasher = Sha256::new(); - hasher.update(node.content_hash); + let mut hasher = blake3::Hasher::new(); + hasher.update(&node.content_hash); for child in &node.children { - hasher.update(child.merkle_hash); + hasher.update(&child.merkle_hash); } node.merkle_hash = hasher.finalize().into(); } From 55a471d15d7b53e0529580cb9f1384365d292562 Mon Sep 17 00:00:00 2001 From: Joshua 'Josh' Long Date: Tue, 12 May 2026 10:38:10 -0400 Subject: [PATCH 5/5] perf(round-3a): tighten SymbolNode integer widths to u32 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fields on `SymbolNode` were `usize` (8 bytes each on 64-bit) when `u32` is plenty: - `byte_range: Range` — a source file fits in 4 GB - `line_range: Range` — line counts fit in u32 by many orders - `estimated_tokens: usize` — same Switch all three to `u32` (Range for the ranges). Saves 8 B × 2 range fields + 4 B = 20 B/symbol nominally; measured 24 B/symbol after factoring in struct alignment changes. `AgentToolCall.target_lines` also moves to `Option>` so the symbol/line-overlap comparison in `app::symbol_lines_match` stays a clean integer comparison without casts. Conversion happens at construction sites in each parser (`byte_range.start as u32..byte_range.end as u32`) and at slice sites (`as usize` to feed into `&src[..]`); tree-sitter still returns `Range` so the boundary lives there. A few `ledger.record()` call sites pass `sym.estimated_tokens as usize` since the ledger's token counter remains `usize` (it sums across many symbols). Measured impact (heap_probe, dhat, same machine): tier peak bytes (R2 → R3a) peak blocks (unchanged) 10k 2,570,839 → 2,330,839 (-9.3%) 14,834 100k 25,691,839 → 23,291,839 (-9.3%) 148,034 500k 128,451,839 → 116,451,839 (-9.3%) 740,034 → 12 MB saved at the 500k tier with zero block-count change (these are pure per-symbol-size reductions, not allocation count) → per-symbol cost: 257 B → 233 B average Time benches (`symbol_pipeline`) all within noise — type changes don't affect the work being measured. Cumulative effect of Round 0 → Round 3a on 500k symbols: peak heap: 155.0 MB → 116.5 MB (-24.9%) blocks at peak: 1.61M → 740k (-54%) merkle pass: 57.13 ms → 26.53 ms (-54%) Co-Authored-By: Claude Opus 4.7 (1M context) --- benches/parser.rs | 2 +- benches/support/fixtures.rs | 2 +- benches/symbol_pipeline.rs | 4 ++-- src/app.rs | 8 ++++---- src/ingest/claude.rs | 4 ++-- src/ingest/mod.rs | 2 +- src/parser/python.rs | 14 +++++++------- src/parser/rust.rs | 12 ++++++------ src/parser/typescript.rs | 6 +++--- src/serena/mod.rs | 8 ++++---- src/symbols/mod.rs | 14 ++++++++++---- tests/helpers/mod.rs | 6 +++--- 12 files changed, 44 insertions(+), 38 deletions(-) diff --git a/benches/parser.rs b/benches/parser.rs index 8a3c57a..510f384 100644 --- a/benches/parser.rs +++ b/benches/parser.rs @@ -67,7 +67,7 @@ fn typescript_parse_and_coverage(bencher: divan::Bencher, size: &&str) { let mut ledger = ContextLedger::new(); let hash = [0u8; 32]; for sym in &file_syms.symbols { - ledger.record(sym.id.clone(), ReadDepth::FullBody, hash, "agent".to_string(), sym.estimated_tokens); + ledger.record(sym.id.clone(), ReadDepth::FullBody, hash, "agent".to_string(), sym.estimated_tokens as usize); } // 3. Build project tree from the single file diff --git a/benches/support/fixtures.rs b/benches/support/fixtures.rs index 357166c..6b4c445 100644 --- a/benches/support/fixtures.rs +++ b/benches/support/fixtures.rs @@ -1089,7 +1089,7 @@ pub fn make_populated_ledger(project: &ProjectTree) -> ContextLedger { let hash = [0u8; 32]; for file in &project.files { for sym in &file.symbols { - ledger.record(sym.id.clone(), ReadDepth::FullBody, hash, "agent-0".to_string(), sym.estimated_tokens); + ledger.record(sym.id.clone(), ReadDepth::FullBody, hash, "agent-0".to_string(), sym.estimated_tokens as usize); } } ledger diff --git a/benches/symbol_pipeline.rs b/benches/symbol_pipeline.rs index 4a5b1ac..041e22d 100644 --- a/benches/symbol_pipeline.rs +++ b/benches/symbol_pipeline.rs @@ -70,7 +70,7 @@ fn ledger_populate_full_project(bencher: divan::Bencher, n_files: &usize) { ReadDepth::Overview, sym.content_hash, "agent-0".to_string(), - sym.estimated_tokens, + sym.estimated_tokens as usize, ); } } @@ -87,7 +87,7 @@ fn populate_full(project: &ProjectTree) -> ContextLedger { ReadDepth::FullBody, sym.content_hash, "agent-0".to_string(), - sym.estimated_tokens, + sym.estimated_tokens as usize, ); } } diff --git a/src/app.rs b/src/app.rs index 60f75d2..9554f86 100644 --- a/src/app.rs +++ b/src/app.rs @@ -675,7 +675,7 @@ fn flatten_symbol( is_expanded, has_children: !sym.children.is_empty(), line_range: format!("L{}-{}", sym.line_range.start, sym.line_range.end), - token_count: sym.estimated_tokens, + token_count: sym.estimated_tokens as usize, read_depth, coverage_status: None, file_coverage_seen: 0, @@ -719,7 +719,7 @@ pub fn mark_file_symbols( event.read_depth, sym.content_hash, event.agent_id.to_string(), - sym.estimated_tokens, + sym.estimated_tokens as usize, ); mark_file_symbols(&sym.children, event, ledger); } @@ -744,7 +744,7 @@ pub fn mark_targeted_symbols( event.read_depth, sym.content_hash, event.agent_id.to_string(), - sym.estimated_tokens, + sym.estimated_tokens as usize, ); // Full body was in the response — bulk-mark all descendants. mark_file_symbols(&sym.children, event, ledger); @@ -755,7 +755,7 @@ pub fn mark_targeted_symbols( event.read_depth, sym.content_hash, event.agent_id.to_string(), - sym.estimated_tokens, + sym.estimated_tokens as usize, ); // Parent container overlaps the read range — recurse precisely so // only children whose ranges also overlap get promoted. diff --git a/src/ingest/claude.rs b/src/ingest/claude.rs index 96bb0e6..f7a35dd 100644 --- a/src/ingest/claude.rs +++ b/src/ingest/claude.rs @@ -664,8 +664,8 @@ pub fn map_tool_call( // Extract target_lines. let target_lines = mapping.target_lines.as_ref().and_then(|spec| { - let offset = input.get(&spec.offset_key)?.as_u64()? as usize; - let limit = input.get(&spec.limit_key)?.as_u64()? as usize; + let offset = input.get(&spec.offset_key)?.as_u64()? as u32; + let limit = input.get(&spec.limit_key)?.as_u64()? as u32; Some(offset..offset + limit) }); diff --git a/src/ingest/mod.rs b/src/ingest/mod.rs index 8c18780..e3dca1f 100644 --- a/src/ingest/mod.rs +++ b/src/ingest/mod.rs @@ -19,7 +19,7 @@ pub struct AgentToolCall { /// Optional symbol name path to target (e.g. "MyClass/my_method"). pub target_symbol: Option, /// Optional line range to target (1-based, e.g. 10..25). - pub target_lines: Option>, + pub target_lines: Option>, /// Human-readable label for the agent (e.g. "Explore parser and symbol types"). /// Falls back to agent_id if no label could be extracted from the session log. pub label: Arc, diff --git a/src/parser/python.rs b/src/parser/python.rs index ae8d6f3..68f58ca 100644 --- a/src/parser/python.rs +++ b/src/parser/python.rs @@ -191,12 +191,12 @@ fn extract_symbols( category: meta.category, label: meta.label, file_path: Arc::clone(file_path), - byte_range, - line_range: start_line..end_line, + byte_range: byte_range.start as u32..byte_range.end as u32, + line_range: start_line as u32..end_line as u32, content_hash: content_hash(text), merkle_hash: [0u8; 32], children: Vec::new(), - estimated_tokens: estimate_tokens(text), + estimated_tokens: estimate_tokens(text) as u32, }; // For classes, recurse into the body block to find methods. @@ -267,12 +267,12 @@ fn extract_decorated( category: meta.category, label: meta.label, file_path: Arc::clone(file_path), - byte_range, - line_range: start_line..end_line, + byte_range: byte_range.start as u32..byte_range.end as u32, + line_range: start_line as u32..end_line as u32, content_hash: content_hash(text), merkle_hash: [0u8; 32], children: Vec::new(), - estimated_tokens: estimate_tokens(text), + estimated_tokens: estimate_tokens(text) as u32, }; if meta.category == SymbolCategory::Type { @@ -711,7 +711,7 @@ mod tests { let syms = parse(src); let range = &syms[0].byte_range; assert!(range.start < range.end); - assert!(range.end <= src.len()); + assert!(range.end as usize <= src.len()); } // --- Variable extraction tests --- diff --git a/src/parser/rust.rs b/src/parser/rust.rs index 3232054..431fb4e 100644 --- a/src/parser/rust.rs +++ b/src/parser/rust.rs @@ -119,12 +119,12 @@ fn extract_symbols( category: meta.category, label: meta.label, file_path: Arc::clone(file_path), - byte_range, - line_range: start_line..end_line, + byte_range: byte_range.start as u32..byte_range.end as u32, + line_range: start_line as u32..end_line as u32, content_hash: content_hash(text), merkle_hash: [0u8; 32], children: Vec::new(), - estimated_tokens: estimate_tokens(text), + estimated_tokens: estimate_tokens(text) as u32, }; // Recurse into container types for their children. @@ -174,12 +174,12 @@ fn extract_body_children( category: meta.category, label: meta.label, file_path: Arc::clone(file_path), - byte_range, - line_range: start_line..end_line, + byte_range: byte_range.start as u32..byte_range.end as u32, + line_range: start_line as u32..end_line as u32, content_hash: content_hash(text), merkle_hash: [0u8; 32], children: Vec::new(), - estimated_tokens: estimate_tokens(text), + estimated_tokens: estimate_tokens(text) as u32, }); } } diff --git a/src/parser/typescript.rs b/src/parser/typescript.rs index 1902c76..a2a7dcd 100644 --- a/src/parser/typescript.rs +++ b/src/parser/typescript.rs @@ -563,12 +563,12 @@ fn make_symbol( category: meta.category, label: meta.label, file_path: Arc::clone(file_path), - byte_range, - line_range: start_line..end_line, + byte_range: byte_range.start as u32..byte_range.end as u32, + line_range: start_line as u32..end_line as u32, content_hash: content_hash(text), merkle_hash: [0u8; 32], children, - estimated_tokens: estimate_tokens(text), + estimated_tokens: estimate_tokens(text) as u32, } } diff --git a/src/serena/mod.rs b/src/serena/mod.rs index 5dbd180..a140188 100644 --- a/src/serena/mod.rs +++ b/src/serena/mod.rs @@ -212,12 +212,12 @@ fn convert_symbol( category, label, file_path: std::sync::Arc::clone(file_path), - byte_range: (start_line * 40 + start_char)..(end_line * 40 + end_char), - line_range: (start_line + 1)..(end_line + 1), // 1-indexed like tree-sitter + byte_range: ((start_line * 40 + start_char) as u32)..((end_line * 40 + end_char) as u32), + line_range: ((start_line + 1) as u32)..((end_line + 1) as u32), // 1-indexed like tree-sitter content_hash, merkle_hash: [0u8; 32], children, - estimated_tokens: line_count * 15, + estimated_tokens: (line_count * 15) as u32, }; compute_merkle_hash(&mut node); Ok(node) @@ -246,7 +246,7 @@ fn extract_range(val: &Value) -> (usize, usize, usize, usize) { fn estimate_total_lines(symbols: &[SymbolNode]) -> usize { symbols .iter() - .map(|s| s.line_range.end) + .map(|s| s.line_range.end as usize) .max() .unwrap_or(0) } diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index 175e61c..1485933 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -64,12 +64,17 @@ pub struct SymbolNode { /// `.to_string_lossy()` all work). Equality checks against `Path` / /// `PathBuf` need `&**arc` or `arc.as_path()` to unwrap. pub file_path: Arc, - pub byte_range: Range, - pub line_range: Range, + /// Byte range of the symbol in its source file. `u32` is plenty for any + /// realistic single-file size (4 GB cap) and saves 8 B/symbol over `usize` + /// on 64-bit targets. Cast to `usize` at slice sites: + /// `&src[sym.byte_range.start as usize..sym.byte_range.end as usize]`. + pub byte_range: Range, + /// 1-based inclusive line range. `u32` for the same reason as `byte_range`. + pub line_range: Range, pub content_hash: [u8; 32], pub merkle_hash: [u8; 32], pub children: Vec, - pub estimated_tokens: usize, + pub estimated_tokens: u32, } /// Per-file interner for symbol names. Real code has heavy name repetition @@ -111,7 +116,8 @@ impl SymbolNode { } pub fn total_tokens(&self) -> usize { - self.estimated_tokens + self.children.iter().map(|c| c.total_tokens()).sum::() + self.estimated_tokens as usize + + self.children.iter().map(|c| c.total_tokens()).sum::() } } diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs index cf221c3..8a46d9e 100644 --- a/tests/helpers/mod.rs +++ b/tests/helpers/mod.rs @@ -34,7 +34,7 @@ pub fn sym_with_children(id: &str, name: &str, children: Vec) -> Sym /// Create a SymbolNode with a custom line range. pub fn sym_with_lines(id: &str, name: &str, start: usize, end: usize) -> SymbolNode { let mut s = sym(id, name); - s.line_range = start..end; + s.line_range = start as u32..end as u32; s } @@ -48,7 +48,7 @@ pub fn sym_with_children_and_lines( ) -> SymbolNode { let mut s = sym(id, name); s.children = children; - s.line_range = start..end; + s.line_range = start as u32..end as u32; s } @@ -103,7 +103,7 @@ pub fn tool_call_targeted(tool: &str, path: &str, depth: ReadDepth, symbol: &str /// Create a tool call targeting a line range. pub fn tool_call_lines(tool: &str, path: &str, depth: ReadDepth, start: usize, end: usize) -> AgentToolCall { let mut tc = tool_call(tool, path, depth); - tc.target_lines = Some(start..end); + tc.target_lines = Some(start as u32..end as u32); tc }