diff --git a/.gitignore b/.gitignore index ae8dea9..7072cdc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ node_modules/ # Dolt database files (added by bd init) .dolt/ *.db + +# Heap probe / perf measurement artifacts — machine-specific. +.perf-baseline/ +dhat-heap.json diff --git a/Cargo.lock b/Cargo.lock index 843e5c9..1809d36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,9 +36,11 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" name = "ambits" version = "0.14.0" dependencies = [ + "blake3", "clap", "color-eyre", "crossterm", + "dhat", "divan", "flume", "ignore", @@ -47,7 +49,6 @@ dependencies = [ "serde", "serde-pickle", "serde_json", - "sha2", "tempfile", "toml", "tree-sitter", @@ -106,6 +107,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "autocfg" version = "1.5.0" @@ -140,12 +153,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] -name = "block-buffer" -version = "0.10.4" +name = "blake3" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ - "generic-array", + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", ] [[package]] @@ -295,11 +313,17 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] @@ -354,16 +378,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "darling" version = "0.23.0" @@ -399,13 +413,19 @@ dependencies = [ ] [[package]] -name = "digest" -version = "0.10.7" +name = "dhat" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "98cd11d84628e233de0ce467de10b8633f4ddaecafadefc86e13b84b8739b827" dependencies = [ - "block-buffer", - "crypto-common", + "backtrace", + "lazy_static", + "mintex", + "parking_lot", + "rustc-hash", + "serde", + "serde_json", + "thousands", ] [[package]] @@ -530,16 +550,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -827,6 +837,12 @@ dependencies = [ "adler2", ] +[[package]] +name = "mintex" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c505b3e17ed6b70a7ed2e67fbb2c560ee327353556120d6e72f5232b6880d536" + [[package]] name = "mio" version = "1.1.1" @@ -1061,6 +1077,12 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustix" version = "0.38.44" @@ -1179,17 +1201,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -1325,6 +1336,12 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "thousands" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" + [[package]] name = "thread_local" version = "1.1.9" @@ -1458,12 +1475,6 @@ dependencies = [ "tree-sitter-language", ] -[[package]] -name = "typenum" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" - [[package]] name = "unicode-ident" version = "1.0.22" @@ -1511,12 +1522,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index 0c773ef..324932e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,11 +23,15 @@ notify = "7" clap = { version = "4", features = ["derive"] } serde = { version = "1", features = ["derive"] } serde_json = "1" -sha2 = "0.10" color-eyre = "0.6" serde-pickle = "1.2" flume = "0.12" toml = { version = "0.8", default-features = false, features = ["parse"] } +blake3 = "1" +dhat = { version = "0.3", optional = true } + +[features] +dhat-heap = ["dep:dhat"] [dev-dependencies] tempfile = "3" @@ -53,6 +57,15 @@ harness = false name = "tracking" harness = false +[[bench]] +name = "symbol_pipeline" +harness = false + +[[bin]] +name = "heap_probe" +path = "src/bin/heap_probe.rs" +required-features = ["dhat-heap"] + [lib] name = "ambits" path = "src/lib.rs" diff --git a/benches/parser.rs b/benches/parser.rs index 8a3c57a..510f384 100644 --- a/benches/parser.rs +++ b/benches/parser.rs @@ -67,7 +67,7 @@ fn typescript_parse_and_coverage(bencher: divan::Bencher, size: &&str) { let mut ledger = ContextLedger::new(); let hash = [0u8; 32]; for sym in &file_syms.symbols { - ledger.record(sym.id.clone(), ReadDepth::FullBody, hash, "agent".to_string(), sym.estimated_tokens); + ledger.record(sym.id.clone(), ReadDepth::FullBody, hash, "agent".to_string(), sym.estimated_tokens as usize); } // 3. Build project tree from the single file diff --git a/benches/support/fixtures.rs b/benches/support/fixtures.rs index 55e8739..6b4c445 100644 --- a/benches/support/fixtures.rs +++ b/benches/support/fixtures.rs @@ -1,9 +1,10 @@ #![allow(dead_code)] use std::path::PathBuf; +use std::sync::Arc; use ambits::symbols::merkle::content_hash; -use ambits::symbols::{FileSymbols, ProjectTree, SymbolCategory, SymbolNode}; +use ambits::symbols::{FileSymbols, NameInterner, ProjectTree, SymbolCategory, SymbolNode}; use ambits::tracking::{ContextLedger, ReadDepth}; // ─── Rust source fixtures ───────────────────────────────────────────────────── @@ -959,10 +960,10 @@ pub fn make_sym(id: &str, name: &str) -> SymbolNode { let hash = content_hash(name); SymbolNode { id: id.to_string(), - name: name.to_string(), + name: Arc::from(name), category: SymbolCategory::Function, label: "fn", - file_path: PathBuf::from("src/bench.rs"), + file_path: Arc::new(PathBuf::from("src/bench.rs")), byte_range: 0..100, line_range: 1..10, content_hash: hash, @@ -996,9 +997,10 @@ pub fn make_flat_symbols(n: usize) -> Vec { /// Build a FileSymbols containing `n_symbols` flat symbols. pub fn make_file(path: &str, n_symbols: usize) -> FileSymbols { let file_path = PathBuf::from(path); + let file_path_arc = Arc::new(file_path.clone()); let symbols = make_flat_symbols(n_symbols) .into_iter() - .map(|mut s| { s.file_path = file_path.clone(); s }) + .map(|mut s| { s.file_path = Arc::clone(&file_path_arc); s }) .collect(); FileSymbols { file_path, symbols, total_lines: n_symbols * 5 } } @@ -1011,13 +1013,83 @@ pub fn make_project(n_files: usize, symbols_per_file: usize) -> ProjectTree { ProjectTree { root: PathBuf::from("/bench/project"), files } } +/// Build a synthetic monorepo `ProjectTree` for project-scale benchmarks. +/// +/// Distributes `n_files` across nested package directories (mimicking realistic +/// path depth) and varies file extensions across `rs`/`ts`/`py`. Within each +/// file the symbol mix is ~80% flat leaves and ~20% one-level-nested groups +/// (a type with four methods), so `compute_merkle_hash`'s leaf-vs-parent code +/// paths both get exercised in proportions roughly matching real source. +/// +/// Pure in-memory construction — no disk I/O, no tree-sitter. Use this for +/// benchmarking the code paths *downstream* of parsing. +pub fn make_monorepo_tree(n_files: usize, symbols_per_file: usize) -> ProjectTree { + const EXTENSIONS: [&str; 3] = ["rs", "ts", "py"]; + let files: Vec = (0..n_files) + .map(|file_idx| { + let ext = EXTENSIONS[file_idx % EXTENSIONS.len()]; + let pkg_idx = file_idx / 20; + let path_str = format!("src/pkg_{pkg_idx:04}/module_{file_idx:06}.{ext}"); + let symbols = build_file_symbols(&path_str, symbols_per_file); + FileSymbols { + file_path: PathBuf::from(&path_str), + symbols, + total_lines: symbols_per_file * 5, + } + }) + .collect(); + ProjectTree { + root: PathBuf::from("/bench/monorepo"), + files, + } +} + +fn build_file_symbols(file_path: &str, n: usize) -> Vec { + let groups = n / 5; + let remainder = n - groups * 5; + let mut out = Vec::with_capacity(groups + remainder); + let file_path_arc = Arc::new(PathBuf::from(file_path)); + // One interner per file — same scope as production parsers. + let names = NameInterner::new(); + for g in 0..groups { + let parent_id = format!("{file_path}::Type_{g:04}"); + let parent_name = format!("Type_{g:04}"); + let mut parent = make_sym(&parent_id, &parent_name); + parent.name = names.intern(&parent_name); + parent.category = SymbolCategory::Type; + parent.label = "struct"; + parent.file_path = Arc::clone(&file_path_arc); + parent.children = (0..4) + .map(|i| { + let child_id = format!("{parent_id}::method_{i}"); + let child_name = format!("method_{i}"); + let mut sym = make_sym(&child_id, &child_name); + sym.name = names.intern(&child_name); + sym.file_path = Arc::clone(&file_path_arc); + sym + }) + .collect(); + ambits::symbols::merkle::compute_merkle_hash(&mut parent); + out.push(parent); + } + for i in 0..remainder { + let leaf_id = format!("{file_path}::fn_{i:04}"); + let leaf_name = format!("fn_{i:04}"); + let mut sym = make_sym(&leaf_id, &leaf_name); + sym.name = names.intern(&leaf_name); + sym.file_path = Arc::clone(&file_path_arc); + out.push(sym); + } + out +} + /// Build a ContextLedger pre-populated with entries for all symbols in `project`. pub fn make_populated_ledger(project: &ProjectTree) -> ContextLedger { let mut ledger = ContextLedger::new(); let hash = [0u8; 32]; for file in &project.files { for sym in &file.symbols { - ledger.record(sym.id.clone(), ReadDepth::FullBody, hash, "agent-0".to_string(), sym.estimated_tokens); + ledger.record(sym.id.clone(), ReadDepth::FullBody, hash, "agent-0".to_string(), sym.estimated_tokens as usize); } } ledger diff --git a/benches/symbol_pipeline.rs b/benches/symbol_pipeline.rs new file mode 100644 index 0000000..041e22d --- /dev/null +++ b/benches/symbol_pipeline.rs @@ -0,0 +1,95 @@ +//! Project-scale benches: simulate operating on a synthetic monorepo's +//! symbol tree (up to ~500k symbols). These do NOT invoke tree-sitter — +//! they target the code paths downstream of parsing: merkle hashing, +//! coverage report construction, and ledger population. +//! +//! Tier sizing (`SYMBOLS_PER_FILE = 100`): +//! 100 files → 10,000 symbols +//! 1,000 files → 100,000 symbols +//! 5,000 files → 500,000 symbols (top tier for monorepo target) + +#[path = "support/fixtures.rs"] +mod fixtures; + +use ambits::coverage::CoverageReport; +use ambits::symbols::ProjectTree; +use ambits::symbols::merkle::compute_merkle_hash; +use ambits::tracking::{ContextLedger, ReadDepth}; +use fixtures::make_monorepo_tree; + +fn main() { + divan::main(); +} + +const SYMBOLS_PER_FILE: usize = 100; + +/// Time `compute_merkle_hash` over every top-level symbol in a project-scale tree. +/// Captures merkle pass cost on monorepo loads — this is the operation parsers +/// perform once per file after symbol extraction. +#[divan::bench(args = [100, 1_000, 5_000], sample_count = 20)] +fn compute_merkle_hash_project(bencher: divan::Bencher, n_files: &usize) { + bencher + .with_inputs(|| make_monorepo_tree(*n_files, SYMBOLS_PER_FILE)) + .bench_local_values(|mut tree| { + for file in tree.files.iter_mut() { + for sym in file.symbols.iter_mut() { + compute_merkle_hash(divan::black_box(sym)); + } + } + divan::black_box(tree) + }); +} + +/// Time `CoverageReport::from_project` on a fully-populated ledger. Models +/// the end-of-session coverage report at monorepo scale. +#[divan::bench(args = [100, 1_000], sample_count = 20)] +fn coverage_full_project(bencher: divan::Bencher, n_files: &usize) { + let project = make_monorepo_tree(*n_files, SYMBOLS_PER_FILE); + let ledger = populate_full(&project); + bencher.bench(|| { + divan::black_box(CoverageReport::from_project( + divan::black_box(&project), + divan::black_box(&ledger), + None, + )) + }); +} + +/// Time recording every symbol in the project tree into a fresh ledger. Models +/// the cost of ingesting a session that read the entire repo once. +#[divan::bench(args = [100, 1_000], sample_count = 20)] +fn ledger_populate_full_project(bencher: divan::Bencher, n_files: &usize) { + let project = make_monorepo_tree(*n_files, SYMBOLS_PER_FILE); + bencher + .with_inputs(ContextLedger::new) + .bench_local_values(|mut ledger| { + for file in &project.files { + for sym in &file.symbols { + ledger.record( + sym.id.clone(), + ReadDepth::Overview, + sym.content_hash, + "agent-0".to_string(), + sym.estimated_tokens as usize, + ); + } + } + divan::black_box(ledger) + }); +} + +fn populate_full(project: &ProjectTree) -> ContextLedger { + let mut ledger = ContextLedger::new(); + for file in &project.files { + for sym in &file.symbols { + ledger.record( + sym.id.clone(), + ReadDepth::FullBody, + sym.content_hash, + "agent-0".to_string(), + sym.estimated_tokens as usize, + ); + } + } + ledger +} diff --git a/src/app.rs b/src/app.rs index 4139c54..9554f86 100644 --- a/src/app.rs +++ b/src/app.rs @@ -668,14 +668,14 @@ fn flatten_symbol( rows.push(TreeRow { symbol_id: sym.id.clone(), - display_name: sym.name.clone(), + display_name: sym.name.to_string(), label: sym.label, depth, is_file: false, is_expanded, has_children: !sym.children.is_empty(), line_range: format!("L{}-{}", sym.line_range.start, sym.line_range.end), - token_count: sym.estimated_tokens, + token_count: sym.estimated_tokens as usize, read_depth, coverage_status: None, file_coverage_seen: 0, @@ -719,7 +719,7 @@ pub fn mark_file_symbols( event.read_depth, sym.content_hash, event.agent_id.to_string(), - sym.estimated_tokens, + sym.estimated_tokens as usize, ); mark_file_symbols(&sym.children, event, ledger); } @@ -744,7 +744,7 @@ pub fn mark_targeted_symbols( event.read_depth, sym.content_hash, event.agent_id.to_string(), - sym.estimated_tokens, + sym.estimated_tokens as usize, ); // Full body was in the response — bulk-mark all descendants. mark_file_symbols(&sym.children, event, ledger); @@ -755,7 +755,7 @@ pub fn mark_targeted_symbols( event.read_depth, sym.content_hash, event.agent_id.to_string(), - sym.estimated_tokens, + sym.estimated_tokens as usize, ); // Parent container overlaps the read range — recurse precisely so // only children whose ranges also overlap get promoted. @@ -825,7 +825,7 @@ pub fn symbol_name_matches(sym: &SymbolNode, event: &AgentToolCall) -> bool { } // Plain name match (e.g. target = "handle_key", sym.name = "handle_key"). let norm_sym_name = normalize_name_path(&sym.name); - sym.name == *target_name || norm_sym_name == norm_target + sym.name.as_ref() == target_name.as_str() || norm_sym_name == norm_target } /// Check if a symbol's line range overlaps with the tool call's `target_lines`. diff --git a/src/bin/heap_probe.rs b/src/bin/heap_probe.rs new file mode 100644 index 0000000..67d14d8 --- /dev/null +++ b/src/bin/heap_probe.rs @@ -0,0 +1,136 @@ +//! Project-scale heap profile harness. +//! +//! Build: cargo build --release --features dhat-heap --bin heap_probe +//! Run: ./target/release/heap_probe +//! +//! Writes `dhat-heap.json` to CWD. View with `dh_view.html` or `dhat` from +//! https://valgrind.org/docs/manual/dh-manual.html — drop the JSON onto the +//! viewer to get an interactive breakdown of allocation sites and bytes. +//! +//! Default tier (omit arg) is 1000 files ≈ 100k symbols. + +use std::path::PathBuf; +use std::sync::Arc; + +use ambits::symbols::merkle::{compute_merkle_hash, content_hash}; +use ambits::symbols::{FileSymbols, NameInterner, ProjectTree, SymbolCategory, SymbolNode}; + +#[global_allocator] +static ALLOC: dhat::Alloc = dhat::Alloc; + +const SYMBOLS_PER_FILE: usize = 100; + +fn main() { + let _profiler = dhat::Profiler::new_heap(); + + let n_files: usize = std::env::args() + .nth(1) + .and_then(|s| s.parse().ok()) + .unwrap_or(1_000); + + eprintln!( + "heap_probe: building tree with {n_files} files × {SYMBOLS_PER_FILE} symbols/file", + ); + + let project = make_monorepo_tree(n_files, SYMBOLS_PER_FILE); + + // Drive a few representative operations against the tree so dhat sees + // the steady-state allocation pattern of real consumers, not just the + // initial construction. + let mut total_symbols = 0usize; + for file in &project.files { + for sym in &file.symbols { + total_symbols += count_subtree(sym); + } + } + eprintln!("heap_probe: total symbols (including nested children) = {total_symbols}"); + + // Keep the tree live until the profiler captures the heap. + std::hint::black_box(&project); +} + +fn count_subtree(node: &SymbolNode) -> usize { + 1 + node.children.iter().map(count_subtree).sum::() +} + +// ─── Inlined monorepo fixture (mirrors benches/support/fixtures.rs) ────────── +// Duplicated here so the binary is self-contained; this lives outside any +// bench harness and is short enough that duplication beats the path-include +// gymnastics needed to share with the bench fixture file. + +fn make_monorepo_tree(n_files: usize, symbols_per_file: usize) -> ProjectTree { + const EXTENSIONS: [&str; 3] = ["rs", "ts", "py"]; + let files: Vec = (0..n_files) + .map(|file_idx| { + let ext = EXTENSIONS[file_idx % EXTENSIONS.len()]; + let pkg_idx = file_idx / 20; + let path_str = format!("src/pkg_{pkg_idx:04}/module_{file_idx:06}.{ext}"); + let symbols = build_file_symbols(&path_str, symbols_per_file); + FileSymbols { + file_path: PathBuf::from(&path_str), + symbols, + total_lines: symbols_per_file * 5, + } + }) + .collect(); + ProjectTree { + root: PathBuf::from("/bench/monorepo"), + files, + } +} + +fn build_file_symbols(file_path: &str, n: usize) -> Vec { + let groups = n / 5; + let remainder = n - groups * 5; + let mut out = Vec::with_capacity(groups + remainder); + let file_path_arc = Arc::new(PathBuf::from(file_path)); + // One interner per file — same scope as production parsers. + let names = NameInterner::new(); + for g in 0..groups { + let parent_id = format!("{file_path}::Type_{g:04}"); + let parent_name = format!("Type_{g:04}"); + let mut parent = make_sym(&parent_id, &parent_name); + parent.name = names.intern(&parent_name); + parent.category = SymbolCategory::Type; + parent.label = "struct"; + parent.file_path = Arc::clone(&file_path_arc); + parent.children = (0..4) + .map(|i| { + let child_id = format!("{parent_id}::method_{i}"); + let child_name = format!("method_{i}"); + let mut sym = make_sym(&child_id, &child_name); + sym.name = names.intern(&child_name); + sym.file_path = Arc::clone(&file_path_arc); + sym + }) + .collect(); + compute_merkle_hash(&mut parent); + out.push(parent); + } + for i in 0..remainder { + let leaf_id = format!("{file_path}::fn_{i:04}"); + let leaf_name = format!("fn_{i:04}"); + let mut sym = make_sym(&leaf_id, &leaf_name); + sym.name = names.intern(&leaf_name); + sym.file_path = Arc::clone(&file_path_arc); + out.push(sym); + } + out +} + +fn make_sym(id: &str, name: &str) -> SymbolNode { + let hash = content_hash(name); + SymbolNode { + id: id.to_string(), + name: Arc::from(name), + category: SymbolCategory::Function, + label: "fn", + file_path: Arc::new(PathBuf::from("src/bench.rs")), + byte_range: 0..100, + line_range: 1..10, + content_hash: hash, + merkle_hash: hash, + children: Vec::new(), + estimated_tokens: 30, + } +} diff --git a/src/ingest/claude.rs b/src/ingest/claude.rs index 96bb0e6..f7a35dd 100644 --- a/src/ingest/claude.rs +++ b/src/ingest/claude.rs @@ -664,8 +664,8 @@ pub fn map_tool_call( // Extract target_lines. let target_lines = mapping.target_lines.as_ref().and_then(|spec| { - let offset = input.get(&spec.offset_key)?.as_u64()? as usize; - let limit = input.get(&spec.limit_key)?.as_u64()? as usize; + let offset = input.get(&spec.offset_key)?.as_u64()? as u32; + let limit = input.get(&spec.limit_key)?.as_u64()? as u32; Some(offset..offset + limit) }); diff --git a/src/ingest/mod.rs b/src/ingest/mod.rs index 8c18780..e3dca1f 100644 --- a/src/ingest/mod.rs +++ b/src/ingest/mod.rs @@ -19,7 +19,7 @@ pub struct AgentToolCall { /// Optional symbol name path to target (e.g. "MyClass/my_method"). pub target_symbol: Option, /// Optional line range to target (1-based, e.g. 10..25). - pub target_lines: Option>, + pub target_lines: Option>, /// Human-readable label for the agent (e.g. "Explore parser and symbol types"). /// Falls back to agent_id if no label could be extracted from the session log. pub label: Arc, diff --git a/src/parser/python.rs b/src/parser/python.rs index f991928..68f58ca 100644 --- a/src/parser/python.rs +++ b/src/parser/python.rs @@ -29,13 +29,14 @@ //! | `MAX_SIZE = ...` (UPPER_SNAKE_CASE) | Variable | `"var"` | //! | class methods | Function | `"def"` | -use std::path::Path; +use std::path::{Path, PathBuf}; +use std::sync::Arc; use color_eyre::eyre::eyre; use tree_sitter::{Node, Parser}; use crate::symbols::merkle::{compute_merkle_hash, content_hash, estimate_tokens}; -use crate::symbols::{FileSymbols, SymbolCategory, SymbolNode}; +use crate::symbols::{FileSymbols, NameInterner, SymbolCategory, SymbolNode}; use super::LanguageParser; @@ -73,8 +74,10 @@ impl LanguageParser for PythonParser { let path_prefix = path.to_string_lossy(); let src = source.as_bytes(); let mut symbols = Vec::new(); + let file_path_arc = Arc::new(path.to_path_buf()); + let names = NameInterner::new(); - extract_symbols(root, src, path, &path_prefix, "", &mut symbols); + extract_symbols(root, src, &file_path_arc, &names, &path_prefix, "", &mut symbols); for sym in symbols.iter_mut() { compute_merkle_hash(sym); @@ -136,7 +139,8 @@ const TYPE_ALIAS: SymbolMeta = SymbolMeta { category: SymbolCategory::Type, labe fn extract_symbols( node: Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -150,7 +154,7 @@ fn extract_symbols( "class_definition" => child_name(&child, src).map(|n| (n, CLASS)), // Decorated definitions: unwrap the decorator to find the inner def/class. "decorated_definition" => { - extract_decorated(&child, src, file_path, path_prefix, parent_name_path, out); + extract_decorated(&child, src, file_path, names, path_prefix, parent_name_path, out); None } // Module-level assignments: annotated or UPPER_SNAKE_CASE only. @@ -161,7 +165,7 @@ fn extract_symbols( "if_statement" | "for_statement" | "while_statement" | "try_statement" | "with_statement" | "match_statement" => { extract_from_compound_bodies( - &child, src, file_path, path_prefix, parent_name_path, out, + &child, src, file_path, names, path_prefix, parent_name_path, out, ); None } @@ -183,22 +187,22 @@ fn extract_symbols( let mut sym = SymbolNode { id, - name: name.clone(), + name: names.intern(&name), category: meta.category, label: meta.label, - file_path: file_path.to_path_buf(), - byte_range, - line_range: start_line..end_line, + file_path: Arc::clone(file_path), + byte_range: byte_range.start as u32..byte_range.end as u32, + line_range: start_line as u32..end_line as u32, content_hash: content_hash(text), merkle_hash: [0u8; 32], children: Vec::new(), - estimated_tokens: estimate_tokens(text), + estimated_tokens: estimate_tokens(text) as u32, }; // For classes, recurse into the body block to find methods. if meta.category == SymbolCategory::Type { if let Some(body) = child.child_by_field_name("body") { - extract_symbols(body, src, file_path, path_prefix, &name_path, &mut sym.children); + extract_symbols(body, src, file_path, names, path_prefix, &name_path, &mut sym.children); } } @@ -224,7 +228,8 @@ fn extract_symbols( fn extract_decorated( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -258,21 +263,21 @@ fn extract_decorated( let mut sym = SymbolNode { id, - name: name.clone(), + name: names.intern(&name), category: meta.category, label: meta.label, - file_path: file_path.to_path_buf(), - byte_range, - line_range: start_line..end_line, + file_path: Arc::clone(file_path), + byte_range: byte_range.start as u32..byte_range.end as u32, + line_range: start_line as u32..end_line as u32, content_hash: content_hash(text), merkle_hash: [0u8; 32], children: Vec::new(), - estimated_tokens: estimate_tokens(text), + estimated_tokens: estimate_tokens(text) as u32, }; if meta.category == SymbolCategory::Type { if let Some(body) = child.child_by_field_name("body") { - extract_symbols(body, src, file_path, path_prefix, &name_path, &mut sym.children); + extract_symbols(body, src, file_path, names, path_prefix, &name_path, &mut sym.children); } } @@ -396,7 +401,8 @@ fn extract_type_alias(node: &Node, src: &[u8]) -> Option<(String, SymbolMeta)> { fn extract_from_compound_bodies( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -404,12 +410,12 @@ fn extract_from_compound_bodies( match node.kind() { "for_statement" | "while_statement" | "with_statement" => { if let Some(body) = node.child_by_field_name("body") { - extract_symbols(body, src, file_path, path_prefix, parent_name_path, out); + extract_symbols(body, src, file_path, names, path_prefix, parent_name_path, out); } } "if_statement" => { if let Some(body) = node.child_by_field_name("consequence") { - extract_symbols(body, src, file_path, path_prefix, parent_name_path, out); + extract_symbols(body, src, file_path, names, path_prefix, parent_name_path, out); } let mut cursor = node.walk(); for child in node.children(&mut cursor) { @@ -417,14 +423,14 @@ fn extract_from_compound_bodies( "elif_clause" => { if let Some(body) = child.child_by_field_name("consequence") { extract_symbols( - body, src, file_path, path_prefix, parent_name_path, out, + body, src, file_path, names, path_prefix, parent_name_path, out, ); } } "else_clause" => { if let Some(body) = child.child_by_field_name("body") { extract_symbols( - body, src, file_path, path_prefix, parent_name_path, out, + body, src, file_path, names, path_prefix, parent_name_path, out, ); } } @@ -434,7 +440,7 @@ fn extract_from_compound_bodies( } "try_statement" => { if let Some(body) = node.child_by_field_name("body") { - extract_symbols(body, src, file_path, path_prefix, parent_name_path, out); + extract_symbols(body, src, file_path, names, path_prefix, parent_name_path, out); } let mut cursor = node.walk(); for child in node.children(&mut cursor) { @@ -444,7 +450,7 @@ fn extract_from_compound_bodies( for inner in child.children(&mut inner_cursor) { if inner.kind() == "block" { extract_symbols( - inner, src, file_path, path_prefix, parent_name_path, out, + inner, src, file_path, names, path_prefix, parent_name_path, out, ); } } @@ -452,7 +458,7 @@ fn extract_from_compound_bodies( "else_clause" => { if let Some(body) = child.child_by_field_name("body") { extract_symbols( - body, src, file_path, path_prefix, parent_name_path, out, + body, src, file_path, names, path_prefix, parent_name_path, out, ); } } @@ -467,7 +473,7 @@ fn extract_from_compound_bodies( if child.kind() == "case_clause" { if let Some(consequence) = child.child_by_field_name("consequence") { extract_symbols( - consequence, src, file_path, path_prefix, parent_name_path, out, + consequence, src, file_path, names, path_prefix, parent_name_path, out, ); } } @@ -506,7 +512,7 @@ mod tests { fn parse_simple_function() { let syms = parse("def foo():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].category, SymbolCategory::Function); assert_eq!(syms[0].label, "def"); assert_eq!(syms[0].id, "test.py::foo"); @@ -517,15 +523,15 @@ mod tests { fn parse_multiple_functions() { let syms = parse("def foo():\n pass\n\ndef bar():\n pass\n"); assert_eq!(syms.len(), 2); - assert_eq!(syms[0].name, "foo"); - assert_eq!(syms[1].name, "bar"); + assert_eq!(syms[0].name.as_ref(), "foo"); + assert_eq!(syms[1].name.as_ref(), "bar"); } #[test] fn parse_simple_class() { let syms = parse("class Foo:\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].label, "class"); assert_eq!(syms[0].id, "test.py::Foo"); @@ -537,13 +543,13 @@ mod tests { "class Foo:\n def __init__(self):\n pass\n def bar(self):\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].children.len(), 2); - assert_eq!(syms[0].children[0].name, "__init__"); + assert_eq!(syms[0].children[0].name.as_ref(), "__init__"); assert_eq!(syms[0].children[0].category, SymbolCategory::Function); assert_eq!(syms[0].children[0].id, "test.py::Foo/__init__"); - assert_eq!(syms[0].children[1].name, "bar"); + assert_eq!(syms[0].children[1].name.as_ref(), "bar"); assert_eq!(syms[0].children[1].id, "test.py::Foo/bar"); } @@ -551,7 +557,7 @@ mod tests { fn parse_decorated_function() { let syms = parse("@staticmethod\ndef foo():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].category, SymbolCategory::Function); assert_eq!(syms[0].label, "def"); // Decorated definitions include the decorator in the byte range @@ -562,7 +568,7 @@ mod tests { fn parse_decorated_class() { let syms = parse("@dataclass\nclass Point:\n x: int\n y: int\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Point"); + assert_eq!(syms[0].name.as_ref(), "Point"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].label, "class"); // Decorated class range includes the decorator @@ -575,11 +581,11 @@ mod tests { "@dataclass\nclass Point:\n def __init__(self):\n pass\n def distance(self):\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Point"); + assert_eq!(syms[0].name.as_ref(), "Point"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].children.len(), 2); - assert_eq!(syms[0].children[0].name, "__init__"); - assert_eq!(syms[0].children[1].name, "distance"); + assert_eq!(syms[0].children[0].name.as_ref(), "__init__"); + assert_eq!(syms[0].children[1].name.as_ref(), "distance"); } #[test] @@ -588,12 +594,12 @@ mod tests { "def helper():\n pass\n\nclass Foo:\n def method(self):\n pass\n\ndef another():\n pass\n", ); assert_eq!(syms.len(), 3); - assert_eq!(syms[0].name, "helper"); + assert_eq!(syms[0].name.as_ref(), "helper"); assert_eq!(syms[0].category, SymbolCategory::Function); - assert_eq!(syms[1].name, "Foo"); + assert_eq!(syms[1].name.as_ref(), "Foo"); assert_eq!(syms[1].category, SymbolCategory::Type); assert_eq!(syms[1].children.len(), 1); - assert_eq!(syms[2].name, "another"); + assert_eq!(syms[2].name.as_ref(), "another"); assert_eq!(syms[2].category, SymbolCategory::Function); } @@ -603,14 +609,14 @@ mod tests { "class Outer:\n class Inner:\n def method(self):\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Outer"); + assert_eq!(syms[0].name.as_ref(), "Outer"); assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "Inner"); + assert_eq!(syms[0].children[0].name.as_ref(), "Inner"); assert_eq!(syms[0].children[0].category, SymbolCategory::Type); assert_eq!(syms[0].children[0].id, "test.py::Outer/Inner"); // Inner class should have its own children assert_eq!(syms[0].children[0].children.len(), 1); - assert_eq!(syms[0].children[0].children[0].name, "method"); + assert_eq!(syms[0].children[0].children[0].name.as_ref(), "method"); assert_eq!(syms[0].children[0].children[0].id, "test.py::Outer/Inner/method"); } @@ -656,7 +662,7 @@ mod tests { .parse_file(Path::new("src/main.py"), "def foo():\n pass\n") .unwrap(); assert_eq!(file.file_path, Path::new("src/main.py")); - assert_eq!(file.symbols[0].file_path, Path::new("src/main.py")); + assert_eq!(file.symbols[0].file_path.as_path(), Path::new("src/main.py")); assert_eq!(file.symbols[0].id, "src/main.py::foo"); } @@ -673,9 +679,9 @@ mod tests { "class Foo:\n @staticmethod\n def bar():\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "bar"); + assert_eq!(syms[0].children[0].name.as_ref(), "bar"); assert_eq!(syms[0].children[0].category, SymbolCategory::Function); } @@ -683,7 +689,7 @@ mod tests { fn parse_multiple_decorators() { let syms = parse("@decorator1\n@decorator2\ndef foo():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].line_range.start, 1); } @@ -695,7 +701,7 @@ mod tests { // async def is not a direct `function_definition` – it is an expression_statement // or may be wrapped. Verify actual behavior either way. assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -705,7 +711,7 @@ mod tests { let syms = parse(src); let range = &syms[0].byte_range; assert!(range.start < range.end); - assert!(range.end <= src.len()); + assert!(range.end as usize <= src.len()); } // --- Variable extraction tests --- @@ -714,7 +720,7 @@ mod tests { fn parse_annotated_variable() { let syms = parse("x: int = 42\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "x"); + assert_eq!(syms[0].name.as_ref(), "x"); assert_eq!(syms[0].category, SymbolCategory::Variable); assert_eq!(syms[0].label, "var"); assert_eq!(syms[0].id, "test.py::x"); @@ -725,7 +731,7 @@ mod tests { fn parse_annotated_variable_no_value() { let syms = parse("x: int\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "x"); + assert_eq!(syms[0].name.as_ref(), "x"); assert_eq!(syms[0].category, SymbolCategory::Variable); } @@ -733,7 +739,7 @@ mod tests { fn parse_upper_case_constant() { let syms = parse("MAX_SIZE = 100\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "MAX_SIZE"); + assert_eq!(syms[0].name.as_ref(), "MAX_SIZE"); assert_eq!(syms[0].category, SymbolCategory::Variable); assert_eq!(syms[0].label, "var"); } @@ -742,7 +748,7 @@ mod tests { fn parse_upper_case_no_underscore() { let syms = parse("DEBUG = True\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "DEBUG"); + assert_eq!(syms[0].name.as_ref(), "DEBUG"); assert_eq!(syms[0].category, SymbolCategory::Variable); } @@ -774,11 +780,11 @@ mod tests { fn parse_variables_mixed_with_functions() { let syms = parse("MAX: int = 100\n\ndef foo():\n pass\n\nDEBUG = True\n"); assert_eq!(syms.len(), 3); - assert_eq!(syms[0].name, "MAX"); + assert_eq!(syms[0].name.as_ref(), "MAX"); assert_eq!(syms[0].category, SymbolCategory::Variable); - assert_eq!(syms[1].name, "foo"); + assert_eq!(syms[1].name.as_ref(), "foo"); assert_eq!(syms[1].category, SymbolCategory::Function); - assert_eq!(syms[2].name, "DEBUG"); + assert_eq!(syms[2].name.as_ref(), "DEBUG"); assert_eq!(syms[2].category, SymbolCategory::Variable); } @@ -788,7 +794,7 @@ mod tests { fn parse_type_alias() { let syms = parse("type Vector = list[float]\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Vector"); + assert_eq!(syms[0].name.as_ref(), "Vector"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].label, "type"); assert_eq!(syms[0].id, "test.py::Vector"); @@ -798,9 +804,9 @@ mod tests { fn parse_type_alias_with_functions() { let syms = parse("type ID = int\n\ndef process(x: ID) -> None:\n pass\n"); assert_eq!(syms.len(), 2); - assert_eq!(syms[0].name, "ID"); + assert_eq!(syms[0].name.as_ref(), "ID"); assert_eq!(syms[0].category, SymbolCategory::Type); - assert_eq!(syms[1].name, "process"); + assert_eq!(syms[1].name.as_ref(), "process"); assert_eq!(syms[1].category, SymbolCategory::Function); } @@ -810,7 +816,7 @@ mod tests { fn parse_function_inside_if() { let syms = parse("if True:\n def foo():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -820,8 +826,8 @@ mod tests { "if sys.platform == 'win32':\n def init():\n pass\nelse:\n def init():\n pass\n", ); assert_eq!(syms.len(), 2); - assert_eq!(syms[0].name, "init"); - assert_eq!(syms[1].name, "init"); + assert_eq!(syms[0].name.as_ref(), "init"); + assert_eq!(syms[1].name.as_ref(), "init"); } #[test] @@ -830,7 +836,7 @@ mod tests { "if False:\n pass\nelif True:\n def handler():\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "handler"); + assert_eq!(syms[0].name.as_ref(), "handler"); } #[test] @@ -839,9 +845,9 @@ mod tests { "try:\n class Foo:\n pass\nexcept Exception:\n class FallbackFoo:\n pass\n", ); assert_eq!(syms.len(), 2); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].category, SymbolCategory::Type); - assert_eq!(syms[1].name, "FallbackFoo"); + assert_eq!(syms[1].name.as_ref(), "FallbackFoo"); assert_eq!(syms[1].category, SymbolCategory::Type); } @@ -851,36 +857,36 @@ mod tests { "try:\n def setup():\n pass\nfinally:\n def cleanup():\n pass\n", ); assert_eq!(syms.len(), 2); - assert_eq!(syms[0].name, "setup"); - assert_eq!(syms[1].name, "cleanup"); + assert_eq!(syms[0].name.as_ref(), "setup"); + assert_eq!(syms[1].name.as_ref(), "cleanup"); } #[test] fn parse_function_inside_for() { let syms = parse("for i in range(1):\n def worker():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "worker"); + assert_eq!(syms[0].name.as_ref(), "worker"); } #[test] fn parse_function_inside_while() { let syms = parse("while True:\n def loop_body():\n pass\n break\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "loop_body"); + assert_eq!(syms[0].name.as_ref(), "loop_body"); } #[test] fn parse_function_inside_with() { let syms = parse("with open('f') as f:\n def process():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "process"); + assert_eq!(syms[0].name.as_ref(), "process"); } #[test] fn parse_decorated_function_inside_if() { let syms = parse("if True:\n @decorator\n def foo():\n pass\n"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -890,7 +896,7 @@ mod tests { "try:\n if True:\n def deeply_nested():\n pass\nexcept:\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "deeply_nested"); + assert_eq!(syms[0].name.as_ref(), "deeply_nested"); } #[test] @@ -899,10 +905,10 @@ mod tests { "if True:\n class Foo:\n def method(self):\n pass\n", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "method"); + assert_eq!(syms[0].children[0].name.as_ref(), "method"); } // --- is_upper_snake_case unit tests --- diff --git a/src/parser/rust.rs b/src/parser/rust.rs index 7ebe8dd..431fb4e 100644 --- a/src/parser/rust.rs +++ b/src/parser/rust.rs @@ -1,10 +1,11 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; +use std::sync::Arc; use color_eyre::eyre::eyre; use tree_sitter::{Node, Parser}; use crate::symbols::merkle::{compute_merkle_hash, content_hash, estimate_tokens}; -use crate::symbols::{FileSymbols, SymbolCategory, SymbolNode}; +use crate::symbols::{FileSymbols, NameInterner, SymbolCategory, SymbolNode}; use super::LanguageParser; @@ -38,8 +39,10 @@ impl LanguageParser for RustParser { let path_prefix = path.to_string_lossy(); let src = source.as_bytes(); let mut symbols = Vec::new(); + let file_path_arc = Arc::new(path.to_path_buf()); + let names = NameInterner::new(); - extract_symbols(root, src, path, &path_prefix, "", &mut symbols); + extract_symbols(root, src, &file_path_arc, &names, &path_prefix, "", &mut symbols); for sym in symbols.iter_mut() { compute_merkle_hash(sym); @@ -75,7 +78,8 @@ const MACRO: SymbolMeta = SymbolMeta { category: SymbolCategory::Macro, label: " fn extract_symbols( node: Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -111,16 +115,16 @@ fn extract_symbols( let mut sym = SymbolNode { id, - name: name.clone(), + name: names.intern(&name), category: meta.category, label: meta.label, - file_path: file_path.to_path_buf(), - byte_range, - line_range: start_line..end_line, + file_path: Arc::clone(file_path), + byte_range: byte_range.start as u32..byte_range.end as u32, + line_range: start_line as u32..end_line as u32, content_hash: content_hash(text), merkle_hash: [0u8; 32], children: Vec::new(), - estimated_tokens: estimate_tokens(text), + estimated_tokens: estimate_tokens(text) as u32, }; // Recurse into container types for their children. @@ -128,7 +132,7 @@ fn extract_symbols( || meta.label == "trait" { if let Some(body) = child_by_kind(&child, "declaration_list") { - extract_body_children(body, src, file_path, path_prefix, &name_path, &mut sym.children); + extract_body_children(body, src, file_path, names, path_prefix, &name_path, &mut sym.children); } } @@ -140,7 +144,8 @@ fn extract_symbols( fn extract_body_children( body: Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -165,16 +170,16 @@ fn extract_body_children( out.push(SymbolNode { id, - name, + name: names.intern(&name), category: meta.category, label: meta.label, - file_path: file_path.to_path_buf(), - byte_range, - line_range: start_line..end_line, + file_path: Arc::clone(file_path), + byte_range: byte_range.start as u32..byte_range.end as u32, + line_range: start_line as u32..end_line as u32, content_hash: content_hash(text), merkle_hash: [0u8; 32], children: Vec::new(), - estimated_tokens: estimate_tokens(text), + estimated_tokens: estimate_tokens(text) as u32, }); } } @@ -248,7 +253,7 @@ mod tests { fn parse_function() { let syms = parse("fn foo() {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "foo"); + assert_eq!(syms[0].name.as_ref(), "foo"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -258,22 +263,22 @@ mod tests { "struct Point { x: i32 }\nimpl Point {\n fn new() -> Self { Self { x: 0 } }\n}", ); assert_eq!(syms.len(), 2); - assert_eq!(syms[0].name, "Point"); + assert_eq!(syms[0].name.as_ref(), "Point"); assert_eq!(syms[0].category, SymbolCategory::Type); - assert_eq!(syms[1].name, "Point"); + assert_eq!(syms[1].name.as_ref(), "Point"); assert_eq!(syms[1].category, SymbolCategory::Implementation); assert_eq!(syms[1].children.len(), 1); - assert_eq!(syms[1].children[0].name, "new"); + assert_eq!(syms[1].children[0].name.as_ref(), "new"); } #[test] fn parse_nested_module() { let syms = parse("mod inner {\n fn bar() {}\n}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "inner"); + assert_eq!(syms[0].name.as_ref(), "inner"); assert_eq!(syms[0].category, SymbolCategory::Module); assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "bar"); + assert_eq!(syms[0].children[0].name.as_ref(), "bar"); } #[test] diff --git a/src/parser/typescript.rs b/src/parser/typescript.rs index 7f9f73b..a2a7dcd 100644 --- a/src/parser/typescript.rs +++ b/src/parser/typescript.rs @@ -32,13 +32,14 @@ //! | class properties / interface props | Variable | `"property"` | //! | `declare ...` | Variable | `"declare"` | -use std::path::Path; +use std::path::{Path, PathBuf}; +use std::sync::Arc; use color_eyre::eyre::eyre; use tree_sitter::{Node, Parser}; use crate::symbols::merkle::{compute_merkle_hash, content_hash, estimate_tokens}; -use crate::symbols::{FileSymbols, SymbolCategory, SymbolNode}; +use crate::symbols::{FileSymbols, NameInterner, SymbolCategory, SymbolNode}; use super::LanguageParser; @@ -76,8 +77,10 @@ impl LanguageParser for TypescriptParser { let path_prefix = path.to_string_lossy(); let src = source.as_bytes(); let mut symbols = Vec::new(); + let file_path_arc = Arc::new(path.to_path_buf()); + let names = NameInterner::new(); - extract_symbols(root, src, path, &path_prefix, "", &mut symbols); + extract_symbols(root, src, &file_path_arc, &names, &path_prefix, "", &mut symbols); for sym in symbols.iter_mut() { compute_merkle_hash(sym); @@ -146,7 +149,8 @@ const DECLARE: SymbolMeta = SymbolMeta { category: SymbolCategory::Variable, lab fn extract_symbols( node: Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -173,7 +177,7 @@ fn extract_symbols( let mut inner_cursor = child.walk(); for inner in child.children(&mut inner_cursor) { if inner.kind() == "internal_module" { - emit_namespace(&inner, src, file_path, path_prefix, parent_name_path, out); + emit_namespace(&inner, src, file_path, names, path_prefix, parent_name_path, out); } } continue; @@ -186,45 +190,45 @@ fn extract_symbols( match target.kind() { // `function foo()` or `function* gen()` - leaf symbol, no children. "function_declaration" | "generator_function_declaration" => { - if let Some(sym) = build_named_symbol(&target, src, file_path, path_prefix, parent_name_path, &FN, byte_range) { + if let Some(sym) = build_named_symbol(&target, src, file_path, names, path_prefix, parent_name_path, &FN, byte_range) { out.push(sym); } } // `class Foo { ... }` - container, recurse into class_body for members. "class_declaration" => { - emit_class(&target, src, file_path, path_prefix, parent_name_path, &CLASS, byte_range, out); + emit_class(&target, src, file_path, names, path_prefix, parent_name_path, &CLASS, byte_range, out); } // `abstract class Base { ... }` - same as class but different label. "abstract_class_declaration" => { - emit_class(&target, src, file_path, path_prefix, parent_name_path, &ABSTRACT_CLASS, byte_range, out); + emit_class(&target, src, file_path, names, path_prefix, parent_name_path, &ABSTRACT_CLASS, byte_range, out); } // `interface Config { ... }` - container, recurse into interface_body. "interface_declaration" => { - emit_interface(&target, src, file_path, path_prefix, parent_name_path, byte_range, out); + emit_interface(&target, src, file_path, names, path_prefix, parent_name_path, byte_range, out); } // `type Alias = ...` - leaf symbol. "type_alias_declaration" => { - if let Some(sym) = build_named_symbol(&target, src, file_path, path_prefix, parent_name_path, &TYPE, byte_range) { + if let Some(sym) = build_named_symbol(&target, src, file_path, names, path_prefix, parent_name_path, &TYPE, byte_range) { out.push(sym); } } // `enum Status { ... }` - leaf (we don't extract enum members). "enum_declaration" => { - if let Some(sym) = build_named_symbol(&target, src, file_path, path_prefix, parent_name_path, &ENUM, byte_range) { + if let Some(sym) = build_named_symbol(&target, src, file_path, names, path_prefix, parent_name_path, &ENUM, byte_range) { out.push(sym); } } // `namespace N { ... }` / `module M { ... }` - container, recurse. "internal_module" | "module" => { - emit_namespace(&target, src, file_path, path_prefix, parent_name_path, out); + emit_namespace(&target, src, file_path, names, path_prefix, parent_name_path, out); } // `const foo = () => {}` or `let bar = function() {}` - detect arrow/fn expressions. "lexical_declaration" | "variable_declaration" => { - extract_arrow_fns(&target, src, file_path, path_prefix, parent_name_path, out); + extract_arrow_fns(&target, src, file_path, names, path_prefix, parent_name_path, out); } // `declare function ...`, `declare class ...`, `declare const ...`, etc. "ambient_declaration" => { - extract_ambient(&target, src, file_path, path_prefix, parent_name_path, out); + extract_ambient(&target, src, file_path, names, path_prefix, parent_name_path, out); } // Imports, comments, expression statements, etc. - ignored. _ => {} @@ -250,7 +254,8 @@ fn extract_symbols( fn extract_arrow_fns( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -281,7 +286,7 @@ fn extract_arrow_fns( // Use the full declaration range (includes const/let keyword). let byte_range = node.byte_range(); - out.push(make_symbol(name, &FN, node, byte_range, src, file_path, path_prefix, parent_name_path, Vec::new())); + out.push(make_symbol(name, &FN, node, byte_range, src, file_path, names, path_prefix, parent_name_path, Vec::new())); } } @@ -297,7 +302,8 @@ fn extract_arrow_fns( fn extract_ambient( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -312,14 +318,14 @@ fn extract_ambient( | "internal_module" | "module" => child_name(&child, src), "lexical_declaration" | "variable_declaration" => { // Extract variable names from declare const/let/var. - extract_ambient_vars(&child, src, file_path, path_prefix, parent_name_path, &ambient_range, out); + extract_ambient_vars(&child, src, file_path, names, path_prefix, parent_name_path, &ambient_range, out); None } _ => None, }; if let Some(name) = name { - out.push(make_symbol(name, &DECLARE, node, ambient_range, src, file_path, path_prefix, parent_name_path, Vec::new())); + out.push(make_symbol(name, &DECLARE, node, ambient_range, src, file_path, names, path_prefix, parent_name_path, Vec::new())); return; // One symbol per ambient_declaration. } } @@ -329,7 +335,8 @@ fn extract_ambient( fn extract_ambient_vars( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, ambient_range: &std::ops::Range, @@ -349,7 +356,7 @@ fn extract_ambient_vars( None => continue, }; - out.push(make_symbol(name, &DECLARE, &child, ambient_range.clone(), src, file_path, path_prefix, parent_name_path, Vec::new())); + out.push(make_symbol(name, &DECLARE, &child, ambient_range.clone(), src, file_path, names, path_prefix, parent_name_path, Vec::new())); } } @@ -364,7 +371,8 @@ fn extract_ambient_vars( fn emit_class( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, meta: &SymbolMeta, @@ -384,10 +392,10 @@ fn emit_class( let mut children = Vec::new(); if let Some(body) = child_by_kind(node, "class_body") { - extract_members(body, src, file_path, path_prefix, &name_path, &mut children); + extract_members(body, src, file_path, names, path_prefix, &name_path, &mut children); } - out.push(make_symbol(name, meta, node, byte_range, src, file_path, path_prefix, parent_name_path, children)); + out.push(make_symbol(name, meta, node, byte_range, src, file_path, names, path_prefix, parent_name_path, children)); } /// Extract members from a `class_body` or `interface_body` node. @@ -402,7 +410,8 @@ fn emit_class( fn extract_members( body: Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -440,7 +449,7 @@ fn extract_members( }; let byte_range = child.byte_range(); - out.push(make_symbol(name, meta, &child, byte_range, src, file_path, path_prefix, parent_name_path, Vec::new())); + out.push(make_symbol(name, meta, &child, byte_range, src, file_path, names, path_prefix, parent_name_path, Vec::new())); } } @@ -451,7 +460,8 @@ fn extract_members( fn emit_interface( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, byte_range: std::ops::Range, @@ -470,10 +480,10 @@ fn emit_interface( let mut children = Vec::new(); if let Some(body) = child_by_kind(node, "interface_body") { - extract_members(body, src, file_path, path_prefix, &name_path, &mut children); + extract_members(body, src, file_path, names, path_prefix, &name_path, &mut children); } - out.push(make_symbol(name, &IFACE, node, byte_range, src, file_path, path_prefix, parent_name_path, children)); + out.push(make_symbol(name, &IFACE, node, byte_range, src, file_path, names, path_prefix, parent_name_path, children)); } /// Emit a `namespace`/`module` symbol and recurse into `statement_block` for nested declarations. @@ -484,7 +494,8 @@ fn emit_interface( fn emit_namespace( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, out: &mut Vec, @@ -502,11 +513,11 @@ fn emit_namespace( let mut children = Vec::new(); if let Some(body) = child_by_kind(node, "statement_block") { - extract_symbols(body, src, file_path, path_prefix, &name_path, &mut children); + extract_symbols(body, src, file_path, names, path_prefix, &name_path, &mut children); } let byte_range = node.byte_range(); - out.push(make_symbol(name, &NS, node, byte_range, src, file_path, path_prefix, parent_name_path, children)); + out.push(make_symbol(name, &NS, node, byte_range, src, file_path, names, path_prefix, parent_name_path, children)); } // --------------------------------------------------------------------------- @@ -530,7 +541,8 @@ fn make_symbol( line_node: &Node, byte_range: std::ops::Range, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, children: Vec, @@ -547,16 +559,16 @@ fn make_symbol( SymbolNode { id, - name, + name: names.intern(&name), category: meta.category, label: meta.label, - file_path: file_path.to_path_buf(), - byte_range, - line_range: start_line..end_line, + file_path: Arc::clone(file_path), + byte_range: byte_range.start as u32..byte_range.end as u32, + line_range: start_line as u32..end_line as u32, content_hash: content_hash(text), merkle_hash: [0u8; 32], children, - estimated_tokens: estimate_tokens(text), + estimated_tokens: estimate_tokens(text) as u32, } } @@ -566,14 +578,15 @@ fn make_symbol( fn build_named_symbol( node: &Node, src: &[u8], - file_path: &Path, + file_path: &Arc, + names: &NameInterner, path_prefix: &str, parent_name_path: &str, meta: &SymbolMeta, byte_range: std::ops::Range, ) -> Option { let name = child_name(node, src)?; - Some(make_symbol(name, meta, node, byte_range, src, file_path, path_prefix, parent_name_path, Vec::new())) + Some(make_symbol(name, meta, node, byte_range, src, file_path, names, path_prefix, parent_name_path, Vec::new())) } // --------------------------------------------------------------------------- @@ -638,7 +651,7 @@ mod tests { fn find<'a>(symbols: &'a [SymbolNode], name: &str) -> &'a SymbolNode { symbols .iter() - .find(|s| s.name == name) + .find(|s| s.name.as_ref() == name) .unwrap_or_else(|| panic!("symbol '{}' not found", name)) } @@ -650,7 +663,7 @@ mod tests { fn function_declaration() { let syms = parse("function greet(name: string): string { return name; }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "greet"); + assert_eq!(syms[0].name.as_ref(), "greet"); assert_eq!(syms[0].label, "function"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -659,7 +672,7 @@ mod tests { fn generator_function() { let syms = parse("function* gen() { yield 1; }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "gen"); + assert_eq!(syms[0].name.as_ref(), "gen"); assert_eq!(syms[0].label, "function"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -668,7 +681,7 @@ mod tests { fn async_function() { let syms = parse("async function fetchData(): Promise {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "fetchData"); + assert_eq!(syms[0].name.as_ref(), "fetchData"); assert_eq!(syms[0].label, "function"); } @@ -680,7 +693,7 @@ mod tests { fn arrow_function_const() { let syms = parse("const handler = () => {};"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "handler"); + assert_eq!(syms[0].name.as_ref(), "handler"); assert_eq!(syms[0].label, "function"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -689,7 +702,7 @@ mod tests { fn arrow_function_with_body() { let syms = parse("const process = (data: string) => {\n return data;\n};"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "process"); + assert_eq!(syms[0].name.as_ref(), "process"); assert_eq!(syms[0].label, "function"); } @@ -697,7 +710,7 @@ mod tests { fn function_expression() { let syms = parse("const handler = function() {};"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "handler"); + assert_eq!(syms[0].name.as_ref(), "handler"); assert_eq!(syms[0].label, "function"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -706,7 +719,7 @@ mod tests { fn arrow_function_let() { let syms = parse("let handler = () => {};"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "handler"); + assert_eq!(syms[0].name.as_ref(), "handler"); assert_eq!(syms[0].label, "function"); } @@ -736,7 +749,7 @@ mod tests { fn class_declaration() { let syms = parse("class Foo {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].label, "class"); assert_eq!(syms[0].category, SymbolCategory::Type); } @@ -751,11 +764,11 @@ mod tests { ); assert_eq!(syms.len(), 1); let cls = &syms[0]; - assert_eq!(cls.name, "Service"); + assert_eq!(cls.name.as_ref(), "Service"); assert_eq!(cls.children.len(), 2); - assert_eq!(cls.children[0].name, "doWork"); + assert_eq!(cls.children[0].name.as_ref(), "doWork"); assert_eq!(cls.children[0].label, "method"); - assert_eq!(cls.children[1].name, "process"); + assert_eq!(cls.children[1].name.as_ref(), "process"); assert_eq!(cls.children[1].label, "method"); } @@ -770,10 +783,10 @@ mod tests { assert_eq!(syms.len(), 1); let cls = &syms[0]; assert_eq!(cls.children.len(), 2); - assert_eq!(cls.children[0].name, "host"); + assert_eq!(cls.children[0].name.as_ref(), "host"); assert_eq!(cls.children[0].label, "property"); assert_eq!(cls.children[0].category, SymbolCategory::Variable); - assert_eq!(cls.children[1].name, "port"); + assert_eq!(cls.children[1].name.as_ref(), "port"); assert_eq!(cls.children[1].label, "property"); } @@ -803,7 +816,7 @@ mod tests { ); assert_eq!(syms.len(), 1); assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "constructor"); + assert_eq!(syms[0].children[0].name.as_ref(), "constructor"); assert_eq!(syms[0].children[0].label, "method"); } @@ -820,13 +833,13 @@ mod tests { }", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Base"); + assert_eq!(syms[0].name.as_ref(), "Base"); assert_eq!(syms[0].label, "abstract class"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].children.len(), 2); - assert_eq!(syms[0].children[0].name, "doWork"); + assert_eq!(syms[0].children[0].name.as_ref(), "doWork"); assert_eq!(syms[0].children[0].label, "method"); - assert_eq!(syms[0].children[1].name, "concrete"); + assert_eq!(syms[0].children[1].name.as_ref(), "concrete"); assert_eq!(syms[0].children[1].label, "method"); } @@ -838,7 +851,7 @@ mod tests { fn interface_declaration() { let syms = parse("interface Foo {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Foo"); + assert_eq!(syms[0].name.as_ref(), "Foo"); assert_eq!(syms[0].label, "interface"); assert_eq!(syms[0].category, SymbolCategory::Type); } @@ -856,15 +869,15 @@ mod tests { assert_eq!(syms.len(), 1); let iface = &syms[0]; assert_eq!(iface.children.len(), 4); - assert_eq!(iface.children[0].name, "host"); + assert_eq!(iface.children[0].name.as_ref(), "host"); assert_eq!(iface.children[0].label, "property"); assert_eq!(iface.children[0].category, SymbolCategory::Variable); - assert_eq!(iface.children[1].name, "port"); + assert_eq!(iface.children[1].name.as_ref(), "port"); assert_eq!(iface.children[1].label, "property"); - assert_eq!(iface.children[2].name, "connect"); + assert_eq!(iface.children[2].name.as_ref(), "connect"); assert_eq!(iface.children[2].label, "method"); assert_eq!(iface.children[2].category, SymbolCategory::Function); - assert_eq!(iface.children[3].name, "disconnect"); + assert_eq!(iface.children[3].name.as_ref(), "disconnect"); assert_eq!(iface.children[3].label, "method"); } @@ -876,7 +889,7 @@ mod tests { fn type_alias() { let syms = parse("type UserId = string;"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "UserId"); + assert_eq!(syms[0].name.as_ref(), "UserId"); assert_eq!(syms[0].label, "type"); assert_eq!(syms[0].category, SymbolCategory::Type); } @@ -885,7 +898,7 @@ mod tests { fn generic_type_alias() { let syms = parse("type Result = { ok: true; value: T } | { ok: false; error: Error };"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Result"); + assert_eq!(syms[0].name.as_ref(), "Result"); assert_eq!(syms[0].label, "type"); } @@ -897,7 +910,7 @@ mod tests { fn enum_declaration() { let syms = parse("enum Status { Active, Inactive, Pending }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Status"); + assert_eq!(syms[0].name.as_ref(), "Status"); assert_eq!(syms[0].label, "enum"); assert_eq!(syms[0].category, SymbolCategory::Type); assert_eq!(syms[0].children.len(), 0, "enum should not recurse into members"); @@ -907,7 +920,7 @@ mod tests { fn const_enum() { let syms = parse("const enum Direction { Up, Down, Left, Right }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Direction"); + assert_eq!(syms[0].name.as_ref(), "Direction"); assert_eq!(syms[0].label, "enum"); } @@ -923,12 +936,12 @@ mod tests { }", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Validation"); + assert_eq!(syms[0].name.as_ref(), "Validation"); assert_eq!(syms[0].label, "namespace"); assert_eq!(syms[0].category, SymbolCategory::Module); // Namespace should recurse into children. assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "isValid"); + assert_eq!(syms[0].children[0].name.as_ref(), "isValid"); assert_eq!(syms[0].children[0].label, "function"); } @@ -940,10 +953,10 @@ mod tests { }", ); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "MyModule"); + assert_eq!(syms[0].name.as_ref(), "MyModule"); assert_eq!(syms[0].label, "namespace"); assert_eq!(syms[0].children.len(), 1); - assert_eq!(syms[0].children[0].name, "Inner"); + assert_eq!(syms[0].children[0].name.as_ref(), "Inner"); assert_eq!(syms[0].children[0].label, "class"); } @@ -955,7 +968,7 @@ mod tests { fn declare_function() { let syms = parse("declare function require(id: string): any;"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "require"); + assert_eq!(syms[0].name.as_ref(), "require"); assert_eq!(syms[0].label, "declare"); assert_eq!(syms[0].category, SymbolCategory::Variable); } @@ -964,7 +977,7 @@ mod tests { fn declare_const() { let syms = parse("declare const __dirname: string;"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "__dirname"); + assert_eq!(syms[0].name.as_ref(), "__dirname"); assert_eq!(syms[0].label, "declare"); } @@ -979,7 +992,7 @@ mod tests { fn declare_class() { let syms = parse("declare class Buffer { constructor(str: string); }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Buffer"); + assert_eq!(syms[0].name.as_ref(), "Buffer"); assert_eq!(syms[0].label, "declare"); } @@ -991,7 +1004,7 @@ mod tests { fn export_function() { let syms = parse("export function greet(): void {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "greet"); + assert_eq!(syms[0].name.as_ref(), "greet"); assert_eq!(syms[0].label, "function"); } @@ -999,7 +1012,7 @@ mod tests { fn export_class() { let syms = parse("export class Service {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Service"); + assert_eq!(syms[0].name.as_ref(), "Service"); assert_eq!(syms[0].label, "class"); } @@ -1007,7 +1020,7 @@ mod tests { fn export_interface() { let syms = parse("export interface Config { host: string; }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Config"); + assert_eq!(syms[0].name.as_ref(), "Config"); assert_eq!(syms[0].label, "interface"); assert_eq!(syms[0].children.len(), 1); } @@ -1016,7 +1029,7 @@ mod tests { fn export_type_alias() { let syms = parse("export type Id = string;"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Id"); + assert_eq!(syms[0].name.as_ref(), "Id"); assert_eq!(syms[0].label, "type"); } @@ -1024,7 +1037,7 @@ mod tests { fn export_enum() { let syms = parse("export enum Color { Red, Green, Blue }"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "Color"); + assert_eq!(syms[0].name.as_ref(), "Color"); assert_eq!(syms[0].label, "enum"); } @@ -1032,7 +1045,7 @@ mod tests { fn export_arrow_function() { let syms = parse("export const handler = () => {};"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "handler"); + assert_eq!(syms[0].name.as_ref(), "handler"); assert_eq!(syms[0].label, "function"); assert_eq!(syms[0].category, SymbolCategory::Function); } @@ -1041,7 +1054,7 @@ mod tests { fn export_default_function() { let syms = parse("export default function main() {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "main"); + assert_eq!(syms[0].name.as_ref(), "main"); assert_eq!(syms[0].label, "function"); } @@ -1049,7 +1062,7 @@ mod tests { fn export_default_class() { let syms = parse("export default class App {}"); assert_eq!(syms.len(), 1); - assert_eq!(syms[0].name, "App"); + assert_eq!(syms[0].name.as_ref(), "App"); assert_eq!(syms[0].label, "class"); } diff --git a/src/serena/mod.rs b/src/serena/mod.rs index 11b7509..a140188 100644 --- a/src/serena/mod.rs +++ b/src/serena/mod.rs @@ -4,7 +4,6 @@ use std::path::{Path, PathBuf}; use color_eyre::eyre::{bail, eyre, Result}; use serde_pickle::value::{HashableValue, Value}; -use sha2::{Digest, Sha256}; use ambits::symbols::merkle::compute_merkle_hash; use ambits::symbols::{FileSymbols, ProjectTree, SymbolCategory, SymbolNode}; @@ -94,9 +93,10 @@ fn parse_raw_pickle(value: &Value) -> Result> { as_list(&items[1]).ok_or_else(|| eyre!("Symbol list not an array for {file_path_str}"))?; let path_prefix = file_path.to_string_lossy(); + let file_path_arc = std::sync::Arc::new(file_path.clone()); let mut symbols = Vec::new(); for sym_val in symbol_list { - if let Ok(node) = convert_symbol(sym_val, &file_path, &path_prefix, "") { + if let Ok(node) = convert_symbol(sym_val, &file_path_arc, &path_prefix, "") { symbols.push(node); } } @@ -136,9 +136,10 @@ fn parse_document_pickle(value: &Value) -> Result> { .ok_or_else(|| eyre!("Cannot find symbols for {file_path_str}"))?; let path_prefix = file_path.to_string_lossy(); + let file_path_arc = std::sync::Arc::new(file_path.clone()); let mut symbols = Vec::new(); for sym_val in symbol_list { - if let Ok(node) = convert_symbol(sym_val, &file_path, &path_prefix, "") { + if let Ok(node) = convert_symbol(sym_val, &file_path_arc, &path_prefix, "") { symbols.push(node); } } @@ -156,7 +157,7 @@ fn parse_document_pickle(value: &Value) -> Result> { /// Convert a pickle Value dict into a SymbolNode. fn convert_symbol( val: &Value, - file_path: &Path, + file_path: &std::sync::Arc, path_prefix: &str, parent_id: &str, ) -> Result { @@ -184,13 +185,14 @@ fn convert_symbol( 1 }; - // Content hash from identity (no source text available in raw format) - let content_hash = { - let mut hasher = Sha256::new(); + // Content hash from identity (no source text available in raw format). + // BLAKE3 matches `symbols::merkle::content_hash` algorithm choice. + let content_hash: [u8; 32] = { + let mut hasher = blake3::Hasher::new(); hasher.update(name.as_bytes()); - hasher.update(kind_int.to_le_bytes()); - hasher.update(start_line.to_le_bytes()); - hasher.update(end_line.to_le_bytes()); + hasher.update(&kind_int.to_le_bytes()); + hasher.update(&start_line.to_le_bytes()); + hasher.update(&end_line.to_le_bytes()); hasher.finalize().into() }; @@ -206,16 +208,16 @@ fn convert_symbol( let mut node = SymbolNode { id, - name, + name: std::sync::Arc::from(name.as_str()), category, label, - file_path: file_path.to_path_buf(), - byte_range: (start_line * 40 + start_char)..(end_line * 40 + end_char), - line_range: (start_line + 1)..(end_line + 1), // 1-indexed like tree-sitter + file_path: std::sync::Arc::clone(file_path), + byte_range: ((start_line * 40 + start_char) as u32)..((end_line * 40 + end_char) as u32), + line_range: ((start_line + 1) as u32)..((end_line + 1) as u32), // 1-indexed like tree-sitter content_hash, merkle_hash: [0u8; 32], children, - estimated_tokens: line_count * 15, + estimated_tokens: (line_count * 15) as u32, }; compute_merkle_hash(&mut node); Ok(node) @@ -244,7 +246,7 @@ fn extract_range(val: &Value) -> (usize, usize, usize, usize) { fn estimate_total_lines(symbols: &[SymbolNode]) -> usize { symbols .iter() - .map(|s| s.line_range.end) + .map(|s| s.line_range.end as usize) .max() .unwrap_or(0) } diff --git a/src/symbols/merkle.rs b/src/symbols/merkle.rs index 736a51e..69c825a 100644 --- a/src/symbols/merkle.rs +++ b/src/symbols/merkle.rs @@ -1,12 +1,16 @@ -use sha2::{Digest, Sha256}; - use super::SymbolNode; /// Compute content hash from the raw source text of a symbol. /// Normalizes whitespace to make hashing resilient to formatting changes. +/// +/// Uses BLAKE3 (256-bit output) — fast, SIMD-accelerated, no adversary threat +/// model required for our change-detection use case. Output width matches the +/// `[u8; 32]` field on `SymbolNode`. Hash values differ from previous SHA-256 +/// runs; no on-disk format references these hashes, so the change is internal +/// only. pub fn content_hash(source: &str) -> [u8; 32] { let normalized = normalize_source(source); - let mut hasher = Sha256::new(); + let mut hasher = blake3::Hasher::new(); hasher.update(normalized.as_bytes()); hasher.finalize().into() } @@ -16,10 +20,10 @@ pub fn content_hash(source: &str) -> [u8; 32] { /// This must be called bottom-up (children first). /// /// Leaf nodes (no children) reuse `content_hash` as their `merkle_hash` directly, -/// skipping a SHA-256 invocation. The input would be a 32-byte cryptographic hash -/// either way, so hashing it again adds no collision resistance. Real source +/// skipping a hash invocation. The input would be a 32-byte cryptographic-quality +/// hash either way, so hashing it again adds no collision resistance. Real source /// projects are leaf-dominated (≈90% leaves in typical symbol trees), so this -/// skips the majority of SHA-256 calls during parse. +/// skips the majority of hash calls during parse. pub fn compute_merkle_hash(node: &mut SymbolNode) { if node.children.is_empty() { node.merkle_hash = node.content_hash; @@ -30,10 +34,10 @@ pub fn compute_merkle_hash(node: &mut SymbolNode) { compute_merkle_hash(child); } - let mut hasher = Sha256::new(); - hasher.update(node.content_hash); + let mut hasher = blake3::Hasher::new(); + hasher.update(&node.content_hash); for child in &node.children { - hasher.update(child.merkle_hash); + hasher.update(&child.merkle_hash); } node.merkle_hash = hasher.finalize().into(); } @@ -150,12 +154,13 @@ mod tests { fn leaf_merkle_hash_equals_content_hash() { use super::super::{SymbolCategory, SymbolNode}; use std::path::PathBuf; + use std::sync::Arc; let mut leaf = SymbolNode { id: "x".into(), name: "x".into(), category: SymbolCategory::Function, label: "fn", - file_path: PathBuf::new(), + file_path: Arc::new(PathBuf::new()), byte_range: 0..0, line_range: 0..0, content_hash: content_hash("fn x() {}"), @@ -172,12 +177,13 @@ mod tests { fn parent_merkle_hash_differs_from_content_hash() { use super::super::{SymbolCategory, SymbolNode}; use std::path::PathBuf; + use std::sync::Arc; let make = |name: &str| SymbolNode { id: name.into(), name: name.into(), category: SymbolCategory::Function, label: "fn", - file_path: PathBuf::new(), + file_path: Arc::new(PathBuf::new()), byte_range: 0..0, line_range: 0..0, content_hash: content_hash(name), diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index f6d35cc..1485933 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -1,6 +1,9 @@ +use std::cell::RefCell; +use std::collections::HashMap; use std::fmt; use std::ops::Range; use std::path::PathBuf; +use std::sync::Arc; pub mod merkle; @@ -43,16 +46,68 @@ impl fmt::Display for SymbolCategory { #[derive(Debug, Clone)] pub struct SymbolNode { pub id: SymbolId, - pub name: String, + /// Symbol's short name (`foo`, `Bar`, `method_0`, etc). Stored as + /// `Arc` so common names (`new`, `default`, `get`, `build`, + /// `method_0` repeating across types in a file) share one heap + /// allocation. Construct via [`NameInterner::intern`] from a parser, or + /// directly with `Arc::from(name)` from a test/fixture. Display, Eq, and + /// `as_ref()` work via deref to `&str`; explicit comparisons against + /// `String` need `&*sym.name == &**other` or `sym.name.as_ref() == other`. + pub name: Arc, pub category: SymbolCategory, pub label: &'static str, // Language-specific label (e.g., "class", "struct", "def") - pub file_path: PathBuf, - pub byte_range: Range, - pub line_range: Range, + /// Path to the source file that produced this symbol. Wrapped in `Arc` so + /// symbols within the same file share one backing `PathBuf` instead of + /// each carrying their own clone — saves real heap at monorepo scale where + /// a typical file has dozens of symbols. Reads go through deref: most + /// consumers don't notice the change (`.as_path()`, `.display()`, + /// `.to_string_lossy()` all work). Equality checks against `Path` / + /// `PathBuf` need `&**arc` or `arc.as_path()` to unwrap. + pub file_path: Arc, + /// Byte range of the symbol in its source file. `u32` is plenty for any + /// realistic single-file size (4 GB cap) and saves 8 B/symbol over `usize` + /// on 64-bit targets. Cast to `usize` at slice sites: + /// `&src[sym.byte_range.start as usize..sym.byte_range.end as usize]`. + pub byte_range: Range, + /// 1-based inclusive line range. `u32` for the same reason as `byte_range`. + pub line_range: Range, pub content_hash: [u8; 32], pub merkle_hash: [u8; 32], pub children: Vec, - pub estimated_tokens: usize, + pub estimated_tokens: u32, +} + +/// Per-file interner for symbol names. Real code has heavy name repetition +/// within a file (think methods named `new`, `default`, `build` across every +/// type in a module), so a small hash cache here lets us share one `Arc` +/// across every reuse instead of allocating one heap buffer per occurrence. +/// +/// Uses interior mutability so it can be threaded as `&NameInterner` alongside +/// other immutable parser state without requiring `&mut` propagation through +/// every recursive helper. +/// +/// Scope is intentionally per-file rather than global: simpler borrow story +/// (no thread safety needed), and the wins from within-file repetition +/// dominate. +#[derive(Debug, Default)] +pub struct NameInterner { + cache: RefCell>>, +} + +impl NameInterner { + pub fn new() -> Self { + Self::default() + } + + pub fn intern(&self, s: &str) -> Arc { + let mut cache = self.cache.borrow_mut(); + if let Some(arc) = cache.get(s) { + return Arc::clone(arc); + } + let arc: Arc = Arc::from(s); + cache.insert(s.to_string(), Arc::clone(&arc)); + arc + } } impl SymbolNode { @@ -61,7 +116,8 @@ impl SymbolNode { } pub fn total_tokens(&self) -> usize { - self.estimated_tokens + self.children.iter().map(|c| c.total_tokens()).sum::() + self.estimated_tokens as usize + + self.children.iter().map(|c| c.total_tokens()).sum::() } } diff --git a/src/ui/stats.rs b/src/ui/stats.rs index 611756c..f5a96f5 100644 --- a/src/ui/stats.rs +++ b/src/ui/stats.rs @@ -295,7 +295,7 @@ mod tests { let hash = ambits::symbols::merkle::content_hash(name); SymbolNode { id: id.into(), name: name.into(), category: SymbolCategory::Function, - label: "fn", file_path: PathBuf::new(), + label: "fn", file_path: std::sync::Arc::new(PathBuf::new()), byte_range: 0..100, line_range: 1..10, content_hash: hash, merkle_hash: hash, children: Vec::new(), estimated_tokens: 30, } diff --git a/src/ui/tree_view.rs b/src/ui/tree_view.rs index 83c2705..5db8588 100644 --- a/src/ui/tree_view.rs +++ b/src/ui/tree_view.rs @@ -120,7 +120,7 @@ mod tests { let hash = ambits::symbols::merkle::content_hash(name); SymbolNode { id: id.into(), name: name.into(), category: SymbolCategory::Function, - label: "fn", file_path: PathBuf::new(), + label: "fn", file_path: std::sync::Arc::new(PathBuf::new()), byte_range: 0..100, line_range: 1..10, content_hash: hash, merkle_hash: hash, children: Vec::new(), estimated_tokens: 30, } diff --git a/tests/e2e.rs b/tests/e2e.rs index 8746edd..68ed49d 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -4,6 +4,7 @@ use std::io::Write; use std::path::PathBuf; +use std::sync::Arc; use ambits::app::App; use ambits::coverage::{CoverageFormatter, CoverageReport, JsonFormatter, TextFormatter}; @@ -39,10 +40,10 @@ fn sym(id: &str, name: &str) -> SymbolNode { let hash = content_hash(name); SymbolNode { id: id.to_string(), - name: name.to_string(), + name: Arc::from(name), category: SymbolCategory::Function, label: "fn", - file_path: PathBuf::new(), + file_path: Arc::new(PathBuf::new()), byte_range: 0..100, line_range: 1..10, content_hash: hash, @@ -54,10 +55,11 @@ fn sym(id: &str, name: &str) -> SymbolNode { fn file(path: &str, symbols: Vec) -> FileSymbols { let file_path = PathBuf::from(path); + let file_path_arc = Arc::new(file_path.clone()); let symbols = symbols .into_iter() .map(|mut s| { - s.file_path = file_path.clone(); + s.file_path = Arc::clone(&file_path_arc); s }) .collect(); diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs index 50f8689..8a46d9e 100644 --- a/tests/helpers/mod.rs +++ b/tests/helpers/mod.rs @@ -1,4 +1,5 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::ingest::AgentToolCall; use crate::symbols::{FileSymbols, ProjectTree, SymbolCategory, SymbolNode}; @@ -10,10 +11,10 @@ pub fn sym(id: &str, name: &str) -> SymbolNode { let hash = crate::symbols::merkle::content_hash(name); SymbolNode { id: id.to_string(), - name: name.to_string(), + name: Arc::from(name), category: SymbolCategory::Function, label: "fn", - file_path: PathBuf::new(), + file_path: Arc::new(PathBuf::new()), byte_range: 0..100, line_range: 1..10, content_hash: hash, @@ -33,7 +34,7 @@ pub fn sym_with_children(id: &str, name: &str, children: Vec) -> Sym /// Create a SymbolNode with a custom line range. pub fn sym_with_lines(id: &str, name: &str, start: usize, end: usize) -> SymbolNode { let mut s = sym(id, name); - s.line_range = start..end; + s.line_range = start as u32..end as u32; s } @@ -47,17 +48,18 @@ pub fn sym_with_children_and_lines( ) -> SymbolNode { let mut s = sym(id, name); s.children = children; - s.line_range = start..end; + s.line_range = start as u32..end as u32; s } /// Create a FileSymbols entry, setting each symbol's `file_path` to match. pub fn file(path: &str, symbols: Vec) -> FileSymbols { let file_path = PathBuf::from(path); + let file_path_arc = Arc::new(file_path.clone()); let symbols = symbols .into_iter() .map(|mut s| { - s.file_path = file_path.clone(); + s.file_path = Arc::clone(&file_path_arc); s }) .collect(); @@ -101,7 +103,7 @@ pub fn tool_call_targeted(tool: &str, path: &str, depth: ReadDepth, symbol: &str /// Create a tool call targeting a line range. pub fn tool_call_lines(tool: &str, path: &str, depth: ReadDepth, start: usize, end: usize) -> AgentToolCall { let mut tc = tool_call(tool, path, depth); - tc.target_lines = Some(start..end); + tc.target_lines = Some(start as u32..end as u32); tc }