Perf/symbol pipeline and tooling

.gitignore

@@ -6,3 +6,7 @@ node_modules/
 # Dolt database files (added by bd init)
 .dolt/
 *.db
+
+# Heap probe / perf measurement artifacts — machine-specific.
+.perf-baseline/
+dhat-heap.json

Cargo.toml

@@ -23,11 +23,15 @@ notify = "7"
 clap = { version = "4", features = ["derive"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-sha2 = "0.10"
 color-eyre = "0.6"
 serde-pickle = "1.2"
 flume = "0.12"
 toml = { version = "0.8", default-features = false, features = ["parse"] }
+blake3 = "1"
+dhat = { version = "0.3", optional = true }
+
+[features]
+dhat-heap = ["dep:dhat"]
 
 [dev-dependencies]
 tempfile = "3"
@@ -53,6 +57,15 @@ harness = false
 name = "tracking"
 harness = false
 
+[[bench]]
+name = "symbol_pipeline"
+harness = false
+
+[[bin]]
+name = "heap_probe"
+path = "src/bin/heap_probe.rs"
+required-features = ["dhat-heap"]
+
 [lib]
 name = "ambits"
 path = "src/lib.rs"

benches/parser.rs

@@ -67,7 +67,7 @@ fn typescript_parse_and_coverage(bencher: divan::Bencher, size: &&str) {
         let mut ledger = ContextLedger::new();
         let hash = [0u8; 32];
         for sym in &file_syms.symbols {
-            ledger.record(sym.id.clone(), ReadDepth::FullBody, hash, "agent".to_string(), sym.estimated_tokens);
+            ledger.record(sym.id.clone(), ReadDepth::FullBody, hash, "agent".to_string(), sym.estimated_tokens as usize);
         }
 
         // 3. Build project tree from the single file

benches/support/fixtures.rs

@@ -1,9 +1,10 @@
 #![allow(dead_code)]
 
 use std::path::PathBuf;
+use std::sync::Arc;
 
 use ambits::symbols::merkle::content_hash;
-use ambits::symbols::{FileSymbols, ProjectTree, SymbolCategory, SymbolNode};
+use ambits::symbols::{FileSymbols, NameInterner, ProjectTree, SymbolCategory, SymbolNode};
 use ambits::tracking::{ContextLedger, ReadDepth};
 
 // ─── Rust source fixtures ─────────────────────────────────────────────────────
@@ -959,10 +960,10 @@ pub fn make_sym(id: &str, name: &str) -> SymbolNode {
     let hash = content_hash(name);
     SymbolNode {
         id: id.to_string(),
-        name: name.to_string(),
+        name: Arc::from(name),
         category: SymbolCategory::Function,
         label: "fn",
-        file_path: PathBuf::from("src/bench.rs"),
+        file_path: Arc::new(PathBuf::from("src/bench.rs")),
         byte_range: 0..100,
         line_range: 1..10,
         content_hash: hash,
@@ -996,9 +997,10 @@ pub fn make_flat_symbols(n: usize) -> Vec<SymbolNode> {
 /// Build a FileSymbols containing `n_symbols` flat symbols.
 pub fn make_file(path: &str, n_symbols: usize) -> FileSymbols {
     let file_path = PathBuf::from(path);
+    let file_path_arc = Arc::new(file_path.clone());
     let symbols = make_flat_symbols(n_symbols)
         .into_iter()
-        .map(|mut s| { s.file_path = file_path.clone(); s })
+        .map(|mut s| { s.file_path = Arc::clone(&file_path_arc); s })
         .collect();
     FileSymbols { file_path, symbols, total_lines: n_symbols * 5 }
 }
@@ -1011,13 +1013,83 @@ pub fn make_project(n_files: usize, symbols_per_file: usize) -> ProjectTree {
     ProjectTree { root: PathBuf::from("/bench/project"), files }
 }
 
+/// Build a synthetic monorepo `ProjectTree` for project-scale benchmarks.
+///
+/// Distributes `n_files` across nested package directories (mimicking realistic
+/// path depth) and varies file extensions across `rs`/`ts`/`py`. Within each
+/// file the symbol mix is ~80% flat leaves and ~20% one-level-nested groups
+/// (a type with four methods), so `compute_merkle_hash`'s leaf-vs-parent code
+/// paths both get exercised in proportions roughly matching real source.
+///
+/// Pure in-memory construction — no disk I/O, no tree-sitter. Use this for
+/// benchmarking the code paths *downstream* of parsing.
+pub fn make_monorepo_tree(n_files: usize, symbols_per_file: usize) -> ProjectTree {
+    const EXTENSIONS: [&str; 3] = ["rs", "ts", "py"];
+    let files: Vec<FileSymbols> = (0..n_files)
+        .map(|file_idx| {
+            let ext = EXTENSIONS[file_idx % EXTENSIONS.len()];
+            let pkg_idx = file_idx / 20;
+            let path_str = format!("src/pkg_{pkg_idx:04}/module_{file_idx:06}.{ext}");
+            let symbols = build_file_symbols(&path_str, symbols_per_file);
+            FileSymbols {
+                file_path: PathBuf::from(&path_str),
+                symbols,
+                total_lines: symbols_per_file * 5,
+            }
+        })
+        .collect();
+    ProjectTree {
+        root: PathBuf::from("/bench/monorepo"),
+        files,
+    }
+}
+
+fn build_file_symbols(file_path: &str, n: usize) -> Vec<SymbolNode> {
+    let groups = n / 5;
+    let remainder = n - groups * 5;
+    let mut out = Vec::with_capacity(groups + remainder);
+    let file_path_arc = Arc::new(PathBuf::from(file_path));
+    // One interner per file — same scope as production parsers.
+    let names = NameInterner::new();
+    for g in 0..groups {
+        let parent_id = format!("{file_path}::Type_{g:04}");
+        let parent_name = format!("Type_{g:04}");
+        let mut parent = make_sym(&parent_id, &parent_name);
+        parent.name = names.intern(&parent_name);
+        parent.category = SymbolCategory::Type;
+        parent.label = "struct";
+        parent.file_path = Arc::clone(&file_path_arc);
+        parent.children = (0..4)
+            .map(|i| {
+                let child_id = format!("{parent_id}::method_{i}");
+                let child_name = format!("method_{i}");
+                let mut sym = make_sym(&child_id, &child_name);
+                sym.name = names.intern(&child_name);
+                sym.file_path = Arc::clone(&file_path_arc);
+                sym
+            })
+            .collect();
+        ambits::symbols::merkle::compute_merkle_hash(&mut parent);
+        out.push(parent);
+    }
+    for i in 0..remainder {
+        let leaf_id = format!("{file_path}::fn_{i:04}");
+        let leaf_name = format!("fn_{i:04}");
+        let mut sym = make_sym(&leaf_id, &leaf_name);
+        sym.name = names.intern(&leaf_name);
+        sym.file_path = Arc::clone(&file_path_arc);
+        out.push(sym);
+    }
+    out
+}
+
 /// Build a ContextLedger pre-populated with entries for all symbols in `project`.
 pub fn make_populated_ledger(project: &ProjectTree) -> ContextLedger {
     let mut ledger = ContextLedger::new();
     let hash = [0u8; 32];
     for file in &project.files {
         for sym in &file.symbols {
-            ledger.record(sym.id.clone(), ReadDepth::FullBody, hash, "agent-0".to_string(), sym.estimated_tokens);
+            ledger.record(sym.id.clone(), ReadDepth::FullBody, hash, "agent-0".to_string(), sym.estimated_tokens as usize);
         }
     }
     ledger