Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .repository-projection.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@
"projection": "deixic-code",
"projectionSchemaVersion": 1,
"sourceRepository": "dx-corp/mono",
"sourceSha": "14cef0cc775bc3b6079b191eef5806015cd3b635",
"sourceSha": "f1457cd20a4b86478bcc70b0152230aaa8a17dea",
"destinationRepository": "dx-corp/code",
"priorProjectedBase": "14a7be00225b633151b0f7b40c904bc67160023f",
"priorProjectedBase": "6a1ee9a7f2d6d82fa7f55c3a1dc7c7d8e95c964f",
"definitionDigest": "cb9d429542ebb0a2de9b42a7aad60d9d8696a648ceba47c30f05c0b285ca0db7",
"toolDigest": "f8cb071b0f27267120ccf45a00d0982f45113bd23535bef6a1555b4933f99f13",
"contentDigest": "1b65d1cc4dac2a27935e1a34cb493b615ddeb7027a2124d22d8b2e0850773d01",
"toolDigest": "c9112982c70b80737d8faac67d58b06d55bde0dc297eb1009c1b02296f34f4a9",
"contentDigest": "2ee4dc7a3e288db3224383edce5c9dede750c4e7733a87436b0fb14c221b58c0",
"publicationEligible": true
}
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

139 changes: 139 additions & 0 deletions packages/codemode-rs/examples/discovery_benchmark.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
//! Run with `cargo run -p agent-codemode --example discovery_benchmark`.
//! Measures discovery separately from fixture generation. No timing CI gates.
use agent_codemode::{Catalog, Event, Session, Store, Tool};
use serde_json::{Value, json};
use std::hint::black_box;
use std::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;

fn fixture(count: usize) -> Vec<Tool> {
let mut tools: Vec<_> = (0..count - 3).map(|i| Tool {
name: format!("inventory.item_{i:05}"),
description: "Look up stock quantity and warehouse location".into(),
namespace: Some("inventory".into()),
schema: json!({"type":"object","properties":{"itemId":{"type":"string"},"notes":{"type":"string","description":"x".repeat(8192)}}}),
..Default::default()
}).collect();
for (name, description, properties) in [
(
"documents.export_pdf",
"Export a document as PDF",
json!({"documentId":{"type":"string"}}),
),
(
"documents.comments",
"Read document comments and archived remarks",
json!({"continuationToken":{"type":"string"}}),
),
(
"documents.export_text",
"Export a document as plain text",
json!({"documentId":{"type":"string"}}),
),
] {
tools.push(Tool {
name: name.into(),
description: description.into(),
namespace: Some("documents".into()),
schema: json!({"type":"object","properties":properties}),
..Default::default()
});
}
tools
}

async fn script(catalog: Catalog) {
let mut session = Session::start_with_catalog(
"text(searchTools('export PDF',{namespace:'documents',limit:1})[0].name);".into(),
catalog,
&CancellationToken::new(),
Duration::from_secs(10),
Store::new(),
);
let Some(Event::Done(report)) = session.next().await else {
panic!("unexpected dispatch")
};
assert!(report.error.is_none(), "{:?}", report.error);
assert_eq!(report.output, vec!["documents_export_pdf"]);
}

async fn measure(count: usize) -> Value {
let tools = fixture(count);
let schema_source_bytes = tools
.iter()
.map(|t| serde_json::to_vec(&t.schema).unwrap().len())
.sum::<usize>();
let start = Instant::now();
let catalog = Catalog::new(tools);
let construct_us = start.elapsed().as_micros();
let queries = [
("export PDF", "documents.export_pdf"),
("continuationToken", "documents.comments"),
("archived remarks", "documents.comments"),
];
for (query, want) in queries {
assert_eq!(catalog.search(query, &[], 1, None)[0].name, want);
}
let start = Instant::now();
for _ in 0..100 {
for (query, _) in queries {
black_box(catalog.search(query, &[], 8, None));
}
}
let warm_search_us = start.elapsed().as_micros() as f64 / 300.0;
let start = Instant::now();
let first = catalog
.schema_page("inventory.item_00000", &json!({"maxBytes":4096}))
.unwrap();
let schema_cold_us = start.elapsed().as_micros();
let start = Instant::now();
for _ in 0..300 {
black_box(
catalog
.schema_page(
"inventory.item_00000",
&json!({"offsetBytes":first["nextOffsetBytes"],"maxBytes":4096}),
)
.unwrap(),
);
}
let schema_warm_us = start.elapsed().as_micros() as f64 / 300.0;
// The prior ownership pattern copied schemas and rebuilt search text on
// every script. This baseline isolates that cost under the current ranking.
let start = Instant::now();
for _ in 0..5 {
black_box(Catalog::new(catalog.iter().cloned().collect()).search(
"export PDF",
&[],
8,
None,
));
}
let rebuilt_search_us = start.elapsed().as_micros() as f64 / 5.0;
let start = Instant::now();
for _ in 0..5 {
script(catalog.clone()).await;
}
let shared_script_us = start.elapsed().as_micros() as f64 / 5.0;
json!({"tools":count,"schema_source_bytes":schema_source_bytes,"catalog_construct_us":construct_us,
"warm_search_us":warm_search_us,"rebuild_and_search_us":rebuilt_search_us,
"schema_cold_page_us":schema_cold_us,"schema_warm_page_us":schema_warm_us,
"cached_schema_allocation_bytes":catalog.cached_schema_bytes(),"shared_script_us":shared_script_us,
"discovery_queries_correct":3,"script_heap_limit_bytes":32*1024*1024,
"discovery_description_bytes":agent_codemode::DESCRIPTION.len()})
}

fn main() {
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let results = runtime.block_on(async {
let mut rows = Vec::new();
for count in [100, 1_000, 10_000] {
rows.push(measure(count).await);
}
rows
});
println!("{}",serde_json::to_string_pretty(&json!({"profile":"debug","timing_thresholds":false,
"memory_note":"schema_source_bytes is serialized source size; retained cache bytes count string capacities. Whole-process peak RSS is measured separately.","results":results})).unwrap());
}
Loading
Loading