Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions crates/webcodex-tool-contracts/src/tests/definitions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -933,6 +933,7 @@ fn adaptive_runtime_direct_declarations_are_visible_ranked_and_unique() {
("run_script", 74),
("run_shell", 75),
("observe_jobs", 80),
("project_build", 94),
("project_validate", 95),
] {
let definition = derived
Expand Down Expand Up @@ -1479,12 +1480,16 @@ fn readiness_tool_is_sequential_outer_only() {
}

#[test]
fn project_build_is_gateway_visible_but_not_adaptive_direct() {
fn project_build_is_adaptive_direct_and_keeps_typed_build_contract() {
let definition = lookup_tool_definition("project_build").expect("project_build definition");
assert!(definition.visibility.is_model_visible());
assert_eq!(definition.category, TOOL_CATEGORY_EXECUTION);
assert_eq!(definition.adaptive_runtime_direct_rank(), None);
assert!(!is_adaptive_runtime_direct_tool("project_build"));
assert_eq!(definition.adaptive_runtime_direct_rank(), Some(94));
assert_eq!(
definition.adaptive_runtime_direct_reason(),
Some(ToolDirectReason::CoreWorkflow)
);
assert!(is_adaptive_runtime_direct_tool("project_build"));
let requirement = runtime_tool_runner_capability("project_build")
.expect("project_build must require its typed Runner capability");
assert_eq!(requirement, RunnerCapabilityRequirement::ProjectBuild);
Expand Down
64 changes: 34 additions & 30 deletions crates/webcodex-tool-contracts/src/tool_definition/jobs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,36 +28,40 @@ pub(super) const EXECUTION_DEFINITIONS: &[ToolDefinition] = &[
71,
super::ToolDirectReason::CoreWorkflow,
),
model_spec(
def(
"project_build",
super::ToolAuditPolicy::TYPED_CANONICAL
.execution(super::ToolAuditExecutionPolicy::DIRECT_ARGV_TEXT),
ModelVisible,
TOOL_CATEGORY_EXECUTION,
Some(ProjectBuild),
TOOL_PROVIDER_RUNNER,
super::ToolSemanticContract {
effect: super::ToolEffect::Execute,
risk: JobRun,
approval: super::ToolApprovalPolicy::Standard,
idempotency: super::ToolIdempotency::NonIdempotent,
},
Some(JOB_RUN),
true,
NoPath,
false,
false,
super::ToolSessionEvidencePolicy::NONE,
),
"Preferred portable project build gateway. Runner resolves the nearest unambiguous Rust/Go recipe and executes canonical cargo build or go build argv. Go project builds run in Runner-owned single-module mode (GO111MODULE=on, GOWORK=off), so ambient module/workspace mode cannot change the planned graph. Optional scope selects either bounded explicit packages or all_packages=true. Rust all-packages is admitted only for a Cargo workspace whose effective root is the registered Project root; Go all-packages keeps the Runner-owned ./... single-module scope. dependency_policy.mode=locked forbids adapters from repairing dependency selection state (Cargo --locked; Go -mod=readonly); it does not imply offline execution. Long builds remain the same durable Job; pending never authorizes retry.",
)
.with_execution(super::ToolExecutionContract::new(
super::ToolExecutionForm::ProjectBuild,
super::ToolExecutionLifetime::Runner,
super::ToolExecutionStart::SyncFirst,
super::ToolExecutionContinuation::ObserveJobs,
)),
adaptive_runtime_direct(
model_spec(
def(
"project_build",
super::ToolAuditPolicy::TYPED_CANONICAL
.execution(super::ToolAuditExecutionPolicy::DIRECT_ARGV_TEXT),
ModelVisible,
TOOL_CATEGORY_EXECUTION,
Some(ProjectBuild),
TOOL_PROVIDER_RUNNER,
super::ToolSemanticContract {
effect: super::ToolEffect::Execute,
risk: JobRun,
approval: super::ToolApprovalPolicy::Standard,
idempotency: super::ToolIdempotency::NonIdempotent,
},
Some(JOB_RUN),
true,
NoPath,
false,
false,
super::ToolSessionEvidencePolicy::NONE,
),
"Preferred portable project build gateway. Runner resolves the nearest unambiguous Rust/Go recipe and executes canonical cargo build or go build argv. Go project builds run in Runner-owned single-module mode (GO111MODULE=on, GOWORK=off), so ambient module/workspace mode cannot change the planned graph. Optional scope selects either bounded explicit packages or all_packages=true. Rust all-packages is admitted only for a Cargo workspace whose effective root is the registered Project root; Go all-packages keeps the Runner-owned ./... single-module scope. dependency_policy.mode=locked forbids adapters from repairing dependency selection state (Cargo --locked; Go -mod=readonly); it does not imply offline execution. Long builds remain the same durable Job; pending never authorizes retry.",
)
.with_execution(super::ToolExecutionContract::new(
super::ToolExecutionForm::ProjectBuild,
super::ToolExecutionLifetime::Runner,
super::ToolExecutionStart::SyncFirst,
super::ToolExecutionContinuation::ObserveJobs,
)),
94,
super::ToolDirectReason::CoreWorkflow,
),
adaptive_runtime_direct(
model_spec(
def(
Expand Down
89 changes: 89 additions & 0 deletions docs/experiments/project-build-direct-measurement.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
# `project_build` Adaptive Direct measurement

## Decision

Promote `project_build` to the Adaptive Runtime Direct surface as a
`CoreWorkflow` tool at rank 94, immediately before `project_validate` at
rank 95.

This changes model-facing discovery only. The canonical
`project_build_v1` Runner capability, Standard approval policy, typed build
planning, Job identity, Runner-owned execution, and `observe_jobs`
continuation remain unchanged.

## Base and scope

The experiment was run from exact benchmark base
`b543a416d9e09ba0d5486219a8bc9bd583bf638e`, after the #822 paired benchmark
lane had passed its local evidence gates, fork CI, and security analysis.

The final candidate changed only model-facing discovery: it declared the existing
`project_build` definition Adaptive Direct with rank 94 and reason
`CoreWorkflow`, and supplied a bounded compact selection description that retains
the existing same-Job/no-redispatch rule when compact `tools/list` omits the full
manifest prose. No build planner, Runner, authority, approval, execution, or
result-projection code differed between the two variants.

The #822 runner itself was not repurposed for this A/B. Its comparison axis is
Direct versus Code Mode, while `project_build` is a durable Job tool and the
benchmark contract keeps Job tools outside nested Code Mode. Reusing that axis
for gateway versus dedicated-descriptor exposure would conflate two different
experiments. Instead, this measurement reused the benchmark lane's
fresh-workspace, paired-order, independent-correctness, and ActionAudit
principles with a real Codex Host.

## Method

Two valid A/B pairs were run in opposite order. Every sample used:

- a fresh isolated Server database, Runner, registered Rust Project, and Git
workspace;
- the same `gpt-6.1-sol` model at medium reasoning effort and the same task
prompt;
- locked dependency selection;
- the same exact Runner binary and build implementation;
- ActionAudit as the authoritative WebCodex call record;
- independent checks that the build artifact existed and the source workspace
remained clean.

The baseline exposed `project_build` through Adaptive discovery and
`call_runtime_tool`. The candidate exposed the same canonical tool as a
dedicated Direct MCP callable.

## Results

| Metric | Gateway baseline | Direct candidate |
| --- | ---: | ---: |
| Compact tools | 30 | 31 |
| Compact serialized tool bytes | 52,762 | 54,819 |
| Dedicated-surface cost | — | +2,057 bytes (+3.899%) |
| Successful samples | 2 / 2 | 2 / 2 |
| WebCodex call sequence | `read_tool_manifest ×2 → project_build` | `project_build ×1` |
| Completed Host MCP calls | 3 | 1 |
| Model responses per sample | 8 | 5 |
| Input tokens | 145,619 / 141,884 | 81,289 / 81,255 |
| Wall time | 25,189 / 16,602 ms | 9,980 / 11,564 ms |
| Median wall time | 20,895.5 ms | 10,772 ms |
| ActionAudit failures | 0 / 0 | 0 / 0 |
| Build artifact | 2 / 2 | 2 / 2 |
| Source workspace clean | 2 / 2 | 2 / 2 |

For this fixed task and Host, the final Direct candidate removed two manifest
lookups plus the gateway indirection and reduced model responses from 8 to 5.
Input-token and wall-time measurements moved in the same direction in both pair
orders, but remain Host- and environment-sensitive supporting observations rather
than a general latency claim. The Results table is from the final descriptor,
including its bounded same-Job/no-redispatch compact guidance.

## Interpretation

The dedicated descriptor adds a measurable permanent compact-surface cost:
2,057 bytes, or 3.899% in this snapshot. In return, the real Host consistently
selected the typed build lifecycle immediately, avoided repeated manifest
discovery, preserved all correctness checks, and materially reduced model
round trips and task time in both A/B orders.

That is sufficient evidence to make `project_build` Direct. Rank 94 keeps the
portable build lifecycle adjacent to, and immediately before, the existing
Direct `project_validate` rank 95 without changing either tool's authority or
execution semantics.
1 change: 1 addition & 0 deletions src/mcp/discovery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ pub(super) fn compact_tool(tool: &mut Value) {
"plugin_tool" => "Use Runner-owned Plugins via opaque bindings. Catalog selects; describe when schema/binding is missing. Reuse retained exact binding for the same Project; reconcile unknown outcomes.",
"call_runtime_tool" => "Call one admitted runtime tool with its exact arguments. Use read_tool_manifest to discover the contract. Prefer an available direct callable; ordinary direct tools may fall back here when unavailable, but MCP App presentation tools must use their direct callable while Apps are enabled. Target validation and authority checks still apply.",
"edit_project_files" => "Read with read_files; send one change per file. Existing edit/delete/rename require expected_read_revision; create needs content. Exact edits fail closed on ambiguity; replace_range uses 1-based inclusive lines from that snapshot. Batches preflight transactionally; Runner rechecks source. Stale: read_files recovery. outcome_unknown: observe before another write. Review and use task-appropriate validation.",
"project_build" => "Portable Rust/Go project build with Runner-owned canonical argv. Supports bounded packages/all_packages and dependency_policy.mode=locked; no arbitrary argv, profile/target or artifact contract. Long builds keep the same durable Job and returned continuation; pending never authorizes retry or redispatch. Use observe_jobs only for logs/details/recovery.",
"run_process" => "Run one native executable with literal argv. Use run_shell for shell grammar/short related chains; run_script for program-like scripts. Pending keeps the same Job and returned continuation; never redispatch. Continue independent work first; when blocked use wait_for_job_readiness. Read logs/details with observe_jobs. Diagnose failures and use task-appropriate validation; outcome_unknown requires observation.",
"run_shell" => "Run shell grammar or a short related command chain. Use run_process for literal argv; run_script for program-like scripts. Pending keeps the same Job and returned continuation; never redispatch. Continue independent work first; when blocked use wait_for_job_readiness. Read logs/details with observe_jobs. Diagnose failures and use task-appropriate validation; outcome_unknown requires observation.",
"run_script" => "Run typed sh/bash/PowerShell/Python/JS/TS script data. Use run_process for literal argv or run_shell for shell grammar; prefer structured editors for source edits. Pending keeps the same Job/continuation; never redispatch. Continue independent work; when blocked use wait_for_job_readiness. Read logs/details with observe_jobs. Diagnose failures and validate; outcome_unknown requires observation.",
Expand Down
28 changes: 28 additions & 0 deletions src/mcp_tests/tools.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2223,6 +2223,14 @@ fn mcp_compact_descriptions_preserve_selection_and_schema_literals() {
let compact = mcp_tools_list_payload_with_compact(true);
let tools = compact["tools"].as_array().unwrap();
for (name, phrases) in [
(
"project_build",
vec![
"Portable Rust/Go project build",
"same durable Job",
"pending never authorizes retry or redispatch",
],
),
(
"run_process",
vec!["native executable", "literal argv", "observe_jobs"],
Expand Down Expand Up @@ -2648,6 +2656,26 @@ async fn mcp_text_json_compat_uses_runtime_snapshot() {
);
}

#[tokio::test]
async fn project_build_is_exposed_directly_on_adaptive_mcp_surface() {
let runtime = test_runtime();
let outcome = handle_mcp_request(
&runtime,
rpc("tools/list", Some(Value::from(30)), json!({})),
None,
)
.await;
let McpOutcome::Ok(value) = outcome else {
panic!("expected Adaptive tools/list success, got {outcome:?}");
};
let tools = value["result"]["tools"].as_array().unwrap();
assert!(tools.iter().any(|tool| tool["name"] == "project_build"));
assert_eq!(
crate::model_surface::adaptive_runtime_tool_invocation_route("project_build"),
("direct", None)
);
}

#[tokio::test]
async fn session_tools_stay_registered_and_follow_adaptive_routes() {
let runtime = test_runtime();
Expand Down
Loading