diff --git a/crates/webcodex-tool-contracts/src/tests/definitions.rs b/crates/webcodex-tool-contracts/src/tests/definitions.rs index cfec2489a..44bddd43e 100644 --- a/crates/webcodex-tool-contracts/src/tests/definitions.rs +++ b/crates/webcodex-tool-contracts/src/tests/definitions.rs @@ -933,6 +933,7 @@ fn adaptive_runtime_direct_declarations_are_visible_ranked_and_unique() { ("run_script", 74), ("run_shell", 75), ("observe_jobs", 80), + ("project_build", 94), ("project_validate", 95), ] { let definition = derived @@ -1479,12 +1480,16 @@ fn readiness_tool_is_sequential_outer_only() { } #[test] -fn project_build_is_gateway_visible_but_not_adaptive_direct() { +fn project_build_is_adaptive_direct_and_keeps_typed_build_contract() { let definition = lookup_tool_definition("project_build").expect("project_build definition"); assert!(definition.visibility.is_model_visible()); assert_eq!(definition.category, TOOL_CATEGORY_EXECUTION); - assert_eq!(definition.adaptive_runtime_direct_rank(), None); - assert!(!is_adaptive_runtime_direct_tool("project_build")); + assert_eq!(definition.adaptive_runtime_direct_rank(), Some(94)); + assert_eq!( + definition.adaptive_runtime_direct_reason(), + Some(ToolDirectReason::CoreWorkflow) + ); + assert!(is_adaptive_runtime_direct_tool("project_build")); let requirement = runtime_tool_runner_capability("project_build") .expect("project_build must require its typed Runner capability"); assert_eq!(requirement, RunnerCapabilityRequirement::ProjectBuild); diff --git a/crates/webcodex-tool-contracts/src/tool_definition/jobs.rs b/crates/webcodex-tool-contracts/src/tool_definition/jobs.rs index f553ea957..d131b4122 100644 --- a/crates/webcodex-tool-contracts/src/tool_definition/jobs.rs +++ b/crates/webcodex-tool-contracts/src/tool_definition/jobs.rs @@ -28,36 +28,40 @@ pub(super) const EXECUTION_DEFINITIONS: &[ToolDefinition] = &[ 71, super::ToolDirectReason::CoreWorkflow, ), - model_spec( - def( - "project_build", - super::ToolAuditPolicy::TYPED_CANONICAL - .execution(super::ToolAuditExecutionPolicy::DIRECT_ARGV_TEXT), - ModelVisible, - TOOL_CATEGORY_EXECUTION, - Some(ProjectBuild), - TOOL_PROVIDER_RUNNER, - super::ToolSemanticContract { - effect: super::ToolEffect::Execute, - risk: JobRun, - approval: super::ToolApprovalPolicy::Standard, - idempotency: super::ToolIdempotency::NonIdempotent, - }, - Some(JOB_RUN), - true, - NoPath, - false, - false, - super::ToolSessionEvidencePolicy::NONE, - ), - "Preferred portable project build gateway. Runner resolves the nearest unambiguous Rust/Go recipe and executes canonical cargo build or go build argv. Go project builds run in Runner-owned single-module mode (GO111MODULE=on, GOWORK=off), so ambient module/workspace mode cannot change the planned graph. Optional scope selects either bounded explicit packages or all_packages=true. Rust all-packages is admitted only for a Cargo workspace whose effective root is the registered Project root; Go all-packages keeps the Runner-owned ./... single-module scope. dependency_policy.mode=locked forbids adapters from repairing dependency selection state (Cargo --locked; Go -mod=readonly); it does not imply offline execution. Long builds remain the same durable Job; pending never authorizes retry.", - ) - .with_execution(super::ToolExecutionContract::new( - super::ToolExecutionForm::ProjectBuild, - super::ToolExecutionLifetime::Runner, - super::ToolExecutionStart::SyncFirst, - super::ToolExecutionContinuation::ObserveJobs, - )), + adaptive_runtime_direct( + model_spec( + def( + "project_build", + super::ToolAuditPolicy::TYPED_CANONICAL + .execution(super::ToolAuditExecutionPolicy::DIRECT_ARGV_TEXT), + ModelVisible, + TOOL_CATEGORY_EXECUTION, + Some(ProjectBuild), + TOOL_PROVIDER_RUNNER, + super::ToolSemanticContract { + effect: super::ToolEffect::Execute, + risk: JobRun, + approval: super::ToolApprovalPolicy::Standard, + idempotency: super::ToolIdempotency::NonIdempotent, + }, + Some(JOB_RUN), + true, + NoPath, + false, + false, + super::ToolSessionEvidencePolicy::NONE, + ), + "Preferred portable project build gateway. Runner resolves the nearest unambiguous Rust/Go recipe and executes canonical cargo build or go build argv. Go project builds run in Runner-owned single-module mode (GO111MODULE=on, GOWORK=off), so ambient module/workspace mode cannot change the planned graph. Optional scope selects either bounded explicit packages or all_packages=true. Rust all-packages is admitted only for a Cargo workspace whose effective root is the registered Project root; Go all-packages keeps the Runner-owned ./... single-module scope. dependency_policy.mode=locked forbids adapters from repairing dependency selection state (Cargo --locked; Go -mod=readonly); it does not imply offline execution. Long builds remain the same durable Job; pending never authorizes retry.", + ) + .with_execution(super::ToolExecutionContract::new( + super::ToolExecutionForm::ProjectBuild, + super::ToolExecutionLifetime::Runner, + super::ToolExecutionStart::SyncFirst, + super::ToolExecutionContinuation::ObserveJobs, + )), + 94, + super::ToolDirectReason::CoreWorkflow, + ), adaptive_runtime_direct( model_spec( def( diff --git a/docs/experiments/project-build-direct-measurement.md b/docs/experiments/project-build-direct-measurement.md new file mode 100644 index 000000000..3fc6595d6 --- /dev/null +++ b/docs/experiments/project-build-direct-measurement.md @@ -0,0 +1,89 @@ +# `project_build` Adaptive Direct measurement + +## Decision + +Promote `project_build` to the Adaptive Runtime Direct surface as a +`CoreWorkflow` tool at rank 94, immediately before `project_validate` at +rank 95. + +This changes model-facing discovery only. The canonical +`project_build_v1` Runner capability, Standard approval policy, typed build +planning, Job identity, Runner-owned execution, and `observe_jobs` +continuation remain unchanged. + +## Base and scope + +The experiment was run from exact benchmark base +`b543a416d9e09ba0d5486219a8bc9bd583bf638e`, after the #822 paired benchmark +lane had passed its local evidence gates, fork CI, and security analysis. + +The final candidate changed only model-facing discovery: it declared the existing +`project_build` definition Adaptive Direct with rank 94 and reason +`CoreWorkflow`, and supplied a bounded compact selection description that retains +the existing same-Job/no-redispatch rule when compact `tools/list` omits the full +manifest prose. No build planner, Runner, authority, approval, execution, or +result-projection code differed between the two variants. + +The #822 runner itself was not repurposed for this A/B. Its comparison axis is +Direct versus Code Mode, while `project_build` is a durable Job tool and the +benchmark contract keeps Job tools outside nested Code Mode. Reusing that axis +for gateway versus dedicated-descriptor exposure would conflate two different +experiments. Instead, this measurement reused the benchmark lane's +fresh-workspace, paired-order, independent-correctness, and ActionAudit +principles with a real Codex Host. + +## Method + +Two valid A/B pairs were run in opposite order. Every sample used: + +- a fresh isolated Server database, Runner, registered Rust Project, and Git + workspace; +- the same `gpt-6.1-sol` model at medium reasoning effort and the same task + prompt; +- locked dependency selection; +- the same exact Runner binary and build implementation; +- ActionAudit as the authoritative WebCodex call record; +- independent checks that the build artifact existed and the source workspace + remained clean. + +The baseline exposed `project_build` through Adaptive discovery and +`call_runtime_tool`. The candidate exposed the same canonical tool as a +dedicated Direct MCP callable. + +## Results + +| Metric | Gateway baseline | Direct candidate | +| --- | ---: | ---: | +| Compact tools | 30 | 31 | +| Compact serialized tool bytes | 52,762 | 54,819 | +| Dedicated-surface cost | — | +2,057 bytes (+3.899%) | +| Successful samples | 2 / 2 | 2 / 2 | +| WebCodex call sequence | `read_tool_manifest ×2 → project_build` | `project_build ×1` | +| Completed Host MCP calls | 3 | 1 | +| Model responses per sample | 8 | 5 | +| Input tokens | 145,619 / 141,884 | 81,289 / 81,255 | +| Wall time | 25,189 / 16,602 ms | 9,980 / 11,564 ms | +| Median wall time | 20,895.5 ms | 10,772 ms | +| ActionAudit failures | 0 / 0 | 0 / 0 | +| Build artifact | 2 / 2 | 2 / 2 | +| Source workspace clean | 2 / 2 | 2 / 2 | + +For this fixed task and Host, the final Direct candidate removed two manifest +lookups plus the gateway indirection and reduced model responses from 8 to 5. +Input-token and wall-time measurements moved in the same direction in both pair +orders, but remain Host- and environment-sensitive supporting observations rather +than a general latency claim. The Results table is from the final descriptor, +including its bounded same-Job/no-redispatch compact guidance. + +## Interpretation + +The dedicated descriptor adds a measurable permanent compact-surface cost: +2,057 bytes, or 3.899% in this snapshot. In return, the real Host consistently +selected the typed build lifecycle immediately, avoided repeated manifest +discovery, preserved all correctness checks, and materially reduced model +round trips and task time in both A/B orders. + +That is sufficient evidence to make `project_build` Direct. Rank 94 keeps the +portable build lifecycle adjacent to, and immediately before, the existing +Direct `project_validate` rank 95 without changing either tool's authority or +execution semantics. diff --git a/src/mcp/discovery.rs b/src/mcp/discovery.rs index 84639861d..61f91a7fc 100644 --- a/src/mcp/discovery.rs +++ b/src/mcp/discovery.rs @@ -21,6 +21,7 @@ pub(super) fn compact_tool(tool: &mut Value) { "plugin_tool" => "Use Runner-owned Plugins via opaque bindings. Catalog selects; describe when schema/binding is missing. Reuse retained exact binding for the same Project; reconcile unknown outcomes.", "call_runtime_tool" => "Call one admitted runtime tool with its exact arguments. Use read_tool_manifest to discover the contract. Prefer an available direct callable; ordinary direct tools may fall back here when unavailable, but MCP App presentation tools must use their direct callable while Apps are enabled. Target validation and authority checks still apply.", "edit_project_files" => "Read with read_files; send one change per file. Existing edit/delete/rename require expected_read_revision; create needs content. Exact edits fail closed on ambiguity; replace_range uses 1-based inclusive lines from that snapshot. Batches preflight transactionally; Runner rechecks source. Stale: read_files recovery. outcome_unknown: observe before another write. Review and use task-appropriate validation.", + "project_build" => "Portable Rust/Go project build with Runner-owned canonical argv. Supports bounded packages/all_packages and dependency_policy.mode=locked; no arbitrary argv, profile/target or artifact contract. Long builds keep the same durable Job and returned continuation; pending never authorizes retry or redispatch. Use observe_jobs only for logs/details/recovery.", "run_process" => "Run one native executable with literal argv. Use run_shell for shell grammar/short related chains; run_script for program-like scripts. Pending keeps the same Job and returned continuation; never redispatch. Continue independent work first; when blocked use wait_for_job_readiness. Read logs/details with observe_jobs. Diagnose failures and use task-appropriate validation; outcome_unknown requires observation.", "run_shell" => "Run shell grammar or a short related command chain. Use run_process for literal argv; run_script for program-like scripts. Pending keeps the same Job and returned continuation; never redispatch. Continue independent work first; when blocked use wait_for_job_readiness. Read logs/details with observe_jobs. Diagnose failures and use task-appropriate validation; outcome_unknown requires observation.", "run_script" => "Run typed sh/bash/PowerShell/Python/JS/TS script data. Use run_process for literal argv or run_shell for shell grammar; prefer structured editors for source edits. Pending keeps the same Job/continuation; never redispatch. Continue independent work; when blocked use wait_for_job_readiness. Read logs/details with observe_jobs. Diagnose failures and validate; outcome_unknown requires observation.", diff --git a/src/mcp_tests/tools.rs b/src/mcp_tests/tools.rs index 2f70f7ed5..a3c7a4d22 100644 --- a/src/mcp_tests/tools.rs +++ b/src/mcp_tests/tools.rs @@ -2223,6 +2223,14 @@ fn mcp_compact_descriptions_preserve_selection_and_schema_literals() { let compact = mcp_tools_list_payload_with_compact(true); let tools = compact["tools"].as_array().unwrap(); for (name, phrases) in [ + ( + "project_build", + vec![ + "Portable Rust/Go project build", + "same durable Job", + "pending never authorizes retry or redispatch", + ], + ), ( "run_process", vec!["native executable", "literal argv", "observe_jobs"], @@ -2648,6 +2656,26 @@ async fn mcp_text_json_compat_uses_runtime_snapshot() { ); } +#[tokio::test] +async fn project_build_is_exposed_directly_on_adaptive_mcp_surface() { + let runtime = test_runtime(); + let outcome = handle_mcp_request( + &runtime, + rpc("tools/list", Some(Value::from(30)), json!({})), + None, + ) + .await; + let McpOutcome::Ok(value) = outcome else { + panic!("expected Adaptive tools/list success, got {outcome:?}"); + }; + let tools = value["result"]["tools"].as_array().unwrap(); + assert!(tools.iter().any(|tool| tool["name"] == "project_build")); + assert_eq!( + crate::model_surface::adaptive_runtime_tool_invocation_route("project_build"), + ("direct", None) + ); +} + #[tokio::test] async fn session_tools_stay_registered_and_follow_adaptive_routes() { let runtime = test_runtime();