From 314d547276d8f569462c5bd2e046bb32edd6eb34 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 15 Sep 2026 01:39:54 +0000 Subject: [PATCH] chore: sync public mirror from internal --- CHANGELOG.md | 34 ++ Cargo.lock | 2 +- deny.toml | 1 + docs/protocols/hosted-runner-contract.md | 86 ++-- package-lock.json | 4 +- package.json | 2 +- packages/ai-rs/src/anthropic.rs | 2 +- packages/ai-rs/src/google.rs | 10 + packages/ai-rs/src/kimi.rs | 47 +- packages/ai-rs/src/lib.rs | 2 + packages/ai-rs/src/openai.rs | 2 +- packages/ai-rs/src/provider_matrix.rs | 434 ++++++++++++++++++ packages/ai-rs/src/transform.rs | 2 +- packages/ai-rs/src/vertex.rs | 10 + packages/local-host-rs/src/agent/mod.rs | 1 + .../src/headless/messages/tests.rs | 14 + packages/local-host-rs/src/headless/proto.rs | 19 + .../local-host-rs/src/headless/session.rs | 126 +++-- .../local-host-rs/src/headless/supervisor.rs | 51 +- .../src/headless/supervisor/tests.rs | 13 +- .../src/headless/workspace_capabilities.rs | 46 ++ .../headless/workspace_capabilities_test.rs | 77 +++- packages/local-host-rs/src/hosted_runner.rs | 6 +- .../local-host-rs/src/hosted_runner/config.rs | 9 +- .../src/hosted_runner/env_aliases.rs | 74 +++ .../local-host-rs/src/hosted_runner/tests.rs | 8 +- .../src/hosted_runner/tests/env_aliases.rs | 151 ++++++ .../local-host-rs/src/hosted_runner_cli.rs | 8 + .../hosted-runner-startup-contract.json | 415 +++++++++++++++++ .../tests/hosted_runner_startup_contract.rs | 346 ++++++++++++++ packages/maestro-rs/Cargo.toml | 2 +- .../fixtures/headless-protocol-v1.json | 8 +- packages/runtime-contracts-rs/src/lib.rs | 2 + packages/runtime-contracts-rs/src/protocol.rs | 77 +++- .../runtime-contracts-rs/src/turn_outcome.rs | 295 ++++++++++++ .../runtime-rs/src/agent/extensions/mod.rs | 9 +- packages/runtime-rs/src/agent/native.rs | 8 +- .../src/agent/native/session_scenarios.rs | 165 ++++++- .../v1/protocol-compatibility-manifest.json | 4 +- scripts/pr-feedback-audit.mjs | 2 +- ...solve-public-mirror-bot-review-threads.mjs | 15 +- ...-public-mirror-bot-review-threads.test.mjs | 37 +- .../compaction-tool-history.json | 65 +++ 43 files changed, 2531 insertions(+), 160 deletions(-) create mode 100644 packages/ai-rs/src/provider_matrix.rs create mode 100644 packages/local-host-rs/src/hosted_runner/env_aliases.rs create mode 100644 packages/local-host-rs/src/hosted_runner/tests/env_aliases.rs create mode 100644 packages/local-host-rs/tests/fixtures/hosted-runner-startup-contract.json create mode 100644 packages/local-host-rs/tests/hosted_runner_startup_contract.rs create mode 100644 packages/runtime-contracts-rs/src/turn_outcome.rs create mode 100644 test/fixtures/session-scenarios/compaction-tool-history.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cc0cb12a..30cd63198 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,40 @@ versioning when releases are cut. and keep scheduled public runs inert so public publishing stays downstream of the internal source-of-truth release. +## [0.10.89] - 2026-09-14 + +### Changed + +- Use GPT-5.6 as the default for OpenAI and Codex provider routes; preserve explicitly configured models. +- Refresh bundled model context and output limits from the catalog source (#9120). +- Require a signed handoff before a public release can be published (#9101). +- Keep build caches complete so builds stay fast (#9093). +- Verify changes that land through the merge queue with the same required checks as direct merges (#9098). +- Start runtime verification faster so pre-release checks finish reliably (#9097). +- Verify every change on the main branch individually so release gates never wait on a skipped run (#9091). +- Treat the hosted-orb smoke test as a hard release gate and unblock supply-chain policy reviews (#9086). +- Promote runtime images only after the Maestro image is published so deployments never reference a missing image (#9087). +- Restore reliable verification of every change on the main branch (#9085). +- Publish releases from GitHub, including native artifact validation and a stable Linux release ABI (#9079). +- Move Maestro's continuous integration to GitHub Actions (#9080). +- Remove files left over from the retired CI system (#9081). +- Remove tooling tied to the retired CI system from the public port (#9078). + +### Fixed + +- Keep pre-merge checks within build-machine capacity so they no longer time out (#9116). +- Keep Rust builds within build-machine memory and CPU limits so they no longer fail mid-build (#9113). +- Verify cerebro row-level security under non-superuser database roles so tenant isolation is tested for real (#9110). +- Keep the public mirror syncing when review bots leave open threads on generated sync PRs (#9107). +- Fix mobile acceptance tests failing on a missing protobuf runtime (#9105). +- Fix the APEX readiness check timing out before it could report (#9103). +- Recover publication retries and refresh app progress (#9090). +- Speed up affected-component checks by avoiding repeated file lookups (#9096). +- Complete the Sandboxwich CI migration (#9095). +- Fix mobile conversation submissions that the platform rejected with HTTP 415 (#9099). +- Retain organization scope in attention events (#9094). +- Include the active workflow inputs in queue reports so stale reports are distinguishable (#9088). + ## [0.10.88] - 2026-09-13 ### Added diff --git a/Cargo.lock b/Cargo.lock index 7f89f7f74..16da691fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3509,7 +3509,7 @@ checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" [[package]] name = "maestro" -version = "0.10.88" +version = "0.10.89" dependencies = [ "anyhow", "ctor", diff --git a/deny.toml b/deny.toml index 43addc266..171a5835b 100644 --- a/deny.toml +++ b/deny.toml @@ -33,6 +33,7 @@ ignore = [ # --- Unsound (no known exploitable vulnerability reported here) --- { id = "RUSTSEC-2026-0002", reason = "lru 0.12.5: IterMut violates Stacked Borrows (Miri/unsafe-code soundness issue), pulled in by ratatui (production). Not a memory-safety issue under normal (non-Miri) execution; tracked for the next ratatui bump that picks up a fixed lru. expires: 2026-10-23" }, + { id = "RUSTSEC-2026-0285", reason = "rustls <0.23.45 accepts TLS 1.3 handshake messages at the wrong encryption level after a key change in the same record (GHSA-2mjx-qc3c-rqvc, CVSS 5.3). Fixed in rustls 0.23.45, published 2026-09-14; that upgrade also resolves aws-lc-rs 1.18.1 and aws-lc-sys 0.45.0, all still inside the 14-day dependency-admission cool-off in security/rust-dependency-admission.toml. Bump the lockfile once they age past it. expires: 2026-09-30" }, ] [licenses] diff --git a/docs/protocols/hosted-runner-contract.md b/docs/protocols/hosted-runner-contract.md index 23652e006..a5e2755bf 100644 --- a/docs/protocols/hosted-runner-contract.md +++ b/docs/protocols/hosted-runner-contract.md @@ -1,6 +1,6 @@ # Hosted Runner Contract -> **Status:** This document predates the Rust-only runtime migration (#3016, #3017, merged 2026-07-22), which deleted Maestro's TypeScript agent runtime and SDK. Hosted runner code now lives in `packages/local-host-rs/src/hosted_runner.rs`, `hosted_runner_cli.rs`, and `packages/tui-rs/src/hosted_runner/`. Some file paths below may be stale; they are kept for design context and updated only where a corresponding Rust module was confirmed. +> **Status:** This document predates the Rust-only runtime migration (#3016, #3017, merged 2026-07-22), which deleted Maestro's TypeScript agent runtime and SDK. Hosted runner code now lives in `packages/local-host-rs/src/hosted_runner/` (config, handle, manifests, snapshots, rendezvous, workload identity) and `packages/local-host-rs/src/hosted_runner_cli.rs` (launch resolution and the `hosted-runner` entrypoint). The startup coordinates in [Configuration](#configuration) are executable: `packages/local-host-rs/tests/hosted_runner_startup_contract.rs` runs every case in `packages/local-host-rs/tests/fixtures/hosted-runner-startup-contract.json` through the real CLI resolver, so a documented coordinate that stops being honored fails `cargo test -p maestro-local-host`. Other file paths below may be stale; they are kept for design context and updated only where a corresponding Rust module was confirmed. Maestro hosted runners are substrate-neutral runtime pods or sandboxes that @@ -38,17 +38,34 @@ flags or environment variables, but the resolved values are the contract. | Contract field | Flags and environment | Required | | --- | --- | --- | -| Runner session id | `--runner-session-id`, `MAESTRO_RUNNER_SESSION_ID`, `REMOTE_RUNNER_SESSION_ID` | yes | -| Workspace root | `--workspace-root`, `MAESTRO_WORKSPACE_ROOT`, `WORKSPACE_ROOT` | yes | -| Listen address | `--listen`, `--host`, `--port`, `MAESTRO_HOSTED_RUNNER_LISTEN`, `MAESTRO_HOSTED_RUNNER_HOST`, `MAESTRO_HOSTED_RUNNER_PORT`, `PORT` | yes | -| Owner generation | `--owner-instance-id`, `MAESTRO_REMOTE_RUNNER_OWNER_INSTANCE_ID`, `REMOTE_RUNNER_OWNER_INSTANCE_ID` | required when Platform fences owners | -| Runtime generation | `MAESTRO_PLACEMENT_GENERATION`, `MAESTRO_SANDBOXWICH_PLACEMENT_GENERATION`, `MAESTRO_REMOTE_RUNNER_GENERATION` | required for managed durable threads; the canonical placement variable takes precedence | -| Snapshot root | `--snapshot-root`, `MAESTRO_REMOTE_RUNNER_SNAPSHOT_ROOT`, `REMOTE_RUNNER_SNAPSHOT_ROOT` | optional | -| Restore manifest | `MAESTRO_REMOTE_RUNNER_RESTORE_MANIFEST`, `REMOTE_RUNNER_RESTORE_MANIFEST` | optional | +| Runner session id | `--runner-session-id`, `MAESTRO_RUNNER_SESSION_ID` (deprecated `REMOTE_RUNNER_SESSION_ID`) | yes | +| Workspace root | `--workspace-root`, `MAESTRO_WORKSPACE_ROOT` (deprecated `WORKSPACE_ROOT`) | yes | +| Listen address | `--listen`, `--host`, `--port`, `MAESTRO_HOSTED_RUNNER_LISTEN`, `MAESTRO_HOSTED_RUNNER_HOST`, `MAESTRO_HOSTED_RUNNER_PORT` (deprecated `PORT`) | yes | +| Owner generation | `--owner-instance-id`, `MAESTRO_REMOTE_RUNNER_OWNER_INSTANCE_ID` (deprecated `REMOTE_RUNNER_OWNER_INSTANCE_ID`) | required when Platform fences owners | +| Runtime generation | `MAESTRO_PLACEMENT_GENERATION`, `MAESTRO_SANDBOXWICH_PLACEMENT_GENERATION`, `MAESTRO_REMOTE_RUNNER_GENERATION` | required for managed durable threads; the canonical placement variable takes precedence and also enters workload-identity mode, so it must arrive with the full identity set below | +| Snapshot root | `--snapshot-root`, `MAESTRO_REMOTE_RUNNER_SNAPSHOT_ROOT` (deprecated `REMOTE_RUNNER_SNAPSHOT_ROOT`) | optional | +| Restore manifest | `MAESTRO_REMOTE_RUNNER_RESTORE_MANIFEST` (deprecated `REMOTE_RUNNER_RESTORE_MANIFEST`) | optional | | Workspace id | `--workspace-id`, `MAESTRO_REMOTE_RUNNER_WORKSPACE_ID`, `MAESTRO_WORKSPACE_ID` | optional | | Agent run id | `--agent-run-id`, `MAESTRO_AGENT_RUN_ID` | optional | | Existing Maestro session | `--maestro-session-id`, `MAESTRO_SESSION_ID` | optional | | Attach audience | `--attach-audience`, `MAESTRO_ATTACH_AUDIENCE` | optional | +| Attach bearer token | `MAESTRO_HOSTED_RUNNER_AUTH_TOKEN` or `MAESTRO_HOSTED_RUNNER_AUTH_TOKEN_FILE` (deprecated `MAESTRO_WEB_API_KEY`, `MAESTRO_WEB_API_KEY_FILE`) | required unless workload identity is present or `MAESTRO_WEB_REQUIRE_KEY=0` on a loopback bind; inline and file forms are mutually exclusive | +| Headless model binding | `MAESTRO_MODEL`; managed `evalops/` and `maestro-managed/` models also require `MAESTRO_RESIDENT_CONTRACT_REVISION` pinned to the current resident revision plus non-empty `MAESTRO_EVALOPS_ACCESS_TOKEN`, `MAESTRO_EVALOPS_BASE_URL`, `MAESTRO_EVALOPS_ORG_ID`, `MAESTRO_EVALOPS_WORKSPACE_ID`, and `MAESTRO_EVALOPS_PROVIDER` | yes | +| Headless child | `--agent-cli-path`, `MAESTRO_HEADLESS_CLI_PATH` | optional | +| Launch spec | `--config` or `MAESTRO_HOSTED_LAUNCH_SPEC_FILE` (not both); when present it replaces the flag/environment coordinates above | optional | +| Rendezvous | `MAESTRO_RENDEZVOUS_MODE` (`inbound`, `outbound_shadow`, `outbound`); outbound modes require workload identity | optional | + +Deprecated environment keys are the entries of `DEPRECATED_ENV_ALIASES` in +`packages/local-host-rs/src/hosted_runner/config.rs`. The canonical key always +wins when both are set; every deprecated key that carries a value produces one +`hosted-runner environment key ... is deprecated` startup warning, and the +deprecated keys stop being read after the next Maestro minor release. Platform +(`runner-host` and `sandboxwich`) already launches with the canonical keys. + +Startup fails closed on a missing or blank runner session id, a missing or +non-directory workspace root, a non-integer runtime generation, a non-loopback +bind without attach authentication, and any partially supplied workload +identity set. Managed Kubernetes workload-identity mode additionally requires all of the following values. Supplying only part of the set fails startup; static hosted @@ -66,7 +83,8 @@ runner bearer authentication is forbidden when the set is present. | Runner session binding | `MAESTRO_RUNNER_SESSION_ID` | There is intentionally no file or environment input for a Runner Host client -CA, Maestro server certificate, or private key. The authenticated exchange +CA, Maestro server certificate, or private key; setting +`MAESTRO_RUNNER_CLIENT_CA_FILE` rejects startup. The authenticated exchange response carries the workload CA in memory. Maestro uses that CA only for the current server chain and Runner Host client verification, pinned to the exact client URI `spiffe://identity.evalops.dev/service/runner-host`. @@ -464,29 +482,33 @@ the operational model. ## Conformance -Every hosted runner implementation should satisfy the shared conformance suite: - -```bash -npm run test -- test/headless/runtime-conformance.test.ts -``` - -Rust hosted-runner wire parity is enforced by the dedicated -`rust-hosted-conformance` CI job. Run the same gate locally with: - -```bash -MAESTRO_RUST_HOSTED_CONFORMANCE=1 npm run test -- test/headless/runtime-conformance.test.ts -``` - -Current coverage includes schema-valid snapshots/envelopes, controller/viewer -roles, explicit controller takeover, cursor replay/reset, approval request and -response resolution, workspace-root file-read enforcement, utility -command/search/watch lifecycle, and disconnect cleanup. - -The TypeScript adapter targets the in-process host. The Rust-hosted adapter -drives the same scenarios through -`maestro_tui::hosted_runner::start_hosted_runner_with_message_executor` and the -external HTTP/SSE surface. The scenario body must remain shared; only adapter -startup and transport details should vary. +Two executable suites back this contract; both run in the Maestro CI Rust lane. + +Startup and lifecycle (`cargo test -p maestro-local-host --test hosted_runner_startup_contract`): + +- `packages/local-host-rs/tests/fixtures/hosted-runner-startup-contract.json` + is the accept/reject matrix for the [Configuration](#configuration) + coordinates. Each case is resolved through + `hosted_runner_cli::resolve_hosted_runner_launch_config`, the same code path + as `maestro hosted-runner`, and asserts either the resolved contract fields or + the fail-closed error. Adding a coordinate means adding a fixture case; the + suite rejects any exercised coordinate that this document does not name. + The guard is one-directional: coordinates named here but absent from the + fixture (for example `MAESTRO_REMOTE_RUNNER_SNAPSHOT_ROOT`, + `MAESTRO_REMOTE_RUNNER_RESTORE_MANIFEST`, and the split `--host`/`--port` + flags) are documented, not mechanically enforced. +- The runtime scenario starts the full CLI runtime behind a stub headless child + and checks the identity endpoint (`ready`, `draining`, sparse fields), a + drain that writes the snapshot manifest under the workspace root, and the + post-drain identity transition. + +Wire surface (`maestro conformance` and the `rust-hosted-conformance` CI job): +schema-valid snapshots/envelopes, controller/viewer roles, explicit controller +takeover, cursor replay/reset, approval request and response resolution, +workspace-root file-read enforcement, utility command/search/watch lifecycle, +and disconnect cleanup, driven through +`maestro_local_host::hosted_runner_conformance` against the external HTTP/SSE +surface. See [Headless runtime conformance](./headless-conformance.md). ## References diff --git a/package-lock.json b/package-lock.json index d7f297cc1..2510fe2b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@evalops/maestro", - "version": "0.10.88", + "version": "0.10.89", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@evalops/maestro", - "version": "0.10.88", + "version": "0.10.89", "license": "BUSL-1.1", "bin": { "deixic-code": "bin/deixic-code", diff --git a/package.json b/package.json index b74f88f84..43890b96a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@evalops/deixic-code", "description": "Deixic Code — native Rust coding agent, CLI, TUI, and web runtime gateway", - "version": "0.10.88", + "version": "0.10.89", "private": false, "type": "module", "bin": { diff --git a/packages/ai-rs/src/anthropic.rs b/packages/ai-rs/src/anthropic.rs index 0cd5ed1a9..6d49ac819 100644 --- a/packages/ai-rs/src/anthropic.rs +++ b/packages/ai-rs/src/anthropic.rs @@ -320,7 +320,7 @@ impl AnthropicClient { /// } /// ] /// ``` - fn build_request_body( + pub(crate) fn build_request_body( &self, messages: &[Message], config: &RequestConfig, diff --git a/packages/ai-rs/src/google.rs b/packages/ai-rs/src/google.rs index 3fdec4fb0..7448f2379 100644 --- a/packages/ai-rs/src/google.rs +++ b/packages/ai-rs/src/google.rs @@ -96,6 +96,16 @@ impl GoogleClient { } /// Build the Gemini API request body + /// Serialized request body, used by the provider conformance matrix. + #[cfg(test)] + pub(crate) fn request_body_json( + &self, + messages: &[Message], + config: &RequestConfig, + ) -> Result { + Ok(serde_json::to_value(self.build_request(messages, config)?)?) + } + fn build_request(&self, messages: &[Message], config: &RequestConfig) -> Result { let messages = crate::cache_topology::messages_with_volatile_tail(messages, config); let messages = super::transform::google_messages_for_wire(&messages); diff --git a/packages/ai-rs/src/kimi.rs b/packages/ai-rs/src/kimi.rs index 8a5d8aaf1..2e9837a9c 100644 --- a/packages/ai-rs/src/kimi.rs +++ b/packages/ai-rs/src/kimi.rs @@ -41,7 +41,7 @@ impl KimiK3Client { }) } - fn build_request_body(&self, messages: &[Message], config: &RequestConfig) -> Value { + pub(crate) fn build_request_body(&self, messages: &[Message], config: &RequestConfig) -> Value { let mut converted = convert_messages(messages); if let Some(system) = config .system @@ -299,7 +299,7 @@ fn tool_definition(tool: &Tool) -> Value { fn convert_messages(messages: &[Message]) -> Vec { let mut converted = Vec::new(); - for message in messages { + for message in &super::transform::repair_tool_sequence(messages.to_vec()) { if let MessageContent::Blocks(blocks) = &message.content { let tool_results = blocks.iter().filter_map(|block| match block { ContentBlock::ToolResult { @@ -309,18 +309,13 @@ fn convert_messages(messages: &[Message]) -> Vec { } => Some((tool_use_id, content)), _ => None, }); - let mut emitted_tool_result = false; for (tool_use_id, content) in tool_results { - emitted_tool_result = true; converted.push(json!({ "role": "tool", "tool_call_id": tool_use_id, "content": content, })); } - if emitted_tool_result { - continue; - } } match &message.content { @@ -636,6 +631,44 @@ mod tests { assert_eq!(body["model"], "kimi-k3"); } + #[test] + fn unmatched_results_in_a_mixed_batch_reach_the_model_as_text() { + let messages = vec![ + Message { + role: Role::Assistant, + content: MessageContent::Blocks(vec![ContentBlock::ToolUse { + id: "call_a".to_string(), + name: "read".to_string(), + input: json!({}), + gemini_context: None, + }]), + }, + Message { + role: Role::User, + content: MessageContent::Blocks(vec![ + ContentBlock::ToolResult { + tool_use_id: "call_a".to_string(), + content: "matched".to_string(), + is_error: None, + }, + ContentBlock::ToolResult { + tool_use_id: "call_stale".to_string(), + content: "retained evidence".to_string(), + is_error: None, + }, + ]), + }, + ]; + + let converted = convert_messages(&messages); + assert_eq!(converted[1]["role"], "tool"); + assert_eq!(converted[1]["tool_call_id"], "call_a"); + assert_eq!(converted[2]["role"], "user"); + let text = converted[2]["content"].as_str().unwrap(); + assert!(text.contains("retained evidence"), "{text}"); + assert_eq!(converted.len(), 3); + } + #[test] fn thinking_budget_maps_to_kimi_supported_effort_levels() { let client = KimiK3Client::new("test-key", "https://api.moonshot.ai/v1").unwrap(); diff --git a/packages/ai-rs/src/lib.rs b/packages/ai-rs/src/lib.rs index 7dc206576..51887bd0a 100644 --- a/packages/ai-rs/src/lib.rs +++ b/packages/ai-rs/src/lib.rs @@ -96,6 +96,8 @@ pub mod op_secret; mod openai; #[path = "openai.rs"] mod openai_base; +#[cfg(test)] +mod provider_matrix; mod providers; pub mod sanitize; mod scripted; diff --git a/packages/ai-rs/src/openai.rs b/packages/ai-rs/src/openai.rs index 1f2b487cf..300fdaf8a 100644 --- a/packages/ai-rs/src/openai.rs +++ b/packages/ai-rs/src/openai.rs @@ -2022,7 +2022,7 @@ impl OpenAiClient { ) } - fn build_request_body_for_api( + pub(crate) fn build_request_body_for_api( &self, messages: &[Message], config: &RequestConfig, diff --git a/packages/ai-rs/src/provider_matrix.rs b/packages/ai-rs/src/provider_matrix.rs new file mode 100644 index 000000000..4940d7389 --- /dev/null +++ b/packages/ai-rs/src/provider_matrix.rs @@ -0,0 +1,434 @@ +//! Provider request-builder conformance matrix. +//! +//! One canonical multi-turn script (system prompt, user turn, assistant tool +//! call, tool result, assistant text, follow-up user turn, plus an interrupted +//! tool call) is rendered through every provider request builder. Each wire +//! shape is reduced to the same observation record and checked against +//! invariants that hold for all providers, so a regression in one adapter's +//! tool-history handling fails here instead of at the provider. + +use crate::anthropic::AnthropicClient; +use crate::google::GoogleClient; +use crate::kimi::KimiK3Client; +use crate::openai_base::OpenAiClient; +use crate::types::{ContentBlock, Message, MessageContent, RequestConfig, Role, Tool}; +use crate::vertex::VertexAiClient; +use serde_json::{Value, json}; +use std::collections::BTreeSet; +use std::sync::Arc; + +const SYSTEM_PROMPT: &str = "You are the matrix system prompt."; +const USER_ONE: &str = "matrix user turn one"; +const ASSISTANT_TEXT: &str = "matrix assistant text after tool"; +const USER_TWO: &str = "matrix user turn two"; +const TOOL_A: &str = "read_file"; +const TOOL_B: &str = "run_shell"; +const CALL_A: &str = "toolu_matrix_call_a"; +const CALL_B: &str = "toolu_matrix_call_b"; +const RESULT_A: &str = "matrix tool result a"; +/// Fragment of the text every adapter must hand the model for a call that never completed. +const INTERRUPTED_MARKER: &str = "interrupted"; + +/// Provider-neutral view of a rendered request. +#[derive(Debug, Default)] +struct Observed { + model: String, + tool_names: BTreeSet, + /// Tool calls in wire order as (call identity, tool name). + calls: Vec<(String, String)>, + /// Tool results in wire order. + results: Vec, + text: String, +} + +#[derive(Debug)] +struct ObservedResult { + /// Call identity (tool name on wires without call ids). + id: String, + /// Explicit error flag where the wire has one. + is_error: Option, + /// Result payload rendered as text. + content: String, +} + +impl ObservedResult { + fn new(id: &str, is_error: Option, content: &Value) -> Self { + let mut text = String::new(); + collect_text(content, &mut text); + Self { + id: id.to_string(), + is_error, + content: text, + } + } +} + +fn script() -> Vec { + vec![ + Message { + role: Role::User, + content: MessageContent::Text(USER_ONE.to_string()), + }, + Message { + role: Role::Assistant, + content: MessageContent::Blocks(vec![ContentBlock::ToolUse { + id: CALL_A.to_string(), + name: TOOL_A.to_string(), + input: json!({"path": "README.md"}), + gemini_context: None, + }]), + }, + Message { + role: Role::User, + content: MessageContent::Blocks(vec![ContentBlock::ToolResult { + tool_use_id: CALL_A.to_string(), + content: RESULT_A.to_string(), + is_error: None, + }]), + }, + Message { + role: Role::Assistant, + content: MessageContent::Blocks(vec![ + ContentBlock::Text { + text: ASSISTANT_TEXT.to_string(), + }, + ContentBlock::ToolUse { + id: CALL_B.to_string(), + name: TOOL_B.to_string(), + input: json!({"command": "ls"}), + gemini_context: None, + }, + ]), + }, + Message { + role: Role::User, + content: MessageContent::Text(USER_TWO.to_string()), + }, + ] +} + +fn config(model: &str) -> RequestConfig { + RequestConfig { + model: model.to_string(), + system: Some(SYSTEM_PROMPT.to_string()), + tools: Arc::new(vec![ + Tool::new(TOOL_A, "read a file"), + Tool::new(TOOL_B, "run a shell command"), + ]), + ..RequestConfig::default() + } +} + +fn collect_text(value: &Value, out: &mut String) { + match value { + Value::String(text) => { + out.push_str(text); + out.push('\n'); + } + Value::Array(items) => items.iter().for_each(|item| collect_text(item, out)), + Value::Object(map) => map.values().for_each(|item| collect_text(item, out)), + _ => {} + } +} + +fn str_at<'a>(value: &'a Value, key: &str) -> &'a str { + value[key].as_str().unwrap_or_default() +} + +fn observe_openai_chat(body: &Value) -> Observed { + let mut observed = Observed { + model: str_at(body, "model").to_string(), + ..Observed::default() + }; + for tool in body["tools"].as_array().into_iter().flatten() { + observed + .tool_names + .insert(str_at(&tool["function"], "name").to_string()); + } + for message in body["messages"].as_array().into_iter().flatten() { + for call in message["tool_calls"].as_array().into_iter().flatten() { + observed.calls.push(( + str_at(call, "id").to_string(), + str_at(&call["function"], "name").to_string(), + )); + } + if message["role"] == "tool" { + observed.results.push(ObservedResult::new( + str_at(message, "tool_call_id"), + None, + &message["content"], + )); + continue; + } + collect_text(&message["content"], &mut observed.text); + } + observed +} + +fn observe_openai_responses(body: &Value) -> Observed { + let mut observed = Observed { + model: str_at(body, "model").to_string(), + ..Observed::default() + }; + for tool in body["tools"].as_array().into_iter().flatten() { + observed.tool_names.insert(str_at(tool, "name").to_string()); + } + collect_text(&body["instructions"], &mut observed.text); + for item in body["input"].as_array().into_iter().flatten() { + match str_at(item, "type") { + "function_call" => observed.calls.push(( + str_at(item, "call_id").to_string(), + str_at(item, "name").to_string(), + )), + "function_call_output" => observed.results.push(ObservedResult::new( + str_at(item, "call_id"), + None, + &item["output"], + )), + _ => collect_text(&item["content"], &mut observed.text), + } + } + observed +} + +fn observe_anthropic(body: &Value) -> Observed { + let mut observed = Observed { + model: str_at(body, "model").to_string(), + ..Observed::default() + }; + for tool in body["tools"].as_array().into_iter().flatten() { + observed.tool_names.insert(str_at(tool, "name").to_string()); + } + collect_text(&body["system"], &mut observed.text); + for message in body["messages"].as_array().into_iter().flatten() { + match &message["content"] { + Value::Array(blocks) => { + for block in blocks { + match str_at(block, "type") { + "tool_use" => observed.calls.push(( + str_at(block, "id").to_string(), + str_at(block, "name").to_string(), + )), + "tool_result" => observed.results.push(ObservedResult::new( + str_at(block, "tool_use_id"), + block["is_error"].as_bool(), + &block["content"], + )), + _ => collect_text(block, &mut observed.text), + } + } + } + other => collect_text(other, &mut observed.text), + } + } + observed +} + +/// Gemini has no call identifiers: a `functionResponse` is paired with its +/// `functionCall` by function name and order, so the name is the identity. +fn observe_gemini(body: &Value) -> Observed { + let mut observed = Observed::default(); + for tool in body["tools"].as_array().into_iter().flatten() { + for declaration in tool["functionDeclarations"] + .as_array() + .or_else(|| tool["function_declarations"].as_array()) + .into_iter() + .flatten() + { + observed + .tool_names + .insert(str_at(declaration, "name").to_string()); + } + } + collect_text(&body["systemInstruction"], &mut observed.text); + collect_text(&body["system_instruction"], &mut observed.text); + for content in body["contents"].as_array().into_iter().flatten() { + for part in content["parts"].as_array().into_iter().flatten() { + if let Some(call) = part.get("functionCall") { + let name = str_at(call, "name").to_string(); + observed.calls.push((name.clone(), name)); + } else if let Some(response) = part.get("functionResponse") { + observed.results.push(ObservedResult::new( + str_at(response, "name"), + None, + &response["response"], + )); + } else { + collect_text(part, &mut observed.text); + } + } + } + observed +} + +fn render_all() -> Vec<(&'static str, Observed)> { + let openai = OpenAiClient::new("test-key").expect("openai client"); + let anthropic = AnthropicClient::new("test-key").expect("anthropic client"); + let google = GoogleClient::new("test-key"); + let vertex = VertexAiClient::new( + "matrix-project", + "us-central1", + Some("test-key".into()), + None, + ); + let kimi = KimiK3Client::new("test-key", "https://kimi.test").expect("kimi client"); + let messages = script(); + + let anthropic_body = anthropic + .build_request_body(&messages, &config("anthropic/claude-sonnet-4-20250514")) + .expect("anthropic body"); + let google_body = google + .request_body_json(&messages, &config("google/gemini-2.5-pro")) + .expect("google request"); + let vertex_body = vertex + .request_body_json(&messages, &config("vertex/gemini-2.5-pro")) + .expect("vertex request"); + + vec![ + ( + "openai-chat", + observe_openai_chat(&openai.build_request_body_for_api( + &messages, + &config("openai/gpt-4.1"), + false, + )), + ), + ( + "openai-responses", + observe_openai_responses(&openai.build_request_body_for_api( + &messages, + &config("openai/gpt-5.5"), + true, + )), + ), + ("anthropic", observe_anthropic(&anthropic_body)), + ("google", observe_gemini(&google_body)), + ("vertex", observe_gemini(&vertex_body)), + ( + "kimi", + observe_openai_chat(&kimi.build_request_body(&messages, &config("moonshot/kimi-k3"))), + ), + ] +} + +#[test] +fn every_provider_strips_the_route_prefix_from_the_model() { + for (provider, observed) in render_all() { + if observed.model.is_empty() { + continue; // Gemini carries the model in the URL, not the body. + } + assert!( + !observed.model.contains('/'), + "{provider}: model `{}` still carries a provider prefix", + observed.model + ); + } +} + +#[test] +fn every_provider_declares_every_tool_from_the_request_config() { + for (provider, observed) in render_all() { + let expected: BTreeSet = [TOOL_A, TOOL_B].into_iter().map(String::from).collect(); + assert_eq!( + observed.tool_names, expected, + "{provider}: tool declarations drifted" + ); + } +} + +#[test] +fn every_provider_pairs_each_tool_call_with_exactly_one_result() { + for (provider, observed) in render_all() { + assert_eq!( + observed.calls.len(), + 2, + "{provider}: expected both scripted tool calls, got {:?}", + observed.calls + ); + assert_eq!( + observed + .calls + .iter() + .map(|(_, name)| name.as_str()) + .collect::>(), + vec![TOOL_A, TOOL_B], + "{provider}: tool call order changed" + ); + let call_ids: Vec<&str> = observed.calls.iter().map(|(id, _)| id.as_str()).collect(); + let result_ids: Vec<&str> = observed + .results + .iter() + .map(|result| result.id.as_str()) + .collect(); + assert_eq!( + result_ids, call_ids, + "{provider}: every call must be answered once, in order, with the same identity \ + (interrupted calls must be completed with an error result, never dropped)" + ); + let unique: BTreeSet<&str> = call_ids.iter().copied().collect(); + assert_eq!( + unique.len(), + call_ids.len(), + "{provider}: distinct calls collapsed onto one wire identity" + ); + } +} + +#[test] +fn every_provider_reports_an_interrupted_call_as_an_error_not_a_success() { + for (provider, observed) in render_all() { + let Some(interrupted) = observed + .results + .iter() + .find(|result| result.id != CALL_A && result.id != TOOL_A) + else { + panic!("{provider}: no result for the interrupted call"); + }; + if let Some(is_error) = interrupted.is_error { + assert!( + is_error, + "{provider}: a synthesized result for an interrupted call must be marked as an error" + ); + } + assert!( + interrupted.content.contains(INTERRUPTED_MARKER), + "{provider}: the model must be told the call was interrupted, wire content was: {}", + interrupted.content + ); + let first = observed + .results + .iter() + .find(|result| result.id == CALL_A || result.id == TOOL_A) + .expect("result for the completed call"); + assert!( + first.content.contains(RESULT_A), + "{provider}: completed tool result payload missing from the wire" + ); + assert!( + !observed.text.contains(RESULT_A), + "{provider}: tool result leaked into plain text" + ); + } +} + +#[test] +fn every_provider_keeps_the_conversation_text_in_order_without_duplication() { + for (provider, observed) in render_all() { + for fragment in [SYSTEM_PROMPT, USER_ONE, ASSISTANT_TEXT, USER_TWO] { + assert_eq!( + observed.text.matches(fragment).count(), + 1, + "{provider}: `{fragment}` must appear exactly once, text was:\n{}", + observed.text + ); + } + let positions: Vec = [USER_ONE, ASSISTANT_TEXT, USER_TWO] + .iter() + .map(|fragment| observed.text.find(fragment).expect("fragment present")) + .collect(); + assert!( + positions.windows(2).all(|pair| pair[0] < pair[1]), + "{provider}: conversation order changed" + ); + } +} diff --git a/packages/ai-rs/src/transform.rs b/packages/ai-rs/src/transform.rs index d2e352772..4f724b40f 100644 --- a/packages/ai-rs/src/transform.rs +++ b/packages/ai-rs/src/transform.rs @@ -96,7 +96,7 @@ pub(crate) fn google_messages_for_wire(messages: &[Message]) -> Vec { /// Complete interrupted tool sequences only in the outgoing request. A missing /// result is reported as an error, never as successful execution. -fn repair_tool_sequence(messages: Vec) -> Vec { +pub(crate) fn repair_tool_sequence(messages: Vec) -> Vec { let mut result = Vec::new(); let mut pending: Vec = Vec::new(); for mut message in messages { diff --git a/packages/ai-rs/src/vertex.rs b/packages/ai-rs/src/vertex.rs index 00feff3fe..cebeaffb4 100644 --- a/packages/ai-rs/src/vertex.rs +++ b/packages/ai-rs/src/vertex.rs @@ -177,6 +177,16 @@ impl VertexAiClient { } /// Build the Vertex AI request body + /// Serialized request body, used by the provider conformance matrix. + #[cfg(test)] + pub(crate) fn request_body_json( + &self, + messages: &[Message], + config: &RequestConfig, + ) -> Result { + Ok(serde_json::to_value(self.build_request(messages, config)?)?) + } + fn build_request(&self, messages: &[Message], config: &RequestConfig) -> Result { let messages = crate::cache_topology::messages_with_volatile_tail(messages, config); let messages = super::transform::google_messages_for_wire(&messages); diff --git a/packages/local-host-rs/src/agent/mod.rs b/packages/local-host-rs/src/agent/mod.rs index 18c2628cb..8b7d62a06 100644 --- a/packages/local-host-rs/src/agent/mod.rs +++ b/packages/local-host-rs/src/agent/mod.rs @@ -184,6 +184,7 @@ impl NativeAgentConfig { max_turn_steps: self.max_turn_steps, allow_unbounded_turn: self.allow_unbounded_turn, retry_config: self.retry_config, + safety_config: crate::agent::safety::SafetyConfig::default(), } } } diff --git a/packages/local-host-rs/src/headless/messages/tests.rs b/packages/local-host-rs/src/headless/messages/tests.rs index b58e2cc39..8d323839a 100644 --- a/packages/local-host-rs/src/headless/messages/tests.rs +++ b/packages/local-host-rs/src/headless/messages/tests.rs @@ -24,6 +24,20 @@ fn headless_receipt_event_contains_record_lineage_and_status_only() { ); } +#[test] +fn managed_gateway_receipt_decodes_as_known_and_rejects_malformed_payloads() { + let raw = r#"{"type":"managed_gateway_receipt","request_id":"request-1","record_id":"record-1","lineage_id":"lineage-1","record_status":"planned"}"#; + assert!(matches!( + decode_from_agent_message(raw).expect("valid producer receipt"), + maestro_runtime::TaggedMessageDecode::Known(FromAgentMessage::ManagedGatewayReceipt { record_id, .. }) + if record_id == "record-1" + )); + assert!(matches!( + decode_from_agent_message(r#"{"type":"managed_gateway_receipt","request_id":"request-1"}"#), + Err(maestro_runtime::TaggedMessageDecodeError::InvalidKnownMessage { .. }) + )); +} + #[test] fn prompt_experiment_assignment_and_exposure_round_trip() { let assignment = PromptExperimentAssignment { diff --git a/packages/local-host-rs/src/headless/proto.rs b/packages/local-host-rs/src/headless/proto.rs index dd2879ae5..bceb7edd0 100644 --- a/packages/local-host-rs/src/headless/proto.rs +++ b/packages/local-host-rs/src/headless/proto.rs @@ -574,6 +574,25 @@ mod tests { resume: true, steering: true, }, + FromAgentMessage::ManagedGatewayReceipt { + request_id: "request-1".into(), + record_id: "record-1".into(), + lineage_id: "lineage-1".into(), + record_status: "planned".into(), + prompt_experiment: None, + }, + FromAgentMessage::ProcessBudgetCheckpoint { + budget: crate::agent::process_budget::ProcessBudgetState::new( + crate::agent::process_budget::ProcessBudgetLimits { + event_id: "event-1".into(), + max_requests: 1, + max_total_tokens: 1, + max_cost_micros: 1, + cost_micros_per_token: 1, + }, + ) + .expect("valid process budget limits"), + }, ] } diff --git a/packages/local-host-rs/src/headless/session.rs b/packages/local-host-rs/src/headless/session.rs index 867257848..e921385aa 100644 --- a/packages/local-host-rs/src/headless/session.rs +++ b/packages/local-host-rs/src/headless/session.rs @@ -99,7 +99,11 @@ use super::messages::{ GovernedClientToolBinding, InitConfig, PendingApproval, ServerRequestType, StreamingResponse, ToAgentMessage, TokenUsage, }; -use super::workspace_capabilities::{ApplyWorkspaceCapabilitySet, WorkspaceCapabilitySetApplied}; +#[cfg(test)] +use super::workspace_capabilities::WorkspaceCapabilitySetApplied; +use super::workspace_capabilities::{ + ApplyWorkspaceCapabilitySet, accept_workspace_capability_receipt, +}; fn workspace_capability_replay_cursor(request: &ApplyWorkspaceCapabilitySet) -> String { format!( @@ -117,27 +121,6 @@ fn record_sent_workspace_capability( } } -fn accept_workspace_capability_receipt( - pending: &mut HashMap, - last_accepted: &mut Option, - receipt: &WorkspaceCapabilitySetApplied, -) -> bool { - let Some(request) = pending.get(&receipt.replay_cursor) else { - return false; - }; - let matches = receipt.organization_id == request.organization_id - && receipt.workspace_id == request.workspace_id - && receipt.runner_session_id == request.runner_session_id - && receipt.runtime_generation == request.runtime_generation - && receipt.activation_generation == request.activation_generation - && receipt.effective_catalog_digest == request.capability_set_digest; - if matches { - *last_accepted = pending.remove(&receipt.replay_cursor); - return true; - } - false -} - fn init_config_from_message(message: &ToAgentMessage) -> Option { match message { ToAgentMessage::Init { @@ -1165,7 +1148,13 @@ impl SessionRecorder { // this never installs it into the live agent or grants authority. self.last_process_budget = Some(budget.clone()); } - let _ = self.replay_state.handle_message(message.clone()); + if !matches!( + message, + FromAgentMessage::WorkspaceCapabilitySetApplied { .. } + ) || workspace_capability_accepted + { + let _ = self.replay_state.handle_message(message.clone()); + } self.entries_since_checkpoint += 1; self.maybe_write_checkpoint( matches!( @@ -1694,13 +1683,16 @@ fn apply_replay_entry( } } SessionEntry::Received { message, .. } => { - if let FromAgentMessage::WorkspaceCapabilitySetApplied { receipt } = message { - let _ = accept_workspace_capability_receipt( - pending_workspace_capability_sets, - last_workspace_capability_set, - receipt, - ); - } + let accepted_capability_receipt = + if let FromAgentMessage::WorkspaceCapabilitySetApplied { receipt } = message { + accept_workspace_capability_receipt( + pending_workspace_capability_sets, + last_workspace_capability_set, + receipt, + ) + } else { + true + }; if let FromAgentMessage::ConversationSnapshot { protocol_version, messages, @@ -1711,7 +1703,9 @@ fn apply_replay_entry( == crate::headless::messages::SEMANTIC_CONVERSATION_PROTOCOL) .then_some(messages.clone()); } - let _ = state.handle_message(message.clone()); + if accepted_capability_receipt { + let _ = state.handle_message(message.clone()); + } } SessionEntry::Checkpoint { state: checkpoint, @@ -1914,13 +1908,17 @@ impl SessionReader { } } SessionEntry::Received { message, .. } => { - if let FromAgentMessage::WorkspaceCapabilitySetApplied { receipt } = message { - let _ = accept_workspace_capability_receipt( - &mut pending_workspace_capability_sets, - &mut last_workspace_capability_set, - receipt, - ); - } + let accepted_capability_receipt = + if let FromAgentMessage::WorkspaceCapabilitySetApplied { receipt } = message + { + accept_workspace_capability_receipt( + &mut pending_workspace_capability_sets, + &mut last_workspace_capability_set, + receipt, + ) + } else { + true + }; if let FromAgentMessage::ConversationSnapshot { protocol_version, messages, @@ -1935,7 +1933,9 @@ impl SessionReader { semantic_conversation = None; } } - let _ = state.handle_message(message.clone()); + if accepted_capability_receipt { + let _ = state.handle_message(message.clone()); + } } SessionEntry::Checkpoint { .. } => {} } @@ -2114,14 +2114,18 @@ mod tests { request: &ApplyWorkspaceCapabilitySet, ) -> WorkspaceCapabilitySetApplied { WorkspaceCapabilitySetApplied { - schema_version: "evalops.maestro.workspace-capability-set.v1".to_string(), + schema_version: "evalops.maestro.workspace-prompt-capability-set.v1".to_string(), organization_id: request.organization_id.clone(), workspace_id: request.workspace_id.clone(), runner_session_id: request.runner_session_id.clone(), runtime_generation: request.runtime_generation, activation_generation: request.activation_generation, effective_catalog_digest: request.capability_set_digest.clone(), - accepted_entry_digests: Vec::new(), + accepted_entry_digests: request + .admitted_catalog + .iter() + .map(|entry| entry.entry_digest.clone()) + .collect(), rejected_entries: Vec::new(), replay_cursor: workspace_capability_replay_cursor(request), applied_at: 123, @@ -2399,6 +2403,46 @@ mod tests { assert_eq!(replay.last_workspace_capability_set, Some(accepted)); } + #[test] + fn session_restart_ignores_partial_and_late_older_capability_receipts() { + let temp = TempDir::new().expect("session root"); + let mut recorder = SessionRecorder::new(temp.path()).expect("session recorder"); + let session_id = recorder.id().to_string(); + let first = workspace_capability_request(1); + let second = workspace_capability_request(2); + for request in [&first, &second] { + recorder + .record_sent(&ToAgentMessage::ApplyWorkspaceCapabilitySet { + request: request.clone(), + }) + .expect("record candidate"); + } + let mut partial = workspace_capability_receipt(&second); + partial.rejected_entries.push("skill.review".to_string()); + recorder + .record_received(&FromAgentMessage::WorkspaceCapabilitySetApplied { receipt: partial }) + .expect("record partial receipt"); + assert!(recorder.replay().last_workspace_capability_set.is_none()); + + recorder + .record_received(&FromAgentMessage::WorkspaceCapabilitySetApplied { + receipt: workspace_capability_receipt(&second), + }) + .expect("record accepted receipt"); + recorder + .record_received(&FromAgentMessage::WorkspaceCapabilitySetApplied { + receipt: workspace_capability_receipt(&first), + }) + .expect("record late older receipt"); + recorder.flush().expect("flush journal"); + drop(recorder); + + let replay = SessionRecorder::resume(temp.path(), &session_id) + .expect("resume recorder") + .replay(); + assert_eq!(replay.last_workspace_capability_set, Some(second)); + } + #[test] fn governed_init_grant_identity_survives_session_replay() { let temp = TempDir::new().unwrap(); diff --git a/packages/local-host-rs/src/headless/supervisor.rs b/packages/local-host-rs/src/headless/supervisor.rs index 7e7f15429..e2e78ff59 100644 --- a/packages/local-host-rs/src/headless/supervisor.rs +++ b/packages/local-host-rs/src/headless/supervisor.rs @@ -30,7 +30,9 @@ use super::remote_transport::{ RemoteAgentTransport, RemoteConnectionResumeAuthority, RemoteIncoming, RemoteTransportConfig, }; use super::session::{SessionRecorder, SessionReplay}; -use super::workspace_capabilities::{ApplyWorkspaceCapabilitySet, WorkspaceCapabilitySetApplied}; +use super::workspace_capabilities::{ + ApplyWorkspaceCapabilitySet, WorkspaceCapabilitySetApplied, accept_workspace_capability_receipt, +}; const MAX_STALE_REMOTE_REFERENCE_RETRIES: u32 = 3; const MIN_RECONNECT_SLEEP: Duration = Duration::from_millis(1); @@ -46,19 +48,6 @@ fn workspace_capability_replay_cursor(request: &ApplyWorkspaceCapabilitySet) -> ) } -fn workspace_capability_receipt_matches( - request: &ApplyWorkspaceCapabilitySet, - receipt: &WorkspaceCapabilitySetApplied, -) -> bool { - receipt.replay_cursor == workspace_capability_replay_cursor(request) - && receipt.organization_id == request.organization_id - && receipt.workspace_id == request.workspace_id - && receipt.runner_session_id == request.runner_session_id - && receipt.runtime_generation == request.runtime_generation - && receipt.activation_generation == request.activation_generation - && receipt.effective_catalog_digest == request.capability_set_digest -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum ResponseAcknowledgement { NotExpected, @@ -1187,10 +1176,15 @@ impl AgentSupervisor { == crate::headless::messages::SEMANTIC_CONVERSATION_PROTOCOL) .then(|| messages.clone()); } - if let FromAgentMessage::WorkspaceCapabilitySetApplied { receipt } = &message { - self.accept_workspace_capability_receipt(receipt); - } - let event = self.state.handle_message(message.clone()); + let accepted_capability_receipt = + if let FromAgentMessage::WorkspaceCapabilitySetApplied { receipt } = &message { + self.accept_workspace_capability_receipt(receipt) + } else { + true + }; + let event = accepted_capability_receipt + .then(|| self.state.handle_message(message.clone())) + .flatten(); if let Some(ref mut recorder) = self.session_recorder { let result = recorder.record_received(&message); self.report_session_recorder_result(result, "record received message"); @@ -1211,18 +1205,15 @@ impl AgentSupervisor { event.map(|event| SupervisorEvent::Agent(Box::new(event))) } - fn accept_workspace_capability_receipt(&mut self, receipt: &WorkspaceCapabilitySetApplied) { - let Some(request) = self - .pending_workspace_capability_sets - .get(&receipt.replay_cursor) - else { - return; - }; - if workspace_capability_receipt_matches(request, receipt) { - self.last_workspace_capability_set = self - .pending_workspace_capability_sets - .remove(&receipt.replay_cursor); - } + fn accept_workspace_capability_receipt( + &mut self, + receipt: &WorkspaceCapabilitySetApplied, + ) -> bool { + accept_workspace_capability_receipt( + &mut self.pending_workspace_capability_sets, + &mut self.last_workspace_capability_set, + receipt, + ) } fn clear_transient_progress_state(&mut self) { diff --git a/packages/local-host-rs/src/headless/supervisor/tests.rs b/packages/local-host-rs/src/headless/supervisor/tests.rs index d28990884..6199a61d9 100644 --- a/packages/local-host-rs/src/headless/supervisor/tests.rs +++ b/packages/local-host-rs/src/headless/supervisor/tests.rs @@ -2031,7 +2031,7 @@ async fn reconnect_replays_last_init_and_matching_accepted_capability_set() { }) .expect("send capability set"); let receipt = WorkspaceCapabilitySetApplied { - schema_version: "evalops.maestro.workspace-capability-set.v1".to_string(), + schema_version: "evalops.maestro.workspace-prompt-capability-set.v1".to_string(), organization_id: capability.organization_id.clone(), workspace_id: capability.workspace_id.clone(), runner_session_id: capability.runner_session_id.clone(), @@ -2047,6 +2047,17 @@ async fn reconnect_replays_last_init_and_matching_accepted_capability_set() { staged_for_next_turn: false, idempotent: false, }; + let mut partial = receipt.clone(); + partial.rejected_entries.push("skill.review".to_string()); + assert!( + supervisor + .apply_agent_message(FromAgentMessage::WorkspaceCapabilitySetApplied { + receipt: partial, + }) + .is_none(), + "a partial receipt must not become a live activation event" + ); + assert!(supervisor.last_workspace_capability_set.is_none()); let _ = supervisor.apply_agent_message(FromAgentMessage::WorkspaceCapabilitySetApplied { receipt }); diff --git a/packages/local-host-rs/src/headless/workspace_capabilities.rs b/packages/local-host-rs/src/headless/workspace_capabilities.rs index 4a277c5d1..bc31e7f41 100644 --- a/packages/local-host-rs/src/headless/workspace_capabilities.rs +++ b/packages/local-host-rs/src/headless/workspace_capabilities.rs @@ -4,6 +4,7 @@ //! prompt. It deliberately does not create tools, credentials, approvals, or //! any other executable authority. +use std::collections::HashMap; use std::time::{SystemTime, UNIX_EPOCH}; use serde::{Deserialize, Serialize}; @@ -77,6 +78,51 @@ pub struct WorkspaceCapabilitySetApplied { pub idempotent: bool, } +/// A runtime receipt may become reconnect authority only for the complete +/// admitted set, and only if it cannot roll back a newer accepted generation. +pub(crate) fn accept_workspace_capability_receipt( + pending: &mut HashMap, + last_accepted: &mut Option, + receipt: &WorkspaceCapabilitySetApplied, +) -> bool { + let Some(request) = pending.get(&receipt.replay_cursor) else { + return false; + }; + let matches = receipt.schema_version == PROMPT_CAPABILITY_SCHEMA_VERSION + && receipt.replay_cursor + == format!( + "{}:{}", + request.activation_generation, request.capability_set_digest + ) + && receipt.organization_id == request.organization_id + && receipt.workspace_id == request.workspace_id + && receipt.runner_session_id == request.runner_session_id + && receipt.runtime_generation == request.runtime_generation + && receipt.activation_generation == request.activation_generation + && receipt.effective_catalog_digest == request.capability_set_digest + && receipt.accepted_entry_digests + == request + .admitted_catalog + .iter() + .map(|entry| entry.entry_digest.as_str()) + .collect::>() + && receipt.rejected_entries.is_empty() + && last_accepted.as_ref().is_none_or(|accepted| { + request.activation_generation > accepted.activation_generation + || (request.activation_generation == accepted.activation_generation + && request.capability_set_digest == accepted.capability_set_digest) + }); + if !matches { + return false; + } + let accepted = pending + .remove(&receipt.replay_cursor) + .expect("matched pending capability set"); + pending.retain(|_, request| request.activation_generation > accepted.activation_generation); + *last_accepted = Some(accepted); + true +} + #[derive(Debug, Error, Eq, PartialEq)] pub(crate) enum WorkspaceCapabilityError { #[error( diff --git a/packages/local-host-rs/src/headless/workspace_capabilities_test.rs b/packages/local-host-rs/src/headless/workspace_capabilities_test.rs index 6648a3add..f36010790 100644 --- a/packages/local-host-rs/src/headless/workspace_capabilities_test.rs +++ b/packages/local-host-rs/src/headless/workspace_capabilities_test.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use sha2::{Digest, Sha256}; use super::controller_binding::{ @@ -6,9 +8,82 @@ use super::controller_binding::{ }; use super::workspace_capabilities::{ ApplyWorkspaceCapabilitySet, WorkspaceCapabilityActivation, WorkspacePromptCapability, - recompute_request_digests, + accept_workspace_capability_receipt, recompute_request_digests, }; +#[test] +fn runtime_receipts_cannot_promote_partial_sets_or_roll_back_replay() { + let mut activation = WorkspaceCapabilityActivation::new("base prompt".to_string()); + let first = request(1, "first generation"); + let second = request(2, "second generation"); + let first_receipt = activation + .apply( + first.clone(), + &binding(), + &resident_context(), + "runner-1", + false, + ) + .expect("first capability set"); + let second_receipt = activation + .apply( + second.clone(), + &binding(), + &resident_context(), + "runner-1", + false, + ) + .expect("second capability set"); + let mut pending = HashMap::from([ + (first_receipt.replay_cursor.clone(), first.clone()), + (second_receipt.replay_cursor.clone(), second.clone()), + ]); + let mut accepted = None; + + for malformed in [ + { + let mut receipt = second_receipt.clone(); + receipt.schema_version = "unknown".to_string(); + receipt + }, + { + let mut receipt = second_receipt.clone(); + receipt.accepted_entry_digests.clear(); + receipt + }, + { + let mut receipt = second_receipt.clone(); + receipt.rejected_entries.push("skill.review".to_string()); + receipt + }, + ] { + assert!(!accept_workspace_capability_receipt( + &mut pending, + &mut accepted, + &malformed, + )); + assert!(accepted.is_none()); + } + + assert!(accept_workspace_capability_receipt( + &mut pending, + &mut accepted, + &second_receipt, + )); + assert_eq!(accepted, Some(second.clone())); + assert!( + pending.is_empty(), + "older pending sets lose replay authority" + ); + pending.insert(first_receipt.replay_cursor.clone(), first); + assert!(!accept_workspace_capability_receipt( + &mut pending, + &mut accepted, + &first_receipt, + )); + assert_eq!(accepted, Some(second)); +} + fn resident_context() -> ControllerContext { ControllerContext { schema_version: CONTROLLER_CONTEXT_SCHEMA_VERSION.to_string(), diff --git a/packages/local-host-rs/src/hosted_runner.rs b/packages/local-host-rs/src/hosted_runner.rs index 7fbad87e7..79e971f97 100644 --- a/packages/local-host-rs/src/hosted_runner.rs +++ b/packages/local-host-rs/src/hosted_runner.rs @@ -51,6 +51,7 @@ use crate::headless::{ use crate::headless_server::{GovernedGrantVerificationContext, verify_governed_tool_grant}; mod config; +mod env_aliases; mod handle; mod initial_actions; mod manifests; @@ -62,11 +63,10 @@ mod snapshots; mod thread_protocol; mod workload_identity; +use self::{manifests::*, snapshots::*, thread_protocol::*}; pub use config::{HostedRunnerConfig, HostedRunnerConfigError}; +pub use env_aliases::{DEPRECATED_ENV_ALIASES, DeprecatedEnvAlias}; pub use handle::{HostedRunnerHandle, HostedRunnerIdentity}; -use manifests::*; -use snapshots::*; -use thread_protocol::*; pub const HOSTED_RUNNER_IDENTITY_PATH: &str = "/.well-known/evalops/remote-runner/identity"; pub const HOSTED_RUNNER_DRAIN_PATH: &str = "/.well-known/evalops/remote-runner/drain"; diff --git a/packages/local-host-rs/src/hosted_runner/config.rs b/packages/local-host-rs/src/hosted_runner/config.rs index 7f8381082..7610c6de8 100644 --- a/packages/local-host-rs/src/hosted_runner/config.rs +++ b/packages/local-host-rs/src/hosted_runner/config.rs @@ -8,6 +8,7 @@ use maestro_runtime::{ HostedRuntimeBoundaryInput, MAX_RUNTIME_RECEIPT_STRING_BYTES, }; +use super::env_aliases::{DeprecatedEnvAlias, deprecated_env_aliases}; use super::manifests::SnapshotManifest; use super::rendezvous_protocol::{RendezvousMode, RendezvousNonce}; @@ -35,6 +36,8 @@ pub struct HostedRunnerConfig { pub auth_token: Option, pub workload_identity: Option, pub rendezvous: Option, + /// Legacy environment keys that supplied a value for this config. + pub deprecated_env_aliases: Vec, } #[derive(Debug, Clone)] @@ -119,6 +122,7 @@ impl HostedRunnerConfig { } pub fn from_env_map(env: &HashMap) -> Result { + let deprecated_env_aliases = deprecated_env_aliases(env); let runner_session_id = first_env( env, &["MAESTRO_RUNNER_SESSION_ID", "REMOTE_RUNNER_SESSION_ID"], @@ -245,6 +249,7 @@ impl HostedRunnerConfig { auth_token, workload_identity, rendezvous, + deprecated_env_aliases, }; config.validate_runtime_receipt_identity(None)?; Ok(config) @@ -423,6 +428,7 @@ impl HostedRunnerConfig { auth_token, workload_identity, rendezvous, + deprecated_env_aliases: Vec::new(), }; config.validate_runtime_receipt_identity(None)?; Ok(config) @@ -452,6 +458,7 @@ impl HostedRunnerConfig { auth_token: None, workload_identity: None, rendezvous: None, + deprecated_env_aliases: Vec::new(), }; config.validate_runtime_receipt_identity(None)?; Ok(config) @@ -816,7 +823,7 @@ fn first_env(env: &HashMap, keys: &[&str]) -> Option { keys.iter().find_map(|key| env_value(env, key)) } -fn env_value(env: &HashMap, key: &str) -> Option { +pub(super) fn env_value(env: &HashMap, key: &str) -> Option { env.get(key).map(|value| value.trim()).and_then(|value| { if value.is_empty() { None diff --git a/packages/local-host-rs/src/hosted_runner/env_aliases.rs b/packages/local-host-rs/src/hosted_runner/env_aliases.rs new file mode 100644 index 000000000..96f649e39 --- /dev/null +++ b/packages/local-host-rs/src/hosted_runner/env_aliases.rs @@ -0,0 +1,74 @@ +//! Legacy hosted-runner environment keys and the warning metadata the +//! startup path emits for them. + +use std::collections::HashMap; + +use super::config::env_value; + +/// Legacy environment keys still honoured for one deprecation window, paired +/// with the canonical `MAESTRO_*` key that replaces each of them. The +/// canonical key always wins; every legacy key that carries a value is +/// reported on the resolved config so the runner can warn at startup. +pub const DEPRECATED_ENV_ALIASES: &[(&str, &str)] = &[ + ("REMOTE_RUNNER_SESSION_ID", "MAESTRO_RUNNER_SESSION_ID"), + ("WORKSPACE_ROOT", "MAESTRO_WORKSPACE_ROOT"), + ("PORT", "MAESTRO_HOSTED_RUNNER_PORT"), + ("MAESTRO_WEB_API_KEY", "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN"), + ( + "MAESTRO_WEB_API_KEY_FILE", + "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN_FILE", + ), + ( + "REMOTE_RUNNER_SNAPSHOT_ROOT", + "MAESTRO_REMOTE_RUNNER_SNAPSHOT_ROOT", + ), + ( + "REMOTE_RUNNER_RESTORE_MANIFEST", + "MAESTRO_REMOTE_RUNNER_RESTORE_MANIFEST", + ), + ( + "REMOTE_RUNNER_OWNER_INSTANCE_ID", + "MAESTRO_REMOTE_RUNNER_OWNER_INSTANCE_ID", + ), +]; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeprecatedEnvAlias { + pub alias: &'static str, + pub canonical: &'static str, + /// The canonical key was also set with a different value, so the legacy + /// value was ignored. + pub shadowed: bool, +} + +impl DeprecatedEnvAlias { + pub fn warning(&self) -> String { + let shadowed = if self.shadowed { + format!( + "; its value disagrees with {} and was ignored", + self.canonical + ) + } else { + String::new() + }; + format!( + "hosted-runner environment key {} is deprecated; set {} instead (the legacy key stops being read after the next Maestro minor release){shadowed}", + self.alias, self.canonical + ) + } +} + +pub(super) fn deprecated_env_aliases(env: &HashMap) -> Vec { + DEPRECATED_ENV_ALIASES + .iter() + .filter_map(|(alias, canonical)| { + let legacy_value = env_value(env, alias)?; + let shadowed = env_value(env, canonical).is_some_and(|value| value != legacy_value); + Some(DeprecatedEnvAlias { + alias, + canonical, + shadowed, + }) + }) + .collect() +} diff --git a/packages/local-host-rs/src/hosted_runner/tests.rs b/packages/local-host-rs/src/hosted_runner/tests.rs index e21de09cc..b1dd68471 100644 --- a/packages/local-host-rs/src/hosted_runner/tests.rs +++ b/packages/local-host-rs/src/hosted_runner/tests.rs @@ -1,7 +1,6 @@ use reqwest::StatusCode; -use std::fs; -use std::sync::Condvar; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::{fs, sync::Condvar}; use tempfile::tempdir; use tokio::sync::Notify; @@ -10,11 +9,11 @@ use std::os::unix::fs::PermissionsExt; use super::config::HostedRunnerWorkloadIdentityConfig; use super::*; -use crate::headless::RemoteTransportConfig; use crate::headless::messages::{CodexSubagentContinuityEdge, ToolRetryDecisionAction}; -use crate::headless::{NativeToolCapability, PendingApproval}; +use crate::headless::{NativeToolCapability, PendingApproval, RemoteTransportConfig}; use crate::hosted_runner::rendezvous_protocol::RendezvousMode; +mod env_aliases; mod transport_flush; #[test] @@ -2690,6 +2689,7 @@ fn test_config(workspace_root: PathBuf) -> HostedRunnerConfig { auth_token: None, workload_identity: None, rendezvous: None, + deprecated_env_aliases: Vec::new(), } } diff --git a/packages/local-host-rs/src/hosted_runner/tests/env_aliases.rs b/packages/local-host-rs/src/hosted_runner/tests/env_aliases.rs new file mode 100644 index 000000000..0b678c77b --- /dev/null +++ b/packages/local-host-rs/src/hosted_runner/tests/env_aliases.rs @@ -0,0 +1,151 @@ +use std::collections::BTreeSet; +use std::fs; + +use tempfile::tempdir; + +use super::*; + +#[test] +fn shadowed_legacy_web_api_key_is_reported_and_ignored() { + let workspace = tempdir().expect("workspace"); + let mut env = base_hosted_runner_env(workspace.path()); + env.insert( + "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN".to_string(), + "runner-secret".to_string(), + ); + env.insert( + "MAESTRO_WEB_API_KEY".to_string(), + "legacy-secret".to_string(), + ); + + let config = HostedRunnerConfig::from_env_map(&env).expect("config"); + + assert_eq!(config.auth_token.as_deref(), Some("runner-secret")); + assert_eq!( + config.deprecated_env_aliases, + vec![DeprecatedEnvAlias { + alias: "MAESTRO_WEB_API_KEY", + canonical: "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN", + shadowed: true, + }] + ); + assert!( + config.deprecated_env_aliases[0] + .warning() + .contains("disagrees with MAESTRO_HOSTED_RUNNER_AUTH_TOKEN and was ignored") + ); +} + +#[test] +fn canonical_hosted_runner_env_reports_no_deprecated_aliases() { + let workspace = tempdir().expect("workspace"); + let mut env = base_hosted_runner_env(workspace.path()); + env.insert( + "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN".to_string(), + "runner-secret".to_string(), + ); + env.insert("MAESTRO_HOSTED_RUNNER_PORT".to_string(), "9091".to_string()); + + let config = HostedRunnerConfig::from_env_map(&env).expect("config"); + + assert!(config.deprecated_env_aliases.is_empty()); +} + +#[test] +fn every_deprecated_hosted_runner_env_alias_resolves_like_its_canonical_key() { + let workspace = tempdir().expect("workspace"); + let snapshot_root = workspace.path().join("snapshots"); + let manifest = workspace.path().join("restore.json"); + let token_file = workspace.path().join("token"); + fs::write(&token_file, "file-secret").expect("token file"); + let values: HashMap<&str, String> = HashMap::from([ + ("MAESTRO_RUNNER_SESSION_ID", "mrs_alias".to_string()), + ( + "MAESTRO_WORKSPACE_ROOT", + workspace.path().display().to_string(), + ), + ("MAESTRO_HOSTED_RUNNER_PORT", "9093".to_string()), + ( + "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN", + "inline-secret".to_string(), + ), + ( + "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN_FILE", + token_file.display().to_string(), + ), + ( + "MAESTRO_REMOTE_RUNNER_SNAPSHOT_ROOT", + snapshot_root.display().to_string(), + ), + ( + "MAESTRO_REMOTE_RUNNER_RESTORE_MANIFEST", + manifest.display().to_string(), + ), + ( + "MAESTRO_REMOTE_RUNNER_OWNER_INSTANCE_ID", + "owner_alias".to_string(), + ), + ]); + let canonical_keys: BTreeSet<&str> = DEPRECATED_ENV_ALIASES + .iter() + .map(|(_, canonical)| *canonical) + .collect(); + assert_eq!( + canonical_keys, + values.keys().copied().collect::>(), + "alias table changed; extend this resolution matrix" + ); + + for (alias, canonical) in DEPRECATED_ENV_ALIASES { + let mut canonical_env = base_hosted_runner_env(workspace.path()); + let mut alias_env = base_hosted_runner_env(workspace.path()); + // Inline and file bearer forms are mutually exclusive, so only the + // form under test (or the inline form otherwise) is present. + let excluded_auth_key = if *canonical == "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN_FILE" { + "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN" + } else { + "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN_FILE" + }; + for (key, value) in &values { + if *key == excluded_auth_key { + continue; + } + canonical_env.insert((*key).to_string(), value.clone()); + alias_env.insert((*key).to_string(), value.clone()); + } + alias_env.remove(*canonical); + alias_env.insert((*alias).to_string(), values[canonical].clone()); + + let expected = HostedRunnerConfig::from_env_map(&canonical_env) + .unwrap_or_else(|error| panic!("canonical {canonical}: {error}")); + let actual = HostedRunnerConfig::from_env_map(&alias_env) + .unwrap_or_else(|error| panic!("alias {alias}: {error}")); + + assert!(expected.deprecated_env_aliases.is_empty(), "{canonical}"); + assert_eq!( + actual.deprecated_env_aliases, + vec![DeprecatedEnvAlias { + alias, + canonical, + shadowed: false, + }], + "{alias}" + ); + assert_eq!( + actual.runner_session_id, expected.runner_session_id, + "{alias}" + ); + assert_eq!(actual.workspace_root, expected.workspace_root, "{alias}"); + assert_eq!(actual.bind_addr, expected.bind_addr, "{alias}"); + assert_eq!(actual.auth_token, expected.auth_token, "{alias}"); + assert_eq!(actual.snapshot_root, expected.snapshot_root, "{alias}"); + assert_eq!( + actual.restore_manifest_path, expected.restore_manifest_path, + "{alias}" + ); + assert_eq!( + actual.owner_instance_id, expected.owner_instance_id, + "{alias}" + ); + } +} diff --git a/packages/local-host-rs/src/hosted_runner_cli.rs b/packages/local-host-rs/src/hosted_runner_cli.rs index e1ed105f6..67d17e332 100644 --- a/packages/local-host-rs/src/hosted_runner_cli.rs +++ b/packages/local-host-rs/src/hosted_runner_cli.rs @@ -484,6 +484,14 @@ async fn start_resolved_hosted_runner_cli_runtime( ) -> Result { let runtime_boundary = config.runtime_boundary()?; let launch_spec = config.launch_spec(&resolved_env)?; + for alias in &config.runner.deprecated_env_aliases { + tracing::warn!( + deprecated_env_key = alias.alias, + canonical_env_key = alias.canonical, + "{}", + alias.warning() + ); + } tracing::info!( runtime_product = maestro_runtime::RUNTIME_PRODUCT_ID, runtime_boundary = %runtime_boundary.schema_version, diff --git a/packages/local-host-rs/tests/fixtures/hosted-runner-startup-contract.json b/packages/local-host-rs/tests/fixtures/hosted-runner-startup-contract.json new file mode 100644 index 000000000..a489352c2 --- /dev/null +++ b/packages/local-host-rs/tests/fixtures/hosted-runner-startup-contract.json @@ -0,0 +1,415 @@ +{ + "schema_version": "evalops.maestro.hosted-runner-startup-contract.v1", + "description": "Executable startup matrix for `deixic-code hosted-runner`. Each case resolves the documented flag/environment coordinates through the real CLI resolver and asserts accept or fail-closed reject. `${workspace}` is a fresh temporary directory, `${agent}` an executable stub headless runtime, and `${file:}` a temporary file declared in `files`.", + "cases": [ + { + "name": "canonical MAESTRO_* environment is accepted", + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_canonical", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43123", + "MAESTRO_REMOTE_RUNNER_OWNER_INSTANCE_ID": "owner_canonical", + "MAESTRO_REMOTE_RUNNER_WORKSPACE_ID": "ws_canonical", + "MAESTRO_AGENT_RUN_ID": "run_canonical", + "MAESTRO_SESSION_ID": "maestro-session-canonical", + "MAESTRO_ATTACH_AUDIENCE": "platform", + "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN": "token-canonical", + "MAESTRO_MODEL": "gpt-5.5", + "MAESTRO_SANDBOXWICH_PLACEMENT_GENERATION": "7" + }, + "expect": { + "accept": { + "runner_session_id": "mrs_canonical", + "owner_instance_id": "owner_canonical", + "runtime_generation": 7, + "workspace_id": "ws_canonical", + "agent_run_id": "run_canonical", + "maestro_session_id": "maestro-session-canonical", + "attach_audience": "platform", + "bind_addr": "127.0.0.1:43123", + "auth_token_present": true, + "workload_identity_present": false + } + } + }, + { + "name": "legacy REMOTE_RUNNER_* aliases resolve to the same contract", + "env": { + "REMOTE_RUNNER_SESSION_ID": "mrs_legacy", + "WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_HOST": "127.0.0.1", + "PORT": "43124", + "REMOTE_RUNNER_OWNER_INSTANCE_ID": "owner_legacy", + "MAESTRO_REMOTE_RUNNER_GENERATION": "3", + "MAESTRO_WORKSPACE_ID": "ws_legacy", + "MAESTRO_WEB_API_KEY": "token-legacy", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "accept": { + "runner_session_id": "mrs_legacy", + "owner_instance_id": "owner_legacy", + "runtime_generation": 3, + "workspace_id": "ws_legacy", + "bind_addr": "127.0.0.1:43124", + "auth_token_present": true + } + } + }, + { + "name": "CLI flags override environment coordinates", + "args": [ + "--runner-session-id", + "mrs_flag", + "--workspace-root", + "${workspace}", + "--listen", + "127.0.0.1:43125", + "--owner-instance-id", + "owner_flag", + "--workspace-id", + "ws_flag", + "--agent-cli-path", + "${agent}" + ], + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_env", + "MAESTRO_REMOTE_RUNNER_OWNER_INSTANCE_ID": "owner_env", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "accept": { + "runner_session_id": "mrs_flag", + "owner_instance_id": "owner_flag", + "workspace_id": "ws_flag", + "bind_addr": "127.0.0.1:43125", + "agent_cli_path": "${agent}" + } + } + }, + { + "name": "sandboxwich placement generation takes precedence over the legacy generation alias", + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_generation", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43126", + "MAESTRO_SANDBOXWICH_PLACEMENT_GENERATION": "8", + "MAESTRO_REMOTE_RUNNER_GENERATION": "1", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "accept": { + "runtime_generation": 8 + } + } + }, + { + "name": "missing runner session id fails closed", + "env": { + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43127", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "requires MAESTRO_RUNNER_SESSION_ID" + } + }, + { + "name": "blank runner session id is treated as missing", + "env": { + "MAESTRO_RUNNER_SESSION_ID": " ", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43128", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "requires MAESTRO_RUNNER_SESSION_ID" + } + }, + { + "name": "missing workspace root fails closed", + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_no_workspace", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43129", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "MAESTRO_WORKSPACE_ROOT" + } + }, + { + "name": "workspace root that is not a directory fails closed", + "files": { + "not-a-dir": "plain file" + }, + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_file_workspace", + "MAESTRO_WORKSPACE_ROOT": "${file:not-a-dir}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43130", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "workspace root" + } + }, + { + "name": "non-loopback bind without attach authentication fails closed", + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_open_bind", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "0.0.0.0:43131", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "non-loopback" + } + }, + { + "name": "loopback bind without attach authentication fails closed unless local testing is explicit", + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_no_token", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43132", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "requires MAESTRO_HOSTED_RUNNER_AUTH_TOKEN or MAESTRO_WEB_API_KEY" + } + }, + { + "name": "inline token and token file together fail closed", + "files": { + "token": "token-from-file" + }, + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_two_tokens", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43133", + "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN": "inline", + "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN_FILE": "${file:token}", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "not both" + } + }, + { + "name": "token file reference is accepted", + "files": { + "token": "token-from-file\n" + }, + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_token_file", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43134", + "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN_FILE": "${file:token}", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "accept": { + "auth_token_present": true + } + } + }, + { + "name": "non-integer runtime generation fails closed", + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_bad_generation", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43135", + "MAESTRO_REMOTE_RUNNER_GENERATION": "seven", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "runtime generation" + } + }, + { + "name": "managed model requires the resident contract revision", + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_managed_no_revision", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43136", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "evalops/gpt-5.5" + }, + "expect": { + "reject": "MAESTRO_RESIDENT_CONTRACT_REVISION" + } + }, + { + "name": "managed model with the pinned resident contract revision and EvalOps credentials is accepted", + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_managed", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43137", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "evalops/gpt-5.5", + "MAESTRO_RESIDENT_CONTRACT_REVISION": "maestro-resident-model-ready-v3", + "MAESTRO_EVALOPS_ACCESS_TOKEN": "evalops-access-token", + "MAESTRO_EVALOPS_BASE_URL": "https://api.evalops.test", + "MAESTRO_EVALOPS_ORG_ID": "org_managed", + "MAESTRO_EVALOPS_WORKSPACE_ID": "ws_managed", + "MAESTRO_EVALOPS_PROVIDER": "openai" + }, + "expect": { + "accept": { + "runner_session_id": "mrs_managed" + } + } + }, + { + "name": "managed model without EvalOps credentials fails closed", + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_managed_no_creds", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43144", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "evalops/gpt-5.5", + "MAESTRO_RESIDENT_CONTRACT_REVISION": "maestro-resident-model-ready-v3" + }, + "expect": { + "reject": "managed headless model requires MAESTRO_EVALOPS_ACCESS_TOKEN" + } + }, + { + "name": "complete workload identity set is accepted without static bearer authentication and pins the placement generation", + "files": { + "token": "projected-token", + "ca": "-----BEGIN CERTIFICATE-----\n" + }, + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_wi", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "0.0.0.0:43142", + "MAESTRO_KUBERNETES_TOKEN_FILE": "${file:token}", + "MAESTRO_IDENTITY_TLS_CA_FILE": "${file:ca}", + "MAESTRO_IDENTITY_EXCHANGE_URL": "https://identity.evalops.test/exchange", + "MAESTRO_ORGANIZATION_ID": "org_wi", + "MAESTRO_WORKSPACE_ID": "ws_wi", + "MAESTRO_SANDBOX_ID": "6f1c2b6e-1e7c-4b6e-9c4e-2f0f4a7e8d21", + "MAESTRO_PLACEMENT_GENERATION": "4", + "MAESTRO_REMOTE_RUNNER_GENERATION": "1", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "accept": { + "runtime_generation": 4, + "workload_identity_present": true, + "auth_token_present": false, + "workspace_id": "ws_wi" + } + } + }, + { + "name": "placement generation alone enters workload identity mode and fails closed", + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_placement_only", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43143", + "MAESTRO_PLACEMENT_GENERATION": "4", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "projected workload identity requires MAESTRO_KUBERNETES_TOKEN_FILE" + } + }, + { + "name": "partial workload identity set fails closed", + "files": { + "token": "projected-token" + }, + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_partial_wi", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43138", + "MAESTRO_KUBERNETES_TOKEN_FILE": "${file:token}", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "projected workload identity requires" + } + }, + { + "name": "workload identity forbids static bearer authentication", + "files": { + "token": "projected-token", + "ca": "-----BEGIN CERTIFICATE-----\n" + }, + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_wi_static", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "0.0.0.0:43139", + "MAESTRO_KUBERNETES_TOKEN_FILE": "${file:token}", + "MAESTRO_IDENTITY_TLS_CA_FILE": "${file:ca}", + "MAESTRO_IDENTITY_EXCHANGE_URL": "https://identity.evalops.test/exchange", + "MAESTRO_ORGANIZATION_ID": "org_wi", + "MAESTRO_WORKSPACE_ID": "ws_wi", + "MAESTRO_SANDBOX_ID": "6f1c2b6e-1e7c-4b6e-9c4e-2f0f4a7e8d21", + "MAESTRO_PLACEMENT_GENERATION": "4", + "MAESTRO_HOSTED_RUNNER_AUTH_TOKEN": "static-token", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "forbids static bearer authentication" + } + }, + { + "name": "runner host client CA input is not part of the contract", + "files": { + "ca": "-----BEGIN CERTIFICATE-----\n" + }, + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_client_ca", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43140", + "MAESTRO_RUNNER_CLIENT_CA_FILE": "${file:ca}", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "MAESTRO_RUNNER_CLIENT_CA_FILE is not supported" + } + }, + { + "name": "outbound rendezvous without workload identity fails closed", + "env": { + "MAESTRO_RUNNER_SESSION_ID": "mrs_rendezvous", + "MAESTRO_WORKSPACE_ROOT": "${workspace}", + "MAESTRO_HOSTED_RUNNER_LISTEN": "127.0.0.1:43141", + "MAESTRO_RENDEZVOUS_MODE": "outbound_shadow", + "MAESTRO_WEB_REQUIRE_KEY": "0", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "outbound rendezvous requires projected workload identity" + } + }, + { + "name": "launch spec file and --config together fail closed", + "files": { + "spec": "{}" + }, + "args": [ + "--config", + "${file:spec}" + ], + "env": { + "MAESTRO_HOSTED_LAUNCH_SPEC_FILE": "${file:spec}", + "MAESTRO_MODEL": "gpt-5.5" + }, + "expect": { + "reject": "not both" + } + } + ] +} diff --git a/packages/local-host-rs/tests/hosted_runner_startup_contract.rs b/packages/local-host-rs/tests/hosted_runner_startup_contract.rs new file mode 100644 index 000000000..6fc673b41 --- /dev/null +++ b/packages/local-host-rs/tests/hosted_runner_startup_contract.rs @@ -0,0 +1,346 @@ +//! Executable form of `docs/protocols/hosted-runner-contract.md`. +//! +//! The startup matrix lives in `fixtures/hosted-runner-startup-contract.json` +//! so the documented flag/environment coordinates, the CLI resolver, and the +//! canary that launches real pods share one table. Every case runs through +//! `resolve_hosted_runner_launch_config`, the same entrypoint +//! `deixic-code hosted-runner` uses; a second test drives the full CLI runtime +//! (stub headless child, identity readiness, drain manifest) through +//! `start_hosted_runner_cli_runtime`. + +use std::collections::{BTreeMap, HashMap}; +use std::fs; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; + +use maestro_local_host::hosted_runner_cli::{ + HostedRunnerLaunchConfig, resolve_hosted_runner_launch_config, +}; +use serde::Deserialize; +use serde_json::Value; +use tempfile::TempDir; + +const FIXTURE: &str = include_str!("fixtures/hosted-runner-startup-contract.json"); +const CONTRACT_DOC: &str = include_str!("../../../docs/protocols/hosted-runner-contract.md"); +const SCHEMA_VERSION: &str = "evalops.maestro.hosted-runner-startup-contract.v1"; + +#[derive(Debug, Deserialize)] +struct Fixture { + schema_version: String, + cases: Vec, +} + +#[derive(Debug, Deserialize)] +struct Case { + name: String, + #[serde(default)] + files: BTreeMap, + #[serde(default)] + args: Vec, + #[serde(default)] + env: BTreeMap, + expect: Expect, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +enum Expect { + Accept(BTreeMap), + Reject(String), +} + +struct Scratch { + workspace: TempDir, + files: TempDir, + agent: PathBuf, +} + +impl Scratch { + fn new(case: &Case) -> Self { + let workspace = TempDir::new().expect("workspace tempdir"); + let files = TempDir::new().expect("files tempdir"); + for (name, content) in &case.files { + fs::write(files.path().join(name), content).expect("fixture file"); + } + let agent = write_stub_agent(workspace.path()); + Self { + workspace, + files, + agent, + } + } + + fn expand(&self, raw: &str) -> String { + let mut value = raw + .replace("${workspace}", &self.workspace.path().to_string_lossy()) + .replace("${agent}", &self.agent.to_string_lossy()); + while let Some(start) = value.find("${file:") { + let end = value[start..] + .find('}') + .map(|offset| start + offset) + .expect("unterminated ${file:} placeholder"); + let name = &value[start + "${file:".len()..end]; + let path = self.files.path().join(name); + assert!( + path.is_file(), + "fixture file {name} is not declared in `files`" + ); + value.replace_range(start..=end, &path.to_string_lossy()); + } + value + } +} + +/// A headless child that announces readiness and then idles until stdin closes. +fn write_stub_agent(dir: &Path) -> PathBuf { + let agent = dir.join("stub-maestro-headless.sh"); + fs::write( + &agent, + "#!/bin/sh\nprintf '%s\\n' '{\"type\":\"ready\",\"model\":\"gpt-5.5\",\"provider\":\"test\"}'\nwhile IFS= read -r line; do :; done\n", + ) + .expect("stub agent"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut permissions = fs::metadata(&agent).expect("metadata").permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).expect("chmod"); + } + agent +} + +fn load_fixture() -> Fixture { + let fixture: Fixture = serde_json::from_str(FIXTURE).expect("fixture parses"); + assert_eq!(fixture.schema_version, SCHEMA_VERSION); + assert!(!fixture.cases.is_empty()); + fixture +} + +fn resolve(case: &Case, scratch: &Scratch) -> anyhow::Result { + let args = std::iter::once("deixic-code hosted-runner".to_string()) + .chain(case.args.iter().map(|arg| scratch.expand(arg))) + .collect::>(); + let env = case + .env + .iter() + .map(|(key, value)| (key.clone(), scratch.expand(value))) + .collect::>(); + resolve_hosted_runner_launch_config(args, &env) +} + +fn observed(config: &HostedRunnerLaunchConfig, field: &str) -> Value { + let runner = &config.runner; + match field { + "runner_session_id" => runner.runner_session_id.clone().into(), + "owner_instance_id" => runner.owner_instance_id.clone().into(), + "runtime_generation" => runner.runtime_generation.into(), + "workspace_id" => runner.workspace_id.clone().into(), + "agent_run_id" => runner.agent_run_id.clone().into(), + "maestro_session_id" => runner.maestro_session_id.clone().into(), + "attach_audience" => runner.attach_audience.clone().into(), + "bind_addr" => runner.bind_addr.to_string().into(), + "auth_token_present" => runner.auth_token.is_some().into(), + "workload_identity_present" => runner.workload_identity.is_some().into(), + "agent_cli_path" => config.supervisor.transport.cli_path.clone().into(), + other => panic!("fixture asserts unknown field {other}"), + } +} + +#[test] +fn every_documented_startup_case_accepts_or_fails_closed() { + for case in load_fixture().cases { + let scratch = Scratch::new(&case); + let result = resolve(&case, &scratch); + match &case.expect { + Expect::Accept(fields) => { + let config = result.unwrap_or_else(|error| { + panic!("case `{}` must be accepted, got: {error:#}", case.name) + }); + for (field, expected) in fields { + let expected = match expected { + Value::String(raw) => Value::String(scratch.expand(raw)), + other => other.clone(), + }; + assert_eq!( + observed(&config, field), + expected, + "case `{}` field `{field}`", + case.name + ); + } + } + Expect::Reject(fragment) => { + let error = match result { + Ok(_) => panic!("case `{}` must fail closed", case.name), + Err(error) => format!("{error:#}"), + }; + assert!( + error.contains(fragment), + "case `{}` rejected for another reason: {error}", + case.name + ); + } + } + } +} + +#[test] +fn fixture_case_names_are_unique() { + let fixture = load_fixture(); + let mut names = fixture + .cases + .iter() + .map(|case| case.name.as_str()) + .collect::>(); + names.sort_unstable(); + names.dedup(); + assert_eq!(names.len(), fixture.cases.len(), "duplicate case names"); +} + +fn is_documented(name: &str) -> bool { + CONTRACT_DOC.contains(&format!("`{name}`")) || CONTRACT_DOC.contains(&format!("`{name}=")) +} + +/// Every coordinate the executable matrix exercises must be documented in the +/// prose contract, so a fixture case cannot introduce an undocumented input. +#[test] +fn every_exercised_coordinate_is_documented() { + let fixture = load_fixture(); + let mut exercised = fixture + .cases + .iter() + .flat_map(|case| case.env.keys().cloned()) + .chain(fixture.cases.iter().flat_map(|case| { + case.args + .iter() + .filter(|arg| arg.starts_with("--")) + .cloned() + })) + .collect::>(); + exercised.sort_unstable(); + exercised.dedup(); + + let missing_from_doc = exercised + .iter() + .filter(|name| !is_documented(name)) + .cloned() + .collect::>(); + assert!( + missing_from_doc.is_empty(), + "fixture coordinates absent from hosted-runner-contract.md: {missing_from_doc:?}" + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn cli_runtime_reports_identity_and_drains_a_manifest() { + use maestro_local_host::hosted_runner_cli::{ + HostedRunnerShutdownSignal, start_hosted_runner_cli_runtime, + }; + + let workspace = TempDir::new().expect("workspace"); + let agent = write_stub_agent(workspace.path()); + let env = HashMap::from([ + ("MAESTRO_WEB_REQUIRE_KEY".to_string(), "0".to_string()), + ("MAESTRO_MODEL".to_string(), "gpt-5.5".to_string()), + ]); + // `--listen` rejects port 0, so reserve an ephemeral port and retry on the + // rare race where another process takes it between release and bind. + let mut runtime = None; + for _ in 0..5 { + let listen = { + let probe = TcpListener::bind("127.0.0.1:0").expect("ephemeral port probe"); + probe.local_addr().expect("probe addr").to_string() + }; + match start_hosted_runner_cli_runtime( + [ + "deixic-code hosted-runner", + "--runner-session-id", + "mrs_contract", + "--owner-instance-id", + "owner_contract", + "--workspace-root", + workspace.path().to_str().expect("workspace path"), + "--listen", + listen.as_str(), + "--agent-cli-path", + agent.to_str().expect("agent path"), + ], + &env, + ) + .await + { + Ok(started) => { + runtime = Some(started); + break; + } + Err(error) if error.to_string().contains("address") => continue, + Err(error) => panic!("hosted runner failed to start: {error}"), + } + } + let runtime = runtime.expect("hosted runner starts behind the stub headless child"); + + let client = reqwest::Client::new(); + let identity: Value = client + .get(format!( + "{}/.well-known/evalops/remote-runner/identity", + runtime.base_url() + )) + .send() + .await + .expect("identity response") + .json() + .await + .expect("identity json"); + assert_eq!( + identity["protocol_version"], + "evalops.remote-runner.identity.v1" + ); + assert_eq!(identity["runner_session_id"], "mrs_contract"); + assert_eq!(identity["owner_instance_id"], "owner_contract"); + assert_eq!(identity["ready"], true); + assert_eq!(identity["draining"], false); + for hidden in ["workspace_id", "agent_run_id", "maestro_session_id"] { + assert!( + identity.get(hidden).is_none(), + "identity must stay sparse; leaked {hidden}" + ); + } + + let drain = runtime + .drain_for_shutdown(HostedRunnerShutdownSignal::Terminate) + .await + .expect("drain succeeds"); + assert_eq!(drain["status"], "drained"); + let manifest_path = drain["manifest_path"] + .as_str() + .unwrap_or_else(|| panic!("drain response carries a manifest path: {drain}")); + assert_eq!( + drain["manifest"]["workspace_export"]["mode"], + "local_path_contract" + ); + assert!( + Path::new(manifest_path).is_file(), + "drain manifest must be written under the workspace: {manifest_path}" + ); + assert!( + Path::new(manifest_path).starts_with(workspace.path()), + "drain manifest must stay under the workspace root" + ); + + let identity: Value = client + .get(format!( + "{}/.well-known/evalops/remote-runner/identity", + runtime.base_url() + )) + .send() + .await + .expect("identity response after drain") + .json() + .await + .expect("identity json after drain"); + assert_eq!(identity["draining"], true); + assert_eq!(identity["ready"], false); + + runtime.shutdown().await; +} diff --git a/packages/maestro-rs/Cargo.toml b/packages/maestro-rs/Cargo.toml index 9a61e1fee..a2283cafa 100644 --- a/packages/maestro-rs/Cargo.toml +++ b/packages/maestro-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "maestro" -version = "0.10.88" +version = "0.10.89" edition = "2021" license = "MIT" description = "Canonical native Rust CLI for Deixic Code" diff --git a/packages/runtime-contracts-rs/fixtures/headless-protocol-v1.json b/packages/runtime-contracts-rs/fixtures/headless-protocol-v1.json index ac3c257e3..79010a9c0 100644 --- a/packages/runtime-contracts-rs/fixtures/headless-protocol-v1.json +++ b/packages/runtime-contracts-rs/fixtures/headless-protocol-v1.json @@ -83,7 +83,9 @@ "codex_session_state", "codex_turn_state", "codex_usage_state", - "codex_compatibility" + "codex_compatibility", + "managed_gateway_receipt", + "process_budget_checkpoint" ], "runtimeOnlyToRuntimeMessages": [ "restore_conversation", @@ -94,7 +96,9 @@ "codex_session_state", "codex_turn_state", "codex_usage_state", - "codex_compatibility" + "codex_compatibility", + "managed_gateway_receipt", + "process_budget_checkpoint" ], "capabilities": { "schemaOnlyServerRequests": [ diff --git a/packages/runtime-contracts-rs/src/lib.rs b/packages/runtime-contracts-rs/src/lib.rs index 3b69c56d0..2527c7a82 100644 --- a/packages/runtime-contracts-rs/src/lib.rs +++ b/packages/runtime-contracts-rs/src/lib.rs @@ -20,6 +20,7 @@ mod receipts; mod session_event; pub mod telemetry; pub mod tool_details; +pub mod turn_outcome; pub use boundary::{ HostedRuntimeAuthMode, HostedRuntimeBoundary, HostedRuntimeBoundaryInput, RuntimeBoundaryError, @@ -85,6 +86,7 @@ pub use tool_details::{ BashDetails, BatchDetails, DiffDetails, EditDetails, GlobDetails, GrepDetails, ImageDetails, InlineToolDetails, ListDetails, ReadDetails, ToolDetails, WebFetchDetails, WriteDetails, }; +pub use turn_outcome::{TurnFailureClass, TurnPhase}; /// Stable product identifier for the native Maestro runtime boundary. pub const RUNTIME_PRODUCT_ID: &str = "evalops.maestro.runtime"; diff --git a/packages/runtime-contracts-rs/src/protocol.rs b/packages/runtime-contracts-rs/src/protocol.rs index 2f097761c..d5698ef58 100644 --- a/packages/runtime-contracts-rs/src/protocol.rs +++ b/packages/runtime-contracts-rs/src/protocol.rs @@ -255,6 +255,25 @@ pub enum FromRuntimeMessageType { /// Private Codex compatibility projection retained outside the generated /// public envelope. CodexCompatibility, + /// Private managed-Gateway inference receipt retained outside the + /// generated public envelope. + ManagedGatewayReceipt, + /// Private process usage checkpoint retained outside the generated public + /// envelope. + ProcessBudgetCheckpoint, +} + +impl FromRuntimeMessageType { + /// Reads the `type` tag of one runtime-to-client message. + /// + /// Returns `None` for a missing tag or a tag outside the current contract + /// so adapters classify unknown additive events explicitly instead of + /// matching visible strings. + #[must_use] + pub fn from_message(message: &Value) -> Option { + let type_name = message.get("type").and_then(Value::as_str)?; + serde_json::from_value(Value::String(type_name.to_string())).ok() + } } /// Server-request capability names advertised by the runtime. @@ -375,6 +394,8 @@ pub const HEADLESS_FROM_RUNTIME_MESSAGE_NAMES: &[&str] = &[ "codex_turn_state", "codex_usage_state", "codex_compatibility", + "managed_gateway_receipt", + "process_budget_checkpoint", ]; /// Serialized names retained by the JSON compatibility edge but absent from @@ -389,6 +410,8 @@ pub const HEADLESS_RUNTIME_ONLY_FROM_RUNTIME_MESSAGE_NAMES: &[&str] = &[ "codex_turn_state", "codex_usage_state", "codex_compatibility", + "managed_gateway_receipt", + "process_budget_checkpoint", ]; /// Typed capability projection used by the producer-owned protocol contract. @@ -538,6 +561,8 @@ const FROM_RUNTIME_MESSAGES: &[FromRuntimeMessageType] = &[ FromRuntimeMessageType::CodexTurnState, FromRuntimeMessageType::CodexUsageState, FromRuntimeMessageType::CodexCompatibility, + FromRuntimeMessageType::ManagedGatewayReceipt, + FromRuntimeMessageType::ProcessBudgetCheckpoint, ]; const SERVER_REQUEST_CAPABILITIES: &[ServerRequestCapability] = &[ @@ -595,6 +620,8 @@ pub const fn headless_protocol_contract() -> HeadlessProtocolContract { FromRuntimeMessageType::CodexTurnState, FromRuntimeMessageType::CodexUsageState, FromRuntimeMessageType::CodexCompatibility, + FromRuntimeMessageType::ManagedGatewayReceipt, + FromRuntimeMessageType::ProcessBudgetCheckpoint, ], capabilities: HeadlessCapabilityProjection { server_requests: SERVER_REQUEST_CAPABILITIES, @@ -1170,6 +1197,54 @@ mod tests { )); } + #[test] + fn from_message_reads_known_tags_and_rejects_unknown_or_missing_tags() { + for (index, name) in HEADLESS_FROM_RUNTIME_MESSAGE_NAMES.iter().enumerate() { + let typed = FromRuntimeMessageType::from_message(&serde_json::json!({ + "type": name, + "payload": index, + })) + .unwrap_or_else(|| panic!("{name} must decode as a known runtime message")); + assert_eq!( + serde_json::to_value(typed).expect("message type serializes"), + serde_json::Value::String((*name).to_string()) + ); + } + assert_eq!( + FromRuntimeMessageType::from_message(&serde_json::json!({"type": "future_event"})), + None + ); + assert_eq!( + FromRuntimeMessageType::from_message(&serde_json::json!({"type": ""})), + None + ); + assert_eq!( + FromRuntimeMessageType::from_message(&serde_json::json!({"type": 7})), + None + ); + assert_eq!( + FromRuntimeMessageType::from_message(&serde_json::json!({"payload": {}})), + None + ); + assert_eq!( + FROM_RUNTIME_MESSAGES.len(), + HEADLESS_FROM_RUNTIME_MESSAGE_NAMES.len() + ); + } + + #[test] + fn decoder_tags_match_the_advertised_typed_contract() { + let contract = serde_json::to_value(headless_protocol_contract()).unwrap(); + assert_eq!( + contract["fromRuntimeMessages"], + serde_json::json!(HEADLESS_FROM_RUNTIME_MESSAGE_NAMES) + ); + assert_eq!( + contract["runtimeOnlyFromRuntimeMessages"], + serde_json::json!(HEADLESS_RUNTIME_ONLY_FROM_RUNTIME_MESSAGE_NAMES) + ); + } + #[test] fn checked_in_fixture_matches_typed_contract() { let fixture: serde_json::Value = @@ -1192,7 +1267,7 @@ mod tests { assert_eq!(headless_protocol_capability_digest(), expected); assert_eq!( headless_protocol_capability_digest(), - "sha256:641d393c709db54d139f93af6a3bcb1f93f97291518a1ef0ecd45e5b11b17c21" + "sha256:43703fc50891d32f11e169729feb0a9eed7b92546a53234c758ba451dc1f4e53" ); } } diff --git a/packages/runtime-contracts-rs/src/turn_outcome.rs b/packages/runtime-contracts-rs/src/turn_outcome.rs new file mode 100644 index 000000000..dd2231540 --- /dev/null +++ b/packages/runtime-contracts-rs/src/turn_outcome.rs @@ -0,0 +1,295 @@ +//! Closed vocabulary for the terminal outcome of one Platform-driven Maestro turn. +//! +//! Platform reports the seam between its durable turn owner and the Maestro +//! runtime with a bounded phase and failure class so dashboards, canaries, and +//! alerts share one typed vocabulary instead of matching free-form error codes. + +use serde::{Deserialize, Serialize}; + +/// Lifecycle phase of one Platform-driven Maestro turn. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnPhase { + /// Ingress, binding, and process or client admission before any protocol traffic. + Admission, + /// Waiting for the runtime `ready` announcement and protocol negotiation. + Ready, + /// Controller `hello` exchange, binding acknowledgement, and tool-catalog validation. + Hello, + /// Session `init` and optional experiment configuration before the prompt. + Init, + /// Prompt submitted; streaming responses and terminal events. + Prompt, + /// A governed Platform tool call requested by the runtime is executing. + ToolBridge, + /// Turn accepted by Platform; shutdown and post-turn bookkeeping. + Completion, +} + +impl TurnPhase { + pub const ALL: [Self; 7] = [ + Self::Admission, + Self::Ready, + Self::Hello, + Self::Init, + Self::Prompt, + Self::ToolBridge, + Self::Completion, + ]; + + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Admission => "admission", + Self::Ready => "ready", + Self::Hello => "hello", + Self::Init => "init", + Self::Prompt => "prompt", + Self::ToolBridge => "tool_bridge", + Self::Completion => "completion", + } + } +} + +/// Bounded classification of why one Maestro turn did not complete. +/// +/// Classes are derived from the stable Platform error-code families for the +/// Maestro seam. Codes outside every family classify as [`Self::Other`] so a +/// new code never silently joins a class that drives retry or alert policy. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnFailureClass { + /// Process spawn, stdio, or native client transport failed or closed. + Transport, + /// Runtime declared a transient failure that Platform may retry. + Transient, + /// Protocol ordering, encoding, version, or unknown-event violation. + Protocol, + /// Protocol or turn deadline elapsed before completion. + Timeout, + /// Traffic, tool-call, or token budget exhausted. + Budget, + /// Model provider reported a typed failure. + Provider, + /// Governed tool bridge refused, failed, or was misused. + Tool, + /// Controller, model, capability, or prompt-exposure binding mismatch. + Binding, + /// Engine, ingress, or experiment configuration is invalid or unavailable. + Configuration, + /// Turn cancelled or interrupted by the runtime or the owner. + Cancelled, + /// Runtime finished without an acceptable result. + Runtime, + /// Code outside the Maestro seam families. + Other, +} + +impl TurnFailureClass { + pub const ALL: [Self; 12] = [ + Self::Transport, + Self::Transient, + Self::Protocol, + Self::Timeout, + Self::Budget, + Self::Provider, + Self::Tool, + Self::Binding, + Self::Configuration, + Self::Cancelled, + Self::Runtime, + Self::Other, + ]; + + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Transport => "transport", + Self::Transient => "transient", + Self::Protocol => "protocol", + Self::Timeout => "timeout", + Self::Budget => "budget", + Self::Provider => "provider", + Self::Tool => "tool", + Self::Binding => "binding", + Self::Configuration => "configuration", + Self::Cancelled => "cancelled", + Self::Runtime => "runtime", + Self::Other => "other", + } + } + + /// Classifies one stable Platform error code from the Maestro seam. + #[must_use] + pub fn from_error_code(code: &str) -> Self { + if code == "runner_turn_timeout" { + return Self::Timeout; + } + let Some(family) = code.strip_prefix("maestro_") else { + return Self::Other; + }; + // The native transport prefixes its codes with `native_`; the failure + // family is the same vocabulary as the process transport. + let family = family.strip_prefix("native_").unwrap_or(family); + match family { + "transient_failure" => return Self::Transient, + "protocol_timeout" | "turn_deadline_exceeded" | "auxiliary_timeout" => { + return Self::Timeout; + } + "turn_budget_exceeded" | "tool_call_limit_exceeded" | "tool_budget_exhausted" => { + return Self::Budget; + } + "turn_cancelled" | "turn_interrupted" => return Self::Cancelled, + "turn_failed" + | "empty_response" + | "auxiliary_empty_payload" + | "prompt_failed" + | "prompt_encode_failed" + | "response_too_large" => return Self::Runtime, + "process_start_failed" + | "start_failed" + | "client_unavailable" + | "client_resolution_failed" => return Self::Transport, + "ingress_invalid" + | "experiment_store_unavailable" + | "prompt_experiment_unsupported" + | "provider_config_invalid" + | "provider_scope_mismatch" + | "managed_client_required" + | "managed_scope_mismatch" + | "tool_approval_mode_invalid" => return Self::Configuration, + "client_tool_unsupported" + | "required_tool_not_selected" + | "auxiliary_tool_call_rejected" + | "server_request_unhandled" + | "duplicate_tool_call" => return Self::Tool, + _ => {} + } + if family.starts_with("transport_") { + Self::Transport + } else if family.starts_with("protocol_") { + Self::Protocol + } else if family.starts_with("provider_") { + Self::Provider + } else if family.starts_with("tool_catalog_") + || family.starts_with("controller_binding_") + || family.starts_with("model_binding_") + || family.starts_with("prompt_exposure_") + || family.starts_with("capability_catalog_") + || family.starts_with("capability_manifest") + { + Self::Binding + } else if family.starts_with("tool_") { + Self::Tool + } else if family.starts_with("prompt_experiment_") { + Self::Configuration + } else { + Self::Other + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_class_round_trips_through_its_wire_name() { + for class in TurnFailureClass::ALL { + let encoded = serde_json::to_value(class).expect("class serializes"); + assert_eq!( + encoded, + serde_json::Value::String(class.as_str().to_string()) + ); + let decoded: TurnFailureClass = + serde_json::from_value(encoded).expect("class deserializes"); + assert_eq!(decoded, class); + } + for phase in TurnPhase::ALL { + let encoded = serde_json::to_value(phase).expect("phase serializes"); + assert_eq!( + encoded, + serde_json::Value::String(phase.as_str().to_string()) + ); + } + } + + #[test] + fn seam_error_codes_classify_into_closed_families() { + use TurnFailureClass as C; + let cases = [ + ("maestro_transport_closed", C::Transport), + ("maestro_transport_failed", C::Transport), + ("maestro_process_start_failed", C::Transport), + ("maestro_native_client_unavailable", C::Transport), + ("maestro_transient_failure", C::Transient), + ("maestro_protocol_order_invalid", C::Protocol), + ("maestro_protocol_unknown_event", C::Protocol), + ("maestro_protocol_version_mismatch", C::Protocol), + ("maestro_protocol_timeout", C::Timeout), + ("maestro_turn_deadline_exceeded", C::Timeout), + ("runner_turn_timeout", C::Timeout), + ("maestro_turn_budget_exceeded", C::Budget), + ("maestro_tool_call_limit_exceeded", C::Budget), + ("maestro_provider_declared_failure", C::Provider), + ("maestro_provider_output_token_exhaustion", C::Provider), + ("maestro_tool_bridge_unavailable", C::Tool), + ("maestro_tool_failed", C::Tool), + ("maestro_tool_ownership_conflict", C::Tool), + ("maestro_required_tool_not_selected", C::Tool), + ("maestro_controller_binding_mismatch", C::Binding), + ("maestro_model_binding_mismatch", C::Binding), + ("maestro_prompt_exposure_missing", C::Binding), + ("maestro_capability_manifest", C::Binding), + ("maestro_native_capability_catalog_invalid", C::Binding), + ("maestro_ingress_invalid", C::Configuration), + ("maestro_prompt_experiment_config_invalid", C::Configuration), + ("maestro_experiment_store_unavailable", C::Configuration), + ("maestro_turn_cancelled", C::Cancelled), + ("maestro_turn_interrupted", C::Cancelled), + ("maestro_turn_failed", C::Runtime), + ("maestro_empty_response", C::Runtime), + ("maestro_native_transport_closed", C::Transport), + ("maestro_native_start_failed", C::Transport), + ("maestro_native_client_resolution_failed", C::Transport), + ("maestro_native_protocol_order_invalid", C::Protocol), + ("maestro_native_provider_error", C::Provider), + ("maestro_native_turn_deadline_exceeded", C::Timeout), + ("maestro_native_tool_call_limit_exceeded", C::Budget), + ("maestro_native_tool_budget_exhausted", C::Budget), + ("maestro_native_turn_interrupted", C::Cancelled), + ("maestro_native_turn_failed", C::Runtime), + ("maestro_native_empty_response", C::Runtime), + ("maestro_native_prompt_failed", C::Runtime), + ("maestro_native_prompt_encode_failed", C::Runtime), + ("maestro_native_response_too_large", C::Runtime), + ("maestro_native_model_binding_mismatch", C::Binding), + ("maestro_native_model_binding_invalid", C::Binding), + ("maestro_native_tool_catalog_invalid", C::Binding), + ("maestro_native_tool_catalog_empty", C::Binding), + ("maestro_native_tool_call_invalid", C::Tool), + ("maestro_native_tool_unsupported", C::Tool), + ("maestro_native_tool_not_admitted", C::Tool), + ("maestro_native_tool_approval_invalid", C::Tool), + ("maestro_native_tool_response_closed", C::Tool), + ("maestro_native_duplicate_tool_call", C::Tool), + ("maestro_native_managed_client_required", C::Configuration), + ("maestro_native_managed_scope_mismatch", C::Configuration), + ("maestro_native_provider_scope_mismatch", C::Configuration), + ( + "maestro_native_prompt_experiment_unsupported", + C::Configuration, + ), + ("swarm_native_scheduler_unavailable", C::Other), + ("maestro_future_code", C::Other), + ("", C::Other), + ]; + for (code, expected) in cases { + assert_eq!( + TurnFailureClass::from_error_code(code), + expected, + "{code} must classify as {expected:?}" + ); + } + } +} diff --git a/packages/runtime-rs/src/agent/extensions/mod.rs b/packages/runtime-rs/src/agent/extensions/mod.rs index 7870bf186..7f8fc072a 100644 --- a/packages/runtime-rs/src/agent/extensions/mod.rs +++ b/packages/runtime-rs/src/agent/extensions/mod.rs @@ -50,6 +50,7 @@ pub(crate) mod model_dynamics; use std::time::Instant; +use crate::agent::safety::SafetyConfig; pub use doom_loop::DoomLoopExtension; /// State handed to [`AgentExtension::on_user_turn_start`]. @@ -310,8 +311,14 @@ impl ExtensionRegistry { /// Registration order here is the order tenants observe every hook. #[must_use] pub fn with_default_tenants() -> Self { + Self::with_default_tenants_and_safety(SafetyConfig::default()) + } + + /// [`Self::with_default_tenants`] with explicit doom-loop thresholds. + #[must_use] + pub fn with_default_tenants_and_safety(safety: SafetyConfig) -> Self { let mut registry = Self::new(); - registry.register(Box::new(DoomLoopExtension::new())); + registry.register(Box::new(DoomLoopExtension::with_config(safety))); registry } diff --git a/packages/runtime-rs/src/agent/native.rs b/packages/runtime-rs/src/agent/native.rs index a622b2e2b..0a80e90a8 100644 --- a/packages/runtime-rs/src/agent/native.rs +++ b/packages/runtime-rs/src/agent/native.rs @@ -670,6 +670,10 @@ pub struct NativeAgentConfig { /// Callers may extend this bounded window, but retries remain inside the /// owning runtime so completed tool effects are never replayed. pub retry_config: super::retry::RetryConfig, + + /// Doom-loop and per-tool rate-limit thresholds enforced by the loop's + /// safety tenant. + pub safety_config: super::safety::SafetyConfig, } impl NativeAgentConfig { @@ -706,6 +710,7 @@ impl Default for NativeAgentConfig { max_turn_steps: DEFAULT_MAX_TURN_STEPS, allow_unbounded_turn: false, retry_config: super::retry::RetryConfig::default(), + safety_config: super::safety::SafetyConfig::default(), } } } @@ -1509,7 +1514,8 @@ impl NativeAgent { let dynamics = Arc::new(std::sync::Mutex::new( super::model_dynamics::DynamicsState::default(), )); - let mut extensions = ExtensionRegistry::with_default_tenants(); + let mut extensions = + ExtensionRegistry::with_default_tenants_and_safety(config.safety_config.clone()); extensions.register(Box::new( super::extensions::model_dynamics::ModelDynamicsExtension::new( Arc::clone(&dynamics), diff --git a/packages/runtime-rs/src/agent/native/session_scenarios.rs b/packages/runtime-rs/src/agent/native/session_scenarios.rs index 57c5435e4..25fae807d 100644 --- a/packages/runtime-rs/src/agent/native/session_scenarios.rs +++ b/packages/runtime-rs/src/agent/native/session_scenarios.rs @@ -28,6 +28,14 @@ enum Step { count: usize, prompt_bytes: usize, }, + /// Turns whose first provider round calls `read` on a workspace file of + /// `tool_output_bytes` and whose second round answers with text. Call ids + /// use the compound `call-N:native-N` shape that #8889 dropped. + ToolTurns { + count: usize, + prompt_bytes: usize, + tool_output_bytes: usize, + }, SeedHistory { messages: usize, message_bytes: usize, @@ -48,7 +56,7 @@ struct Bounds { max: Option, } -#[derive(Default, Serialize)] +#[derive(Debug, Default, Serialize)] pub(crate) struct Metrics { messages: usize, message_slots: usize, @@ -62,6 +70,40 @@ pub(crate) struct Metrics { pending_prompts: usize, deferred_commands: usize, queued_system_prompts: usize, + tool_calls: usize, + tool_results: usize, + /// Tool calls in history whose id has no result, plus results whose id has + /// no call. Always zero after every step; exposed so fixtures can bound it. + orphan_tool_ids: usize, +} + +fn tool_identity_metrics(messages: &[Message]) -> (usize, usize, usize) { + use std::collections::BTreeSet; + let mut calls = BTreeSet::new(); + let mut results = BTreeSet::new(); + let (mut call_count, mut result_count) = (0usize, 0usize); + for message in messages { + let MessageContent::Blocks(blocks) = &message.content else { + continue; + }; + for block in blocks { + match block { + ContentBlock::ToolUse { id, .. } => { + call_count += 1; + calls.insert(id.as_str()); + } + ContentBlock::ToolResult { tool_use_id, .. } => { + result_count += 1; + results.insert(tool_use_id.as_str()); + } + _ => {} + } + } + } + let orphans = calls.symmetric_difference(&results).count() + + (call_count - calls.len()) + + (result_count - results.len()); + (call_count, result_count, orphans) } type Checkpoint = ( @@ -121,6 +163,11 @@ pub(crate) fn inspect(runner: &NativeAgentRunner, capture: bool) -> SessionState queued_system_prompts: runner.queued_system_prompts.len(), ..Default::default() }; + ( + metrics.tool_calls, + metrics.tool_results, + metrics.orphan_tool_ids, + ) = tool_identity_metrics(&runner.messages); if let Some(record) = &runner.semantic_continuation { metrics.continuation_bytes = json_bytes(record); metrics.retained_requests = record.user_requests.len(); @@ -196,6 +243,17 @@ fn validate(scenario: &Scenario) -> Result<()> { count, prompt_bytes, } => (*count, *prompt_bytes, 0), + Step::ToolTurns { + count, + prompt_bytes, + tool_output_bytes, + } => { + ensure!( + (1..=64 * 1024).contains(tool_output_bytes), + "tool_output_bytes must be 1..65536" + ); + (*count, *prompt_bytes, *tool_output_bytes) + } Step::Save => { saved = true; continue; @@ -270,19 +328,43 @@ struct EventStats { events: usize, compactions: usize, continuation_bytes_emitted: usize, + /// `tokens_before` of the most recent compaction not yet checked against + /// the post-turn history size. + #[serde(skip)] + unchecked_compaction_tokens: Option, } impl EventStats { fn observe(&mut self, event: &FromAgent) { self.events += 1; if let FromAgent::Compaction { - continuation: Some(record), + continuation, + tokens_before, .. } = event { self.compactions += 1; - self.continuation_bytes_emitted += json_bytes(record); + self.unchecked_compaction_tokens = Some(*tokens_before); + if let Some(record) = continuation { + self.continuation_bytes_emitted += json_bytes(record); + } } } + + /// Every compaction must shrink the history it measured: the post-turn + /// estimate is strictly below `tokens_before`, and `tokens_before` is a + /// real measurement rather than the zero of a missing cut point. + fn check_compaction_accounting(&mut self, state: &SessionState) -> Result<()> { + if let Some(tokens_before) = self.unchecked_compaction_tokens.take() { + ensure!(tokens_before > 0, "compaction reported tokens_before=0"); + ensure!( + state.metrics.message_tokens < tokens_before, + "compaction did not reduce history: {} tokens before, {} after", + tokens_before, + state.metrics.message_tokens + ); + } + Ok(()) + } } async fn terminal( @@ -333,6 +415,13 @@ async fn run(scenario: Scenario) -> Result<()> { model: "openai/gpt-4o".into(), cwd: workspace.path().display().to_string(), context_window: Some(scenario.context_window), + approval_mode: ApprovalMode::Yolo, + // Scripted turns finish in microseconds; the production per-tool rate + // limit would otherwise block the compaction scenario. + safety_config: crate::agent::safety::SafetyConfig { + rate_limit: usize::MAX, + ..Default::default() + }, ..Default::default() }; let host = RuntimeTestHost::new( @@ -372,6 +461,32 @@ async fn run(scenario: Scenario) -> Result<()> { } } } + Step::ToolTurns { count, prompt_bytes, tool_output_bytes } => { + for _ in 0..*count { + // Distinct paths keep the doom-loop guard (identical + // repeated arguments) out of the scenario. + let file = format!("evidence-{turns}.txt"); + std::fs::write(workspace.path().join(&file), "e".repeat(*tool_output_bytes))?; + scripted.push_response(crate::ai::ScriptedResponse { + blocks: vec![crate::ai::ScriptedBlock::ToolUse { + id: format!("call-{turns}:native-{turns}"), + name: "read".into(), + input: serde_json::json!({"path": file}), + }], + stop_reason: crate::ai::StopReason::ToolUse, + error: None, + }); + scripted.push_response(crate::ai::ScriptedResponse::text(format!("read-{turns}"))); + let prompt = format!("request-{turns}: {}", "read the evidence ".repeat(prompt_bytes.div_ceil(18))); + agent.prompt(prompt[..*prompt_bytes].to_owned(), vec![]).await?; + terminal(&mut events, &agent, false, &mut event_stats).await?; + turns += 1; + ensure!(scripted.remaining() == 0, "unconsumed response at turn {turns}"); + let state = observe(&agent, false).await?; + event_stats.check_compaction_accounting(&state)?; + ensure!(state.metrics.orphan_tool_ids == 0, "turn {turns}: tool call/result identities diverged: {:?}", state.metrics); + } + } Step::SeedHistory { messages, message_bytes } => agent.replace_history( (0..*messages).map(|index| Message { role: if index % 2 == 0 { Role::User } else { Role::Assistant }, @@ -401,6 +516,8 @@ async fn run(scenario: Scenario) -> Result<()> { let state = observe(&agent, false).await?; // Drain trailing notifications too; measurements must not retain snapshots. while let Ok(event) = events.try_recv() { event_stats.observe(&event); } + event_stats.check_compaction_accounting(&state)?; + ensure!(state.metrics.orphan_tool_ids == 0, "step {index}: tool call/result identities diverged: {:?}", state.metrics); eprintln!("SESSION_STEP {}", serde_json::json!({"scenario": scenario.name, "step": index, "turn": turns, "events": event_stats, "elapsed_ms": started.elapsed().as_millis(), "checkpoint_bytes": saved.as_ref().map_or(0, json_bytes), "metrics": state.metrics})); } Ok::<_, anyhow::Error>(()) @@ -446,6 +563,47 @@ async fn interrupted_turns_remain_within_context_window() { run(load(&path).unwrap()).await.unwrap(); } +#[tokio::test] +async fn compaction_preserves_tool_identities_across_restore() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../test/fixtures/session-scenarios/compaction-tool-history.json"); + run(load(&path).unwrap()).await.unwrap(); +} + +#[test] +fn tool_identity_metrics_count_orphans_and_duplicates() { + let call = |id: &str| Message { + role: Role::Assistant, + content: MessageContent::Blocks(vec![ContentBlock::ToolUse { + id: id.into(), + name: "read".into(), + input: serde_json::json!({}), + gemini_context: None, + }]), + }; + let result = |id: &str| Message { + role: Role::User, + content: MessageContent::Blocks(vec![ContentBlock::ToolResult { + tool_use_id: id.into(), + content: "ok".into(), + is_error: Some(false), + }]), + }; + assert_eq!( + tool_identity_metrics(&[call("a:1"), result("a:1")]), + (1, 1, 0) + ); + assert_eq!( + tool_identity_metrics(&[call("a:1"), result("a")]), + (1, 1, 2), + "a truncated compound id is one orphan call and one orphan result" + ); + assert_eq!( + tool_identity_metrics(&[call("a"), result("a"), result("a")]), + (1, 2, 1) + ); +} + #[test] fn session_scenarios_reject_invalid_contracts() { let valid = serde_json::json!({"schema":SCHEMA,"name":"validation","context_window":4096,"steps":[{"action":"assert","bounds":{"messages":{"max":0}}}]}); @@ -463,6 +621,7 @@ fn session_scenarios_reject_invalid_contracts() { serde_json::json!([{"action":"assert","bounds":{"typo":{"max":0}}}]), serde_json::json!([{"action":"assert","bounds":{"messages":{"min":2,"max":1}}}]), serde_json::json!([{"action":"turns","count":100001,"prompt_bytes":32,"response_bytes":1}]), + serde_json::json!([{"action":"tool_turns","count":1,"prompt_bytes":32,"tool_output_bytes":0}]), ] { let mut input = valid.clone(); input["steps"] = steps; diff --git a/proto/maestro/v1/protocol-compatibility-manifest.json b/proto/maestro/v1/protocol-compatibility-manifest.json index a0f4b11fa..fa6aaed2a 100644 --- a/proto/maestro/v1/protocol-compatibility-manifest.json +++ b/proto/maestro/v1/protocol-compatibility-manifest.json @@ -5,7 +5,7 @@ "canonicalization": "rfc8785", "encoding": "utf-8" }, - "compatibilityDigest": "sha256:d0d5b77ac9165ebb4d2fb96529af3eed31854473b855fc4a4a5ad424e3168b2e", + "compatibilityDigest": "sha256:ee9afab8484ffd1551d962acfac2ee955e0db4621d93fff0d462be207c9e38f1", "buildIdentity": { "sourceSha": null, "buildDigest": null, @@ -201,7 +201,7 @@ }, "runtime": { "schemaVersion": "evalops.maestro.headless-protocol.v1", - "contractDigest": "sha256:641d393c709db54d139f93af6a3bcb1f93f97291518a1ef0ecd45e5b11b17c21", + "contractDigest": "sha256:43703fc50891d32f11e169729feb0a9eed7b92546a53234c758ba451dc1f4e53", "receipt": { "schemaVersion": "evalops.maestro.runtime-receipt.v1", "sourceDigest": "sha256:8fbcb58770dccabaf0d87b2532121addafd35e04f5a09f9dffdd2dea8897b921", diff --git a/scripts/pr-feedback-audit.mjs b/scripts/pr-feedback-audit.mjs index 00f3dd6d6..45c15f87d 100644 --- a/scripts/pr-feedback-audit.mjs +++ b/scripts/pr-feedback-audit.mjs @@ -297,7 +297,7 @@ function firstNonblankLine(body) { export function informationalReviewFeedback(body, author) { const firstLine = firstNonblankLine(body); const trustedReviewBot = - /^(?:cursor|coderabbitai|chatgpt-codex-connector|devin-ai-integration)\b/iu.test( + /^(?:cursor|coderabbitai|chatgpt-codex-connector|devin-ai-integration)(?:\[bot\])?$/iu.test( String(author ?? ""), ); return ( diff --git a/scripts/resolve-public-mirror-bot-review-threads.mjs b/scripts/resolve-public-mirror-bot-review-threads.mjs index 816df6079..47df1aed1 100644 --- a/scripts/resolve-public-mirror-bot-review-threads.mjs +++ b/scripts/resolve-public-mirror-bot-review-threads.mjs @@ -7,8 +7,9 @@ const DEFAULT_REPO = "evalops/maestro"; // GitHub Apps usually have login suffix "[bot]". Some review apps post // without that suffix; keep this list aligned with pr-feedback-audit.mjs. +// Unknown actors remain unresolved so their feedback is not mistaken for advisory review. const KNOWN_REVIEW_BOT_LOGINS = - /^(?:cursor|coderabbitai|chatgpt-codex-connector|devin-ai-integration)\b/iu; + /^(?:cursor|coderabbitai|chatgpt-codex-connector|devin-ai-integration)(?:\[bot\])?$/iu; const LIST_REVIEW_THREADS_QUERY = `query($owner:String!,$repo:String!,$number:Int!,$after:String){ repository(owner:$owner,name:$repo){ @@ -17,10 +18,7 @@ const LIST_REVIEW_THREADS_QUERY = `query($owner:String!,$repo:String!,$number:In nodes{ id isResolved - isOutdated - path - line - comments(first:20){nodes{url body author{__typename login}}} + comments(first:1){nodes{author{__typename login}}} } pageInfo{ hasNextPage @@ -100,17 +98,10 @@ export function isReviewBotAuthor(author) { if (!author || typeof author !== "object") { return false; } - const typename = String(author.__typename ?? "").trim(); - if (typename === "Bot") { - return true; - } const login = String(author.login ?? "").trim(); if (!login) { return false; } - if (login.toLowerCase().endsWith("[bot]")) { - return true; - } return KNOWN_REVIEW_BOT_LOGINS.test(login); } diff --git a/scripts/resolve-public-mirror-bot-review-threads.test.mjs b/scripts/resolve-public-mirror-bot-review-threads.test.mjs index 434455e9a..d17ec62f7 100644 --- a/scripts/resolve-public-mirror-bot-review-threads.test.mjs +++ b/scripts/resolve-public-mirror-bot-review-threads.test.mjs @@ -3,11 +3,14 @@ import { existsSync, readFileSync } from "node:fs"; import test from "node:test"; import { fileURLToPath } from "node:url"; +import { informationalReviewFeedback } from "./pr-feedback-audit.mjs"; + import { flagValue, formatSummaryComment, isResolveReviewThreadRequest, isReviewBotAuthor, + LIST_REVIEW_THREADS_QUERY, resolvePublicMirrorBotReviewThreads, shouldResolveReviewThread, } from "./resolve-public-mirror-bot-review-threads.mjs"; @@ -91,15 +94,17 @@ function makeGhJson(pages) { }; } -test("review-bot authors include [bot] suffix, known apps, and GraphQL Bot actors", () => { +test("only exact known review-bot logins qualify for automatic resolution", () => { assert.equal(isReviewBotAuthor({ login: "chatgpt-codex-connector" }), true); assert.equal(isReviewBotAuthor({ login: "devin-ai-integration" }), true); assert.equal(isReviewBotAuthor({ login: "cursor[bot]" }), true); assert.equal(isReviewBotAuthor({ login: "coderabbitai" }), true); - assert.equal(isReviewBotAuthor({ login: "github-actions[bot]" }), true); + assert.equal(isReviewBotAuthor({ login: "cursor-reviewer" }), false); + assert.equal(isReviewBotAuthor({ login: "coderabbitai-dev" }), false); + assert.equal(isReviewBotAuthor({ login: "github-actions[bot]" }), false); assert.equal( isReviewBotAuthor({ __typename: "Bot", login: "some-review-app" }), - true, + false, ); assert.equal(isReviewBotAuthor({ login: "alice" }), false); assert.equal(isReviewBotAuthor({ __typename: "User", login: "alice" }), false); @@ -107,6 +112,20 @@ test("review-bot authors include [bot] suffix, known apps, and GraphQL Bot actor assert.equal(isReviewBotAuthor({ login: "" }), false); }); +test("feedback audit does not discard informational-looking human comments", () => { + assert.equal(informationalReviewFeedback("## Summary", "cursor[bot]"), true); + assert.equal(informationalReviewFeedback("## Summary", "cursor-reviewer"), false); + assert.equal(informationalReviewFeedback("**Info:** Please review", "coderabbitai-dev"), false); +}); + +test("thread query transfers only the first author needed for classification", () => { + assert.match( + LIST_REVIEW_THREADS_QUERY, + /comments\(first:1\)\{nodes\{author\{__typename login\}\}\}/, + ); + assert.doesNotMatch(LIST_REVIEW_THREADS_QUERY, /\b(?:body|url)\b/); +}); + test("resolved threads are never selected even when the author is a bot", () => { assert.equal( shouldResolveReviewThread( @@ -124,7 +143,7 @@ test("resolved threads are never selected even when the author is a bot", () => shouldResolveReviewThread( thread({ id: "PRRT_outdated_bot", - login: "github-actions[bot]", + login: "cursor[bot]", isOutdated: true, }), ), @@ -143,7 +162,7 @@ test("resolveReviewThread is called only for unresolved bot-started threads", () }), thread({ id: "PRRT_outdated_bot", - login: "github-actions[bot]", + login: "cursor[bot]", isOutdated: true, }), thread({ @@ -151,6 +170,7 @@ test("resolveReviewThread is called only for unresolved bot-started threads", () login: "bob", isOutdated: true, }), + thread({ id: "PRRT_prefixed_human", login: "cursor-reviewer" }), thread({ id: "PRRT_devin", login: "devin-ai-integration" }), thread({ id: "PRRT_typename_bot", @@ -192,13 +212,12 @@ test("resolveReviewThread is called only for unresolved bot-started threads", () "PRRT_bot", "PRRT_outdated_bot", "PRRT_devin", - "PRRT_typename_bot", ]); assert.deepEqual(fake.resolvedIds, result.resolvedIds); - assert.equal(result.humanCount, 4); + assert.equal(result.humanCount, 6); assert.equal(fake.comments.length, 1); - assert.match(fake.comments[0], /Resolved 4 review-bot thread\(s\)/); - assert.match(fake.comments[0], /Left 4 human-started thread\(s\) unresolved/); + assert.match(fake.comments[0], /Resolved 3 review-bot thread\(s\)/); + assert.match(fake.comments[0], /Left 6 human-started thread\(s\) unresolved/); assert.match(fake.comments[0], /Review of mirrored code belongs on the mono source PR/); assert.match(fake.comments[0], /https:\/\/example\.test\/triage\/9100/); assert.doesNotMatch(fake.comments[0], /PRRT_human/); diff --git a/test/fixtures/session-scenarios/compaction-tool-history.json b/test/fixtures/session-scenarios/compaction-tool-history.json new file mode 100644 index 000000000..d2bb80b95 --- /dev/null +++ b/test/fixtures/session-scenarios/compaction-tool-history.json @@ -0,0 +1,65 @@ +{ + "schema": "evalops.maestro.session-scenario.v1", + "name": "compaction-preserves-tool-identities", + "context_window": 4096, + "steps": [ + { + "action": "tool_turns", + "count": 60, + "prompt_bytes": 128, + "tool_output_bytes": 512 + }, + { + "action": "assert", + "bounds": { + "tool_calls": { + "min": 1 + }, + "orphan_tool_ids": { + "max": 0 + }, + "message_tokens": { + "max": 4096 + }, + "commands": { + "min": 1 + } + } + }, + { + "action": "save" + }, + { + "action": "clear" + }, + { + "action": "restore" + }, + { + "action": "tool_turns", + "count": 40, + "prompt_bytes": 128, + "tool_output_bytes": 768 + }, + { + "action": "turns", + "count": 20, + "prompt_bytes": 384, + "response_bytes": 8 + }, + { + "action": "assert", + "bounds": { + "orphan_tool_ids": { + "max": 0 + }, + "message_tokens": { + "max": 4096 + }, + "pending_prompts": { + "max": 0 + } + } + } + ] +}