From 40806905546daa82c01fba7659d56dc4ea80b2cd Mon Sep 17 00:00:00 2001 From: Miya Date: Wed, 23 Sep 2026 00:30:15 +0200 Subject: [PATCH 1/3] wip(delivery): phase 1 Codex native delivery Stacked on PR #1825. Deterministic parity, real Codex evaluation, and unlaunched-session E2E are green on finn-mini. Dual-provider final signoff remains pending Claude CLI capacity. --- .agentworkforce/features/manifest.yaml | 16 +- .../active/traj_nd5a2dd4waas/trajectory.json | 19 + .agentworkforce/trajectories/index.json | 5 + .gitignore | 2 + CHANGELOG.md | 8 + crates/broker/Cargo.toml | 8 + .../src/broker/delivery_verification.rs | 216 +++- crates/broker/src/codex_thread.rs | 581 +++++++++ crates/broker/src/delivery/backend.rs | 100 +- crates/broker/src/delivery/codex_queue.rs | 1057 +++++++++++++++++ crates/broker/src/delivery/mod.rs | 3 + crates/broker/src/delivery/pty.rs | 85 +- crates/broker/src/listen_api.rs | 162 +++ crates/broker/src/node_delivery_probe.rs | 17 + crates/broker/src/runtime/api.rs | 204 +++- crates/broker/src/runtime/dead_letter.rs | 1 + crates/broker/src/runtime/degraded.rs | 1 + crates/broker/src/runtime/delivery.rs | 416 ++++++- crates/broker/src/runtime/fleet.rs | 22 + crates/broker/src/runtime/init.rs | 18 +- crates/broker/src/runtime/maintenance.rs | 156 ++- crates/broker/src/runtime/mod.rs | 1 + crates/broker/src/runtime/relaycast_events.rs | 22 +- crates/broker/src/runtime/tests.rs | 756 ++++++++++++ crates/broker/src/worker.rs | 220 +++- .../broker/tests/delivery_seam_invariants.rs | 20 + crates/relay-pty/src/codex_session.rs | 192 ++- docs/native-delivery-migration.md | 33 + flows/migrate/native-delivery.spec.ts | 92 +- package.json | 2 +- packages/cli/src/cli/agent-relay-mcp.ts | 58 +- .../fleet-lifecycle-integration.test.ts | 4 + scripts/migrate/mutation-proof.mjs | 645 ++++++++++ scripts/migrate/native-delivery-gates.mjs | 623 +++++++++- tests/e2e/unlaunched/codex-session-host.ts | 469 ++++++++ .../unlaunched-codex-delivery.test.ts | 486 ++++++++ tests/e2e/vitest.unlaunched.config.ts | 56 + ...elivery-contract-evals.codex-queue.test.ts | 314 +++++ tests/integration/broker/evals/runner.ts | 86 +- tests/parity/broadcast.ts | 3 +- tests/parity/continuity-handoff.ts | 5 +- tests/parity/multi-worker.ts | 3 +- tests/parity/orch-to-worker.ts | 4 +- tests/parity/stability-soak.ts | 3 +- tests/relayflows/cleanroom/relay.matrix.json | 22 + 45 files changed, 6970 insertions(+), 246 deletions(-) create mode 100644 .agentworkforce/trajectories/active/traj_nd5a2dd4waas/trajectory.json create mode 100644 .agentworkforce/trajectories/index.json create mode 100644 crates/broker/src/codex_thread.rs create mode 100644 crates/broker/src/delivery/codex_queue.rs create mode 100644 scripts/migrate/mutation-proof.mjs create mode 100644 tests/e2e/unlaunched/codex-session-host.ts create mode 100644 tests/e2e/unlaunched/unlaunched-codex-delivery.test.ts create mode 100644 tests/e2e/vitest.unlaunched.config.ts create mode 100644 tests/fixtures/delivery-contract-evals.codex-queue.test.ts diff --git a/.agentworkforce/features/manifest.yaml b/.agentworkforce/features/manifest.yaml index 3e02e28502..66326fc9d4 100644 --- a/.agentworkforce/features/manifest.yaml +++ b/.agentworkforce/features/manifest.yaml @@ -1,5 +1,5 @@ version: '1.1' -updated: '2026-09-20' +updated: '2026-09-21' # Every user-facing feature in this repo, categorized and scored. # @@ -108,9 +108,21 @@ categories: accepted route recorded and settled by its own rules, and no acknowledgement reported that was not observed. The PTY injector is its first backend. - location: crates/broker/src/delivery/, crates/broker/src/broker/delivery_verification.rs, crates/broker/src/lib.rs, crates/broker/src/pty_worker.rs, crates/broker/src/runtime/, crates/broker/src/node_control.rs, crates/broker/src/node_delivery_probe.rs, crates/broker/src/worker.rs, crates/broker/tests/, packages/contracts/fixtures/event-fixtures.json, packages/harness-driver/src/protocol.ts, tests/benchmarks/harness.ts, tests/benchmarks/reliability.ts, tests/benchmarks/stress.ts, tests/e2e/unlaunched/, tests/fixtures/delivery-contract-evals.test.ts, tests/fixtures/targeted-feature-verification.test.ts, tests/integration/broker/cli-spawn.test.ts, tests/integration/broker/evals/delivery/observation-ledger.unit.test.ts, tests/integration/broker/infra-failures.test.ts, tests/integration/broker/stress.test.ts, tests/integration/broker/utils/assert-helpers.ts, tests/integration/broker/utils/obligation-conformance.ts, tests/parity/ + location: crates/broker/src/delivery/, crates/broker/src/broker/delivery_verification.rs, crates/broker/src/lib.rs, crates/broker/src/pty_worker.rs, crates/broker/src/wrap.rs, crates/broker/src/runtime/, crates/broker/src/node_control.rs, crates/broker/src/node_delivery_probe.rs, crates/broker/src/worker.rs, crates/broker/tests/, packages/contracts/fixtures/event-fixtures.json, packages/harness-driver/src/protocol.ts, tests/benchmarks/harness.ts, tests/benchmarks/reliability.ts, tests/benchmarks/stress.ts, tests/e2e/unlaunched/, tests/fixtures/delivery-contract-evals.test.ts, tests/fixtures/targeted-feature-verification.test.ts, tests/integration/broker/cli-spawn.test.ts, tests/integration/broker/evals/delivery/observation-ledger.unit.test.ts, tests/integration/broker/infra-failures.test.ts, tests/integration/broker/stress.test.ts, tests/integration/broker/utils/assert-helpers.ts, tests/integration/broker/utils/obligation-conformance.ts, tests/parity/ verify_tier: 6 + - id: codex-queue-delivery + name: Codex Queue Delivery + description: >- + Native Codex delivery over the public `codex queue` command. Relay + resolves the target thread id, carries a `relay-delivery-id` marker in + the message body because Codex assigns its own client id, and settles + only by observing that marker in the thread's own session file. A + capability probe decides selectability at send time, so an installed + Codex without `codex queue` falls back before any write. + location: crates/broker/src/delivery/codex_queue.rs, crates/broker/src/codex_thread.rs, crates/broker/src/listen_api.rs, crates/relay-pty/src/codex_session.rs, crates/broker/Cargo.toml, packages/cli/src/cli/agent-relay-mcp.ts, tests/e2e/unlaunched/, tests/fixtures/delivery-contract-evals.codex-queue.test.ts, tests/integration/broker/evals/runner.ts + verify_tier: 4 + agent-management: name: Agent Management description: Register and manage agents in the workspace diff --git a/.agentworkforce/trajectories/active/traj_nd5a2dd4waas/trajectory.json b/.agentworkforce/trajectories/active/traj_nd5a2dd4waas/trajectory.json new file mode 100644 index 0000000000..19e44b00f5 --- /dev/null +++ b/.agentworkforce/trajectories/active/traj_nd5a2dd4waas/trajectory.json @@ -0,0 +1,19 @@ +{ + "id": "traj_nd5a2dd4waas", + "version": 1, + "task": { + "title": "Implement native delivery phase 1 with Relayflows" + }, + "status": "active", + "startedAt": "2026-09-21T06:13:08.229Z", + "agents": [], + "chapters": [], + "commits": [], + "filesChanged": [], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "b2b166423560925aa43ddd06fcccd4d78d1afbb1", + "endRef": "b2b166423560925aa43ddd06fcccd4d78d1afbb1" + } +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/index.json b/.agentworkforce/trajectories/index.json new file mode 100644 index 0000000000..e736ceed3b --- /dev/null +++ b/.agentworkforce/trajectories/index.json @@ -0,0 +1,5 @@ +{ + "version": 1, + "lastUpdated": "2026-07-16T20:05:37.749Z", + "trajectories": {} +} diff --git a/.gitignore b/.gitignore index 85bc4b5102..14f81b6521 100644 --- a/.gitignore +++ b/.gitignore @@ -106,3 +106,5 @@ tests/integration/broker/evals-reports/ # Relayflows v2 daemon runtime state (journals, sockets, locks) .relayflowd/ +.relayflowd-continuation/ +.relayflowd-phase1-signoff-repair/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 08bd4d97c5..2302de7959 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Devin CLI is available through Relay PTY, fleet and MCP spawning with isolated worker MCP configuration, preserved approvals, and reliable initial and follow-up message submission. - `agent-relay fleet nodes list --pretty` renders the fleet roster as a human-readable table; `agent-relay fleet nodes --pretty` is available as a shorter equivalent, while JSON remains the default. +### Changed + +- Codex native delivery appends a `relay-delivery-id` marker to queued messages so Relay can confirm delivery from the Codex thread record without fabricating an acknowledgement. +- A Codex delivery is acknowledged (and its Relaycast read receipt published) only once the marker appears in the thread rollout as a consumed user item. A message still sitting in Codex's own queue is reported as handed over, so an idle or exited session no longer produces a read receipt for a message nobody read. +- Setting `manual_flush` on an agent reachable only over a native route (an attached Codex thread) is refused with `manual_flush_unsupported` (HTTP 409); inbound messages for such an agent always drain over the native route instead of parking in a queue the flush path cannot empty. + ### Fixed - Broker PTY delivery now fails closed when a handoff deadline races an admitted write, preventing fleet retries from injecting the same message twice and retaining cursor-purged siblings as non-redeliverable dead letters. +- Releasing an agent, or a worker dying, no longer marks a delivery that already reached a durable native route as freely redeliverable: it is dead-lettered in doubt and the withheld engine acknowledgement is recorded on the node delivery probe rather than dropped. +- A broker restart no longer re-queues a Codex message that a previous broker lifetime already handed to `codex queue`: the accepted route is persisted with the pending delivery and restored into the delivery seam before the first retry. - MCP `spawn`, `add_agent`, and direct-message calls now coalesce in-flight JSON-RPC replays and accept an `idempotency_key` for safe retries after a lost response, preventing duplicate workers, false duplicate-name failures, and duplicate DMs while keeping later JSON-RPC ID reuse distinct. - `node agent attach --node` now replaces an expired remote terminal session once and reconnects existing local terminal clients, while transient terminal failures retain the bounded resume budget without minting additional sessions. diff --git a/crates/broker/Cargo.toml b/crates/broker/Cargo.toml index 767ac131b6..e095e75d57 100644 --- a/crates/broker/Cargo.toml +++ b/crates/broker/Cargo.toml @@ -8,10 +8,18 @@ autobins = false name = "relay_broker" path = "src/lib.rs" +[features] +default = ["seam-probe"] +seam-probe = [] + [[bin]] name = "agent-relay-broker" path = "src/main.rs" +[[test]] +name = "delivery_seam_invariants" +required-features = ["seam-probe"] + [dependencies] anyhow = "1.0" axum = { version = "0.8", features = ["ws"] } diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 6a113546c1..ef3e4c6286 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -8,7 +8,7 @@ use serde_json::{json, Value}; use crate::{ ids::{DeliveryId, EventId, MessageTarget, RequestId, WorkspaceAlias, WorkspaceId}, - util::ansi::strip_ansi, + util::ansi::{floor_char_boundary, strip_ansi}, worker::detection::ActivityDetector, }; @@ -193,15 +193,46 @@ impl VerificationOutput { /// Retained output read after the supplied producer sequence. pub(crate) fn since(&self, boundary: u64) -> Cow<'_, str> { + self.since_with_overlap(boundary, 0) + } + + /// Retained output read after the supplied producer sequence, plus a + /// bounded suffix from the boundary segment. + /// + /// Unix PTY reads can straddle a verified-write admission: bytes already + /// readable before the write keep the boundary sequence, while bytes from + /// the just-submitted echo may be coalesced into the same kernel read or + /// the immediately following one. Echo verification needs enough context + /// to reconstruct a match that begins just before the fresh segment and + /// ends after it, without accepting a complete stale match. Callers pass + /// `expected.len() - 1`, so a full expected echo wholly before the boundary + /// is still one byte too long to match. + pub(crate) fn since_with_overlap(&self, boundary: u64, overlap_bytes: usize) -> Cow<'_, str> { + self.since_with_overlap_and_fresh_start(boundary, overlap_bytes) + .0 + } + + fn since_with_overlap_and_fresh_start( + &self, + boundary: u64, + overlap_bytes: usize, + ) -> (Cow<'_, str>, usize) { let Some(segment) = self .segments .iter() .find(|segment| segment.sequence > boundary) else { - return Cow::Borrowed(""); + return (Cow::Borrowed(""), 0); }; - let start = segment.start_offset.max(self.base_offset) - self.base_offset; - String::from_utf8_lossy(&self.buffer[start..]) + let fresh_start = segment.start_offset.max(self.base_offset); + let start_offset = fresh_start + .saturating_sub(overlap_bytes) + .max(self.base_offset); + let start = start_offset - self.base_offset; + ( + String::from_utf8_lossy(&self.buffer[start..]), + fresh_start.saturating_sub(start_offset), + ) } pub(crate) fn retained(&self) -> Cow<'_, str> { @@ -214,8 +245,15 @@ pub(crate) fn pending_verification_echo_seen( output: &VerificationOutput, verification: &PendingVerification, ) -> bool { - let observed = output.since(verification.output_boundary); - check_echo_in_output(&observed, &verification.expected_echo) + let fresh = output.since(verification.output_boundary); + if full_echo_match(&fresh, &verification.expected_echo) { + return true; + } + let (straddling, fresh_start) = output.since_with_overlap_and_fresh_start( + verification.output_boundary, + verification.expected_echo.len().saturating_sub(1), + ); + full_echo_match_ending_after(&straddling, &verification.expected_echo, fresh_start) } /// Return a verification whose echo arrived before the PTY write ack was @@ -346,13 +384,44 @@ pub(crate) fn verification_timeout_frames( } /// Check if the expected echo string appears in PTY output (after stripping ANSI). +#[cfg(test)] pub(crate) fn check_echo_in_output(output: &str, expected: &str) -> bool { + full_echo_match(output, expected) +} + +fn full_echo_match(output: &str, expected: &str) -> bool { + full_echo_match_ending_after(output, expected, 0) +} + +fn full_echo_match_ending_after(output: &str, expected: &str, min_end: usize) -> bool { let clean = strip_ansi(output); - if clean.contains(expected) { + let clean_min_end = + strip_ansi(&output[..floor_char_boundary(output, min_end.min(output.len()))]).len(); + if contains_match_ending_after(&clean, expected, clean_min_end) { return true; } let normalize = |value: &str| value.replace("\r\n", "\n"); - normalize(&clean).contains(&normalize(expected)) + let clean_min_end = normalize(&clean[..floor_char_boundary(&clean, clean_min_end)]).len(); + let clean = normalize(&clean); + let expected = normalize(expected); + if contains_match_ending_after(&clean, &expected, clean_min_end) { + return true; + } + // PTYs can report visual soft wraps as line feeds when a long logical line + // crosses the terminal width. The delivery still landed if the full + // character sequence appears with only those line feeds inserted. + let without_lf = |value: &str| value.chars().filter(|ch| *ch != '\n').collect::(); + let clean_min_end = without_lf(&clean[..floor_char_boundary(&clean, clean_min_end)]).len(); + contains_match_ending_after(&without_lf(&clean), &without_lf(&expected), clean_min_end) +} + +fn contains_match_ending_after(haystack: &str, needle: &str, min_end: usize) -> bool { + if needle.is_empty() { + return false; + } + haystack + .match_indices(needle) + .any(|(start, _)| start.saturating_add(needle.len()) > min_end) } pub(crate) fn current_timestamp_ms() -> u64 { @@ -446,6 +515,31 @@ mod tests { assert!(check_echo_in_output(output, expected)); } + #[test] + fn check_echo_tolerates_terminal_soft_wraps() { + let output = "Relay message from Alice [workspace-1234567890 / evt_12345678901234\n567890]: hello world"; + let expected = + "Relay message from Alice [workspace-1234567890 / evt_12345678901234567890]: hello world"; + + assert!(check_echo_in_output(output, expected)); + } + + #[test] + fn check_echo_rejects_interleaved_reminder_even_with_intact_relay_line() { + let expected = "\nAgent Relay MCP tools are available for replies.\n\nRelay message from Alice [evt_1]: hello world"; + let output = "\nAgent Relay MCP tools are available for replies.\nRelay message from Alice [evt_1]: hello world"; + + assert!(!check_echo_in_output(output, expected)); + } + + #[test] + fn check_echo_rejects_partial_wrapped_injection() { + let expected = "\nAgent Relay MCP tools are available for replies.\n\nRelay message from Alice [evt_1]: hello world"; + let output = "\nAgent Relay MCP tools are available for replies.\n\nRelay message from Alice [evt_1]: hello"; + + assert!(!check_echo_in_output(output, expected)); + } + #[test] fn check_echo_does_not_turn_bare_cr_into_line_break() { assert!(!check_echo_in_output("foo\rbar", "foo\nbar")); @@ -507,6 +601,74 @@ mod tests { assert!(pending_verification_echo_seen(&output, &verification)); } + #[test] + fn normalized_full_echo_cannot_match_stale_overlap() { + let expected = format!( + "\nAgent Relay MCP tools are available for replies.\n\n{}", + "Relay message from Alice [evt_repeat]: same body" + ); + let mut output = VerificationOutput::default(); + output.push_str(&expected.replace('\n', "")); + let output_boundary = output.boundary(); + let verification = PendingVerification { + delivery_id: "delivery-repeat".into(), + event_id: "evt-repeat".into(), + expected_echo: expected, + output_boundary, + injected_at: Instant::now(), + attempts: 1, + max_attempts: 1, + request_id: None, + workspace_id: None, + workspace_alias: None, + from: "Alice".to_string(), + body: "same body".to_string(), + target: "Worker".into(), + }; + + output.push_str("\nfresh prompt only\n"); + assert!( + !pending_verification_echo_seen(&output, &verification), + "a stale normalized echo inside the overlap must not verify a later write" + ); + output.push_str(&verification.expected_echo); + assert!( + pending_verification_echo_seen(&output, &verification), + "a full echo observed after the submission boundary still verifies" + ); + } + + #[test] + fn relay_line_alone_never_verifies_a_wrapped_injection() { + let relay_line = "Relay message from Alice [evt_real]: same body"; + let expected = format!( + "\nAgent Relay MCP tools are available for replies.\n\n{relay_line}" + ); + let mut output = VerificationOutput::default(); + let output_boundary = output.boundary(); + let verification = PendingVerification { + delivery_id: "delivery-real".into(), + event_id: "evt-real".into(), + expected_echo: expected, + output_boundary, + injected_at: Instant::now(), + attempts: 1, + max_attempts: 1, + request_id: None, + workspace_id: None, + workspace_alias: None, + from: "Alice".to_string(), + body: "same body".to_string(), + target: "Worker".into(), + }; + + output.push_str(relay_line); + assert!( + !pending_verification_echo_seen(&output, &verification), + "real harnesses must echo the full submitted payload, not only the Relay line" + ); + } + #[test] fn verification_ignores_matching_output_queued_before_write_submission() { let expected = "Relay message from Alice [evt-queued]: same body"; @@ -546,6 +708,44 @@ mod tests { ); } + #[test] + fn verification_reconstructs_echo_split_across_the_boundary() { + let expected = + "Relay message from Alice [workspace-1234567890 / evt-1234567890]: wrapped body"; + let split_at = expected + .find("evt-") + .expect("expected text contains split marker") + + 4; + let mut output = VerificationOutput::default(); + output.push_output(1, &expected.as_bytes()[..split_at]); + let output_boundary = output.boundary(); + let verification = PendingVerification { + delivery_id: "delivery-split".into(), + event_id: "evt-split".into(), + expected_echo: expected.to_string(), + output_boundary, + injected_at: Instant::now(), + attempts: 1, + max_attempts: 1, + request_id: None, + workspace_id: None, + workspace_alias: None, + from: "Alice".to_string(), + body: "wrapped body".to_string(), + target: "Worker".into(), + }; + + assert!( + !pending_verification_echo_seen(&output, &verification), + "a complete pre-boundary echo must not verify" + ); + output.push_output(2, &expected.as_bytes()[split_at..]); + assert!( + pending_verification_echo_seen(&output, &verification), + "a match that starts in the boundary segment and finishes after it must verify" + ); + } + #[test] fn verification_offsets_remain_monotonic_when_the_tail_is_trimmed() { let mut output = VerificationOutput::default(); diff --git a/crates/broker/src/codex_thread.rs b/crates/broker/src/codex_thread.rs new file mode 100644 index 0000000000..abdf53499c --- /dev/null +++ b/crates/broker/src/codex_thread.rs @@ -0,0 +1,581 @@ +use std::path::{Path, PathBuf}; +use std::process::Stdio; + +use crate::ids::DeliveryId; +use serde_json::Value; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CodexThreadSession { + thread_id: String, + rollout_path: Option, + codex_home: Option, +} + +impl CodexThreadSession { + pub(crate) fn new(thread_id: impl Into, rollout_path: Option) -> Option { + let thread_id = thread_id.into(); + let thread_id = thread_id.trim(); + if !safe_thread_id(thread_id) { + return None; + } + Some(Self { + thread_id: thread_id.to_string(), + rollout_path, + codex_home: None, + }) + } + + pub(crate) fn with_codex_home(mut self, codex_home: Option) -> Self { + self.codex_home = codex_home; + self + } + + pub(crate) fn thread_id(&self) -> &str { + &self.thread_id + } + + pub(crate) fn has_rollout_path(&self) -> bool { + self.rollout_path.is_some() + } + + pub(crate) fn marker_for(delivery_id: &DeliveryId) -> String { + format!("relay-delivery-id:{}", delivery_id.as_str()) + } + + pub(crate) fn body_with_marker(body: &str, delivery_id: &DeliveryId) -> String { + let marker = Self::marker_for(delivery_id); + if body.contains(&marker) { + return body.to_string(); + } + format!("{body}\n\n") + } + + /// What Codex's own records say about a delivery this route queued. + /// + /// Two stores, and the difference between them is the difference between + /// "delivered" and "read": + /// + /// * `$CODEX_HOME/queue_1.sqlite` — the table `codex queue` writes to. A + /// row here is positive proof the message is durably enqueued against + /// the thread. It is NOT proof anybody read it. + /// * the thread's rollout JSONL — where Codex records the items a turn + /// actually consumed. A queued message does not appear here at all until + /// the live session picks it up. + /// + /// Verified against `codex-cli 0.155.0-alpha.9.2`: after `codex queue` the + /// marker is present in `queued_items` and absent from every rollout + /// record, and `thread/items/list` likewise returns only consumed items. + /// The capture is in + /// `.workflow-artifacts/migrate-native-delivery/phase-1-codex-queue-20260921a/evidence/codex-capture/`. + /// + /// Seam rule 4 is why the two are not collapsed: an acknowledgement has to + /// name what was observed, and "a row exists in a queue table" is not an + /// observation that the recipient read anything. + pub(crate) async fn observe_marker(&self, delivery_id: &DeliveryId) -> CodexMarkerObservation { + let marker = Self::marker_for(delivery_id); + let rollout_path = match self.rollout_path.clone() { + Some(path) => Some(path), + None => { + match lookup_thread_record(self.thread_id(), self.codex_home.as_deref()).await { + Ok(record) => record.and_then(|record| record.rollout_path), + Err(error) => { + tracing::warn!( + target = "agent_relay::broker", + thread_id = %self.thread_id(), + error = %error, + "Codex state lookup failed while settling queued delivery" + ); + None + } + } + } + }; + if let Some(path) = rollout_path { + if let Some(offset) = find_consumed_marker_offset(&path, &marker).await { + return CodexMarkerObservation::Consumed { + source: path.display().to_string(), + offset, + }; + } + } + // Absence from the rollout is not absence from Codex. Ask the queue + // store before answering, so "still waiting in the queue" is reported + // as the distinct fact it is rather than as "nothing is known". + match lookup_queued_marker(self.thread_id(), self.codex_home.as_deref(), &marker).await { + Ok(Some(source)) => CodexMarkerObservation::Queued { source }, + Ok(None) => CodexMarkerObservation::Unknown, + Err(error) => { + tracing::warn!( + target = "agent_relay::broker", + thread_id = %self.thread_id(), + error = %error, + "Codex queue lookup failed while settling queued delivery" + ); + CodexMarkerObservation::Unknown + } + } + } +} + +/// What Codex's records show for one queued delivery. +/// +/// There is deliberately no variant meaning "the message is not there": a +/// queue store this broker cannot read and a message that was never written +/// look identical from here, and treating the second as the first re-sends a +/// delivered message (seam rule 2). `Unknown` is the floor. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum CodexMarkerObservation { + /// The marker appears in the thread's rollout inside a record Codex writes + /// for a USER INPUT item — i.e. a turn consumed it. This is the only + /// observation that supports an acknowledgement. + Consumed { source: String, offset: u64 }, + /// The marker is in Codex's durable queue for this thread and has not been + /// consumed. Delivered to the transport; not read by anyone. + Queued { source: String }, + /// Neither store answered. Not absence. + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CodexThreadRecord { + pub(crate) rollout_path: Option, + pub(crate) cwd: Option, +} + +async fn find_consumed_marker_offset(path: &Path, marker: &str) -> Option { + let bytes = tokio::fs::read(path).await.ok()?; + let mut offset = 0u64; + for line in bytes.split_inclusive(|byte| *byte == b'\n') { + if line + .windows(marker.len()) + .any(|window| window == marker.as_bytes()) + && marker_line_is_consumed_user_input(line, marker) + { + return Some(offset); + } + offset = offset.saturating_add(line.len() as u64); + } + None +} + +/// Read Codex's own queue store for a message carrying `marker`. +/// +/// This is the table `codex queue` writes: one row per queued item, keyed by +/// thread, with the message body inside `payload_json`. Reading it is what +/// lets settlement say "durably enqueued and not yet read" instead of +/// collapsing that onto "nothing is known". +/// +/// Returns the store path when the marker is present, `Ok(None)` when the +/// store is readable and the marker is not in it, and `Err` when the store +/// could not be read at all — which the caller must NOT treat as absence. +async fn lookup_queued_marker( + thread_id: &str, + codex_home: Option<&Path>, + marker: &str, +) -> Result, String> { + if !safe_thread_id(thread_id) { + return Err("Codex thread id contains unsupported characters".to_string()); + } + let Some(db) = codex_queue_db_path(codex_home) else { + return Ok(None); + }; + let query = format!( + "SELECT COALESCE(payload_json, '') FROM queued_items WHERE thread_id = '{}';", + sql_quote(thread_id) + ); + let output = tokio::process::Command::new("sqlite3") + .arg("-readonly") + .arg(&db) + .arg(query) + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .output() + .await + .map_err(|_| "sqlite3 is unavailable".to_string())?; + if !output.status.success() { + return Err("Codex queue lookup failed".to_string()); + } + let stdout = String::from_utf8(output.stdout).map_err(|_| "sqlite3 output was not utf-8")?; + // The marker is relay-generated and ASCII, so a substring test over the + // stored payload is sufficient and needs no JSON parse of a vendor shape + // that may change. + Ok(stdout.contains(marker).then(|| db.display().to_string())) +} + +fn codex_queue_db_path(codex_home: Option<&Path>) -> Option { + codex_store_path(codex_home, "queue_1.sqlite") +} + +pub(crate) async fn lookup_thread_record( + thread_id: &str, + codex_home: Option<&Path>, +) -> Result, String> { + if !safe_thread_id(thread_id) { + return Err("Codex thread id contains unsupported characters".to_string()); + } + let Some(db) = codex_state_db_path(codex_home) else { + return Ok(None); + }; + tracing::debug!( + target = "agent_relay::broker", + thread_id = %thread_id, + db = %db.display(), + "looking up Codex rollout path" + ); + let query = format!( + "SELECT COALESCE(rollout_path, ''), COALESCE(cwd, '') FROM threads WHERE id = '{}';", + sql_quote(thread_id) + ); + let output = tokio::process::Command::new("sqlite3") + .arg("-readonly") + .arg("-separator") + .arg("\t") + .arg(db) + .arg(query) + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .output() + .await + .map_err(|_| "sqlite3 is unavailable".to_string())?; + if !output.status.success() { + return Err("Codex state lookup failed".to_string()); + } + let stdout = String::from_utf8(output.stdout).map_err(|_| "sqlite3 output was not utf-8")?; + let mut rows = stdout + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()); + let Some(first) = rows.next() else { + return Ok(None); + }; + if rows.next().is_some() { + return Err("Codex state returned multiple rows for one thread id".to_string()); + } + let mut fields = first.splitn(3, '\t'); + let rollout_path = fields + .next() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(PathBuf::from); + let cwd = fields + .next() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(PathBuf::from); + Ok(Some(CodexThreadRecord { rollout_path, cwd })) +} + +fn codex_state_db_path(codex_home: Option<&Path>) -> Option { + codex_store_path(codex_home, "state_5.sqlite") +} + +fn codex_store_path(codex_home: Option<&Path>, file_name: &str) -> Option { + let home = codex_home.map(PathBuf::from).or_else(|| { + std::env::var_os("CODEX_HOME") + .map(PathBuf::from) + .or_else(|| dirs::home_dir().map(|home| home.join(".codex"))) + })?; + let root = home.join(file_name); + if root.is_file() { + return Some(root); + } + let nested = home.join("sqlite").join(file_name); + nested.is_file().then_some(nested) +} + +fn safe_thread_id(thread_id: &str) -> bool { + !thread_id.is_empty() + && !thread_id.starts_with('-') + && thread_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b':' | b'.')) +} + +fn sql_quote(value: &str) -> String { + value.replace('\'', "''") +} + +/// Whether a rollout line is a record Codex wrote for a CONSUMED user input +/// item carrying `marker`. +/// +/// Positive, not negative. A filter that only rejects assistant / reasoning / +/// summary records still accepts any future record shape that happens to +/// quote the message back, and seam rule 4 forbids acknowledging on an +/// observation that was never positively identified. The two accepted shapes +/// are the two projections Codex writes for one consumed user item, captured +/// live from `codex-cli 0.155.0-alpha.9.2` (see +/// `evidence/codex-capture/rollout-consumed-user-item.jsonl`): +/// +/// ```text +/// {"type":"response_item","payload":{"type":"message","role":"user", +/// "content":[{"type":"input_text","text":"……"}]}} +/// {"type":"event_msg","payload":{"type":"item_completed", +/// "item":{"type":"UserMessage","content":[{"type":"text","text":"……"}]}}} +/// ``` +/// +/// The older un-wrapped form (`{"type":"item_completed","item":{…,"role":"user"}}`) +/// is accepted too, so a Codex that predates the `payload` envelope still +/// settles. +fn marker_line_is_consumed_user_input(line: &[u8], marker: &str) -> bool { + let Ok(value) = serde_json::from_slice::(line) else { + return false; + }; + // The rollout wraps every record in an envelope whose own `type` is the + // projection (`response_item` / `event_msg`), never the item kind. + let record = value.get("payload").unwrap_or(&value); + for node in [Some(record), record.get("item")].into_iter().flatten() { + if !json_value_contains_marker(node, marker) { + continue; + } + if json_value_is_non_delivery_artifact(node) { + continue; + } + if node_is_user_input(node) { + return true; + } + } + false +} + +/// The kinds Codex uses for a user-authored input item, across the +/// projections above. Compared lower-cased because the rollout writes +/// `UserMessage` in the event projection and `message` + `role: user` in the +/// response-item projection. +fn node_is_user_input(node: &Value) -> bool { + let lower = |key: &str| { + node.get(key) + .and_then(Value::as_str) + .map(str::to_ascii_lowercase) + }; + if lower("role").as_deref() == Some("user") { + return true; + } + matches!( + lower("type").as_deref(), + Some("usermessage" | "user_message" | "userinput" | "user_input") + ) +} + +fn json_value_contains_marker(value: &Value, marker: &str) -> bool { + match value { + Value::String(text) => text.contains(marker), + Value::Array(items) => items + .iter() + .any(|item| json_value_contains_marker(item, marker)), + Value::Object(map) => map + .values() + .any(|item| json_value_contains_marker(item, marker)), + _ => false, + } +} + +fn json_value_is_non_delivery_artifact(value: &Value) -> bool { + let mut strings = Vec::new(); + collect_schema_strings(value, &mut strings); + strings.iter().any(|text| { + matches!( + text.as_str(), + "assistant" | "system" | "summary" | "reasoning" | "compacted" + ) + }) +} + +fn collect_schema_strings(value: &Value, out: &mut Vec) { + match value { + Value::Object(map) => { + for key in ["type", "role", "item_type", "payload_type"] { + if let Some(text) = map.get(key).and_then(Value::as_str) { + out.push(text.to_ascii_lowercase()); + } + } + for value in map.values() { + collect_schema_strings(value, out); + } + } + Value::Array(items) => { + for value in items { + collect_schema_strings(value, out); + } + } + _ => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn marker_is_stable_and_idempotent() { + let delivery_id = DeliveryId::new("del_123"); + let body = CodexThreadSession::body_with_marker("hello", &delivery_id); + assert!(body.contains("hello")); + assert!(body.contains("relay-delivery-id:del_123")); + assert_eq!( + CodexThreadSession::body_with_marker(&body, &delivery_id), + body + ); + } + + #[test] + fn thread_ids_that_look_like_flags_are_rejected() { + assert!(CodexThreadSession::new("-config", None).is_none()); + assert!(CodexThreadSession::new("thread-1", None).is_some()); + } + + #[test] + fn codex_state_db_path_uses_target_home_first() { + let target = tempfile::tempdir().expect("target codex home"); + let broker = tempfile::tempdir().expect("broker codex home"); + let target_db = target.path().join("state_5.sqlite"); + let broker_db = broker.path().join("state_5.sqlite"); + std::fs::write(&target_db, b"target").expect("target db"); + std::fs::write(&broker_db, b"broker").expect("broker db"); + + assert_eq!( + codex_state_db_path(Some(target.path())).as_deref(), + Some(target_db.as_path()) + ); + } + + /// The two real projections Codex writes for ONE consumed user input item, + /// captured live from `codex-cli 0.155.0-alpha.9.2` (`codex app-server` → + /// `thread/start` → `turn/start`). Verbatim apart from the marker text and + /// shortened ids; see + /// `evidence/codex-capture/rollout-consumed-user-item.jsonl`. + fn consumed_user_item_records(marker: &str) -> String { + format!( + concat!( + r#"{{"timestamp":"2026-09-22T18:29:35.389Z","ordinal":8,"type":"response_item","payload":{{"type":"message","id":"msg_01a0ca61","role":"user","content":[{{"type":"input_text","text":"hello from relay\n\n"}}],"internal_chat_message_metadata_passthrough":{{"turn_id":"01a0ca61-6208","create_time":1790101775.388333,"content_item_kinds":["user.text"]}}}}}}"#, + "\n", + r#"{{"timestamp":"2026-09-22T18:29:35.389Z","ordinal":9,"type":"event_msg","payload":{{"type":"item_completed","thread_id":"01a0ca61-60fd","turn_id":"01a0ca61-6208","item":{{"type":"UserMessage","id":"01a0ca61-641d","content":[{{"type":"text","text":"hello from relay\n\n","text_elements":[]}}]}},"started_at_ms":1790101775389,"completed_at_ms":1790101775389}}}}"#, + "\n", + ), + marker = marker + ) + } + + /// An isolated `CODEX_HOME` with no queue store, so a test that reaches the + /// queue lookup cannot read the operator's real `~/.codex`. + fn empty_codex_home() -> tempfile::TempDir { + tempfile::tempdir().expect("codex home") + } + + /// Write a `queue_1.sqlite` shaped like the one `codex queue` writes. + fn write_queue_db(home: &std::path::Path, thread_id: &str, payload: &str) { + let db = home.join("queue_1.sqlite"); + let status = std::process::Command::new("sqlite3") + .arg(&db) + .arg(format!( + "CREATE TABLE queued_items (id TEXT PRIMARY KEY NOT NULL, thread_id TEXT NOT NULL, \ + payload_json TEXT NOT NULL, queue_order INTEGER NOT NULL, created_at_ms INTEGER NOT NULL, \ + updated_at_ms INTEGER NOT NULL); \ + INSERT INTO queued_items VALUES ('item-1', '{thread_id}', '{payload}', 0, 0, 0);" + )) + .status() + .expect("sqlite3 must be available to build the queue fixture"); + assert!(status.success(), "failed to build queue fixture"); + } + + #[tokio::test] + async fn a_consumed_user_item_is_observed_in_both_real_projections() { + for (index, record) in consumed_user_item_records("relay-delivery-id:del_seen") + .lines() + .enumerate() + { + let home = empty_codex_home(); + let path = home.path().join("session.jsonl"); + std::fs::write(&path, format!("{{\"text\":\"before\"}}\n{record}\n")) + .expect("write session"); + let session = CodexThreadSession::new("thread-1", Some(path)) + .expect("session") + .with_codex_home(Some(home.path().to_path_buf())); + + let observed = session.observe_marker(&DeliveryId::new("del_seen")).await; + + let CodexMarkerObservation::Consumed { offset, .. } = observed else { + panic!("real consumed projection {index} must be observed, got {observed:?}"); + }; + assert!(offset > 0); + } + } + + /// The shape `codex queue` ACTUALLY produces before anybody reads the + /// message: a row in `queued_items`, and nothing in the rollout. + /// + /// This is the negative case F3 asks for. Settlement must report the + /// message as queued — never as acknowledged — because no turn has + /// consumed it and a read receipt would name a reader who does not exist. + #[tokio::test] + async fn a_queued_but_unconsumed_message_is_queued_not_consumed() { + let home = empty_codex_home(); + let thread_id = "01a0ca61-60fd-7492-a02d-a014d386db12"; + let path = home.path().join("session.jsonl"); + // The real rollout after `codex queue`: session records only, no marker. + std::fs::write( + &path, + b"{\"type\":\"event_msg\",\"payload\":{\"type\":\"task_started\"}}\n", + ) + .expect("write session"); + write_queue_db( + home.path(), + thread_id, + r#"{"UserInput":{"content":[{"type":"text","text":"hello from relay\n\n"}],"client_id":"01a0ca61-627d"}}"#, + ); + let session = CodexThreadSession::new(thread_id, Some(path)) + .expect("session") + .with_codex_home(Some(home.path().to_path_buf())); + + assert_eq!( + session.observe_marker(&DeliveryId::new("del_queued")).await, + CodexMarkerObservation::Queued { + source: home.path().join("queue_1.sqlite").display().to_string(), + } + ); + } + + /// Rule 4: an assistant record that quotes the delivery back is not an + /// observation that the recipient read it, and neither is any record whose + /// kind the matcher cannot positively identify as a user input item. + #[tokio::test] + async fn a_quoted_marker_in_a_non_user_record_is_not_an_acknowledgement() { + for record in [ + r#"{"type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"you said "}]}}"#, + r#"{"type":"response_item","payload":{"type":"reasoning","summary":[{"type":"summary_text","text":""}]}}"#, + // The synthetic shape the phase's original settlement fixture used. + // No Codex version emits it, and it names no item kind, so it must + // not settle either. + r#"{"text":""}"#, + ] { + let home = empty_codex_home(); + let path = home.path().join("session.jsonl"); + std::fs::write(&path, format!("{record}\n")).expect("write session"); + let session = CodexThreadSession::new("thread-1", Some(path)) + .expect("session") + .with_codex_home(Some(home.path().to_path_buf())); + + assert_eq!( + session.observe_marker(&DeliveryId::new("del_quoted")).await, + CodexMarkerObservation::Unknown, + "record must not acknowledge: {record}" + ); + } + } + + #[tokio::test] + async fn missing_marker_is_unknown_not_absent() { + let home = empty_codex_home(); + let path = home.path().join("session.jsonl"); + std::fs::write(&path, br#"{"text":"different"}"#).expect("write session"); + let session = CodexThreadSession::new("thread-1", Some(path)) + .expect("session") + .with_codex_home(Some(home.path().to_path_buf())); + + assert_eq!( + session + .observe_marker(&DeliveryId::new("del_missing")) + .await, + CodexMarkerObservation::Unknown + ); + } +} diff --git a/crates/broker/src/delivery/backend.rs b/crates/broker/src/delivery/backend.rs index 1893a18358..ef67e1f9ed 100644 --- a/crates/broker/src/delivery/backend.rs +++ b/crates/broker/src/delivery/backend.rs @@ -2,6 +2,7 @@ use std::collections::{HashSet, VecDeque}; use std::fmt; use std::future::Future; use std::pin::Pin; +use std::time::{Duration, Instant}; use crate::ids::{DeliveryId, WorkerName}; use crate::protocol::RelayDelivery; @@ -13,6 +14,11 @@ pub type DeliveryBackendFuture<'a, T> = Pin + Send + pub struct RouteId(String); impl RouteId { + /// The broker-owned PTY route. Named so the one place that has to reason + /// about "does this transport outlive the broker" cannot drift from the + /// string the PTY backend reports. + pub const PTY: &'static str = "pty"; + pub fn new(value: impl Into) -> Self { Self(value.into()) } @@ -20,6 +26,20 @@ impl RouteId { pub fn as_str(&self) -> &str { &self.0 } + + /// Whether a message this route accepted can still reach the recipient + /// after the broker process that wrote it is gone. + /// + /// The PTY route writes into a child the broker owns: the child dies with + /// the broker, so an un-acknowledged write provably never arrived and + /// redelivering it after a restart is correct. Every native route hands the + /// message to a durable store owned by the vendor's own session — Codex's + /// `queued_items` table, for instance — which outlives both the worker and + /// the broker. Redelivering there is a double delivery, so a restart must + /// remember it (seam rule 2). + pub fn survives_broker_restart(&self) -> bool { + self.0 != Self::PTY + } } impl fmt::Display for RouteId { @@ -201,21 +221,40 @@ pub enum SendStatus { } /// Result recorded for a send that was accepted by a specific route. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct SendReceipt { pub delivery_id: DeliveryId, pub route: RouteId, pub status: SendStatus, + recorded_at: Instant, +} + +// `recorded_at` is internal settlement bookkeeping, not part of a receipt's +// externally observable identity. Keep equality stable for callers and tests +// that construct equivalent receipts at different instants. +impl PartialEq for SendReceipt { + fn eq(&self, other: &Self) -> bool { + self.delivery_id == other.delivery_id + && self.route == other.route + && self.status == other.status + } } +impl Eq for SendReceipt {} + impl SendReceipt { pub fn new(delivery_id: DeliveryId, route: RouteId, status: SendStatus) -> Self { Self { delivery_id, route, status, + recorded_at: Instant::now(), } } + + pub fn age(&self) -> Duration { + self.recorded_at.elapsed() + } } /// Result of asking the seam to send one delivery. @@ -485,6 +524,54 @@ impl DeliverySeam { self.recorded_route(delivery_id).is_some() || self.evicted.contains(delivery_id) } + /// Re-seed a receipt for a delivery a PREVIOUS broker lifetime handed to a + /// route that outlives the broker. + /// + /// The seam's memory is process-local and starts empty, but "already handed + /// to a transport" is not a process-local fact for a native route: the + /// message sits in the vendor's own durable queue whether this broker is + /// running or not. Without this, a reloaded pending snapshot classifies + /// `Fresh` and the backend queues the same body a second time — rule 2, + /// broken by a restart rather than by a transport fault. + /// + /// Deliberately restores the receipt as `HandedOver` and not as an + /// acknowledgement: nothing was observed, and the settlement poll must + /// still run. `recorded_at` necessarily restarts from now — an `Instant` + /// has no meaning across processes — so the settlement window is measured + /// from the reload, not from the original write. + /// + /// Callers must only pass a route for which + /// [`RouteId::survives_broker_restart`] is true; a PTY receipt restored + /// here would strand a message the dead child never received. + pub fn restore_handed_over(&mut self, delivery_id: DeliveryId, route: RouteId) { + debug_assert!( + route.survives_broker_restart(), + "restoring a receipt for a route that died with the broker would strand the message" + ); + if self.was_sent(&delivery_id) { + return; + } + self.record_receipt(SendReceipt::new( + delivery_id, + route, + SendStatus::HandedOver(HandoverState::HandedOver), + )); + } + + /// Re-seed the tombstone for a delivery a previous broker lifetime handed + /// to a transport whose route is no longer known. + /// + /// Same fact as [`Self::restore_handed_over`] with the route missing, which + /// is the shape an eviction leaves behind. [`Self::was_sent`] answers true + /// and [`Self::send`] answers [`SendOutcome::Forgotten`], so the caller + /// settles in doubt instead of writing again. + pub fn restore_forgotten(&mut self, delivery_id: DeliveryId) { + if self.was_sent(&delivery_id) { + return; + } + self.evicted.insert(delivery_id); + } + pub fn recorded_route(&self, delivery_id: &DeliveryId) -> Option<&RouteId> { self.receipts .iter() @@ -493,6 +580,14 @@ impl DeliverySeam { .map(|receipt| &receipt.route) } + pub fn recorded_age(&self, delivery_id: &DeliveryId) -> Option { + self.receipts + .iter() + .rev() + .find(|receipt| &receipt.delivery_id == delivery_id) + .map(SendReceipt::age) + } + fn record_receipt(&mut self, receipt: SendReceipt) { while self.receipts.len() >= Self::MAX_RECEIPTS { if let Some(dropped) = self.receipts.pop_front() { @@ -505,13 +600,14 @@ impl DeliverySeam { self.receipts.push_back(receipt); } - fn replace_receipt(&mut self, receipt: SendReceipt) { + fn replace_receipt(&mut self, mut receipt: SendReceipt) { if let Some(recorded) = self .receipts .iter_mut() .rev() .find(|recorded| recorded.delivery_id == receipt.delivery_id) { + receipt.recorded_at = recorded.recorded_at; *recorded = receipt; } else { // Defensive fallback: the seam is exclusively borrowed while a diff --git a/crates/broker/src/delivery/codex_queue.rs b/crates/broker/src/delivery/codex_queue.rs new file mode 100644 index 0000000000..596a7a904a --- /dev/null +++ b/crates/broker/src/delivery/codex_queue.rs @@ -0,0 +1,1057 @@ +use std::collections::HashMap; +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; + +use serde_json::Value; + +use super::backend::{ + DeliveryBackend, DeliveryBackendFuture, DeliveryError, HandoverState, ObservedAck, RouteId, + SendRequest, SendStatus, SettleRequest, SettleStatus, TransportStatus, +}; +use crate::cli::command_parse::{normalize_cli_name, parse_cli_command}; +use crate::delivery::codex_thread::{CodexMarkerObservation, CodexThreadSession}; +use crate::ids::WorkerName; +use crate::protocol::{AgentSpec, ResolvedHarnessConfig}; +use crate::worker::WorkerRegistry; + +const CODEX_QUEUE_TIMEOUT: Duration = Duration::from_secs(15); +const CODEX_QUEUE_ERROR_MAX_BYTES: usize = 2_048; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CodexQueueTarget { + command: String, + global_args: Vec, + env: Vec<(String, String)>, + cwd: Option, + thread: CodexThreadSession, + allow_bundle_fallbacks: bool, +} + +impl CodexQueueTarget { + pub(crate) fn attached( + thread_id: impl Into, + codex_home: Option, + rollout_path: PathBuf, + cwd: Option, + ) -> Result { + let mut env = Vec::new(); + if let Some(home) = codex_home.as_ref() { + env.push(( + "CODEX_HOME".to_string(), + home.to_string_lossy().into_owned(), + )); + } + let thread = CodexThreadSession::new(thread_id, Some(rollout_path)) + .ok_or_else(|| "invalid Codex thread id".to_string())? + .with_codex_home(codex_home); + Ok(Self { + // The authenticated broker, not the MCP caller, resolves the + // executable. A bare name also enables the broker's trusted app + // bundle fallbacks when the desktop Codex is newer than PATH. + command: "codex".to_string(), + global_args: Vec::new(), + env, + cwd, + thread, + allow_bundle_fallbacks: true, + }) + } + + pub(crate) fn thread_id(&self) -> &str { + self.thread.thread_id() + } + + pub(crate) fn has_verified_rollout_path(&self) -> bool { + self.thread.has_rollout_path() + } + + #[cfg(test)] + pub(crate) fn new_for_test( + command: impl Into, + global_args: Vec, + cwd: Option, + thread_id: impl Into, + rollout_path: Option, + ) -> Self { + Self { + command: command.into(), + global_args, + env: Vec::new(), + cwd, + thread: CodexThreadSession::new(thread_id, rollout_path).expect("test thread id"), + allow_bundle_fallbacks: false, + } + } + + fn route_id(&self) -> RouteId { + RouteId::new(format!("codex-queue:{}", self.thread.thread_id())) + } +} + +/// Native Codex delivery over the public `codex queue` command. +/// +/// This backend is intentionally selectable only when Relay already has a +/// stable thread id for the worker. It does not infer ownership from Codex's +/// SQLite state; that file is only a settlement/verification index. +pub(crate) struct CodexQueueBackend { + target: Option, +} + +impl CodexQueueBackend { + pub(crate) fn for_worker(workers: &WorkerRegistry, worker_name: &WorkerName) -> Self { + Self { + target: workers + .native_codex_target(worker_name) + .cloned() + .or_else(|| target_for_worker(workers, worker_name)), + } + } + + pub(crate) fn is_selectable(&self) -> bool { + self.target.is_some() + } + + #[cfg(test)] + pub(crate) fn for_target(target: CodexQueueTarget) -> Self { + Self { + target: Some(target), + } + } +} + +impl DeliveryBackend for CodexQueueBackend { + fn route_id(&self) -> RouteId { + self.target + .as_ref() + .map(CodexQueueTarget::route_id) + .unwrap_or_else(|| RouteId::new("codex-queue")) + } + + fn transport_status(&mut self) -> TransportStatus { + match self.target.as_ref() { + Some(_) => TransportStatus::Available, + None => TransportStatus::Unavailable( + "Codex queue route requires a Codex worker with a known thread id".to_string(), + ), + } + } + + fn send<'a>( + &'a mut self, + request: &'a SendRequest, + ) -> DeliveryBackendFuture<'a, Result> { + Box::pin(async move { + let target = self.target.as_ref().ok_or_else(|| { + DeliveryError::unavailable( + "Codex queue route requires a Codex worker with a known thread id", + ) + })?; + let queue_command = target.ensure_queue_capability().await?; + target.queue_message(request, &queue_command).await?; + Ok(SendStatus::HandedOver(HandoverState::HandedOver)) + }) + } + + /// Settle from Codex's own records. + /// + /// Only a CONSUMED user input item in the thread's rollout produces an + /// acknowledgement. A message still sitting in `queued_items` is durably + /// delivered to the transport and read by nobody, so it settles as + /// `HandedOver` — the same answer an unreadable store gives, because both + /// mean "not observed to have been read", and neither licenses the read + /// receipt `runtime/maintenance.rs` publishes off an ack. Seam rule 4. + fn settle<'a>( + &'a mut self, + request: &'a SettleRequest, + ) -> DeliveryBackendFuture<'a, SettleStatus> { + Box::pin(async move { + let Some(target) = self.target.as_ref() else { + return SettleStatus::HandedOver(HandoverState::HandedOver); + }; + match target.thread.observe_marker(&request.delivery_id).await { + CodexMarkerObservation::Consumed { source, offset } => { + SettleStatus::Acked(ObservedAck::transcript(source, offset)) + } + CodexMarkerObservation::Queued { source } => { + tracing::debug!( + target = "agent_relay::broker", + thread_id = %target.thread_id(), + delivery_id = %request.delivery_id, + source = %source, + "Codex holds this delivery in its durable queue; not acknowledged \ + until a turn consumes it" + ); + SettleStatus::HandedOver(HandoverState::HandedOver) + } + CodexMarkerObservation::Unknown => { + SettleStatus::HandedOver(HandoverState::HandedOver) + } + } + }) + } +} + +impl CodexQueueTarget { + async fn ensure_queue_capability(&self) -> Result { + crate::codex_session::resolve_queue_capable_codex_command( + &self.command, + &self.global_args, + self.cwd.as_deref(), + &self.env, + self.allow_bundle_fallbacks, + ) + .await + .ok_or_else(|| { + DeliveryError::unavailable( + "installed Codex does not expose `codex queue --thread --message`", + ) + }) + } + + async fn queue_message( + &self, + request: &SendRequest, + queue_command: &str, + ) -> Result<(), DeliveryError> { + let body = CodexThreadSession::body_with_marker(&request.body, &request.delivery_id); + let mut command = self.command(queue_command); + command + .arg("queue") + .arg("--thread") + .arg(self.thread.thread_id()) + .arg(format!("--message={body}")) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let child = command + .spawn() + .map_err(|_| DeliveryError::unavailable("Codex queue command could not start"))?; + let output = tokio::time::timeout(CODEX_QUEUE_TIMEOUT, child.wait_with_output()) + .await + .map_err(|_| { + DeliveryError::committed(format!( + "Codex queue did not exit within {}ms", + CODEX_QUEUE_TIMEOUT.as_millis() + )) + })? + .map_err(|_| DeliveryError::committed("Codex queue wait failed"))?; + if output.status.success() { + Ok(()) + } else { + let detail = bounded_redacted_stderr(&output.stderr); + Err(DeliveryError::committed(format!( + "Codex queue exited with status {}{}", + output.status.code().unwrap_or(-1), + if detail.is_empty() { + String::new() + } else { + format!(": {detail}") + } + ))) + } + } + + fn command(&self, program: &str) -> tokio::process::Command { + let mut command = tokio::process::Command::new(program); + command.args(&self.global_args); + for (key, value) in &self.env { + command.env(key, value); + } + if let Some(cwd) = self.cwd.as_deref() { + command.current_dir(cwd); + } + command.stdin(Stdio::null()); + command + } +} + +fn bounded_redacted_stderr(stderr: &[u8]) -> String { + let mut detail = crate::redact::redact(String::from_utf8_lossy(stderr).trim()); + if detail.len() > CODEX_QUEUE_ERROR_MAX_BYTES { + let mut end = CODEX_QUEUE_ERROR_MAX_BYTES; + while end > 0 && !detail.is_char_boundary(end) { + end -= 1; + } + detail.truncate(end); + detail.push('…'); + } + detail +} + +fn target_for_worker( + workers: &WorkerRegistry, + worker_name: &WorkerName, +) -> Option { + target_for_spec(&workers.workers.get(worker_name)?.spec) +} + +/// The selection guard, over the one thing it actually depends on. +/// +/// Two independent conditions, both required: the worker's resolved command +/// must normalize to `codex`, and relay must already hold a stable thread id +/// for it. Everything the parity answer claims about non-Codex CLIs rests on +/// this being structural rather than on per-CLI runs, so it is lifted out of +/// the registry lookup where a test can drive it directly. +fn target_for_spec(spec: &AgentSpec) -> Option { + let CommandParts { + command, + cli_args, + metadata, + env, + } = codex_command_parts(spec)?; + let (command, cli_args) = metadata + .and_then(metadata_codex_queue_command) + .unwrap_or((command, cli_args)); + let normalized = normalize_cli_name(&command).to_lowercase(); + if normalized != "codex" && normalized != "codex.exe" { + return None; + } + let thread_id = spec.session_id.as_deref().or_else(|| { + spec.harness_config + .as_ref() + .and_then(ResolvedHarnessConfig::session_id) + })?; + let rollout_path = metadata.and_then(metadata_rollout_path); + let env = codex_queue_env(metadata, env); + let codex_home = env + .iter() + .find(|(key, _)| key == "CODEX_HOME") + .map(|(_, value)| PathBuf::from(value)); + Some(CodexQueueTarget { + command, + global_args: codex_queue_global_args(&cli_args), + env, + cwd: spec.cwd.as_deref().map(PathBuf::from), + thread: CodexThreadSession::new(thread_id, rollout_path)?.with_codex_home(codex_home), + // A broker-owned terminal must only use the exact Codex binary it + // launched. A different app-bundle binary may expose `queue` while the + // running TUI does not consume that queue, which silently strands the + // message. Attached desktop sessions deliberately allow the trusted + // app-bundle fallback in `CodexQueueTarget::attached`. + allow_bundle_fallbacks: false, + }) +} + +struct CommandParts<'a> { + command: String, + cli_args: Vec, + metadata: Option<&'a HashMap>, + env: Option<&'a HashMap>, +} + +fn codex_command_parts(spec: &AgentSpec) -> Option> { + match spec.harness_config.as_ref() { + Some(ResolvedHarnessConfig::Pty(config)) => { + let (command, mut args) = parse_cli_command(&config.command).ok()?; + args.extend(config.args.clone()); + Some(CommandParts { + command, + cli_args: args, + metadata: config.metadata.as_ref(), + env: config.env.as_ref(), + }) + } + Some(ResolvedHarnessConfig::Headless(config)) => { + let cli = spec.cli.as_deref()?; + let (command, mut args) = parse_cli_command(cli).ok()?; + args.extend(spec.args.clone()); + Some(CommandParts { + command, + cli_args: args, + metadata: config.metadata.as_ref(), + env: None, + }) + } + Some(ResolvedHarnessConfig::Native(config)) => { + let (command, mut args) = parse_cli_command(&config.command).ok()?; + args.extend(config.args.clone()); + Some(CommandParts { + command, + cli_args: args, + metadata: config.metadata.as_ref(), + env: config.env.as_ref(), + }) + } + _ => { + let cli = spec.cli.as_deref()?; + let (command, mut args) = parse_cli_command(cli).ok()?; + args.extend(spec.args.clone()); + Some(CommandParts { + command, + cli_args: args, + metadata: None, + env: None, + }) + } + } +} + +fn metadata_rollout_path(metadata: &HashMap) -> Option { + metadata + .get("rollout_path") + .or_else(|| metadata.get("rolloutPath")) + .or_else(|| metadata.get("codex_rollout_path")) + .or_else(|| metadata.get("codexRolloutPath")) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(PathBuf::from) +} + +fn metadata_codex_queue_command( + metadata: &HashMap, +) -> Option<(String, Vec)> { + metadata + .get("codex_queue_command") + .or_else(|| metadata.get("codexQueueCommand")) + .or_else(|| metadata.get("codex_command")) + .or_else(|| metadata.get("codexCommand")) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .and_then(|value| parse_cli_command(value).ok()) +} + +fn metadata_codex_home(metadata: &HashMap) -> Option { + metadata + .get("codex_home") + .or_else(|| metadata.get("codexHome")) + .or_else(|| metadata.get("CODEX_HOME")) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) +} + +fn codex_queue_env( + metadata: Option<&HashMap>, + env: Option<&HashMap>, +) -> Vec<(String, String)> { + if let Some(value) = env + .and_then(|env| env.get("CODEX_HOME")) + .map(String::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + { + return vec![("CODEX_HOME".to_string(), value.to_string())]; + } + metadata + .and_then(metadata_codex_home) + .map(|value| vec![("CODEX_HOME".to_string(), value)]) + .unwrap_or_default() +} + +pub(crate) fn codex_queue_global_args(args: &[String]) -> Vec { + const VALUE_FLAGS: &[&str] = &[ + "--profile", + "--config", + "-c", + "--cd", + "--cwd", + "--sandbox", + "-s", + "--ask-for-approval", + "--approval-policy", + ]; + const BOOL_FLAGS: &[&str] = &[ + "--dangerously-bypass-approvals-and-sandbox", + "--dangerously-bypass-hook-trust", + "--full-auto", + "--strict-config", + ]; + let mut out = Vec::new(); + let mut index = 0; + while index < args.len() { + let arg = args[index].as_str(); + if arg == "--" || arg == "resume" || arg == "fork" { + break; + } + if let Some((flag, _)) = arg.split_once('=') { + if VALUE_FLAGS.contains(&flag) { + out.push(arg.to_string()); + } + index += 1; + continue; + } + if VALUE_FLAGS.contains(&arg) { + if let Some(value) = args.get(index + 1) { + out.push(arg.to_string()); + out.push(value.clone()); + index += 2; + continue; + } + break; + } + if BOOL_FLAGS.contains(&arg) { + out.push(arg.to_string()); + index += 1; + continue; + } + index += 1; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::delivery::{DeliverySeam, SendOutcome}; + use std::os::unix::fs::PermissionsExt; + + #[cfg(unix)] + fn fake_codex(script: &str) -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("codex"); + std::fs::write(&path, script).expect("write fake codex"); + let mut permissions = std::fs::metadata(&path).expect("metadata").permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&path, permissions).expect("chmod"); + (dir, path) + } + + struct FallbackProbe { + sends: usize, + } + + impl DeliveryBackend for FallbackProbe { + fn route_id(&self) -> RouteId { + RouteId::new("pty") + } + + fn transport_status(&mut self) -> TransportStatus { + TransportStatus::Available + } + + fn send<'a>( + &'a mut self, + _request: &'a SendRequest, + ) -> DeliveryBackendFuture<'a, Result> { + self.sends += 1; + Box::pin(async { Ok(SendStatus::HandedOver(HandoverState::HandedOver)) }) + } + + fn settle<'a>( + &'a mut self, + _request: &'a SettleRequest, + ) -> DeliveryBackendFuture<'a, SettleStatus> { + Box::pin(async { SettleStatus::HandedOver(HandoverState::HandedOver) }) + } + } + + #[cfg(unix)] + #[tokio::test] + async fn unavailable_queue_capability_falls_back_before_write() { + let (_dir, codex) = fake_codex( + r#"#!/bin/sh +exit 2 +"#, + ); + let target = CodexQueueTarget::new_for_test( + codex.display().to_string(), + vec![], + None, + "thread-1", + None, + ); + let mut codex = CodexQueueBackend::for_target(target); + let mut fallback = FallbackProbe { sends: 0 }; + let mut seam = DeliverySeam::new(); + + let outcome = seam + .send( + &mut [&mut codex, &mut fallback], + SendRequest::new("del_capability", "hello"), + ) + .await + .expect("fallback accepts"); + + assert!(matches!( + outcome, + SendOutcome::Fresh(ref receipt) if receipt.route.as_str() == "pty" + )); + assert_eq!(fallback.sends, 1); + } + + #[cfg(unix)] + #[tokio::test] + async fn queue_process_failure_is_committed_and_does_not_fall_back() { + let (_dir, codex) = fake_codex( + r#"#!/bin/sh +if [ "$1" = "queue" ] && [ "$2" = "--help" ]; then + printf '%s\n' 'Usage: codex queue --thread --message=' + exit 0 +fi +printf '%s\n' 'api_key=do-not-leak queue target was rejected' >&2 +exit 7 +"#, + ); + let target = CodexQueueTarget::new_for_test( + codex.display().to_string(), + vec![], + None, + "thread-1", + None, + ); + let mut codex = CodexQueueBackend::for_target(target); + let mut fallback = FallbackProbe { sends: 0 }; + let mut seam = DeliverySeam::new(); + + let error = seam + .send( + &mut [&mut codex, &mut fallback], + SendRequest::new("del_committed", "hello"), + ) + .await + .expect_err("queue process started, so failure is committed"); + + let DeliveryError::CommittedError { reason } = error else { + panic!("queue process failure must be committed"); + }; + assert!(reason.contains("queue target was rejected")); + assert!(reason.contains("[REDACTED]")); + assert!(!reason.contains("do-not-leak")); + assert_eq!(fallback.sends, 0); + assert!(seam + .recorded_route(&crate::ids::DeliveryId::new("del_committed")) + .is_some()); + } + + #[cfg(unix)] + #[tokio::test] + async fn successful_queue_send_is_handed_over_not_acked() { + let dir = tempfile::tempdir().expect("temp dir"); + let record = dir.path().join("queued.txt"); + let (_script_dir, codex) = fake_codex(&format!( + r#"#!/bin/sh +if [ "$1" = "queue" ] && [ "$2" = "--help" ]; then + printf '%s\n' 'Usage: codex queue --thread --message=' + exit 0 +fi +printf '%s\n' "$@" > '{}' +exit 0 +"#, + record.display() + )); + let target = CodexQueueTarget::new_for_test( + codex.display().to_string(), + vec![], + None, + "thread-1", + None, + ); + let mut codex = CodexQueueBackend::for_target(target); + let mut seam = DeliverySeam::new(); + + let outcome = seam + .send(&mut [&mut codex], SendRequest::new("del_ok", "hello")) + .await + .expect("queue succeeds"); + + let SendOutcome::Fresh(receipt) = outcome else { + panic!("first send must be fresh"); + }; + assert_eq!( + receipt.status, + SendStatus::HandedOver(HandoverState::HandedOver) + ); + assert!(std::fs::read_to_string(record) + .expect("recorded args") + .contains("relay-delivery-id:del_ok")); + } + + /// Seam rule 2, on the real transport: one delivery id, one `codex queue` + /// child. The recorded receipt — not the absence of an error — is what + /// stops the second attempt, so the count is read off the fake Codex's own + /// log rather than off the seam's answer. + #[cfg(unix)] + #[tokio::test] + async fn a_repeated_send_never_queues_the_same_delivery_twice() { + let dir = tempfile::tempdir().expect("temp dir"); + let log = dir.path().join("invocations.log"); + let (_script_dir, codex) = fake_codex(&format!( + r#"#!/bin/sh +if [ "$1" = "queue" ] && [ "$2" = "--help" ]; then + printf '%s\n' 'Usage: codex queue --thread --message=' + exit 0 +fi +printf '%s\n' "$*" >> '{}' +exit 0 +"#, + log.display() + )); + let queue_writes = || { + std::fs::read_to_string(&log) + .map(|text| { + text.lines() + .filter(|line| line.starts_with("queue ")) + .count() + }) + .unwrap_or(0) + }; + let target = CodexQueueTarget::new_for_test( + codex.display().to_string(), + vec![], + None, + "thread-dup", + None, + ); + let mut backend = CodexQueueBackend::for_target(target); + let mut seam = DeliverySeam::new(); + + let first = seam + .send(&mut [&mut backend], SendRequest::new("del_dup", "hello")) + .await + .expect("first queue send succeeds"); + assert!(matches!(first, SendOutcome::Fresh(_)), "{first:?}"); + assert_eq!(queue_writes(), 1, "precondition: the first send wrote once"); + + let second = seam + .send(&mut [&mut backend], SendRequest::new("del_dup", "hello")) + .await + .expect("a repeat of a recorded delivery is not an error"); + + assert!( + matches!(second, SendOutcome::AlreadySent(_)), + "a delivery id the seam already routed must not reach the backend again, got {second:?}" + ); + assert_eq!( + queue_writes(), + 1, + "a second `codex queue` child for one delivery id is a double delivery" + ); + } + + /// Rule 2's cancellation half, on the real transport. A dropped send future + /// cannot prove the child never wrote, so the provisional receipt stands + /// and the next attempt must not spawn a second `codex queue`. + #[cfg(unix)] + #[tokio::test] + async fn a_cancelled_queue_send_is_not_retried_on_the_codex_route() { + let dir = tempfile::tempdir().expect("temp dir"); + let log = dir.path().join("invocations.log"); + let (_script_dir, codex) = fake_codex(&format!( + r#"#!/bin/sh +if [ "$1" = "queue" ] && [ "$2" = "--help" ]; then + printf '%s\n' 'Usage: codex queue --thread --message=' + exit 0 +fi +printf '%s\n' "$*" >> '{}' +sleep 30 +exit 0 +"#, + log.display() + )); + let queue_writes = || { + std::fs::read_to_string(&log) + .map(|text| { + text.lines() + .filter(|line| line.starts_with("queue ")) + .count() + }) + .unwrap_or(0) + }; + let target = CodexQueueTarget::new_for_test( + codex.display().to_string(), + vec![], + None, + "thread-cancel", + None, + ); + let mut backend = CodexQueueBackend::for_target(target); + let mut seam = DeliverySeam::new(); + + let cancelled = tokio::time::timeout( + Duration::from_millis(250), + seam.send( + &mut [&mut backend], + SendRequest::new("del_cancelled", "hello"), + ), + ) + .await; + assert!( + cancelled.is_err(), + "fixture must actually cancel the send mid-write" + ); + + let retried = seam + .send( + &mut [&mut backend], + SendRequest::new("del_cancelled", "hello"), + ) + .await + .expect("a cancelled delivery is recorded, not an error"); + + assert!( + matches!(retried, SendOutcome::AlreadySent(_)), + "a cancelled send may have written; it must never be handed to the route again, \ + got {retried:?}" + ); + assert!( + queue_writes() <= 1, + "a cancelled send must not produce a second `codex queue` child" + ); + } + + /// Seam rule 3, on the real transport: settlement asks the route that + /// accepted the send, identified by its thread, and never a + /// differently-threaded Codex. + /// + /// The decoy thread's rollout carries the same marker, so a settlement that + /// resolved by "whatever codex backend is in the slice" would acknowledge + /// from the wrong session's file. + #[cfg(unix)] + #[tokio::test] + async fn settlement_uses_the_recorded_thread_route_and_never_another_codex() { + let dir = tempfile::tempdir().expect("temp dir"); + let (_script_dir, codex) = fake_codex( + r#"#!/bin/sh +if [ "$1" = "queue" ] && [ "$2" = "--help" ]; then + printf '%s\n' 'Usage: codex queue --thread --message=' + exit 0 +fi +exit 0 +"#, + ); + let consumed = |path: &std::path::Path| { + std::fs::write( + path, + concat!( + r#"{"type":"response_item","payload":{"type":"message","role":"user","#, + r#""content":[{"type":"input_text","text":"hello\n\n"}]}}"#, + "\n", + ), + ) + .expect("write rollout"); + }; + let sent_rollout = dir.path().join("sent-thread.jsonl"); + let decoy_rollout = dir.path().join("decoy-thread.jsonl"); + consumed(&sent_rollout); + consumed(&decoy_rollout); + + let mut sent = CodexQueueBackend::for_target(CodexQueueTarget::new_for_test( + codex.display().to_string(), + vec![], + None, + "thread-sent", + Some(sent_rollout.clone()), + )); + let mut decoy = CodexQueueBackend::for_target(CodexQueueTarget::new_for_test( + codex.display().to_string(), + vec![], + None, + "thread-decoy", + Some(decoy_rollout), + )); + let mut seam = DeliverySeam::new(); + let delivery_id = crate::ids::DeliveryId::new("del_route"); + + seam.send( + &mut [&mut sent], + SendRequest::new(delivery_id.clone(), "hello"), + ) + .await + .expect("queue send succeeds"); + assert_eq!( + seam.recorded_route(&delivery_id).map(RouteId::as_str), + Some("codex-queue:thread-sent"), + "precondition: the send is recorded against its own thread" + ); + + // Only the decoy is offered. The recorded route is not in the slice, so + // settlement must report that — not settle against the other thread. + assert_eq!( + seam.settle(&mut [&mut decoy], &delivery_id).await, + crate::delivery::SettleOutcome::RouteUnavailable(RouteId::new( + "codex-queue:thread-sent" + )), + "settling through a differently-threaded Codex would acknowledge from a session \ + this delivery was never sent to" + ); + + // With the recorded route present, settlement resolves through it. + let settled = seam + .settle(&mut [&mut decoy, &mut sent], &delivery_id) + .await; + let crate::delivery::SettleOutcome::Settled(SettleStatus::Acked(ack)) = settled else { + panic!("the recorded route observed the consumed marker, got {settled:?}"); + }; + let crate::delivery::AckEvidence::Transcript { source, .. } = ack.evidence() else { + panic!("a codex-queue acknowledgement must name the transcript it read"); + }; + assert_eq!( + source, + &sent_rollout.display().to_string(), + "settlement must read the thread it sent to" + ); + } + + fn spec_for(cli: &str, session_id: Option<&str>) -> AgentSpec { + let mut value = serde_json::json!({ + "name": "selection-guard", + "runtime": "pty", + "cli": cli, + "args": [], + "channels": [], + }); + if let Some(session_id) = session_id { + value["sessionId"] = Value::String(session_id.to_string()); + } + serde_json::from_value(value).expect("agent spec") + } + + /// The selection guard the parity answer rests on: a worker is only routed + /// over `codex queue` when it IS codex and relay already knows its thread. + /// The positive case is asserted in the same test so a guard that refused + /// everything could not pass it. + #[test] + fn only_a_codex_worker_with_a_known_thread_selects_the_codex_queue_route() { + for cli in [ + "claude", + "gemini", + "opencode", + "grok", + "droid", + "cursor-agent", + ] { + let backend = CodexQueueBackend { + target: target_for_spec(&spec_for(cli, Some("thread-1"))), + }; + assert!( + !backend.is_selectable(), + "{cli} is not codex and must never take the codex queue route" + ); + } + + let no_session = CodexQueueBackend { + target: target_for_spec(&spec_for("codex", None)), + }; + assert!( + !no_session.is_selectable(), + "a codex worker with no known thread id has nothing to queue against" + ); + + let selectable = CodexQueueBackend { + target: target_for_spec(&spec_for("codex", Some("thread-1"))), + }; + assert!( + selectable.is_selectable(), + "control: a codex worker with a known thread id IS selectable, so the two \ + refusals above are about the guard and not about the fixture" + ); + } + + /// An unselectable backend must refuse BEFORE any write, so the seam falls + /// back to the PTY instead of failing the message (rule 1). + #[tokio::test] + async fn an_unselectable_codex_backend_refuses_before_any_write() { + let mut backend = CodexQueueBackend { + target: target_for_spec(&spec_for("claude", Some("thread-1"))), + }; + assert!(matches!( + backend.transport_status(), + TransportStatus::Unavailable(_) + )); + + let error = backend + .send(&SendRequest::new("del_unselectable", "hello")) + .await + .expect_err("an unselectable route cannot accept a send"); + assert!( + error.is_pre_write(), + "refusing for lack of a target is strictly pre-write, got {error:?}" + ); + } + + #[test] + fn queue_global_args_keep_only_codex_global_options() { + let args = vec![ + "--profile".to_string(), + "work".to_string(), + "--config=model=\"gpt-5\"".to_string(), + "resume".to_string(), + "thread-1".to_string(), + ]; + assert_eq!( + codex_queue_global_args(&args), + vec![ + "--profile".to_string(), + "work".to_string(), + "--config=model=\"gpt-5\"".to_string(), + ] + ); + } + + #[test] + fn headless_codex_config_supplies_session_metadata_to_queue_route() { + let spec: AgentSpec = serde_json::from_value(serde_json::json!({ + "name": "codex-attached", + "runtime": "headless", + "cli": "codex", + "args": [], + "channels": [], + "harnessConfig": { + "runtime": "headless", + "driver": "app_server", + "protocol": "codex", + "endpoint": "stdio://codex-app-server/12345", + "sessionId": "thread-123", + "host": {"ownership": "attached", "pid": 12345}, + "release": "detach", + "metadata": { + "rollout_path": "/tmp/codex-rollout.jsonl", + "codex_home": "/tmp/codex-home" + } + } + })) + .expect("agent spec"); + + let parts = codex_command_parts(&spec).expect("codex command parts"); + + assert_eq!(parts.command, "codex"); + assert_eq!( + parts.metadata.and_then(metadata_rollout_path), + Some(PathBuf::from("/tmp/codex-rollout.jsonl")) + ); + assert_eq!( + codex_queue_env(parts.metadata, parts.env), + vec![("CODEX_HOME".to_string(), "/tmp/codex-home".to_string())] + ); + } + + #[test] + fn native_handle_can_name_a_separate_codex_queue_command() { + let spec: AgentSpec = serde_json::from_value(serde_json::json!({ + "name": "codex-attached", + "runtime": "headless", + "cli": "codex", + "args": [], + "channels": [], + "harnessConfig": { + "runtime": "native", + "command": "node", + "args": ["synthetic-native-codex-sidecar"], + "sessionId": "thread-123", + "metadata": { + "codex_queue_command": "/opt/homebrew/bin/codex", + "codex_home": "/tmp/codex-home" + } + } + })) + .expect("agent spec"); + + let parts = codex_command_parts(&spec).expect("codex command parts"); + let metadata = parts.metadata; + let env = parts.env; + let (command, args) = metadata + .and_then(metadata_codex_queue_command) + .unwrap_or((parts.command, parts.cli_args)); + + assert_eq!(command, "/opt/homebrew/bin/codex"); + assert!(args.is_empty()); + assert_eq!( + codex_queue_env(metadata, env), + vec![("CODEX_HOME".to_string(), "/tmp/codex-home".to_string())] + ); + } +} diff --git a/crates/broker/src/delivery/mod.rs b/crates/broker/src/delivery/mod.rs index 30ab8db986..c38314135f 100644 --- a/crates/broker/src/delivery/mod.rs +++ b/crates/broker/src/delivery/mod.rs @@ -7,6 +7,9 @@ //! acknowledgements are only reported after direct observation. pub mod backend; +pub mod codex_queue; +#[path = "../codex_thread.rs"] +pub(crate) mod codex_thread; pub mod pty; pub use backend::{ diff --git a/crates/broker/src/delivery/pty.rs b/crates/broker/src/delivery/pty.rs index 6de517cdb1..485b73cf67 100644 --- a/crates/broker/src/delivery/pty.rs +++ b/crates/broker/src/delivery/pty.rs @@ -18,7 +18,7 @@ pub(crate) struct PtyDeliveryBackend<'a> { impl<'a> PtyDeliveryBackend<'a> { pub(crate) fn new(workers: &'a mut WorkerRegistry) -> Self { Self { - route: RouteId::new("pty"), + route: RouteId::new(RouteId::PTY), workers, } } @@ -68,21 +68,9 @@ impl DeliveryBackend for PtyDeliveryBackend<'_> { } } -/// The seam's contract invariants, held against the route that actually ships. -/// -/// The four named invariants in `tests/delivery_seam_invariants.rs` all run -/// against a `ScriptedBackend`, so they assert properties of a mock whose -/// behaviour the test itself chose. `PtyDeliveryBackend` is the only backend -/// that exists today, and none of them touched it: a scripted backend can be -/// made to return `Committed`, but nothing proved the PTY route ever does. -/// -/// These drive a real `WorkerRegistry` and a real worker-writer task, so the -/// pre-write / committed classification comes from production code -/// (`send_to_worker_with_commit_boundary` -> `classify_write_failure`) rather -/// than from a script. -#[cfg(test)] -#[cfg(unix)] -mod real_route_invariants { +#[cfg(all(unix, feature = "seam-probe"))] +#[doc(hidden)] +pub mod real_route_probe { use super::*; use crate::delivery::backend::{DeliverySeam, SendOutcome}; use crate::ids::{DeliveryId, EventId, MessageTarget, WorkerName}; @@ -97,9 +85,6 @@ mod real_route_invariants { use tokio::sync::mpsc; use uuid::Uuid; - /// A stand-in for a second route, used only to observe whether the seam - /// fell back. It never has to be realistic: what is under test is the PTY - /// route's classification and the seam's reaction to it. struct FallbackProbe { route: RouteId, sends: usize, @@ -126,7 +111,7 @@ mod real_route_invariants { fn send<'a>( &'a mut self, _request: &'a SendRequest, - ) -> super::super::backend::DeliveryBackendFuture<'a, Result> + ) -> crate::delivery::backend::DeliveryBackendFuture<'a, Result> { self.sends += 1; Box::pin(async move { Ok(SendStatus::HandedOver(HandoverState::HandedOver)) }) @@ -135,14 +120,11 @@ mod real_route_invariants { fn settle<'a>( &'a mut self, _request: &'a SettleRequest, - ) -> super::super::backend::DeliveryBackendFuture<'a, SettleStatus> { + ) -> crate::delivery::backend::DeliveryBackendFuture<'a, SettleStatus> { Box::pin(async move { SettleStatus::HandedOver(HandoverState::HandedOver) }) } } - /// Returns the registry alongside the event sender and receiver. The - /// receiver is returned rather than dropped so writer-failure reporting - /// cannot fail for the wrong reason (a closed channel). fn registry() -> ( WorkerRegistry, mpsc::Sender, @@ -192,9 +174,6 @@ mod real_route_invariants { } } - /// Register a worker whose child has already exited, so the very next - /// write to its stdin fails at the pipe — after the frame has crossed the - /// writer queue's commit boundary. async fn register_dead_child( reg: &mut WorkerRegistry, event_tx: &mpsc::Sender, @@ -235,15 +214,11 @@ mod real_route_invariants { ); } - /// Rule 1, on the shipping route: a refusal raised strictly before any - /// write is the one case that may fall back. - #[tokio::test] - async fn real_pty_route_unknown_worker_is_pre_write_and_may_fall_back() { + pub async fn unknown_worker_is_pre_write_and_may_fall_back() { let (mut reg, _event_tx, _event_rx) = registry(); let mut pty = PtyDeliveryBackend::new(&mut reg); let mut fallback = FallbackProbe::new(); let mut seam = DeliverySeam::new(); - let request = SendRequest::relay( WorkerName::from("no-such-worker"), delivery("del_unknown", "no-such-worker"), @@ -257,26 +232,16 @@ mod real_route_invariants { let SendOutcome::Fresh(receipt) = outcome else { panic!("a first successful send must be fresh"); }; - assert_eq!( - receipt.route.as_str(), - "fallback-probe", - "the fallback route must be the one recorded" - ); - assert_eq!(fallback.sends, 1, "the seam must have tried the fallback"); + assert_eq!(receipt.route.as_str(), "fallback-probe"); + assert_eq!(fallback.sends, 1); } - /// Rule 1's other half, on the shipping route: once the frame has entered - /// the sole writer's queue the write may have partially happened, so a - /// failure MUST NOT fall back. This is the case a scripted backend can - /// only assert by construction. - #[tokio::test] - async fn real_pty_route_write_failure_after_commit_does_not_fall_back() { + pub async fn write_failure_after_commit_does_not_fall_back() { let (mut reg, event_tx, _event_rx) = registry(); register_dead_child(&mut reg, &event_tx, "dead-child").await; let mut pty = PtyDeliveryBackend::new(&mut reg); let mut fallback = FallbackProbe::new(); let mut seam = DeliverySeam::new(); - let request = SendRequest::relay( WorkerName::from("dead-child"), delivery("del_committed", "dead-child"), @@ -291,40 +256,28 @@ mod real_route_invariants { matches!(error, DeliveryError::CommittedError { .. }), "a real EPIPE after the commit boundary must classify as committed, got {error:?}" ); - assert_eq!( - fallback.sends, 0, - "seam rule 1: a committed failure must never be re-sent over another route" - ); + assert_eq!(fallback.sends, 0); assert_eq!( seam.recorded_route(&DeliveryId::new("del_committed")) .map(RouteId::as_str), - Some("pty"), - "the route that took the risk must stay recorded against the delivery" + Some("pty") ); } - /// Rule 4, on the shipping route: the PTY route writes into a terminal and - /// cannot see the child read it, so neither its send nor its settle may - /// ever report an observed acknowledgement. - #[tokio::test] - async fn real_pty_route_never_reports_an_observed_ack() { + pub async fn never_reports_an_observed_ack() { let (mut reg, _event_tx, _event_rx) = registry(); let mut pty = PtyDeliveryBackend::new(&mut reg); - let settled = pty .settle(&SettleRequest { delivery_id: DeliveryId::new("del_settle"), - route: RouteId::new("pty"), + route: RouteId::new(RouteId::PTY), }) .await; - assert!( - matches!(settled, SettleStatus::HandedOver(_)), - "the PTY route observes nothing, so it must settle as a hand-over, got {settled:?}" - ); - assert!( - !matches!(settled, SettleStatus::Acked(_)), - "seam rule 4: never claim an acknowledgement nobody observed" - ); + assert!(matches!( + settled, + SettleStatus::HandedOver(HandoverState::HandedOver) + )); + assert!(!matches!(settled, SettleStatus::Acked(_))); } } diff --git a/crates/broker/src/listen_api.rs b/crates/broker/src/listen_api.rs index a6ea529309..87d0265c89 100644 --- a/crates/broker/src/listen_api.rs +++ b/crates/broker/src/listen_api.rs @@ -82,6 +82,17 @@ pub enum ListenApiRequest { List { reply: tokio::sync::oneshot::Sender>, }, + /// Bind a Codex thread that registered itself through MCP to this broker's + /// native queue delivery route. The broker verifies the supplied agent + /// token before publishing the target in its fleet inventory. + AttachNativeCodex { + name: WorkerName, + agent_token: String, + thread_id: String, + codex_home: Option, + cwd: Option, + reply: tokio::sync::oneshot::Sender>, + }, /// `GET /api/fleet-inventory` — snapshot of the in-process `fleet_inventory` /// map (what the broker last published to the engine via `inventory.sync`). /// Callers use this alongside `List` to detect the workers-vs-inventory @@ -262,6 +273,16 @@ pub enum DeliveryRouteError { CapabilityDisabled, /// No worker with that name is currently registered with the broker. WorkerNotFound(WorkerName), + /// The name resolves only to a native delivery route (an attached Codex + /// thread), which relay reaches by handing the message to the vendor's own + /// durable queue rather than by writing into a broker-owned terminal. + /// + /// Manual flush is a PTY affordance: it parks a message so a human can + /// drive the terminal, and its drain writes through + /// `WorkerRegistry::deliver`. That drain cannot reach a native route, so a + /// message parked for one would never leave the queue. Refused here rather + /// than accepted into a queue nothing can empty. + ManualFlushUnsupportedForNativeRoute(WorkerName), } impl std::fmt::Display for DeliveryRouteError { @@ -271,6 +292,11 @@ impl std::fmt::Display for DeliveryRouteError { DeliveryRouteError::WorkerNotFound(name) => { write!(f, "agent_not_found: no worker named '{name}'") } + DeliveryRouteError::ManualFlushUnsupportedForNativeRoute(name) => write!( + f, + "manual_flush is not available for '{name}': it is reachable only over a native \ + delivery route, whose messages are handed to the agent's own durable queue" + ), } } } @@ -495,6 +521,10 @@ pub(crate) fn listen_api_router_with_auth( .route("/api/session", routing::get(listen_api_session)) .route("/api/session/renew", routing::post(listen_api_renew_lease)) .route("/api/spawn", routing::post(listen_api_spawn)) + .route( + "/api/native-delivery/codex/attach", + routing::post(listen_api_attach_native_codex), + ) .route("/api/spawned", routing::get(listen_api_list)) .route( "/api/fleet-inventory", @@ -1230,6 +1260,72 @@ async fn listen_api_spawn( } } +async fn listen_api_attach_native_codex( + axum::extract::State(state): axum::extract::State, + axum::Json(body): axum::Json, +) -> (axum::http::StatusCode, axum::Json) { + fn required(body: &Value, snake: &str, camel: &str) -> Result { + body.get(snake) + .or_else(|| body.get(camel)) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .ok_or_else(|| format!("missing required field '{snake}'")) + } + fn optional(body: &Value, snake: &str, camel: &str) -> Option { + body.get(snake) + .or_else(|| body.get(camel)) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + } + + let parsed = (|| { + Ok::<_, String>(( + required(&body, "name", "name")?, + required(&body, "agent_token", "agentToken")?, + required(&body, "thread_id", "threadId")?, + optional(&body, "codex_home", "codexHome"), + Some(required(&body, "cwd", "cwd")?), + )) + })(); + let (name, agent_token, thread_id, codex_home, cwd) = match parsed { + Ok(values) => values, + Err(error) => { + return ( + axum::http::StatusCode::BAD_REQUEST, + axum::Json(json!({"success": false, "error": error})), + ) + } + }; + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); + if state + .tx + .send(ListenApiRequest::AttachNativeCodex { + name: WorkerName::new(name), + agent_token, + thread_id, + codex_home, + cwd, + reply: reply_tx, + }) + .await + .is_err() + { + return internal_error(); + } + match reply_rx.await { + Ok(Ok(value)) => (axum::http::StatusCode::OK, axum::Json(value)), + Ok(Err(error)) => ( + axum::http::StatusCode::BAD_REQUEST, + axum::Json(json!({"success": false, "error": error})), + ), + Err(_) => internal_error(), + } +} + async fn listen_api_list( axum::extract::State(state): axum::extract::State, ) -> axum::Json { @@ -2662,6 +2758,11 @@ fn delivery_route_error_to_response( "agent_not_found", err.to_string(), ), + DeliveryRouteError::ManualFlushUnsupportedForNativeRoute(_) => api_error( + axum::http::StatusCode::CONFLICT, + "manual_flush_unsupported", + err.to_string(), + ), } } @@ -4174,6 +4275,67 @@ mod auth_tests { ); } + #[tokio::test] + async fn native_codex_attach_requires_the_api_key_when_auth_enabled() { + let (router, _rx) = test_router(Some("secret")); + let response = router + .oneshot( + Request::builder() + .uri("/api/native-delivery/codex/attach") + .method("POST") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "name": "codex-a", + "agent_token": "agent-token", + "thread_id": "thread-1", + "cwd": "/tmp", + }) + .to_string(), + )) + .expect("request should build"), + ) + .await + .expect("request should succeed"); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn native_codex_attach_rejects_missing_cwd_before_runtime_dispatch() { + let (router, mut rx) = test_router(Some("secret")); + let response = router + .oneshot( + Request::builder() + .uri("/api/native-delivery/codex/attach") + .method("POST") + .header("content-type", "application/json") + .header("x-api-key", "secret") + .body(Body::from( + json!({ + "name": "codex-a", + "agent_token": "agent-token", + "thread_id": "thread-1", + }) + .to_string(), + )) + .expect("request should build"), + ) + .await + .expect("request should succeed"); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = response_json(response).await; + assert_eq!(body["success"], false); + assert!(body["error"] + .as_str() + .is_some_and(|error| error.contains("missing required field 'cwd'"))); + assert!( + rx.try_recv().is_err(), + "invalid attach requests must fail before runtime dispatch" + ); + } + #[tokio::test] async fn api_route_accepts_valid_api_key() { let (router, mut rx) = test_router(Some("secret")); diff --git a/crates/broker/src/node_delivery_probe.rs b/crates/broker/src/node_delivery_probe.rs index 1d901e4b64..faf300dbf9 100644 --- a/crates/broker/src/node_delivery_probe.rs +++ b/crates/broker/src/node_delivery_probe.rs @@ -116,12 +116,23 @@ pub(crate) enum DeliverDisposition { /// be visible. Silently advancing and telling nobody is the one option with /// nothing to recommend it. AdvancedPastUnobserved, + /// A worker teardown removed a pending delivery that had ALREADY been + /// handed to a transport, so the withheld ack could neither be released + /// (nothing was observed) nor safely allowed to drive a redelivery (the + /// message may already be sitting in the recipient's own durable queue). + /// + /// Distinct from `SurfaceFailed`: that frame never crossed a transport + /// boundary. This one did, and the dead letter it produces is marked + /// non-redeliverable for exactly that reason — so the count here is the + /// operator-visible record of an ack that will never be sent. + DroppedInDoubt, } impl DeliverDisposition { fn as_str(self) -> &'static str { match self { Self::AdvancedPastUnobserved => "advanced_past_unobserved", + Self::DroppedInDoubt => "dropped_in_doubt", Self::QueuedForInjection => "queued_for_injection", Self::SurfacedAndAcked => "surfaced_and_acked", Self::HeldForManualFlush => "held_for_manual_flush", @@ -236,6 +247,7 @@ struct AgentStats { rejected_identity: u64, rejected_sequence_gap: u64, advanced_past_unobserved: u64, + dropped_in_doubt: u64, last_deliver_at_ms: u64, last_queued_for_injection_at_ms: u64, /// Strictly increasing rank of the last time this row was touched. See @@ -265,6 +277,7 @@ impl AgentStats { "rejected_identity": self.rejected_identity, "rejected_sequence_gap": self.rejected_sequence_gap, "advanced_past_unobserved": self.advanced_past_unobserved, + "dropped_in_doubt": self.dropped_in_doubt, }, "last_deliver_at_ms": non_zero(self.last_deliver_at_ms), "last_queued_for_injection_at_ms": non_zero(self.last_queued_for_injection_at_ms), @@ -294,6 +307,7 @@ struct Counters { rejected_identity: AtomicU64, rejected_sequence_gap: AtomicU64, advanced_past_unobserved: AtomicU64, + dropped_in_doubt: AtomicU64, connects: AtomicU64, disconnects: AtomicU64, /// Whether a node-control session is currently established. @@ -481,6 +495,7 @@ impl NodeDeliveryProbe { DeliverDisposition::RejectedIdentity => &self.counters.rejected_identity, DeliverDisposition::RejectedSequenceGap => &self.counters.rejected_sequence_gap, DeliverDisposition::AdvancedPastUnobserved => &self.counters.advanced_past_unobserved, + DeliverDisposition::DroppedInDoubt => &self.counters.dropped_in_doubt, }; counter.fetch_add(1, Ordering::Relaxed); let touch = self.next_agent_touch(); @@ -510,6 +525,7 @@ impl NodeDeliveryProbe { DeliverDisposition::RejectedIdentity => stats.rejected_identity += 1, DeliverDisposition::RejectedSequenceGap => stats.rejected_sequence_gap += 1, DeliverDisposition::AdvancedPastUnobserved => stats.advanced_past_unobserved += 1, + DeliverDisposition::DroppedInDoubt => stats.dropped_in_doubt += 1, } } } @@ -641,6 +657,7 @@ impl NodeDeliveryProbe { "rejected_identity": load(&c.rejected_identity), "rejected_sequence_gap": load(&c.rejected_sequence_gap), "advanced_past_unobserved": load(&c.advanced_past_unobserved), + "dropped_in_doubt": load(&c.dropped_in_doubt), }, "acks": { "enqueued": load(&c.ack_enqueued), diff --git a/crates/broker/src/runtime/api.rs b/crates/broker/src/runtime/api.rs index 332d0a2157..7004dcdd8d 100644 --- a/crates/broker/src/runtime/api.rs +++ b/crates/broker/src/runtime/api.rs @@ -31,6 +31,10 @@ const PTY_INPUT_ACK_TIMEOUT: std::time::Duration = std::time::Duration::from_sec /// Never let the runtime actor wait on its completion without a deadline. const DEFAULT_SET_MODEL_TIMEOUT: Duration = Duration::from_secs(5); +fn canonical_or_self(path: &std::path::Path) -> std::path::PathBuf { + path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) +} + fn set_model_write_timeout(timeout_ms: Option) -> Duration { timeout_ms .map(Duration::from_millis) @@ -340,6 +344,7 @@ impl BrokerRuntime { let agent_spawn_count = &mut self.agent_spawn_count; let pending_deliveries = &mut self.pending_deliveries; let dead_letters = &mut self.dead_letters; + let delivery_seam = &mut self.delivery_seam; let obligation_store = &mut self.obligation_store; let pending_requests = &mut self.pending_requests; let resize_owners = &mut self.resize_owners; @@ -434,9 +439,10 @@ impl BrokerRuntime { return; } let mut preregistration_warning: Option = None; - // Caller-supplied agent_token is authoritative. In fleet mode it - // was minted by the node control connection, and the worker must - // receive that exact token before its harness starts. + // A caller-supplied agent_token is authoritative, but it may + // belong to a session that registered itself over HTTP rather + // than one minted by this node. Resolve it first, then bind that + // exact identity to this node before admitting the handle. // // Otherwise create a fresh identity over HTTP, then bind it to // this node. The minted token is injected as RELAY_AGENT_TOKEN @@ -448,6 +454,8 @@ impl BrokerRuntime { None } else if let Some(token) = agent_token { seed_supplied_agent_token(relaycast_http, &name, &token); + let already_node_bound = + fleet_delivery_book.active_agent_id(name.as_str()).is_some(); match super::fleet::resolve_fleet_agent_token_identity( relaycast_http, fleet_delivery_book, @@ -457,6 +465,19 @@ impl BrokerRuntime { .await { Ok(registration) => { + if !already_node_bound { + if let Some(warning) = + super::relaycast_events::bind_http_registered_agent_to_node( + relaycast_http, + fleet_node_name, + &name, + session_ref.as_deref(), + ) + .await + { + preregistration_warning = Some(warning); + } + } fleet_registration = Some((registration, None, session_ref.clone())); } Err(error) => { @@ -485,6 +506,7 @@ impl BrokerRuntime { relaycast_http, fleet_node_name, &name, + session_ref.as_deref(), ) .await; if let Some(warning) = bind_warning { @@ -1172,9 +1194,11 @@ impl BrokerRuntime { sdk_out_tx, json!({"kind":"delivery_dropped","name":&name,"count":dropped.len(),"reason":"agent_released"}), ).await; - let _ = emit_dropped_delivery_failures( + let _ = dispose_pending_deliveries_for_teardown( sdk_out_tx, dead_letters, + delivery_seam, + &self.node_delivery_probe, &dropped, "agent_released", ) @@ -1718,6 +1742,166 @@ impl BrokerRuntime { "HTTP API send runtime actor released" ); } + ListenApiRequest::AttachNativeCodex { + name, + agent_token, + thread_id, + codex_home, + cwd, + reply, + } => { + if local_only { + let _ = reply.send(Err( + "native Codex attach requires an active Relaycast fleet connection" + .to_string(), + )); + return; + } + let absolute_path = |label: &str, value: Option| { + value + .map(PathBuf::from) + .map(|path| { + if path.is_absolute() { + Ok(path) + } else { + Err(format!("{label} must be an absolute path")) + } + }) + .transpose() + }; + let codex_home = match absolute_path("codex_home", codex_home) { + Ok(path) => path, + Err(error) => { + let _ = reply.send(Err(error)); + return; + } + }; + let cwd = match absolute_path("cwd", cwd) { + Ok(Some(path)) => path, + Ok(None) => { + let _ = reply.send(Err("cwd is required".to_string())); + return; + } + Err(error) => { + let _ = reply.send(Err(error)); + return; + } + }; + let thread_record = match crate::delivery::codex_thread::lookup_thread_record( + &thread_id, + codex_home.as_deref(), + ) + .await + { + Ok(Some(record)) => record, + Ok(None) => { + let _ = reply.send(Err( + "Codex thread id was not found in the selected Codex state".to_string(), + )); + return; + } + Err(error) => { + let _ = + reply.send(Err(format!("could not verify Codex thread id: {error}"))); + return; + } + }; + let Some(record_cwd) = thread_record.cwd.as_ref() else { + let _ = reply.send(Err( + "Codex thread record does not include a cwd to verify".to_string() + )); + return; + }; + if canonical_or_self(record_cwd) != canonical_or_self(&cwd) { + let _ = reply.send(Err( + "Codex thread cwd does not match the registering MCP session".to_string(), + )); + return; + } + let Some(rollout_path) = thread_record.rollout_path else { + let _ = reply.send(Err( + "Codex thread record does not include a rollout path".to_string() + )); + return; + }; + let target = match crate::delivery::codex_queue::CodexQueueTarget::attached( + thread_id.clone(), + codex_home, + rollout_path, + Some(cwd), + ) { + Ok(target) => target, + Err(error) => { + let _ = reply.send(Err(error)); + return; + } + }; + let already_node_bound = + fleet_delivery_book.active_agent_id(name.as_str()).is_some(); + let identity = match super::fleet::resolve_fleet_agent_token_identity( + relaycast_http, + fleet_delivery_book, + &name, + &agent_token, + ) + .await + { + Ok(identity) => identity, + Err(error) => { + let _ = reply.send(Err(format!( + "could not verify the self-registered Codex identity: {error}" + ))); + return; + } + }; + if !already_node_bound { + if let Some(warning) = + super::relaycast_events::bind_http_registered_agent_to_node( + relaycast_http, + fleet_node_name, + &name, + Some(&thread_id), + ) + .await + { + let _ = reply.send(Err(format!( + "could not bind the self-registered Codex identity to this broker: {warning}" + ))); + return; + } + } + if let Err(error) = workers.attach_native_codex(name.clone(), target) { + let _ = reply.send(Err(error.to_string())); + return; + } + super::fleet::record_fleet_inventory_agent( + fleet_control_tx, + fleet_inventory, + &identity, + None, + Some(thread_id.clone()), + ) + .await; + super::fleet::publish_fleet_load_snapshot( + fleet_control_tx, + fleet_inventory.len() as u32, + fleet_inventory + .values() + .map(|agent| agent.name.clone()) + .collect(), + fleet_max_agents, + fleet_handlers_live, + true, + ) + .await; + let _ = reply.send(Ok(json!({ + "success": true, + "name": name, + "thread_id": thread_id, + "route": "codex-queue", + "owns_session": false, + }))); + } ListenApiRequest::List { reply } => { let counts = super::delivery::pending_message_counts(delivery_states, pending_deliveries); @@ -2608,7 +2792,17 @@ impl BrokerRuntime { expected_revision, reply, } => { - if !workers.has_worker(&name) { + // A native-only target has no broker-owned terminal to hold a + // message in front of, and the manual-flush drain cannot reach + // its route. Refuse by intent, so the queue can never + // accumulate messages the flush path could not deliver. + if workers.is_native_only_delivery_target(&name) + && mode == InboundDeliveryMode::ManualFlush + { + let _ = reply.send(Err( + DeliveryRouteError::ManualFlushUnsupportedForNativeRoute(name), + )); + } else if !workers.has_worker(&name) { let _ = reply.send(Err(DeliveryRouteError::WorkerNotFound(name))); } else { // Compare-and-set guard: when the caller supplied an diff --git a/crates/broker/src/runtime/dead_letter.rs b/crates/broker/src/runtime/dead_letter.rs index eb6ad5c7e5..c0ff2dbf56 100644 --- a/crates/broker/src/runtime/dead_letter.rs +++ b/crates/broker/src/runtime/dead_letter.rs @@ -246,6 +246,7 @@ pub(crate) fn requeue_dead_letter( // fresh redelivery attempt, not a continuation of that withheld ack. withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }; pending_deliveries.insert(pending.delivery.delivery_id.clone(), pending.clone()); Some(pending) diff --git a/crates/broker/src/runtime/degraded.rs b/crates/broker/src/runtime/degraded.rs index 59e7c30a7a..3917b251af 100644 --- a/crates/broker/src/runtime/degraded.rs +++ b/crates/broker/src/runtime/degraded.rs @@ -406,6 +406,7 @@ impl BrokerRuntime { last_error: None, withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }, ); if let Err(error) = save_pending_deliveries(&self.paths.pending, &self.pending_deliveries) { diff --git a/crates/broker/src/runtime/delivery.rs b/crates/broker/src/runtime/delivery.rs index 2163710dc5..3b6c0c928c 100644 --- a/crates/broker/src/runtime/delivery.rs +++ b/crates/broker/src/runtime/delivery.rs @@ -42,6 +42,145 @@ impl std::fmt::Display for TerminalInDoubtError { impl std::error::Error for TerminalInDoubtError {} +/// What the [`DeliverySeam`](crate::delivery::DeliverySeam) knew about this +/// delivery's transport when the pending snapshot was written. +/// +/// The seam's receipt memory is process-local and starts empty, but for a +/// native route "already handed to a transport" is not a process-local fact: +/// the body sits in the vendor's own durable queue whether this broker is +/// running or not. Carrying the route on the `PendingDelivery` is what lets a +/// reloaded snapshot answer `AlreadySent` instead of `Fresh`, and what lets a +/// teardown that happens with no seam in hand still tell a possible write from +/// one that never started. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "kind")] +pub(crate) enum PersistedDeliveryRoute { + /// The seam recorded this route for the delivery. + Route { route: String }, + /// The seam recorded a route and has since evicted the receipt, so the + /// route is unknown. NOT the same as never-sent (seam rule 2). + Forgotten, +} + +/// Label used wherever a delivery is known to have been written but the route +/// it took is no longer recoverable. +pub(crate) const FORGOTTEN_ROUTE_LABEL: &str = "a route the seam has since forgotten"; + +impl PersistedDeliveryRoute { + /// The route name, when one is still known. + pub(crate) fn route(&self) -> Option<&str> { + match self { + Self::Route { route } => Some(route.as_str()), + Self::Forgotten => None, + } + } + + /// Operator-facing label for this record. + pub(crate) fn label(&self) -> &str { + self.route().unwrap_or(FORGOTTEN_ROUTE_LABEL) + } + + /// Whether the message this record describes can still reach the recipient + /// after the broker that wrote it is gone. + /// + /// A forgotten route fails closed: not knowing where a message went is not + /// evidence it did not go. + pub(crate) fn survives_broker_restart(&self) -> bool { + match self { + Self::Route { route } => { + crate::delivery::RouteId::new(route.as_str()).survives_broker_restart() + } + Self::Forgotten => true, + } + } +} + +/// The seam's current answer for one delivery, in persistable form. +pub(crate) fn seam_send_record( + seam: &crate::delivery::DeliverySeam, + delivery_id: &DeliveryId, +) -> Option { + if let Some(route) = seam.recorded_route(delivery_id) { + return Some(PersistedDeliveryRoute::Route { + route: route.as_str().to_string(), + }); + } + seam.was_sent(delivery_id) + .then_some(PersistedDeliveryRoute::Forgotten) +} + +/// Stamp the seam's answer onto a pending delivery, never downgrading a known +/// route to `Forgotten` or to nothing. Eviction loses the route from the seam's +/// bounded memory; it does not make a route the snapshot already recorded less +/// true. +pub(crate) fn record_sent_route( + pending: &mut PendingDelivery, + record: Option, +) { + let Some(record) = record else { return }; + if matches!( + pending.sent_route, + Some(PersistedDeliveryRoute::Route { .. }) + ) { + return; + } + pending.sent_route = Some(record); +} + +/// Re-seed a freshly built [`DeliverySeam`](crate::delivery::DeliverySeam) from +/// a reloaded pending snapshot. +/// +/// Only routes that outlive the broker are restored. A PTY receipt is +/// deliberately NOT restored: that child died with the broker, so its un-acked +/// write provably never arrived and redelivering it is the correct, and the +/// pre-existing, behaviour. Restoring it would silently convert every +/// interrupted PTY delivery into an in-doubt dead letter. +pub(crate) fn rehydrate_delivery_seam( + seam: &mut crate::delivery::DeliverySeam, + deliveries: &HashMap, +) -> usize { + let mut restored = 0usize; + for (delivery_id, pending) in deliveries { + let Some(record) = pending.sent_route.as_ref() else { + continue; + }; + if !record.survives_broker_restart() { + continue; + } + match record.route() { + Some(route) => { + seam.restore_handed_over(delivery_id.clone(), crate::delivery::RouteId::new(route)) + } + None => seam.restore_forgotten(delivery_id.clone()), + } + restored += 1; + } + restored +} + +/// Whether this delivery was ever handed to a transport, and under what label. +/// +/// `Some(label)` is the one answer that forbids treating the message as never +/// having arrived. Three sources, all of which mean "a write may have +/// happened": the seam still holds a receipt, the seam remembers evicting one, +/// or the snapshot recorded a route in an earlier broker lifetime. +pub(crate) fn handed_over_route_label( + seam: &crate::delivery::DeliverySeam, + pending: &PendingDelivery, +) -> Option { + let delivery_id = &pending.delivery.delivery_id; + if let Some(route) = seam.recorded_route(delivery_id) { + return Some(route.as_str().to_string()); + } + if seam.was_sent(delivery_id) { + return Some(FORGOTTEN_ROUTE_LABEL.to_string()); + } + pending + .sent_route + .as_ref() + .map(|record| record.label().to_string()) +} + #[derive(Debug, Clone, PartialEq)] pub(crate) struct PendingDelivery { pub(super) worker_name: WorkerName, @@ -72,6 +211,10 @@ pub(crate) struct PendingDelivery { /// restart cannot make a higher pending sequence look like a safe new /// baseline. pub(super) withheld_fleet_ack_floor: Option, + /// The route the seam handed this delivery to, if any. See + /// [`PersistedDeliveryRoute`]. `None` means no transport has ever been + /// offered this body. + pub(super) sent_route: Option, } /// Serializable snapshot of pending deliveries for crash recovery. @@ -95,6 +238,13 @@ pub(crate) struct PersistedPendingDelivery { pub(super) withheld_fleet_ack: Option, #[serde(default)] pub(super) withheld_fleet_ack_floor: Option, + /// See [`PendingDelivery::sent_route`]. `#[serde(default)]` so a snapshot + /// written before this field existed loads as "no transport was ever + /// offered this body" — which is what those snapshots meant, because the + /// only route that existed when they were written was the PTY, and a PTY + /// write never survives the restart that produced the snapshot. + #[serde(default)] + pub(super) sent_route: Option, } /// Return the immutable fleet identity and the earliest sequence this pending @@ -286,6 +436,7 @@ pub(crate) fn save_pending_deliveries( last_error: pd.last_error.clone(), withheld_fleet_ack: pd.withheld_fleet_ack.clone(), withheld_fleet_ack_floor: pd.withheld_fleet_ack_floor, + sent_route: pd.sent_route.clone(), }) .collect(); crate::util::fs::write_json_atomic(path, &persisted) @@ -336,6 +487,11 @@ pub(crate) fn load_pending_deliveries(path: &Path) -> HashMap, ) -> InboundQueueResult { - if !workers.has_worker(worker_name) { + if !workers.has_delivery_target(worker_name) { return InboundQueueResult { outcome: InboundQueueOutcome::WorkerMissing, evicted_from: None, @@ -711,7 +867,25 @@ pub(crate) fn queue_inbound_for_delivery_mode( evicted_from: None, }; } - let should_drain = state.should_drain_immediately(); + // A native-only target must never park. The manual-flush drain is + // `try_inject_pending_relay_message_once`, which calls + // `WorkerRegistry::deliver` — a path that knows only broker-owned + // PTY/headless workers and answers pre-write "unknown worker" for an + // attached Codex session. A parked message would therefore fail on every + // flush, stay at the head of the FIFO, and block everything behind it: + // permanently undeliverable rather than held. `DrainNow` goes through the + // seam (`try_inject_pending_relay_message` → `retry_pending_delivery`), + // which is the only path that can select the codex queue route. + let native_only = workers.is_native_only_delivery_target(worker_name); + if native_only && state.mode == crate::types::InboundDeliveryMode::ManualFlush { + tracing::warn!( + target = "agent_relay::broker", + worker = %worker_name, + from = %ctx.from, + "draining inbound message for a native-only delivery target despite manual_flush: the manual-flush drain cannot reach a native route" + ); + } + let should_drain = native_only || state.should_drain_immediately(); let queued_at_ms = chrono::Utc::now().timestamp_millis().max(0) as u64; let msg = PendingRelayMessage { from: ctx.from.to_string(), @@ -994,6 +1168,7 @@ pub(crate) async fn insert_and_attempt_delivery( last_error: None, withheld_fleet_ack, withheld_fleet_ack_floor, + sent_route: None, }, ); @@ -1063,16 +1238,25 @@ pub(crate) async fn retry_pending_delivery( // recording and bounded eviction inert. seam: &mut crate::delivery::DeliverySeam, ) -> Result { - let pending = match pending_deliveries.get(delivery_id) { + let mut pending = match pending_deliveries.get(delivery_id) { Some(pending) => pending.clone(), None => return Ok(DeliveryAttemptOutcome::Noop), }; + // Mirror whatever the seam already knows onto the entry before doing + // anything with it, so a snapshot written at any point below carries the + // route. A delivery reloaded from disk arrives with its route already + // stamped and the seam re-seeded from it, so the two agree. + let known_route = seam_send_record(seam, delivery_id); + record_sent_route(&mut pending, known_route.clone()); + if let Some(current) = pending_deliveries.get_mut(delivery_id) { + record_sent_route(current, known_route); + } // A local queue can outlive its broker and worker. Check absence before // retry exhaustion, and give a respawned recipient a fresh handoff budget. // Explicit release still moves its pending deliveries to dead letters. if pending.delivery.event_id.as_str().starts_with("local_") - && !workers.has_worker(&pending.worker_name) + && !workers.has_delivery_target(&pending.worker_name) { if let Some(current) = pending_deliveries.get_mut(delivery_id) { current.failed_attempts = 0; @@ -1098,18 +1282,16 @@ pub(crate) async fn retry_pending_delivery( // whose ack never arrives returns `AlreadySent` on every later tick, // counting `failed_attempts` up to the cap without re-writing. It then // arrived here and was dead-lettered as freely redeliverable. - if seam.was_sent(delivery_id) { - // `was_sent`, not `recorded_route`. A receipt that has aged out of - // the seam's bounded memory leaves `recorded_route` answering - // `None` — the same answer it gives for a delivery that never - // reached a transport. Branching on route presence sent an - // evicted-but-written delivery down the freely-redeliverable path, - // restoring the double delivery this branch exists to prevent, - // under exactly the sustained load that causes eviction. - let route = seam - .recorded_route(delivery_id) - .map(|route| route.as_str().to_string()) - .unwrap_or_else(|| "a route the seam has since forgotten".to_string()); + // `was_sent`/`sent_route`, not `recorded_route`. A receipt that has + // aged out of the seam's bounded memory leaves `recorded_route` + // answering `None` — the same answer it gives for a delivery that + // never reached a transport. Branching on route presence sent an + // evicted-but-written delivery down the freely-redeliverable path, + // restoring the double delivery this branch exists to prevent, under + // exactly the sustained load that causes eviction. The snapshotted + // `sent_route` carries the same fact across a broker restart, where + // the seam's own memory started empty. + if let Some(route) = handed_over_route_label(seam, &removed) { let last_error = removed.last_error.clone().unwrap_or_else(|| { format!( "handed over to {route} and never acknowledged within {MAX_DELIVERY_RETRIES} retries" @@ -1130,7 +1312,7 @@ pub(crate) async fn retry_pending_delivery( }); } - if !workers.has_worker(&pending.worker_name) { + if !workers.has_delivery_target(&pending.worker_name) { let removed = pending_deliveries.remove(delivery_id).unwrap_or(pending); return Ok(DeliveryAttemptOutcome::Failed { pending: Box::new(removed), @@ -1147,10 +1329,44 @@ pub(crate) async fn retry_pending_delivery( return Ok(DeliveryAttemptOutcome::Noop); } + let pty_fallback_available = workers.workers.contains_key(&pending.worker_name); + let pty_worker_can_steer = workers + .workers + .get(&pending.worker_name) + .is_some_and(|handle| matches!(handle.spec.runtime, AgentRuntime::Pty)); + let mut codex_backend = + crate::delivery::codex_queue::CodexQueueBackend::for_worker(workers, &pending.worker_name); + let codex_selectable = codex_backend.is_selectable(); let mut pty_backend = crate::delivery::pty::PtyDeliveryBackend::new(workers); let request = crate::delivery::SendRequest::relay(pending.worker_name.clone(), pending.delivery.clone()); - match seam.send(&mut [&mut pty_backend], request).await { + let steer_requires_pty = matches!(pending.delivery.injection_mode, MessageInjectionMode::Steer) + && pty_worker_can_steer; + let send_result = if steer_requires_pty { + seam.send(&mut [&mut pty_backend], request).await + } else if codex_selectable && pty_fallback_available { + seam.send(&mut [&mut codex_backend, &mut pty_backend], request) + .await + } else if codex_selectable { + // Attached Codex sessions have no broker-owned worker route. The seam + // still treats an unavailable queue capability as pre-write, so this + // remains a refusal rather than a retry through a guessed transport. + seam.send(&mut [&mut codex_backend], request).await + } else { + seam.send(&mut [&mut pty_backend], request).await + }; + // The seam's answer is authoritative the instant `send` returns — including + // for a committed error, where the provisional in-doubt receipt is + // deliberately kept. Stamp it before any arm below removes the entry, so + // every `removed`/`pending` copy that leaves this function carries it and + // no disposal path has to ask the seam a second time. + let sent_route = seam_send_record(seam, delivery_id); + record_sent_route(&mut pending, sent_route.clone()); + if let Some(current) = pending_deliveries.get_mut(delivery_id) { + record_sent_route(current, sent_route); + } + + match send_result { Ok(crate::delivery::SendOutcome::AlreadySent(_)) => { // NOT `Noop`. `Noop` leaves `next_retry_at` untouched, so once the // seam outlives a single call — which is the whole point of hoisting @@ -1190,12 +1406,21 @@ pub(crate) async fn retry_pending_delivery( last_error: "delivery route is in doubt after possible write".to_string(), }) } - Ok(crate::delivery::SendOutcome::Fresh(_)) => { + Ok(crate::delivery::SendOutcome::Fresh(receipt)) => { if let Some(current) = pending_deliveries.get_mut(delivery_id) { current.attempts = current.attempts.saturating_add(1); current.failed_attempts = 0; - current.next_retry_at = Instant::now() - + delivery_ack_timeout(¤t.delivery.injection_mode, retry_interval); + // Native routes settle by polling the recipient's durable + // record. Start that poll on the normal retry cadence instead + // of inheriting the PTY Wait-mode acknowledgement timeout + // (five minutes), which would leave a landed queue message + // pending long after it was visible in the Codex thread. + let settlement_delay = post_send_settlement_delay( + receipt.route.as_str(), + ¤t.delivery.injection_mode, + retry_interval, + ); + current.next_retry_at = Instant::now() + settlement_delay; current.last_error = None; return Ok(DeliveryAttemptOutcome::Attempted { worker_name: current.worker_name.clone(), @@ -1273,6 +1498,18 @@ pub(crate) fn delivery_ack_timeout( std::cmp::max(retry_interval, minimum) } +fn post_send_settlement_delay( + route: &str, + injection_mode: &MessageInjectionMode, + retry_interval: Duration, +) -> Duration { + if route == "pty" { + delivery_ack_timeout(injection_mode, retry_interval) + } else { + retry_interval + } +} + pub(crate) async fn emit_delivery_attempt_outcome( sdk_out_tx: &mpsc::Sender>, dead_letters: &mut DeadLetterStore, @@ -1481,36 +1718,106 @@ pub(crate) async fn emit_dropped_delivery_failures( reason: &str, ) -> Result<()> { for pending in dropped { - if pending.withheld_fleet_ack.is_some() { - tracing::info!( - target = "relay_broker::fleet", - worker = %pending.worker_name, - delivery_id = %pending.delivery.delivery_id, - reason = reason, - "dropping withheld fleet delivery_ack for a delivery dropped from the pending map" + emit_dropped_delivery_failure(sdk_out_tx, dead_letters, pending, reason).await; + } + Ok(()) +} + +/// Worker-teardown disposal that keeps the seam's in-doubt semantics. +/// +/// The four teardown sites (agent release over the HTTP API and over Relaycast, +/// permanent worker death, unsupervised worker exit) used to dead-letter every +/// pending delivery with a bare reason string. That carries no +/// [`IN_DOUBT_REASON_PREFIX`](crate::runtime::dead_letter::IN_DOUBT_REASON_PREFIX), +/// so [`is_auto_redeliverable`](crate::runtime::dead_letter::is_auto_redeliverable) +/// answers true and an operator — or the engine, via the dropped withheld ack — +/// re-sends it. +/// +/// That was sound while every route was a PTY child that died with the worker: +/// the transport was gone, so "dropped" really did mean "never arrived". A +/// native route breaks the assumption. A `codex queue` message is a row in +/// Codex's own `queued_items` table; the session is not a broker child and +/// outlives both the worker and the broker, so the message may well be +/// delivered after relay has torn its record of it down. Re-sending it then is +/// the double delivery seam rule 2 exists to forbid, and +/// `docs/native-delivery-migration.md` names a release blocker. +/// +/// So teardown now asks the same question the retry-cap branch asks — did this +/// delivery ever reach a transport — and dead-letters a yes under the in-doubt +/// prefix, recording the withheld fleet ack's fate on the node delivery probe +/// instead of dropping it with nothing but a log line. +pub(crate) async fn dispose_pending_deliveries_for_teardown( + sdk_out_tx: &mpsc::Sender>, + dead_letters: &mut DeadLetterStore, + seam: &crate::delivery::DeliverySeam, + node_delivery_probe: &crate::node_delivery_probe::NodeDeliveryProbe, + dropped: &[PendingDelivery], + reason: &str, +) -> Result<()> { + for pending in dropped { + let Some(route) = handed_over_route_label(seam, pending) else { + emit_dropped_delivery_failure(sdk_out_tx, dead_letters, pending, reason).await; + continue; + }; + if let Some(deliver) = pending.withheld_fleet_ack.as_ref() { + node_delivery_probe.record_disposition( + deliver, + crate::node_delivery_probe::DeliverDisposition::DroppedInDoubt, ); } - // Notify best-effort: a send failure must not `?`-abort the loop and - // strand the remaining dropped deliveries out of the dead-letter store. - // The DLQ capture below runs regardless of the send's outcome. - let _ = send_broker_event( - sdk_out_tx, - BrokerEvent::MessageDeliveryFailed { - name: pending.worker_name.clone(), - delivery_id: Some(pending.delivery.delivery_id.clone()), - event_id: Some(pending.delivery.event_id.clone()), - from: pending.delivery.from.clone(), - to: pending.delivery.target.clone(), - attempts: pending.attempts, - last_error: reason.to_string(), - }, - ) - .await; - dead_letter_pending_delivery(sdk_out_tx, dead_letters, pending, reason).await; + let in_doubt_reason = format!( + "{}{reason} after the delivery was handed over to {route}", + crate::runtime::dead_letter::IN_DOUBT_REASON_PREFIX + ); + tracing::warn!( + target = "agent_relay::broker", + worker = %pending.worker_name, + delivery_id = %pending.delivery.delivery_id, + event_id = %pending.delivery.event_id, + route = %route, + reason = %reason, + "worker teardown dropped a delivery that had already reached a transport; \ + dead-lettered in doubt without auto-redelivery" + ); + emit_dropped_delivery_failure(sdk_out_tx, dead_letters, pending, &in_doubt_reason).await; } Ok(()) } +async fn emit_dropped_delivery_failure( + sdk_out_tx: &mpsc::Sender>, + dead_letters: &mut DeadLetterStore, + pending: &PendingDelivery, + reason: &str, +) { + if pending.withheld_fleet_ack.is_some() { + tracing::info!( + target = "relay_broker::fleet", + worker = %pending.worker_name, + delivery_id = %pending.delivery.delivery_id, + reason = reason, + "dropping withheld fleet delivery_ack for a delivery dropped from the pending map" + ); + } + // Notify best-effort: a send failure must not `?`-abort the loop and + // strand the remaining dropped deliveries out of the dead-letter store. + // The DLQ capture below runs regardless of the send's outcome. + let _ = send_broker_event( + sdk_out_tx, + BrokerEvent::MessageDeliveryFailed { + name: pending.worker_name.clone(), + delivery_id: Some(pending.delivery.delivery_id.clone()), + event_id: Some(pending.delivery.event_id.clone()), + from: pending.delivery.from.clone(), + to: pending.delivery.target.clone(), + attempts: pending.attempts, + last_error: reason.to_string(), + }, + ) + .await; + dead_letter_pending_delivery(sdk_out_tx, dead_letters, pending, reason).await; +} + /// Drain every in-flight worker request targeting `worker_name` and /// notify each awaiter with [`worker_request::RequestWorkerError::WorkerDisappeared`]. /// Called from every worker-teardown path (explicit release or @@ -1612,7 +1919,7 @@ mod reply_target_tests { #[cfg(test)] mod steer_timing_invariants { - use super::delivery_ack_timeout; + use super::{delivery_ack_timeout, post_send_settlement_delay}; use crate::broker::delivery_verification::{max_verification_window, VERIFICATION_TICK}; use crate::protocol::MessageInjectionMode; use std::time::Duration; @@ -1636,4 +1943,21 @@ mod steer_timing_invariants { max_verification_window() ); } + + #[test] + fn native_routes_poll_without_inheriting_the_five_minute_wait_timeout() { + let retry_interval = Duration::from_secs(1); + assert_eq!( + post_send_settlement_delay( + "codex-queue:thread-1", + &MessageInjectionMode::Wait, + retry_interval, + ), + retry_interval, + ); + assert_eq!( + post_send_settlement_delay("pty", &MessageInjectionMode::Wait, retry_interval), + super::WAIT_DELIVERY_ACK_TIMEOUT, + ); + } } diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index 8aad59aca2..2614644e91 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -641,6 +641,16 @@ impl BrokerRuntime { request_id, }); } + Ok(Err( + error @ DeliveryRouteError::ManualFlushUnsupportedForNativeRoute(_), + )) => { + self.send_terminal(TerminalToCloud::Error { + session_id, + code: "manual_flush_unsupported".into(), + message: error.to_string(), + request_id, + }); + } Err(_) => { self.send_terminal(TerminalToCloud::Error { session_id, @@ -753,6 +763,16 @@ impl BrokerRuntime { request_id, }); } + Ok(Err( + error @ DeliveryRouteError::ManualFlushUnsupportedForNativeRoute(_), + )) => { + self.send_terminal(TerminalToCloud::Error { + session_id, + code: "manual_flush_unsupported".into(), + message: error.to_string(), + request_id, + }); + } Err(_) => { tracing::warn!( target = "relay_broker::terminal", @@ -1610,6 +1630,8 @@ impl BrokerRuntime { &self.sdk_out_tx, &mut self.pending_deliveries, &mut self.dead_letters, + &self.delivery_seam, + &self.node_delivery_probe, &mut self.pending_requests, &mut self.delivery_states, &mut self.agent_result_tokens, diff --git a/crates/broker/src/runtime/init.rs b/crates/broker/src/runtime/init.rs index 8429081316..0da49f9646 100644 --- a/crates/broker/src/runtime/init.rs +++ b/crates/broker/src/runtime/init.rs @@ -772,8 +772,24 @@ pub(crate) async fn run_init(cmd: InitCommand, telemetry: TelemetryClient) -> Re #[cfg(windows)] let mut sigterm = tokio::signal::windows::ctrl_shutdown()?; + // A delivery that reached a native route in a previous broker lifetime is + // still sitting in the recipient's own durable queue. Re-seed the seam from + // the snapshot BEFORE the first maintenance tick, so that delivery answers + // `AlreadySent` (or `Forgotten`) instead of `Fresh` and is never queued a + // second time. PTY receipts are deliberately not restored — that child died + // with the broker, so its un-acked write provably never arrived. + let mut delivery_seam = crate::delivery::DeliverySeam::new(); + let restored_receipts = rehydrate_delivery_seam(&mut delivery_seam, &pending_deliveries); + if restored_receipts > 0 { + tracing::info!( + restored = restored_receipts, + "restored native delivery receipts from the pending snapshot; \ + those deliveries will settle, not re-send" + ); + } + let mut runtime = BrokerRuntime { - delivery_seam: crate::delivery::DeliverySeam::new(), + delivery_seam, degraded, persist: paths.persist, broker_start, diff --git a/crates/broker/src/runtime/maintenance.rs b/crates/broker/src/runtime/maintenance.rs index 7d5148ceb5..cf25499057 100644 --- a/crates/broker/src/runtime/maintenance.rs +++ b/crates/broker/src/runtime/maintenance.rs @@ -1,6 +1,7 @@ use super::fleet::{release_terminal_resize_ownership, try_send_terminal}; use super::*; use crate::terminal_control::TerminalToCloud; +use crate::worker::AgentWorkState; impl BrokerRuntime { pub(super) async fn handle_maintenance_tick(&mut self) { @@ -16,6 +17,7 @@ impl BrokerRuntime { let workers = &mut self.workers; let delivery_seam = &mut self.delivery_seam; let fleet_control_tx = &self.fleet_control_tx; + let node_delivery_probe = &self.node_delivery_probe; let fleet_inventory = &mut self.fleet_inventory; let fleet_inventory_reconcile_retry_after = &mut self.fleet_inventory_reconcile_retry_after; let fleet_delivery_book = &mut self.fleet_delivery_book; @@ -36,6 +38,8 @@ impl BrokerRuntime { let terminal_sessions = &mut self.terminal_sessions; let terminal_snapshot_requests = &mut self.terminal_snapshot_requests; let terminal_input_requests = &mut self.terminal_input_requests; + let dedup = &mut self.dedup; + let terminal_failed_deliveries = &mut self.terminal_failed_deliveries; let delivery_retry_interval = self.delivery_retry_interval; let shutdown = &self.shutdown; let default_workspace = &self.default_workspace; @@ -173,6 +177,148 @@ impl BrokerRuntime { .map(|pending| pending.attempts > 0) .unwrap_or(false); + if delivery_seam + .recorded_route(&delivery_id) + .is_some_and(|route| route.as_str() != "pty") + { + let settle_target = pending_deliveries.get(&delivery_id).map(|pending| { + ( + pending.worker_name.clone(), + pending.delivery.event_id.clone(), + ) + }); + if let Some((worker_name, event_id)) = settle_target { + let mut codex_backend = + crate::delivery::codex_queue::CodexQueueBackend::for_worker( + workers, + &worker_name, + ); + let settled = match delivery_seam + .settle(&mut [&mut codex_backend], &delivery_id) + .await + { + crate::delivery::SettleOutcome::Settled( + crate::delivery::SettleStatus::Acked(_), + ) => true, + crate::delivery::SettleOutcome::Settled( + crate::delivery::SettleStatus::Failed(reason), + ) => { + if let Some(pending) = pending_deliveries.remove(&delivery_id) { + let _ = emit_delivery_attempt_outcome( + sdk_out_tx, + dead_letters, + &delivery_id, + was_retry, + DeliveryAttemptOutcome::TerminalInDoubt { + pending: Box::new(pending), + last_error: reason, + }, + ) + .await; + } + continue; + } + _ => false, + }; + + if settled { + let (pending_for_confirmation, resolved_fleet_ack) = + super::fleet::confirm_pending_delivery_and_resolve_fleet_ack( + pending_deliveries, + delivery_id.as_str(), + Some(event_id.as_str()), + worker_name.as_str(), + "codex_queue_settle", + fleet_delivery_book, + ); + if pending_for_confirmation.is_some() { + terminal_failed_deliveries.remove(&delivery_id); + } + if let Some((agent, up_to_seq)) = resolved_fleet_ack { + super::fleet::enqueue_delivery_ack( + fleet_control_tx, + node_delivery_probe, + agent, + up_to_seq, + ) + .await; + } + if let Some(pending) = pending_for_confirmation { + let _ = send_event( + sdk_out_tx, + json!({ + "kind": "delivery_ack", + "name": worker_name, + "delivery_id": delivery_id, + "event_id": event_id, + "timestamp": crate::broker::delivery_verification::current_timestamp_ms(), + }), + ) + .await; + let read_ack_delivery_id = pending.delivery.delivery_id.clone(); + let read_ack_event_id = pending.delivery.event_id.clone(); + let cli_hint = workers + .workers + .get(&worker_name) + .and_then(|handle| handle.spec.cli.as_deref()) + .map(str::to_string); + if let Some(handle) = workers.workers.get_mut(&worker_name) { + handle.last_activity_at = Instant::now(); + handle.state = AgentWorkState::Working; + } + let _ = send_broker_event( + sdk_out_tx, + BrokerEvent::MessageDeliveryConfirmed { + name: worker_name.clone(), + delivery_id: pending.delivery.delivery_id, + event_id: pending.delivery.event_id, + from: pending.delivery.from, + to: pending.delivery.target, + }, + ) + .await; + mark_delivery_read_ack( + relaycast_http, + sdk_out_tx, + dedup, + &worker_name, + cli_hint.as_deref(), + &read_ack_delivery_id, + &read_ack_event_id, + ); + } + continue; + } + + if delivery_seam + .recorded_age(&delivery_id) + .is_some_and(|age| age >= NATIVE_DELIVERY_SETTLEMENT_TIMEOUT) + { + if let Some(pending) = pending_deliveries.remove(&delivery_id) { + let _ = emit_delivery_attempt_outcome( + sdk_out_tx, + dead_letters, + &delivery_id, + was_retry, + DeliveryAttemptOutcome::TerminalInDoubt { + pending: Box::new(pending), + last_error: format!( + "native delivery was not observed within {}s", + NATIVE_DELIVERY_SETTLEMENT_TIMEOUT.as_secs() + ), + }, + ) + .await; + } + } else if let Some(pending) = pending_deliveries.get_mut(&delivery_id) { + // Settlement is read-only. Poll it without calling send + // again or consuming the transport retry budget. + pending.next_retry_at = Instant::now() + delivery_retry_interval; + } + continue; + } + } + match retry_pending_delivery( &delivery_id, workers, @@ -223,6 +369,8 @@ impl BrokerRuntime { sdk_out_tx, pending_deliveries, dead_letters, + delivery_seam, + node_delivery_probe, pending_requests, delivery_states, agent_result_tokens, @@ -448,9 +596,11 @@ impl BrokerRuntime { }), ) .await; - let _ = emit_dropped_delivery_failures( + let _ = dispose_pending_deliveries_for_teardown( sdk_out_tx, dead_letters, + delivery_seam, + node_delivery_probe, &dropped, "worker_permanently_dead", ) @@ -520,9 +670,11 @@ impl BrokerRuntime { }), ) .await; - let _ = emit_dropped_delivery_failures( + let _ = dispose_pending_deliveries_for_teardown( sdk_out_tx, dead_letters, + delivery_seam, + node_delivery_probe, &dropped, "worker_exited", ) diff --git a/crates/broker/src/runtime/mod.rs b/crates/broker/src/runtime/mod.rs index fb9d1c949d..01861c069d 100644 --- a/crates/broker/src/runtime/mod.rs +++ b/crates/broker/src/runtime/mod.rs @@ -60,6 +60,7 @@ use crate::{broker, listen_api, worker_request}; const DEFAULT_DELIVERY_RETRY_MS: u64 = 1_000; const MAX_DELIVERY_RETRIES: u32 = 10; const WAIT_DELIVERY_ACK_TIMEOUT: Duration = Duration::from_secs(5 * 60); +const NATIVE_DELIVERY_SETTLEMENT_TIMEOUT: Duration = Duration::from_secs(2 * 60); const THREAD_HISTORY_LIMIT: usize = 1_000; #[allow(dead_code)] // only http_api_local_delivery_timeout's default; see its own allow const DEFAULT_HTTP_API_LOCAL_DELIVERY_TIMEOUT_MS: u64 = 3_000; diff --git a/crates/broker/src/runtime/relaycast_events.rs b/crates/broker/src/runtime/relaycast_events.rs index 23f5702172..24eb3eaf7b 100644 --- a/crates/broker/src/runtime/relaycast_events.rs +++ b/crates/broker/src/runtime/relaycast_events.rs @@ -274,8 +274,8 @@ pub(super) fn relaycast_spawn_verifies_ready(value: &Value) -> bool { .is_some_and(|config| harness_metadata_flag(config, "verify_ready", "verifyReady")) } -/// Bind a freshly HTTP-registered agent to this broker's relaycast node so it -/// becomes `locationType='via_node'`. +/// Bind an existing Relaycast agent to this broker's node so it becomes +/// `locationType='via_node'`. /// /// In node-only delivery the engine only delivers to `via_node` agents. The HTTP /// `register_agent_token` fallback (taken when node-control `agent.register` is @@ -286,6 +286,7 @@ pub(super) async fn bind_http_registered_agent_to_node( relaycast_http: &RelaycastHttpClient, node_name: &str, agent_name: &str, + session_ref: Option<&str>, ) -> Option { let Some(relay) = relaycast_http.relay_client() else { let message = format!( @@ -297,7 +298,7 @@ pub(super) async fn bind_http_registered_agent_to_node( }; let request = relaycast::BindAgentToNodeRequest { agent_name: agent_name.to_string(), - session_ref: None, + session_ref: session_ref.map(ToOwned::to_owned), priority: None, }; match relay.bind_agent_to_node(node_name, request).await { @@ -357,6 +358,12 @@ pub(super) async fn release_worker_locally( sdk_out_tx: &mpsc::Sender>, pending_deliveries: &mut HashMap, dead_letters: &mut DeadLetterStore, + // Read-only: release disposes of pending deliveries and must be able to + // ask whether each one already reached a transport. Without the seam this + // path cannot tell a never-written message from one already sitting in + // Codex's durable queue, and dead-letters both as freely redeliverable. + delivery_seam: &crate::delivery::DeliverySeam, + node_delivery_probe: &crate::node_delivery_probe::NodeDeliveryProbe, pending_requests: &mut HashMap, delivery_states: &mut HashMap, agent_result_tokens: &mut HashMap, @@ -392,9 +399,11 @@ pub(super) async fn release_worker_locally( sdk_out_tx, json!({"kind":"delivery_dropped","name":name,"count":dropped.len(),"reason":"agent_released"}), ).await; - let _ = emit_dropped_delivery_failures( + let _ = dispose_pending_deliveries_for_teardown( sdk_out_tx, dead_letters, + delivery_seam, + node_delivery_probe, &dropped, "agent_released", ) @@ -755,6 +764,7 @@ pub(super) async fn spawn_worker_from_request( workspace_http, node_name, &name, + session_ref.as_deref(), ) .await; if bind_warning.is_none() { @@ -1138,6 +1148,8 @@ mod tests { ), ]); let (terminal_control_tx, mut terminal_control_rx) = mpsc::channel(8); + let delivery_seam = crate::delivery::DeliverySeam::new(); + let node_delivery_probe = crate::node_delivery_probe::NodeDeliveryProbe::new(); let outcome = release_worker_locally( released_agent.clone(), @@ -1149,6 +1161,8 @@ mod tests { &sdk_out_tx, &mut pending_deliveries, &mut dead_letters, + &delivery_seam, + &node_delivery_probe, &mut pending_requests, &mut delivery_states, &mut agent_result_tokens, diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index f6c7b4368c..afb4ae3b61 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -885,6 +885,7 @@ fn pending_delivery(worker_name: &str, delivery_id: &str, event_id: &str) -> Pen last_error: None, withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, } } @@ -1634,6 +1635,7 @@ fn make_pending_delivery(delivery_id: &str, worker: &str) -> PendingDelivery { last_error: None, withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, } } @@ -2943,6 +2945,280 @@ async fn timeout_fallback_never_confirms_or_acks_an_unobserved_delivery() { ); } +/// A rollout in the shape Codex ACTUALLY writes once a turn has consumed a +/// queued user input item. +/// +/// Both records are the projections captured live from `codex-cli +/// 0.155.0-alpha.9.2` (`codex app-server` → `thread/start` → `turn/start` → +/// `codex queue`), verbatim apart from the marker text and shortened ids; the +/// capture is in +/// `.workflow-artifacts/migrate-native-delivery/phase-1-codex-queue-20260921a/evidence/codex-capture/rollout-consumed-user-item.jsonl`. +/// +/// The fixture this replaced wrote `{"text":""}`, +/// which no Codex version emits and which names no item kind, so it pinned no +/// real record shape and settled only because the matcher was negative. +/// `crates/broker/src/codex_thread.rs` now refuses it by name. +fn captured_codex_consumed_rollout(delivery_id: &DeliveryId) -> String { + let marker = format!("relay-delivery-id:{}", delivery_id.as_str()); + format!( + concat!( + r#"{{"timestamp":"2026-09-22T18:29:35.389Z","ordinal":8,"type":"response_item","payload":{{"type":"message","id":"msg_01a0ca61","role":"user","content":[{{"type":"input_text","text":"hello from relay\n\n"}}]}}}}"#, + "\n", + r#"{{"timestamp":"2026-09-22T18:29:35.389Z","ordinal":9,"type":"event_msg","payload":{{"type":"item_completed","thread_id":"01a0ca61-60fd","turn_id":"01a0ca61-6208","item":{{"type":"UserMessage","id":"01a0ca61-641d","content":[{{"type":"text","text":"hello from relay\n\n","text_elements":[]}}]}}}}}}"#, + "\n", + ), + marker = marker + ) +} + +/// A native Codex hand-off is not complete until the marker appears in the +/// target thread's own rollout. Once it does, the maintenance path must apply +/// the same terminal effects as an observed PTY acknowledgement: remove the +/// pending entry and publish exactly one confirmation rather than letting the +/// retry cap turn a delivered message into an in-doubt dead letter. +#[tokio::test] +async fn codex_queue_settlement_confirms_and_removes_the_pending_delivery() { + use crate::delivery::{ + DeliveryBackend, DeliveryBackendFuture, DeliveryError, HandoverState, RouteId, SendRequest, + SendStatus, SettleRequest, SettleStatus, TransportStatus, + }; + + struct RecordedCodexRoute; + + impl DeliveryBackend for RecordedCodexRoute { + fn route_id(&self) -> RouteId { + RouteId::new("codex-queue:thread-settle") + } + + fn transport_status(&mut self) -> TransportStatus { + TransportStatus::Available + } + + fn send<'a>( + &'a mut self, + _request: &'a SendRequest, + ) -> DeliveryBackendFuture<'a, Result> { + Box::pin(async { Ok(SendStatus::HandedOver(HandoverState::HandedOver)) }) + } + + fn settle<'a>( + &'a mut self, + _request: &'a SettleRequest, + ) -> DeliveryBackendFuture<'a, SettleStatus> { + Box::pin(async { SettleStatus::HandedOver(HandoverState::HandedOver) }) + } + } + + let worker_name = "codex-attached"; + let delivery_id = DeliveryId::new("del_codex_settle"); + let event_id = EventId::new("evt_del_codex_settle"); + let rollout_dir = tempfile::tempdir().expect("rollout temp dir"); + let rollout_path = rollout_dir.path().join("rollout.jsonl"); + std::fs::write(&rollout_path, captured_codex_consumed_rollout(&delivery_id)) + .expect("write observed Codex marker"); + + let mut registry = make_worker_registry_with_worker(worker_name).await; + let handle = registry + .workers + .get_mut(worker_name) + .expect("fixture worker"); + handle.spec.runtime = AgentRuntime::Headless; + handle.spec.cli = Some("codex".to_string()); + handle.spec.session_id = Some("thread-settle".to_string()); + handle.spec.harness_config = Some(ResolvedHarnessConfig::Native(NativeHarnessConfig { + command: "codex".to_string(), + args: Vec::new(), + cwd: None, + env: None, + session_id: "thread-settle".to_string(), + metadata: Some(HashMap::from([( + "rollout_path".to_string(), + Value::String(rollout_path.display().to_string()), + )])), + })); + + let mut pending = make_pending_delivery(delivery_id.as_str(), worker_name); + pending.delivery.event_id = event_id.clone(); + pending.delivery.target = MessageTarget::new(worker_name); + pending.next_retry_at = Instant::now(); + let mut fixture = worker_event_runtime_fixture( + registry, + HashMap::from([(delivery_id.clone(), pending.clone())]), + ); + + let mut route = RecordedCodexRoute; + fixture + .runtime + .delivery_seam + .send( + &mut [&mut route], + SendRequest::relay(WorkerName::from(worker_name), pending.delivery), + ) + .await + .expect("precondition: the native hand-off receipt is recorded"); + + fixture.runtime.handle_maintenance_tick().await; + + assert!( + !fixture + .runtime + .pending_deliveries + .contains_key(&delivery_id), + "an observed Codex marker must settle the pending delivery" + ); + assert!( + fixture + .runtime + .dead_letters + .get(delivery_id.as_str()) + .is_none(), + "a settled Codex delivery must not be dead-lettered" + ); + + let mut confirmations = 0; + let mut failures = 0; + while let Ok(frame) = fixture._sdk_out_rx.try_recv() { + match frame.payload.get("kind").and_then(Value::as_str) { + Some("message_delivery_confirmed") + if frame.payload["delivery_id"] == delivery_id.as_str() + && frame.payload["event_id"] == event_id.as_str() => + { + confirmations += 1; + } + Some("message_delivery_failed") + if frame.payload["delivery_id"] == delivery_id.as_str() => + { + failures += 1; + } + _ => {} + } + } + assert_eq!(confirmations, 1, "settlement must publish one confirmation"); + assert_eq!(failures, 0, "settlement must not publish a failure"); + + cleanup_worker_registry(fixture.runtime.workers).await; +} + +#[tokio::test] +async fn codex_queue_settlement_does_not_reemit_ack_while_fleet_confirmation_is_held() { + use crate::delivery::{ + DeliveryBackend, DeliveryBackendFuture, DeliveryError, HandoverState, RouteId, SendRequest, + SendStatus, SettleRequest, SettleStatus, TransportStatus, + }; + + struct RecordedCodexRoute; + + impl DeliveryBackend for RecordedCodexRoute { + fn route_id(&self) -> RouteId { + RouteId::new("codex-queue:thread-held") + } + + fn transport_status(&mut self) -> TransportStatus { + TransportStatus::Available + } + + fn send<'a>( + &'a mut self, + _request: &'a SendRequest, + ) -> DeliveryBackendFuture<'a, Result> { + Box::pin(async { Ok(SendStatus::HandedOver(HandoverState::HandedOver)) }) + } + + fn settle<'a>( + &'a mut self, + _request: &'a SettleRequest, + ) -> DeliveryBackendFuture<'a, SettleStatus> { + Box::pin(async { SettleStatus::HandedOver(HandoverState::HandedOver) }) + } + } + + let worker_name = "codex-held"; + let lower_deliver = fleet_deliver(5); + let deliver = fleet_deliver(6); + let delivery_id = DeliveryId::from(&deliver.delivery_id); + let lower_delivery_id = DeliveryId::from(&lower_deliver.delivery_id); + let event_id = EventId::from(&deliver.msg_id); + let rollout_dir = tempfile::tempdir().expect("rollout temp dir"); + let rollout_path = rollout_dir.path().join("rollout.jsonl"); + std::fs::write(&rollout_path, captured_codex_consumed_rollout(&delivery_id)) + .expect("write observed Codex marker"); + + let mut registry = make_worker_registry_with_worker(worker_name).await; + let handle = registry + .workers + .get_mut(worker_name) + .expect("fixture worker"); + handle.spec.runtime = AgentRuntime::Headless; + handle.spec.cli = Some("codex".to_string()); + handle.spec.session_id = Some("thread-held".to_string()); + handle.spec.harness_config = Some(ResolvedHarnessConfig::Native(NativeHarnessConfig { + command: "codex".to_string(), + args: Vec::new(), + cwd: None, + env: None, + session_id: "thread-held".to_string(), + metadata: Some(HashMap::from([( + "rollout_path".to_string(), + Value::String(rollout_path.display().to_string()), + )])), + })); + + let mut pending = make_pending_delivery(delivery_id.as_str(), worker_name); + pending.delivery.event_id = event_id.clone(); + pending.delivery.target = MessageTarget::new(worker_name); + pending.withheld_fleet_ack = Some(deliver); + pending.next_retry_at = Instant::now(); + let mut lower_pending = make_pending_delivery(lower_delivery_id.as_str(), worker_name); + lower_pending.delivery.event_id = EventId::from(&lower_deliver.msg_id); + lower_pending.delivery.target = MessageTarget::new(worker_name); + lower_pending.withheld_fleet_ack = Some(lower_deliver); + lower_pending.next_retry_at = Instant::now() + Duration::from_secs(60); + let mut fixture = worker_event_runtime_fixture( + registry, + HashMap::from([ + (lower_delivery_id.clone(), lower_pending), + (delivery_id.clone(), pending.clone()), + ]), + ); + + let mut route = RecordedCodexRoute; + fixture + .runtime + .delivery_seam + .send( + &mut [&mut route], + SendRequest::relay(WorkerName::from(worker_name), pending.delivery), + ) + .await + .expect("precondition: the native hand-off receipt is recorded"); + + fixture.runtime.handle_maintenance_tick().await; + fixture.runtime.handle_maintenance_tick().await; + + let mut sdk_delivery_acks = 0; + while let Ok(frame) = fixture._sdk_out_rx.try_recv() { + if frame.payload.get("kind").and_then(Value::as_str) == Some("delivery_ack") + && frame.payload["delivery_id"] == delivery_id.as_str() + { + sdk_delivery_acks += 1; + } + } + + assert_eq!( + sdk_delivery_acks, 1, + "a held native confirmation must not re-emit delivery_ack on every tick" + ); + assert!( + fixture + .runtime + .pending_deliveries + .contains_key(&delivery_id), + "the held confirmation stays pending until the lower fleet cursor releases it" + ); + + cleanup_worker_registry(fixture.runtime.workers).await; +} + // relay#1310 MUST-NOT-FIRE: once the worker confirms the injection landed // (echo-verified — the ONLY case in which pty_worker.rs sends the internal // `delivery_ack`; its bounded timeout fallback deliberately does not, see @@ -3431,6 +3707,7 @@ async fn delivery_retry_fails_promptly_when_recipient_is_gone() { last_error: Some("failed writing frame".to_string()), withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }, )]); @@ -3583,6 +3860,7 @@ async fn delivery_retry_committed_writer_failure_stops_without_dead_letter() { last_error: None, withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }, )]); @@ -3641,6 +3919,7 @@ async fn delivery_retry_success_clears_stale_last_error() { last_error: Some("old transient failure".to_string()), withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }, )]); @@ -4762,6 +5041,7 @@ fn drop_pending_for_worker_removes_only_matching_entries() { last_error: None, withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }, ); pending.insert( @@ -4787,6 +5067,7 @@ fn drop_pending_for_worker_removes_only_matching_entries() { last_error: None, withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }, ); @@ -4819,6 +5100,7 @@ async fn dropped_pending_deliveries_emit_terminal_message_failures() { last_error: Some("previous blip".to_string()), withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }; let (sdk_out_tx, mut sdk_out_rx) = mpsc::channel(4); let mut dead_letters = DeadLetterStore::default(); @@ -4891,6 +5173,7 @@ fn should_clear_pending_delivery_when_event_id_matches() { last_error: None, withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }; assert!(should_clear_pending_delivery_for_event( @@ -4928,6 +5211,7 @@ fn clear_pending_delivery_returns_none_for_stale_event_id() { last_error: None, withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }, )]); @@ -5347,6 +5631,7 @@ fn should_clear_pending_delivery_without_event_id_for_compatibility() { last_error: None, withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }; assert!(should_clear_pending_delivery_for_event( @@ -8694,6 +8979,7 @@ async fn delivery_retry_walks_the_cap_from_zero_and_then_dead_letters() { last_error: None, withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }, )]); @@ -8820,6 +9106,7 @@ async fn a_handed_over_delivery_that_exhausts_retries_is_in_doubt_not_failed() { last_error: None, withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }, )]); @@ -8939,6 +9226,7 @@ async fn an_evicted_receipt_still_terminates_in_doubt_not_failed() { last_error: None, withheld_fleet_ack: None, withheld_fleet_ack_floor: None, + sent_route: None, }, )]); @@ -9096,3 +9384,471 @@ async fn an_in_doubt_delivery_on_the_raw_queue_path_is_retained_not_dropped() { "retained at the cap, so the next pass terminates it instead of re-sending a possible write" ); } + +/// A backend that reports a native route's hand-over without touching a real +/// Codex install. Named for the route it claims, because the route string is +/// what `survives_broker_restart` and the teardown label both key off. +struct HandedOverNativeRoute { + route: &'static str, +} + +impl crate::delivery::DeliveryBackend for HandedOverNativeRoute { + fn route_id(&self) -> crate::delivery::RouteId { + crate::delivery::RouteId::new(self.route) + } + + fn transport_status(&mut self) -> crate::delivery::TransportStatus { + crate::delivery::TransportStatus::Available + } + + fn send<'a>( + &'a mut self, + _request: &'a crate::delivery::SendRequest, + ) -> crate::delivery::DeliveryBackendFuture< + 'a, + Result, + > { + Box::pin(async { + Ok(crate::delivery::SendStatus::HandedOver( + crate::delivery::HandoverState::HandedOver, + )) + }) + } + + fn settle<'a>( + &'a mut self, + _request: &'a crate::delivery::SettleRequest, + ) -> crate::delivery::DeliveryBackendFuture<'a, crate::delivery::SettleStatus> { + Box::pin(async { + crate::delivery::SettleStatus::HandedOver(crate::delivery::HandoverState::HandedOver) + }) + } +} + +/// Releasing an agent that still owes a handed-over native delivery must dead +/// letter it IN DOUBT, not as a freely redeliverable failure. +/// +/// The four worker-teardown sites (agent release over the HTTP API and over +/// Relaycast, permanent worker death, unsupervised worker exit) all funnel into +/// `dispose_pending_deliveries_for_teardown`. Before this they dead-lettered +/// with a bare reason string, so `is_auto_redeliverable` answered true. That was +/// sound while every route was a PTY child that died with the worker. A +/// `codex queue` message is a row in Codex's own durable store and the session +/// outlives both the worker and the broker, so re-sending it is the double +/// delivery seam rule 2 forbids. +/// +/// The unsent control in the same test is what makes the assertion mean +/// something: a disposal path that marked EVERYTHING in doubt would pass the +/// first half and fail the second. +#[tokio::test] +async fn releasing_an_agent_with_a_handed_over_native_delivery_dead_letters_it_in_doubt() { + let worker_name = "codex-attached"; + let handed_over_id = DeliveryId::new("del_released_handed_over"); + let never_sent_id = DeliveryId::new("del_released_never_sent"); + + let mut seam = crate::delivery::DeliverySeam::new(); + let mut route = HandedOverNativeRoute { + route: "codex-queue:thread-released", + }; + let mut handed_over = make_pending_delivery(handed_over_id.as_str(), worker_name); + handed_over.withheld_fleet_ack = Some(withheld_ack_for(handed_over_id.as_str())); + seam.send( + &mut [&mut route], + crate::delivery::SendRequest::relay( + WorkerName::from(worker_name), + handed_over.delivery.clone(), + ), + ) + .await + .expect("precondition: the native hand-off is recorded"); + assert!( + seam.was_sent(&handed_over_id), + "precondition: the seam must remember handing this to a transport" + ); + + let never_sent = make_pending_delivery(never_sent_id.as_str(), worker_name); + let mut pending_deliveries = HashMap::from([ + (handed_over_id.clone(), handed_over), + (never_sent_id.clone(), never_sent), + ]); + + // The same removal every release / reap site performs. + let dropped = take_pending_for_worker(&mut pending_deliveries, worker_name); + assert_eq!(dropped.len(), 2, "both deliveries are torn down together"); + + let (sdk_out_tx, _sdk_out_rx) = mpsc::channel(16); + let mut dead_letters = DeadLetterStore::default(); + let probe = crate::node_delivery_probe::NodeDeliveryProbe::new(); + super::delivery::dispose_pending_deliveries_for_teardown( + &sdk_out_tx, + &mut dead_letters, + &seam, + &probe, + &dropped, + "agent_released", + ) + .await + .expect("teardown disposal is infallible"); + + let handed = dead_letters + .get(handed_over_id.as_str()) + .expect("a released delivery must stay operator-visible"); + assert!( + handed + .reason + .starts_with(crate::runtime::dead_letter::IN_DOUBT_REASON_PREFIX), + "a delivery already handed to a durable native route must be dead-lettered in doubt, got {}", + handed.reason + ); + assert!( + !crate::runtime::dead_letter::is_auto_redeliverable(&handed.reason), + "an in-doubt native delivery must never be auto-redelivered: {}", + handed.reason + ); + assert!( + handed.reason.contains("codex-queue:thread-released"), + "the dead letter must name the route that has the message: {}", + handed.reason + ); + assert_eq!( + probe.snapshot_with_token(true)["dispositions"]["dropped_in_doubt"], + serde_json::json!(1), + "the withheld fleet ack's fate must be recorded, not dropped with a log line" + ); + + let unsent = dead_letters + .get(never_sent_id.as_str()) + .expect("the control delivery must also be recorded"); + assert!( + !unsent + .reason + .starts_with(crate::runtime::dead_letter::IN_DOUBT_REASON_PREFIX), + "a delivery no transport ever saw must stay redeliverable, got {}", + unsent.reason + ); + assert!( + crate::runtime::dead_letter::is_auto_redeliverable(&unsent.reason), + "control: an un-sent delivery is still safe to redeliver: {}", + unsent.reason + ); +} + +/// Every worker-teardown site must dispose through the seam-aware path. +/// +/// The in-doubt classification lives in +/// `dispose_pending_deliveries_for_teardown`; a site that keeps calling +/// `emit_dropped_delivery_failures` directly after `take_pending_for_worker` +/// re-opens the same hole for its own path only, which is exactly how this +/// started — one disposal site consulted the seam and four did not. +#[test] +fn every_worker_teardown_site_disposes_through_the_seam_aware_path() { + for (file, source) in [ + ("api.rs", include_str!("api.rs")), + ("relaycast_events.rs", include_str!("relaycast_events.rs")), + ("maintenance.rs", include_str!("maintenance.rs")), + ] { + let sites = source.matches("take_pending_for_worker(").count(); + assert!( + sites > 0, + "{file} is declared a worker-teardown site but no longer removes pending deliveries" + ); + assert_eq!( + source + .matches("dispose_pending_deliveries_for_teardown(") + .count(), + sites, + "{file} removes pending deliveries at {sites} site(s) but does not dispose all of \ + them through the seam-aware path" + ); + assert!( + !source.contains("emit_dropped_delivery_failures("), + "{file} must not bypass the seam-aware teardown disposal" + ); + } +} + +/// A broker restart must not queue a landed native delivery a second time. +/// +/// Driven end to end through the real backend: a fake `codex` on disk records +/// every invocation, so "the backend was not invoked again" is observed from +/// the transport's own side rather than from a mock's bookkeeping. Lifetime one +/// hands the message over and snapshots the pending map; lifetime two loads +/// that snapshot into a brand-new `DeliverySeam` — the empty-at-startup state +/// that made a reloaded delivery classify `Fresh` — and runs the same retry the +/// first maintenance tick runs. +#[cfg(unix)] +#[tokio::test] +async fn a_restarted_broker_does_not_queue_a_handed_over_codex_delivery_again() { + use std::os::unix::fs::PermissionsExt; + + let script_dir = tempfile::tempdir().expect("fake codex dir"); + let invocations = script_dir.path().join("invocations.log"); + let codex = script_dir.path().join("codex"); + std::fs::write( + &codex, + format!( + r#"#!/bin/sh +if [ "$1" = "queue" ] && [ "$2" = "--help" ]; then + printf '%s\n' 'Usage: codex queue --thread --message=' + exit 0 +fi +printf '%s\n' "$*" >> '{}' +exit 0 +"#, + invocations.display() + ), + ) + .expect("write fake codex"); + let mut permissions = std::fs::metadata(&codex).expect("metadata").permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&codex, permissions).expect("chmod"); + let queue_writes = || { + std::fs::read_to_string(&invocations) + .map(|log| log.lines().filter(|line| line.contains("queue ")).count()) + .unwrap_or(0) + }; + + let worker_name = "codex-restart"; + let delivery_id = DeliveryId::new("del_codex_restart"); + let mut workers = make_worker_registry_with_worker(worker_name).await; + { + let handle = workers + .workers + .get_mut(worker_name) + .expect("fixture worker"); + handle.spec.runtime = AgentRuntime::Headless; + handle.spec.cli = Some(codex.display().to_string()); + handle.spec.session_id = Some("thread-restart".to_string()); + } + + // --- broker lifetime one ------------------------------------------------- + let mut seam = crate::delivery::DeliverySeam::new(); + let mut pending_deliveries = HashMap::from([(delivery_id.clone(), { + let mut pending = make_pending_delivery(delivery_id.as_str(), worker_name); + pending.next_retry_at = Instant::now(); + pending + })]); + let outcome = retry_pending_delivery( + &delivery_id, + &mut workers, + &mut pending_deliveries, + Duration::from_secs(5), + &mut seam, + ) + .await + .expect("the first hand-off must not error"); + assert!( + matches!(outcome, DeliveryAttemptOutcome::Attempted { .. }), + "the codex queue route must accept the first send, got {outcome:?}" + ); + assert_eq!(queue_writes(), 1, "exactly one `codex queue` write so far"); + assert_eq!( + pending_deliveries + .get(&delivery_id) + .and_then(|pending| pending.sent_route.as_ref()) + .and_then(super::delivery::PersistedDeliveryRoute::route), + Some("codex-queue:thread-restart"), + "the snapshot must carry the route, or the restart has nothing to restore from" + ); + + let snapshot_dir = tempfile::tempdir().expect("snapshot dir"); + let snapshot = snapshot_dir.path().join("pending-deliveries.json"); + super::delivery::save_pending_deliveries(&snapshot, &pending_deliveries) + .expect("persist the pending snapshot"); + + // --- broker lifetime two ------------------------------------------------- + // Exactly what `init.rs` does: load the snapshot, then re-seed a brand-new + // seam from it before the first maintenance tick. + let mut reloaded = super::delivery::load_pending_deliveries(&snapshot); + assert!( + reloaded.contains_key(&delivery_id), + "precondition: the delivery survives the restart" + ); + let mut restarted_seam = crate::delivery::DeliverySeam::new(); + assert_eq!( + super::delivery::rehydrate_delivery_seam(&mut restarted_seam, &reloaded), + 1, + "the reloaded native receipt must be restored into the new seam" + ); + + let restarted_outcome = retry_pending_delivery( + &delivery_id, + &mut workers, + &mut reloaded, + Duration::from_secs(5), + &mut restarted_seam, + ) + .await + .expect("the post-restart tick must not error"); + + assert_eq!( + queue_writes(), + 1, + "a delivery already sitting in Codex's durable queue must not be queued again \ + after a restart, got {restarted_outcome:?}" + ); + assert!( + matches!(restarted_outcome, DeliveryAttemptOutcome::Noop), + "the restored receipt must answer AlreadySent and back off, got {restarted_outcome:?}" + ); + + cleanup_worker_registry(workers).await; +} + +/// Build a registry whose only delivery target for `name` is an attached Codex +/// thread — no broker-owned worker process, which is the whole point of the +/// native route. +fn registry_with_attached_codex(name: &str) -> (WorkerRegistry, tempfile::NamedTempFile) { + let (tx, _rx) = mpsc::channel::(16); + let mut registry = WorkerRegistry::new( + tx, + Vec::new(), + PathBuf::from("/tmp/agent-relay-broker-tests"), + Instant::now(), + ); + let rollout = tempfile::NamedTempFile::new().expect("rollout file"); + let target = crate::delivery::codex_queue::CodexQueueTarget::new_for_test( + "codex", + Vec::new(), + None, + "thread-attached", + Some(rollout.path().to_path_buf()), + ); + registry + .attach_native_codex(WorkerName::from(name), target) + .expect("attach the native Codex target"); + (registry, rollout) +} + +/// `has_delivery_target` was widened to accept native Codex targets, which let +/// an inbound message for an attached Codex session be PARKED in manual flush. +/// The manual-flush drain is `try_inject_pending_relay_message_once` → +/// `WorkerRegistry::deliver`, which knows only broker-owned workers: it refuses +/// pre-write for a native target, the flush loop breaks, and the message stays +/// at the head of the FIFO forever, blocking everything queued behind it. +/// +/// A native-only target must therefore never park. The PTY control in the same +/// test keeps manual flush working where it means something. +#[tokio::test] +async fn a_native_only_delivery_target_never_parks_an_inbound_message() { + let worker_name = "codex-attached-flush"; + let (workers, _rollout) = registry_with_attached_codex(worker_name); + let mut delivery_states: HashMap = HashMap::from([( + WorkerName::from(worker_name), + InboundDeliveryState::new(crate::types::InboundDeliveryMode::ManualFlush), + )]); + + let result = super::delivery::queue_inbound_for_delivery_mode( + &mut delivery_states, + &workers, + worker_name, + super::delivery::InboundContext { + from: "Lead", + body: "hello codex", + target: worker_name, + thread_id: None, + workspace_id: None, + workspace_alias: None, + priority: 2, + mode: MessageInjectionMode::Wait, + event_id: Some("evt_native_flush"), + relaycast_receipt: None, + }, + ); + + let InboundQueueOutcome::DrainNow(to_drain) = result.outcome else { + panic!( + "a message parked for a native-only target can never be flushed, got {:?}", + result.outcome + ); + }; + assert_eq!( + to_drain.len(), + 1, + "the message must be handed to the seam-backed drain" + ); + assert_eq!( + delivery_states + .get(worker_name) + .map(InboundDeliveryState::pending_len), + Some(0), + "nothing may be left parked behind it" + ); + + // Control: a broker-owned PTY worker still honours manual flush, so the + // assertion above is about the native route and not about the mode being + // ignored everywhere. + let pty_name = "pty-manual-flush"; + let pty_workers = make_worker_registry_with_worker(pty_name).await; + let mut pty_states: HashMap = HashMap::from([( + WorkerName::from(pty_name), + InboundDeliveryState::new(crate::types::InboundDeliveryMode::ManualFlush), + )]); + let pty_result = super::delivery::queue_inbound_for_delivery_mode( + &mut pty_states, + &pty_workers, + pty_name, + super::delivery::InboundContext { + from: "Lead", + body: "hello pty", + target: pty_name, + thread_id: None, + workspace_id: None, + workspace_alias: None, + priority: 2, + mode: MessageInjectionMode::Wait, + event_id: Some("evt_pty_flush"), + relaycast_receipt: None, + }, + ); + assert_eq!( + pty_result.outcome, + InboundQueueOutcome::Queued, + "control: manual flush must still hold a PTY worker's inbound messages" + ); + cleanup_worker_registry(pty_workers).await; +} + +/// The other half of the same guarantee: manual flush is refused for a +/// native-only target at the setter, so the mode a message could be parked +/// under cannot be reached in the first place. +#[tokio::test] +async fn manual_flush_is_refused_for_a_native_only_delivery_target() { + let worker_name = "codex-attached-mode"; + let (workers, _rollout) = registry_with_attached_codex(worker_name); + let mut fixture = worker_event_runtime_fixture(workers, HashMap::new()); + + let (reply, rx) = tokio::sync::oneshot::channel(); + fixture + .runtime + .handle_api_request(ListenApiRequest::SetInboundDeliveryMode { + name: WorkerName::from(worker_name), + mode: crate::types::InboundDeliveryMode::ManualFlush, + expected_mode: None, + expected_revision: None, + reply, + }) + .await; + + let error = rx + .await + .expect("the handler must answer") + .expect_err("manual flush must be refused for a native-only delivery target"); + assert!( + matches!( + error, + crate::listen_api::DeliveryRouteError::ManualFlushUnsupportedForNativeRoute(_) + ), + "the refusal must name its reason rather than pretend the agent does not exist, got {error}" + ); + assert!( + !fixture + .runtime + .delivery_states + .get(worker_name) + .is_some_and(|state| state.mode == crate::types::InboundDeliveryMode::ManualFlush), + "a refused transition must not leave the worker in manual flush" + ); + + cleanup_worker_registry(fixture.runtime.workers).await; +} diff --git a/crates/broker/src/worker.rs b/crates/broker/src/worker.rs index 8bd44fecdd..8a590f81c1 100644 --- a/crates/broker/src/worker.rs +++ b/crates/broker/src/worker.rs @@ -283,6 +283,9 @@ pub(crate) struct LiveFleetInventoryCandidate { pub(crate) struct WorkerRegistry { pub(crate) workers: HashMap, + /// Existing Codex sessions attached by their own MCP process. These are + /// delivery targets, not broker-owned child processes. + native_codex_targets: HashMap, event_tx: mpsc::Sender, worker_env: Vec<(String, String)>, worker_logs_dir: PathBuf, @@ -397,6 +400,7 @@ impl WorkerRegistry { Self { workers: HashMap::new(), + native_codex_targets: HashMap::new(), event_tx, worker_env, worker_logs_dir, @@ -439,7 +443,7 @@ impl WorkerRegistry { /// `pending_messages` comes from [`crate::runtime::pending_message_counts`]; /// a worker missing from the map has nothing waiting. pub(crate) fn list(&self, pending_messages: &HashMap) -> Vec { - self.workers + let mut listed: Vec = self.workers .iter() .map(|(name, handle)| { let native_harness = native_harness_metadata(&handle.spec); @@ -469,7 +473,26 @@ impl WorkerRegistry { "native_harness_capabilities": native_harness.and_then(|(_, capabilities)| capabilities), }) }) - .collect() + .collect(); + listed.extend(self.native_codex_targets.iter().map(|(name, target)| { + let ready = target.has_verified_rollout_path(); + json!({ + "name": name, + "runtime": "headless", + "provider": "codex", + "cli": "codex", + "sessionId": target.thread_id(), + "pid": Value::Null, + "workerPid": Value::Null, + "current_state": if ready { "attached" } else { "unverified" }, + "ready": ready, + "pending_messages": pending_messages.get(name).copied().unwrap_or(0), + "runtime_kind": "native", + "native_harness_protocol_version": 1, + "native_harness_capabilities": {"delivery": "codex-queue", "ownsSession": false}, + }) + })); + listed } pub(crate) fn env_value(&self, key: &str) -> Option<&str> { @@ -517,6 +540,53 @@ impl WorkerRegistry { self.workers.contains_key(name) } + pub(crate) fn has_delivery_target(&self, name: &str) -> bool { + self.has_worker(name) || self.native_codex_targets.contains_key(name) + } + + pub(crate) fn native_codex_target( + &self, + name: &str, + ) -> Option<&crate::delivery::codex_queue::CodexQueueTarget> { + self.native_codex_targets.get(name) + } + + /// A delivery target relay can reach ONLY over a native route, because it + /// owns no PTY/headless worker process for that name. + /// + /// The manual-flush drain (`try_inject_pending_relay_message_once` → + /// [`WorkerRegistry::deliver`]) knows only broker-owned workers, so for one + /// of these it fails pre-write forever and the parked message can never + /// leave the head of the FIFO. Callers that decide whether a message may be + /// parked ask this first. + pub(crate) fn is_native_only_delivery_target(&self, name: &str) -> bool { + !self.has_worker(name) && self.native_codex_targets.contains_key(name) + } + + pub(crate) fn attach_native_codex( + &mut self, + name: WorkerName, + target: crate::delivery::codex_queue::CodexQueueTarget, + ) -> Result<()> { + anyhow::ensure!( + !self.workers.contains_key(&name), + "agent '{name}' is already a broker-owned worker" + ); + if let Some(existing) = self.native_codex_targets.get(&name) { + anyhow::ensure!( + existing.thread_id() == target.thread_id(), + "agent '{name}' is already attached to a different Codex thread" + ); + return Ok(()); + } + self.native_codex_targets.insert(name, target); + Ok(()) + } + + pub(crate) fn detach_native_codex(&mut self, name: &str) -> bool { + self.native_codex_targets.remove(name).is_some() + } + /// True when a worker is registered AND its child process is still alive. /// Registration alone (`has_worker`) can lag a dead child until the periodic /// `reap_exited` sweep removes it, so callers that must not act on a @@ -718,9 +788,37 @@ impl WorkerRegistry { let (resolved_cli, inline_cli_args) = parse_cli_command(&config.command) .with_context(|| format!("invalid harness command '{}'", config.command))?; - let normalized_cli = normalize_cli_name(&resolved_cli); let mut effective_args = inline_cli_args; effective_args.extend(config.args.clone()); + let normalized_cli = normalize_cli_name(&resolved_cli); + let cli_lower = normalized_cli.to_lowercase(); + let mut codex_spawn_env = self.worker_env.clone(); + codex_spawn_env.extend(harness_env.clone()); + let mut codex_exact_queue_capable = false; + if cli_lower == "codex" { + let queue_global_args = + crate::delivery::codex_queue::codex_queue_global_args(&effective_args); + if let Some(queue_cli) = + crate::codex_session::resolve_queue_capable_codex_command( + &resolved_cli, + &queue_global_args, + spec.cwd.as_deref().map(Path::new), + &codex_spawn_env, + false, + ) + .await + { + codex_exact_queue_capable = true; + if queue_cli != resolved_cli { + tracing::warn!( + worker = %spec.name, + command = %queue_cli, + "using alternate queue-capable Codex command for native delivery side-channel" + ); + remember_codex_queue_command(&mut spec, queue_cli); + } + } + } command.arg("pty"); command.arg("--agent-name").arg(&spec.name); @@ -740,7 +838,6 @@ impl WorkerRegistry { } command.arg(&resolved_cli); - let cli_lower = normalized_cli.to_lowercase(); let is_claude = cli_lower == "claude" || cli_lower.starts_with("claude:"); let is_codex = cli_lower == "codex"; let is_gemini = cli_lower == "gemini"; @@ -776,7 +873,12 @@ impl WorkerRegistry { | CodexSessionReference::AmbiguousVariadicImage | CodexSessionReference::Unknown => {} CodexSessionReference::None | CodexSessionReference::VariadicImage => { - if codex_has_positional_arg(&effective_args) { + if !codex_exact_queue_capable { + tracing::debug!( + worker = %spec.name, + "not pre-creating Codex session because the launched binary does not expose queue delivery" + ); + } else if codex_has_positional_arg(&effective_args) { tracing::debug!( worker = %spec.name, "not pre-creating Codex session because args contain a positional prompt or subcommand" @@ -786,7 +888,7 @@ impl WorkerRegistry { match crate::codex_session::create_resumable_codex_thread( &resolved_cli, cwd, - &self.worker_env, + &codex_spawn_env, &effective_args, crate::util::version::broker_version(), ) @@ -961,9 +1063,10 @@ impl WorkerRegistry { let cli = spec.cli.as_deref().context("pty runtime requires `cli`")?; let (resolved_cli, inline_cli_args) = parse_cli_command(cli) .with_context(|| format!("invalid CLI command '{cli}'"))?; - let normalized_cli = normalize_cli_name(&resolved_cli); let mut effective_args = inline_cli_args; effective_args.extend(spec.args.clone()); + let normalized_cli = normalize_cli_name(&resolved_cli); + let cli_lower = normalized_cli.to_lowercase(); command.arg("pty"); command.arg("--agent-name").arg(&spec.name); @@ -983,12 +1086,28 @@ impl WorkerRegistry { } command.arg(&resolved_cli); - let cli_lower = normalized_cli.to_lowercase(); let is_claude = cli_lower == "claude" || cli_lower.starts_with("claude:"); let is_codex = cli_lower == "codex"; let is_gemini = cli_lower == "gemini"; let is_grok = cli_lower == "grok"; let trust_flag = muse_trust_flag(&cli_lower, &effective_args); + let mut codex_spawn_env = self.worker_env.clone(); + codex_spawn_env.extend(harness_env.clone()); + let mut codex_exact_queue_capable = false; + if cli_lower == "codex" { + let queue_global_args = + crate::delivery::codex_queue::codex_queue_global_args(&effective_args); + codex_exact_queue_capable = + crate::codex_session::resolve_queue_capable_codex_command( + &resolved_cli, + &queue_global_args, + spec.cwd.as_deref().map(Path::new), + &codex_spawn_env, + false, + ) + .await + .is_some(); + } if let Some(model) = apply_codex_model_arg_fallback( &resolved_cli, &cli_lower, @@ -1020,7 +1139,12 @@ impl WorkerRegistry { | CodexSessionReference::Unknown => {} CodexSessionReference::None | CodexSessionReference::VariadicImage => { - if codex_has_positional_arg(&effective_args) { + if !codex_exact_queue_capable { + tracing::debug!( + worker = %spec.name, + "not pre-creating Codex session because the launched binary does not expose queue delivery" + ); + } else if codex_has_positional_arg(&effective_args) { tracing::debug!( worker = %spec.name, "not pre-creating Codex session because args contain a positional prompt or subcommand" @@ -1030,7 +1154,7 @@ impl WorkerRegistry { match crate::codex_session::create_resumable_codex_thread( &resolved_cli, cwd, - &self.worker_env, + &codex_spawn_env, &effective_args, crate::util::version::broker_version(), ) @@ -1657,6 +1781,10 @@ impl WorkerRegistry { pub(crate) async fn release(&mut self, name: &str) -> Result<()> { tracing::info!(target = "broker::release", name = %name, "releasing worker"); self.initial_tasks.remove(name); + if self.detach_native_codex(name) { + tracing::info!(target = "broker::release", name = %name, "native Codex target detached"); + return Ok(()); + } // An explicit release is terminal even when the process already exited // and disappeared from `workers`. Cancel any pending restart before // looking up the handle so maintenance cannot resurrect the released @@ -2287,6 +2415,16 @@ fn codex_session_reference(args: &[String]) -> CodexSessionReference { CodexSessionReference::None } +fn remember_codex_queue_command(spec: &mut AgentSpec, queue_command: String) { + let Some(ResolvedHarnessConfig::Pty(config)) = spec.harness_config.as_mut() else { + return; + }; + config.metadata.get_or_insert_with(HashMap::new).insert( + "codex_queue_command".to_string(), + Value::String(queue_command), + ); +} + fn codex_has_positional_arg(args: &[String]) -> bool { let mut skip_next = false; for arg in args { @@ -3069,6 +3207,68 @@ sleep 30 assert!(reg.list(&HashMap::new()).is_empty()); } + #[test] + fn native_codex_attach_is_observable_and_release_detaches_it() { + let mut reg = make_registry(vec![]); + let rollout = tempfile::NamedTempFile::new().expect("rollout file"); + let target = crate::delivery::codex_queue::CodexQueueTarget::new_for_test( + "codex", + Vec::new(), + None, + "thread-visible", + Some(rollout.path().to_path_buf()), + ); + + reg.attach_native_codex(WorkerName::from("native-visible"), target) + .expect("attach native Codex target"); + + let listed = reg.list(&HashMap::new()); + let entry = listed + .iter() + .find(|entry| entry.get("name").and_then(Value::as_str) == Some("native-visible")) + .expect("attached target is listed"); + assert_eq!( + entry.get("current_state").and_then(Value::as_str), + Some("attached") + ); + assert_eq!(entry.get("ready").and_then(Value::as_bool), Some(true)); + + assert!(reg.detach_native_codex("native-visible")); + assert!(!reg.has_delivery_target("native-visible")); + } + + #[test] + fn native_codex_attach_rejects_thread_reassignment() { + let mut reg = make_registry(vec![]); + let rollout = tempfile::NamedTempFile::new().expect("rollout file"); + let first = crate::delivery::codex_queue::CodexQueueTarget::new_for_test( + "codex", + Vec::new(), + None, + "thread-one", + Some(rollout.path().to_path_buf()), + ); + let second = crate::delivery::codex_queue::CodexQueueTarget::new_for_test( + "codex", + Vec::new(), + None, + "thread-two", + Some(rollout.path().to_path_buf()), + ); + + reg.attach_native_codex(WorkerName::from("native-visible"), first) + .expect("initial attach"); + let error = reg + .attach_native_codex(WorkerName::from("native-visible"), second) + .expect_err("same name cannot be moved to a different thread") + .to_string(); + + assert!( + error.contains("already attached to a different Codex thread"), + "{error}" + ); + } + #[cfg(unix)] #[tokio::test] async fn spawn_confirmation_rejects_a_process_that_exits_immediately() { diff --git a/crates/broker/tests/delivery_seam_invariants.rs b/crates/broker/tests/delivery_seam_invariants.rs index 3bf741390b..1657063a9a 100644 --- a/crates/broker/tests/delivery_seam_invariants.rs +++ b/crates/broker/tests/delivery_seam_invariants.rs @@ -521,3 +521,23 @@ fn an_acknowledgement_must_name_the_observation_behind_it() { // offset it can re-read must be able to tell them apart. assert_ne!(ObservedAck::peer_ack("ok"), ObservedAck::echo("ok")); } + +#[cfg(unix)] +#[tokio::test] +async fn real_pty_route_unknown_worker_is_pre_write_and_may_fall_back() { + relay_broker::delivery::pty::real_route_probe::unknown_worker_is_pre_write_and_may_fall_back() + .await; +} + +#[cfg(unix)] +#[tokio::test] +async fn real_pty_route_write_failure_after_commit_does_not_fall_back() { + relay_broker::delivery::pty::real_route_probe::write_failure_after_commit_does_not_fall_back() + .await; +} + +#[cfg(unix)] +#[tokio::test] +async fn real_pty_route_never_reports_an_observed_ack() { + relay_broker::delivery::pty::real_route_probe::never_reports_an_observed_ack().await; +} diff --git a/crates/relay-pty/src/codex_session.rs b/crates/relay-pty/src/codex_session.rs index 0c65439204..c7784fb0c3 100644 --- a/crates/relay-pty/src/codex_session.rs +++ b/crates/relay-pty/src/codex_session.rs @@ -5,7 +5,13 @@ //! that a later PTY spawn can `codex resume` into, so the session survives //! agent restarts. -use std::{path::Path, process::Stdio, time::Duration}; +use std::{ + collections::HashMap, + path::Path, + process::Stdio, + sync::{Mutex, OnceLock}, + time::{Duration, Instant}, +}; use anyhow::{bail, Context, Result}; use serde_json::{json, Value}; @@ -16,6 +22,160 @@ use tokio::{ }; const CODEX_BOOTSTRAP_TIMEOUT: Duration = Duration::from_secs(15); +const CODEX_QUEUE_PROBE_TIMEOUT: Duration = Duration::from_secs(1); +const CODEX_QUEUE_PROBE_TTL: Duration = Duration::from_secs(60); +const CODEX_QUEUE_FALLBACKS: [&str; 3] = [ + "/Applications/ChatGPT.app/Contents/Resources/codex", + "/opt/homebrew/bin/codex", + "/usr/local/bin/codex", +]; + +#[derive(Debug, Clone)] +struct QueueProbeCacheEntry { + resolved: Option, + observed_at: Instant, +} + +static CODEX_QUEUE_PROBE_CACHE: OnceLock>> = + OnceLock::new(); + +/// Resolve a Codex executable that exposes `codex queue`. +/// +/// A bare `codex` can lag behind the desktop app bundle on macOS, so PATH is +/// only the first candidate. Explicit paths stay explicit: if the caller named +/// one, do not silently replace it with a different installation. +pub async fn resolve_queue_capable_codex_command( + primary: &str, + global_args: &[String], + cwd: Option<&Path>, + env: &[(String, String)], + allow_bundle_fallbacks: bool, +) -> Option { + let cache_key = + codex_queue_probe_cache_key(primary, global_args, cwd, env, allow_bundle_fallbacks); + if let Some(entry) = CODEX_QUEUE_PROBE_CACHE + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .ok() + .and_then(|cache| cache.get(&cache_key).cloned()) + { + if entry.observed_at.elapsed() <= CODEX_QUEUE_PROBE_TTL { + return entry.resolved; + } + } + + let mut resolved = None; + for candidate in codex_queue_command_candidates(primary, allow_bundle_fallbacks) { + if codex_command_has_queue(&candidate, global_args, cwd, env).await { + resolved = Some(candidate); + break; + } + } + if let Ok(mut cache) = CODEX_QUEUE_PROBE_CACHE + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + { + cache.insert( + cache_key, + QueueProbeCacheEntry { + resolved: resolved.clone(), + observed_at: Instant::now(), + }, + ); + } + resolved +} + +fn codex_queue_probe_cache_key( + primary: &str, + global_args: &[String], + cwd: Option<&Path>, + env: &[(String, String)], + allow_bundle_fallbacks: bool, +) -> String { + let primary_fingerprint = explicit_command_fingerprint(primary); + let codex_home = env + .iter() + .rev() + .find(|(key, _)| key == "CODEX_HOME") + .map(|(_, value)| value.as_str()) + .unwrap_or(""); + format!( + "{primary}\nprimary_fingerprint={primary_fingerprint}\nargs={}\ncwd={}\nCODEX_HOME={codex_home}\nbundle_fallbacks={allow_bundle_fallbacks}", + global_args.join("\u{1f}"), + cwd.map(|path| path.display().to_string()) + .unwrap_or_default() + ) +} + +fn explicit_command_fingerprint(command: &str) -> String { + let path = Path::new(command); + if path.components().count() == 1 { + return String::new(); + } + let Ok(metadata) = std::fs::metadata(path) else { + return "missing".to_string(); + }; + let modified = metadata + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|duration| duration.as_nanos().to_string()) + .unwrap_or_else(|| "unknown".to_string()); + format!("len={};modified={modified}", metadata.len()) +} + +pub fn codex_queue_command_candidates(primary: &str, allow_bundle_fallbacks: bool) -> Vec { + let mut candidates = vec![primary.to_string()]; + if allow_bundle_fallbacks && is_bare_codex_command(primary) { + for fallback in CODEX_QUEUE_FALLBACKS { + if Path::new(fallback).is_file() && !candidates.iter().any(|item| item == fallback) { + candidates.push(fallback.to_string()); + } + } + } + candidates +} + +async fn codex_command_has_queue( + command: &str, + global_args: &[String], + cwd: Option<&Path>, + env: &[(String, String)], +) -> bool { + let mut probe = Command::new(command); + probe + .args(global_args) + .arg("queue") + .arg("--help") + .stdin(Stdio::null()) + .kill_on_drop(true); + if let Some(cwd) = cwd { + probe.current_dir(cwd); + } + for (key, value) in env { + probe.env(key, value); + } + let Ok(Ok(output)) = timeout(CODEX_QUEUE_PROBE_TIMEOUT, probe.output()).await else { + return false; + }; + if !output.status.success() { + return false; + } + let mut help = output.stdout; + help.extend_from_slice(&output.stderr); + let help = String::from_utf8_lossy(&help); + help.contains("--thread") && help.contains("--message") +} + +fn is_bare_codex_command(command: &str) -> bool { + let path = Path::new(command); + path.components().count() == 1 + && path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name == "codex" || name == "codex.exe") +} /// Create a resumable Codex thread and return its id. /// @@ -49,6 +209,11 @@ async fn create_resumable_codex_thread_inner( let thread_cwd = cwd.canonicalize().unwrap_or_else(|_| cwd.to_path_buf()); let mut command = Command::new(codex_bin); command + // These are top-level Codex flags. They must precede the subcommand: + // `codex app-server --dangerously-...` is rejected by clap, while + // `codex --dangerously-... app-server` preserves the PTY launch's + // config/profile context and starts normally. + .args(codex_app_server_passthrough_args(cli_args)) .arg("app-server") .arg("--listen") .arg("stdio://") @@ -57,9 +222,6 @@ async fn create_resumable_codex_thread_inner( .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - for arg in codex_app_server_passthrough_args(cli_args) { - command.arg(arg); - } for (key, value) in env { command.env(key, value); } @@ -281,7 +443,7 @@ mod tests { std::fs::write( &fake_codex, r#"#!/bin/sh -if [ "$1" != "app-server" ]; then +if [ "$1" != "--dangerously-bypass-approvals-and-sandbox" ] || [ "$2" != "app-server" ]; then exit 2 fi read line @@ -304,7 +466,7 @@ while read line; do :; done fake_codex.to_str().expect("utf-8 fake codex path"), dir.path(), &[], - &[], + &["--dangerously-bypass-approvals-and-sandbox".to_string()], "0.0.0-test", ) .await @@ -345,4 +507,22 @@ while read line; do :; done ] ); } + + #[test] + fn queue_candidates_try_app_bundle_for_bare_codex_only() { + let bare = codex_queue_command_candidates("codex", true); + assert_eq!(bare.first().map(String::as_str), Some("codex")); + assert!( + bare.iter() + .any(|candidate| candidate == "/Applications/ChatGPT.app/Contents/Resources/codex") + || !Path::new("/Applications/ChatGPT.app/Contents/Resources/codex").is_file(), + "the desktop app codex should be considered when it exists" + ); + + let exact_bare = codex_queue_command_candidates("codex", false); + assert_eq!(exact_bare, vec!["codex".to_string()]); + + let explicit = codex_queue_command_candidates("/custom/bin/codex", true); + assert_eq!(explicit, vec!["/custom/bin/codex".to_string()]); + } } diff --git a/docs/native-delivery-migration.md b/docs/native-delivery-migration.md index 2f1f03e2dc..bb8a5c1fa8 100644 --- a/docs/native-delivery-migration.md +++ b/docs/native-delivery-migration.md @@ -151,6 +151,39 @@ to queued messages, so carry a marker in the message text and match on that. _Effort: medium, mostly discovery. Exit: parity + `eval:matrix` for codex._ +> **Revised after phase 1's signoff (read this before phase 2).** +> +> Two things the seam's four rules turn out to require from every route that +> outlives the broker, both found by adversarial signoff after the phase was +> otherwise green: +> +> 1. **"Already handed to a transport" is not a process-local fact.** The +> `DeliverySeam`'s receipt memory is rebuilt empty at startup and was +> consulted at exactly one disposal site. For a PTY child that was sound — +> the transport died with the broker, so an un-acked write provably never +> arrived. A native route breaks it: the message is durable in the vendor's +> own store and the session outlives both the worker and the broker, so a +> worker teardown or a restart that treats the delivery as never-arrived +> re-sends a landed message. Phase 1 persists the accepted route on the +> pending delivery, rehydrates the seam from that snapshot before the first +> maintenance tick, and routes every `take_pending_for_worker` site through +> the same `was_sent` test the retry-cap branch uses. **Phase 2's Claude +> routes have the same shape and inherit both requirements.** +> 2. **Durable presence is not a read receipt.** Captured against +> `codex-cli 0.155.0-alpha.9.2`: `codex queue` writes a row to +> `queue_1.sqlite` and NOTHING to the thread rollout until a live turn +> consumes the item. So the queue store answers "delivered" and the rollout +> answers "read", and only the second may publish a Relaycast read receipt +> (rule 4). A negative filter — reject assistant/reasoning/summary records — +> cannot make that distinction; the matcher has to positively identify the +> record the vendor writes for a consumed user input. See decision D4 in the +> phase-1 artifact set. +> +> Mechanically: mutation transcripts are now recorded by +> `scripts/migrate/mutation-proof.mjs` and carry the sha256 of every source +> they guard, because phase 1's transcripts were recorded before a later repair +> round rewrote the code they were proving and no gate could see it. + ### Phase 2 — Claude Code Two targets that do not overlap: terminal sessions via the inbox socket, cloud diff --git a/flows/migrate/native-delivery.spec.ts b/flows/migrate/native-delivery.spec.ts index d86358bba0..b135dad59d 100644 --- a/flows/migrate/native-delivery.spec.ts +++ b/flows/migrate/native-delivery.spec.ts @@ -56,7 +56,7 @@ import path from 'node:path'; import { specWorkflow, type V1StepOptions } from '../spec-builder.ts'; // @ts-expect-error JavaScript module intentionally has no declaration file. -import { PHASES } from '../../scripts/migrate/native-delivery-gates.mjs'; +import { PHASES, PARITY, unlaunchedCommand } from '../../scripts/migrate/native-delivery-gates.mjs'; type PhaseConfig = { slug: string; @@ -177,13 +177,17 @@ function gate(action: string, extra = ''): string { * did, so a red result flows into the repair owner built to answer it; the * verdict is journaled in `evidence/.json` for the `*-final` gate. */ -const parityCommands: Record = { - 'parity-orch-to-worker': 'npx tsx tests/parity/orch-to-worker.ts', - 'parity-multi-worker': 'npx tsx tests/parity/multi-worker.ts', - 'parity-broadcast': 'npx tsx tests/parity/broadcast.ts', - 'parity-continuity-handoff': 'npx tsx tests/parity/continuity-handoff.ts', - 'parity-stability-soak': 'npx tsx tests/parity/stability-soak.ts', -}; +/** + * Imported, never restated. `require-green`/`accept` compare each recorded + * evidence file against the command the contract names, so a second copy of + * these strings here would be a gate that fails on a typo rather than on a + * regression. Same reason `unlaunchedCommand` is imported below. + */ +const parityCommands = PARITY as Record; + +function shellArg(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} function record( name: string, @@ -191,8 +195,8 @@ function record( markers?: { expect?: string[]; forbid?: string[]; retryOnRed?: number } ): string { const encoded = Buffer.from(command, 'utf8').toString('base64'); - const expect = markers?.expect?.length ? ` --expect ${markers.expect.join(',')}` : ''; - const forbid = markers?.forbid?.length ? ` --forbid ${markers.forbid.join(',')}` : ''; + const expect = markers?.expect?.length ? ` --expect ${shellArg(markers.expect.join(','))}` : ''; + const forbid = markers?.forbid?.length ? ` --forbid ${shellArg(markers.forbid.join(','))}` : ''; const retry = markers?.retryOnRed ? ` --retry-on-red ${markers.retryOnRed}` : ''; return gate('record', `--name ${name}${expect}${forbid}${retry} --command-base64 ${encoded}`); } @@ -622,7 +626,7 @@ if (CONFIG.rust) { 'invariant-tests', record( 'invariant-tests', - `${CARGO} test -p agent-relay-broker --test ${path + `${CARGO} test -p agent-relay-broker --features seam-probe --test ${path .basename(CONFIG.invariantTestFile ?? 'crates/broker/tests/delivery_seam_invariants.rs') .replace(/\.rs$/, '')}`, { forbid: ['0 passed'] } @@ -651,7 +655,7 @@ if (CONFIG.rust) { record('rust-fmt', `${CARGO} fmt --all -- --check`), record('rust-clippy', `${CARGO} clippy --all-targets -- -D warnings`), record('rust-build', `${CARGO} build --release --bin agent-relay-broker`), - record('invariant-tests', `${CARGO} test -p agent-relay-broker`), + record('invariant-tests', `${CARGO} test -p agent-relay-broker --features seam-probe --test delivery_seam_invariants`), ].join('\n'), ['repair-rust'], 5_400_000 @@ -757,14 +761,16 @@ det('parity-assert', gate('require-green', `--names ${parityNames.join(',')}`), let evidenceReady = 'parity-assert'; const nativeNames = [...Object.keys(CONFIG.evals ?? {}), ...Object.keys(CONFIG.e2e ?? {})]; +const nativeCommands = { ...(CONFIG.evals ?? {}), ...(CONFIG.e2e ?? {}) }; +const nativeRecord = (name: string): string => + record(name, nativeCommands[name]!, { + ...(Object.hasOwn(CONFIG.evals ?? {}, name) + ? { expect: ['delivery=100%', 'scenarios='] } + : {}), + forbid: ['# SKIP', 'skipped'], + }); if (nativeNames.length > 0) { - const commands = { ...(CONFIG.evals ?? {}), ...(CONFIG.e2e ?? {}) }; - det( - 'native-evidence', - nativeNames.map((name) => record(name, commands[name]!, { forbid: ['# SKIP'] })).join('\n'), - ['parity-assert'], - 10_800_000 - ); + det('native-evidence', nativeNames.map(nativeRecord).join('\n'), ['parity-assert'], 10_800_000); agentStep({ id: 'repair-native-evidence', agent: 'codex-fixer', @@ -773,16 +779,21 @@ if (nativeNames.length > 0) { task: [ ...HOUSE_RULES, `Read ${nativeNames.map((name) => `${ART}/evidence/${name}.json`).join(', ')}.`, + 'A configured delivery-rate or scenario floor failure is a phase failure for this gate. The deterministic', + 'native-evidence-final step reruns the real suites after this repair step, so', + 'never rerun a green native suite from inside the agent step.', 'These run against real CLIs (RELAY_INTEGRATION_REAL_CLI=1). A skipped case is a red case here:', 'a suite that skipped is a suite that proved nothing.', 'If a vendor CLI is genuinely unavailable or its credential is exhausted, that is an external', `blocker: write ${ART}/BLOCKED_NO_COMMIT.md naming the exact CLI, version and error, and stop.`, + 'For Codex, do not equate the first PATH binary with the queue transport: the desktop app can', + 'bundle a newer queue-capable codex, and the unlaunched-session resolver is authoritative.', 'Do not stub the vendor to manufacture a pass.', ], }); det( 'native-evidence-final', - nativeNames.map((name) => record(name, commands[name]!, { forbid: ['# SKIP'] })).join('\n'), + nativeNames.map(nativeRecord).join('\n'), ['repair-native-evidence'], 10_800_000 ); @@ -795,7 +806,20 @@ if (nativeNames.length > 0) { evidenceReady = 'native-evidence-assert'; } -det('unlaunched-gate', recordedGate('unlaunched-gate', 'unlaunched-gate'), [evidenceReady], 900_000); +const unlaunchedScenarioNames = Array.isArray(CONFIG.unlaunched) + ? CONFIG.unlaunched.map((cli) => `unlaunched-${cli}-delivery`) + : []; +const unlaunchedScenarioRecord = (name: string): string => { + const cli = name.slice('unlaunched-'.length, -'-delivery'.length); + return record(name, unlaunchedCommand(cli) as string, { forbid: ['# SKIP'] }); +}; +const unlaunchedEvidenceBlock = (structuralName: string): string => + [ + ...unlaunchedScenarioNames.map(unlaunchedScenarioRecord), + recordedGate(structuralName, 'unlaunched-gate'), + ].join('\n'); + +det('unlaunched-gate', unlaunchedEvidenceBlock('unlaunched-gate'), [evidenceReady], 3_600_000); agentStep({ id: 'repair-unlaunched', agent: 'claude-fixer', @@ -803,7 +827,10 @@ agentStep({ retries: 2, task: [ ...HOUSE_RULES, - `Read ${ART}/evidence/unlaunched-gate.json. If its verdict is green, DO NOTHING and say so.`, + `Read ${[ + ...unlaunchedScenarioNames.map((name) => `${ART}/evidence/${name}.json`), + `${ART}/evidence/unlaunched-gate.json`, + ].join(', ')}. If every verdict is green, DO NOTHING and say so.`, 'A phase with no native route yet reports `not-required`, and that is the correct answer for it —', 'there is no unlaunched session to deliver into until a backend exists. Do not invent a scenario', 'to satisfy a gate that is already satisfied.', @@ -814,13 +841,13 @@ agentStep({ }); det( 'unlaunched-gate-final', - recordedGate('unlaunched-gate-final', 'unlaunched-gate'), + unlaunchedEvidenceBlock('unlaunched-gate-final'), ['repair-unlaunched'], - 900_000 + 3_600_000 ); det( 'unlaunched-assert', - gate('require-green', '--names unlaunched-gate-final'), + gate('require-green', `--names ${['unlaunched-gate-final', ...unlaunchedScenarioNames].join(',')}`), ['unlaunched-gate-final'], 300_000 ); @@ -947,15 +974,14 @@ det( record('rust-fmt', `${CARGO} fmt --all -- --check`), record('rust-clippy', `${CARGO} clippy --all-targets -- -D warnings`), record('rust-build', `${CARGO} build --release --bin agent-relay-broker`), - record('invariant-tests', `${CARGO} test -p agent-relay-broker`), + record('invariant-tests', `${CARGO} test -p agent-relay-broker --features seam-probe --test delivery_seam_invariants`), ] : []), record('ts-typecheck', 'npm run typecheck'), record('unit-tests', VITEST), ...parityNames.map(parityRecord), - ...nativeNames.map((name) => - record(name, { ...(CONFIG.evals ?? {}), ...(CONFIG.e2e ?? {}) }[name]!, { forbid: ['# SKIP'] }) - ), + ...nativeNames.map(nativeRecord), + ...unlaunchedScenarioNames.map(unlaunchedScenarioRecord), recordedGate('edit-gate-final', 'edit-gate'), recordedGate('manifest-gate-final', 'manifest-gate'), recordedGate('targeted-gate-final', 'targeted-gate'), @@ -996,7 +1022,7 @@ const signoffTask = (provider: 'claude' | 'codex'): string[] => [ ' "parityAssessment": "non-empty",', ' "findings": [{ "id": "stable-id", "severity": "critical|high|medium|low", "issue": "...", "requiredFix": "..." }] }', 'Use verdict pass only with an empty findings array.', - `Finish by printing NATIVE_DELIVERY_SIGNOFF provider=${provider}.`, + 'The JSON artifact is the signoff result. No prose marker is required.', ]; for (const provider of ['claude', 'codex'] as const) { @@ -1007,14 +1033,16 @@ for (const provider of ['claude', 'codex'] as const) { retries: 1, recoveryMode: 'inspect', permissions: permissions(`${provider}-signoff`), - verification: { type: 'output_contains', value: `NATIVE_DELIVERY_SIGNOFF provider=${provider}` }, }); } // ─────────────────────────── 10. accept, then commit ─────────────────────────── /** - * Acceptance recomputes the verdict from evidence and both signoffs. The repair + * Acceptance recomputes the verdict from evidence and both signoff JSON files, + * including their schema, provider, sealed digest, verdict, and findings. The + * agent's stdout is deliberately not a gate: a reviewer that writes a valid + * artifact but omits a ceremonial marker must not burn its retry budget. The repair * loop ends here on purpose: a failed final gate is never handed back to a * reviewer, because that would mutate evidence after independent review. */ diff --git a/package.json b/package.json index a21728722c..15479cab13 100644 --- a/package.json +++ b/package.json @@ -124,7 +124,7 @@ "verify:fleet-daytona:validate": "node scripts/verify-features/fleet-daytona.mjs validate", "verify:fleet-daytona:check": "npm run flows:spec:fleet-daytona && flows check .workflow-artifacts/flows/relay.verify.fleet-daytona.json", "test:e2e": "vitest run --config vitest.e2e.config.ts", - "test:e2e:unlaunched": "vitest run --config vitest.e2e.config.ts tests/e2e/unlaunched/**/*.test.ts", + "test:e2e:unlaunched": "vitest run --config tests/e2e/vitest.unlaunched.config.ts", "smoke:prod": "node tests/e2e/prod-smoke/prod-smoke.mjs", "test:subscriptions:proof": "node --test tests/e2e/github-subscriptions/*.test.mjs", "build:cli-surface": "npm --prefix packages/cli-surface run build" diff --git a/packages/cli/src/cli/agent-relay-mcp.ts b/packages/cli/src/cli/agent-relay-mcp.ts index 662ff448e0..4e18b3fd88 100644 --- a/packages/cli/src/cli/agent-relay-mcp.ts +++ b/packages/cli/src/cli/agent-relay-mcp.ts @@ -40,6 +40,11 @@ import { initTelemetry, shutdown as shutdownTelemetry } from './telemetry/index. import { RealtimeResourceBridge, SubscriptionManager, registerResourceDefinitions } from './mcp/resources.js'; import { jsonContent, jsonResult, textContent } from './mcp/tool-results.js'; import { observerUrl, resolveObserverBaseUrl } from './lib/observer-url.js'; +import { + defaultStateDir, + readConnectionFileFromDisk, + resolveBrokerConnection, +} from './lib/broker-connection.js'; import { createWorkspace, extractWorkspaceKey, @@ -919,8 +924,7 @@ async function verifyMetadataLanded( } const record = agents.find((agent) => (agent as { name?: string } | null)?.name === name) as - | { metadata?: Record } - | undefined; + { metadata?: Record } | undefined; if (!record) { return { @@ -1107,11 +1111,20 @@ function registerAgentRelayTools( 'workspace listing. Use when writing durable identity you intend to rely on; ' + 'the response reports metadata_verified as true, false, or "unchecked".' ), + native_delivery: z + .object({ + provider: z.literal('codex'), + thread_id: z.string().min(1).describe('Codex thread id returned by the current session'), + }) + .optional() + .describe( + 'Attach this already-running Codex thread to the authenticated local broker so inbound messages are injected without polling.' + ), }, outputSchema: jsonResult, annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true }, }, - async ({ name, type, persona, metadata, verify_metadata }: any) => { + async ({ name, type, persona, metadata, verify_metadata, native_delivery }: any) => { const payload = await registerAgentWithRebind({ session: getSession(), setSession, @@ -1137,6 +1150,45 @@ function registerAgentRelayTools( const nextAgents = new Map(getSession().agents); nextAgents.set(registeredName, createRegisteredAgent(registeredName, token)); setSession({ agentToken: token, agentName: registeredName, agents: nextAgents }); + if (native_delivery?.provider === 'codex') { + const connection = resolveBrokerConnection( + { stateDir: process.env.AGENT_RELAY_STATE_DIR }, + { + env: process.env, + getDefaultStateDir: defaultStateDir, + readConnectionFile: readConnectionFileFromDisk, + } + ); + if (!connection) { + throw new Error( + 'Agent registered, but native Codex delivery could not attach: no local broker connection was found.' + ); + } + const codexHome = process.env.CODEX_HOME?.trim(); + const response = await fetch(`${connection.url}/api/native-delivery/codex/attach`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(connection.apiKey ? { 'X-API-Key': connection.apiKey } : {}), + }, + body: JSON.stringify({ + name: registeredName, + agent_token: token, + thread_id: native_delivery.thread_id, + ...(codexHome ? { codex_home: codexHome } : {}), + cwd: process.cwd(), + }), + }); + const attach = (await response.json().catch(() => ({}))) as Record; + if (!response.ok) { + throw new Error( + `Agent registered, but native Codex delivery could not attach: ${String( + attach.error ?? `broker returned HTTP ${response.status}` + )}` + ); + } + payload.native_delivery = attach; + } } return jsonContent(payload); diff --git a/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts b/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts index 5840fbf539..99a33f813b 100644 --- a/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts +++ b/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts @@ -41,6 +41,10 @@ describe('fleet CLI lifecycle routing', () => { relayHome = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-fleet-lifecycle-home-')); vi.stubEnv('AGENT_RELAY_PROJECT', projectRoot); vi.stubEnv('AGENT_RELAY_HOME', relayHome); + // This scenario proves that the persisted Cloud target wins after spawn. + // A developer's ambient credential must not silently turn it into a test + // of the process-wide default workspace instead. + vi.stubEnv('RELAY_API_KEY', ''); const dataDir = path.join(projectRoot, '.agentworkforce', 'relay'); writeProjectWorkspaceKey(dataDir, 'rk_live_workspace', { workspaceId: TARGET.workspaceId }); diff --git a/scripts/migrate/mutation-proof.mjs b/scripts/migrate/mutation-proof.mjs new file mode 100644 index 0000000000..c18fbfb901 --- /dev/null +++ b/scripts/migrate/mutation-proof.mjs @@ -0,0 +1,645 @@ +#!/usr/bin/env node +/** + * Re-record this phase's mutation proof against the CURRENT tree. + * + * Signoff finding F2: every transcript in `evidence/mutation-proof.md` had been + * recorded against a tree that later moved. The codex-route transcripts panicked + * at `codex_queue.rs:465/503/546` while the shipped expectations sat ~76 lines + * lower, because the fix round after them rewrote the very code those mutations + * were proving. The prose gate could not see it: it only asks whether each + * invariant's section contains the word FAILED. + * + * Re-running mutations by hand is what let that drift in, so it is automated + * here instead. Every run: + * + * 1. applies one textual mutation to product source (never to a test), + * 2. runs the invariant test(s) that mutation must break, + * 3. restores the file from the byte-for-byte original, always, including on + * error or Ctrl-C, + * 4. writes the transcript verbatim, + * + * and then emits `evidence/mutation-proof.json` carrying the sha256 of every + * guarded source at recording time. `native-delivery-gates.mjs seam-rules` + * recomputes those digests, so a later repair round that touches guarded code + * without re-running this script fails the gate instead of shipping a proof + * about a tree that no longer exists. + * + * Usage: node scripts/migrate/mutation-proof.mjs --art [--only ] + */ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; + +/** + * One test per transcript, always filtered by the invariant's own name. + * + * `seam-rules` slices `mutation-proof.md` per invariant by scanning forward to + * the next invariant NAME, so an unfiltered transcript — which lists every test + * in the file — ends its own section before the failure and reads as no + * evidence at all. Filtering also makes each transcript evidence about exactly + * one claim. + */ +const seamTest = (filter) => [ + 'cargo', + 'test', + '-p', + 'agent-relay-broker', + '--features', + 'seam-probe', + '--test', + 'delivery_seam_invariants', + filter, +]; +const libTest = (filter) => ['cargo', 'test', '-p', 'agent-relay-broker', '--lib', filter]; + +const BACKEND = 'crates/broker/src/delivery/backend.rs'; +const PTY = 'crates/broker/src/delivery/pty.rs'; +const CODEX_QUEUE = 'crates/broker/src/delivery/codex_queue.rs'; +const CODEX_THREAD = 'crates/broker/src/codex_thread.rs'; +const RELAY_PTY_CODEX_SESSION = 'crates/relay-pty/src/codex_session.rs'; +const RUNTIME_DELIVERY = 'crates/broker/src/runtime/delivery.rs'; +const RUNTIME_API = 'crates/broker/src/runtime/api.rs'; +const RUNTIME_RELAYCAST = 'crates/broker/src/runtime/relaycast_events.rs'; +const RUNTIME_TESTS = 'crates/broker/src/runtime/tests.rs'; +const SEAM_TEST_FILE = 'crates/broker/tests/delivery_seam_invariants.rs'; + +/** + * One entry per transcript. `invariants` are the contract names this transcript + * is evidence for; `guards` are the files whose content the evidence depends on + * — the mutated source AND the file the invariant test lives in, because a + * rewritten test invalidates "this test bites" just as surely as rewritten + * product code does. + */ +const MUTATIONS = [ + { + transcript: 'mutation-01-prewrite.txt', + invariants: ['falls_back_only_before_write'], + summary: + '`DeliveryError::is_pre_write` always returned `false`, so a pre-write refusal was classified as committed.', + file: BACKEND, + guards: [BACKEND, SEAM_TEST_FILE], + find: ` pub fn is_pre_write(&self) -> bool { + matches!(self, Self::Unavailable { .. }) + }`, + replace: ` pub fn is_pre_write(&self) -> bool { + false + }`, + command: seamTest('falls_back_only_before_write'), + }, + { + transcript: 'mutation-02-duplicate.txt', + invariants: ['never_resends_on_doubt'], + summary: + 'An existing receipt was classified `Fresh` instead of `AlreadySent`, so a message in doubt — including a cancelled one — was handed to a backend again.', + file: BACKEND, + guards: [BACKEND, SEAM_TEST_FILE], + find: ' return Ok(SendOutcome::AlreadySent(receipt));', + replace: ' return Ok(SendOutcome::Fresh(receipt));', + command: seamTest('never_resends_on_doubt'), + }, + { + transcript: 'mutation-02b-cancelled.txt', + invariants: ['a_cancelled_send_remains_in_doubt_and_is_not_retried'], + summary: + 'The same duplicate-guard mutation seen through cancellation: a cancelled send whose provisional receipt still stood was re-classified `Fresh` and handed to a backend again.', + file: BACKEND, + guards: [BACKEND, SEAM_TEST_FILE], + find: ' return Ok(SendOutcome::AlreadySent(receipt));', + replace: ' return Ok(SendOutcome::Fresh(receipt));', + command: seamTest('a_cancelled_send_remains_in_doubt_and_is_not_retried'), + }, + { + transcript: 'mutation-03-route.txt', + invariants: ['records_route_for_each_send'], + summary: '`settle` ignored the recorded route and settled through the first offered backend.', + file: BACKEND, + guards: [BACKEND, SEAM_TEST_FILE], + find: ` let Some(backend) = backends + .iter_mut() + .find(|backend| backend.route_id() == route) + else { + return SettleOutcome::RouteUnavailable(route); + };`, + replace: ` let Some(backend) = backends.iter_mut().next() else { + return SettleOutcome::RouteUnavailable(route); + };`, + command: seamTest('records_route_for_each_send'), + }, + { + transcript: 'mutation-04-ack.txt', + invariants: ['never_acks_without_observation'], + summary: '`settle` upgraded `HandedOver` into `Acked` with no observation behind it.', + file: BACKEND, + guards: [BACKEND, SEAM_TEST_FILE], + find: ' SettleOutcome::Settled(backend.settle(&request).await)', + replace: ` let status = backend.settle(&request).await; + SettleOutcome::Settled(match status { + SettleStatus::HandedOver(_) => SettleStatus::Acked(ObservedAck::peer_ack("fabricated")), + other => other, + })`, + command: seamTest('never_acks_without_observation'), + }, + { + transcript: 'mutation-05-eviction.txt', + invariants: ['an_evicted_receipt_does_not_become_a_fresh_send'], + summary: + 'Eviction dropped receipts without leaving a tombstone, so a forgotten send looked new to `send` and absent to `settle`.', + file: BACKEND, + guards: [BACKEND, SEAM_TEST_FILE], + find: ' self.evicted.insert(dropped.delivery_id);', + replace: ' let _ = dropped;', + command: seamTest('an_evicted_receipt_does_not_become_a_fresh_send'), + }, + { + transcript: 'mutation-05b-eviction-settle.txt', + invariants: ['settle_reports_an_evicted_receipt_as_unknown_not_absent'], + summary: + 'The same eviction mutation seen through `settle`: a forgotten receipt reported as absence, which licenses a re-send of a message that may already have landed.', + file: BACKEND, + guards: [BACKEND, SEAM_TEST_FILE], + find: ' self.evicted.insert(dropped.delivery_id);', + replace: ' let _ = dropped;', + command: seamTest('settle_reports_an_evicted_receipt_as_unknown_not_absent'), + }, + { + transcript: 'mutation-06-unreachable.txt', + invariants: ['settle_distinguishes_absence_from_an_unreachable_route'], + summary: 'An unreachable recorded route reported `NoReceipt` — positive evidence of absence — instead of `RouteUnavailable`.', + file: BACKEND, + guards: [BACKEND, SEAM_TEST_FILE], + find: ' return SettleOutcome::RouteUnavailable(route);', + replace: ' return SettleOutcome::NoReceipt;', + command: seamTest('settle_distinguishes_absence_from_an_unreachable_route'), + }, + { + transcript: 'mutation-07-ack-evidence.txt', + invariants: ['an_acknowledgement_must_name_the_observation_behind_it'], + summary: '`ObservedAck::echo` stored `PeerAck` evidence, naming an observation that never happened.', + file: BACKEND, + guards: [BACKEND, SEAM_TEST_FILE], + find: ` pub fn echo(matched: impl Into) -> Self { + Self { + evidence: AckEvidence::Echo { + matched: matched.into(), + }, + } + }`, + replace: ` pub fn echo(matched: impl Into) -> Self { + Self { + evidence: AckEvidence::PeerAck { + detail: matched.into(), + }, + } + }`, + command: seamTest('an_acknowledgement_must_name_the_observation_behind_it'), + }, + { + transcript: 'mutation-08-pty-prewrite.txt', + invariants: ['real_pty_route_unknown_worker_is_pre_write_and_may_fall_back'], + summary: 'The shipping PTY route mapped a pre-write failure (unknown worker) to a committed error.', + file: PTY, + guards: [PTY, SEAM_TEST_FILE], + find: ` crate::worker::WorkerDeliverError::PreWrite(reason) => { + DeliveryError::unavailable(reason) + }`, + replace: ` crate::worker::WorkerDeliverError::PreWrite(reason) => { + DeliveryError::committed(reason) + }`, + command: seamTest('real_pty_route_unknown_worker_is_pre_write_and_may_fall_back'), + }, + { + transcript: 'mutation-09-pty-committed.txt', + invariants: ['real_pty_route_write_failure_after_commit_does_not_fall_back'], + summary: + 'The shipping PTY route mapped a committed write failure to a pre-write `Unavailable`, re-opening fallback after a possible write.', + file: PTY, + guards: [PTY, SEAM_TEST_FILE], + find: ` crate::worker::WorkerDeliverError::Committed(reason) => { + DeliveryError::committed(reason) + }`, + replace: ` crate::worker::WorkerDeliverError::Committed(reason) => { + DeliveryError::unavailable(reason) + }`, + command: seamTest('real_pty_route_write_failure_after_commit_does_not_fall_back'), + }, + { + transcript: 'mutation-10-pty-ack.txt', + invariants: ['real_pty_route_never_reports_an_observed_ack'], + summary: + 'The shipping PTY route fabricated an observed ack out of a hand-over at settlement, which is the only place that route could claim one.', + file: PTY, + guards: [PTY, SEAM_TEST_FILE], + find: ` fn settle<'a>( + &'a mut self, + _request: &'a SettleRequest, + ) -> super::backend::DeliveryBackendFuture<'a, SettleStatus> { + Box::pin(async move { SettleStatus::HandedOver(HandoverState::HandedOver) }) + }`, + replace: ` fn settle<'a>( + &'a mut self, + _request: &'a SettleRequest, + ) -> super::backend::DeliveryBackendFuture<'a, SettleStatus> { + Box::pin(async move { + SettleStatus::Acked(super::backend::ObservedAck::echo("fabricated")) + }) + }`, + command: seamTest('real_pty_route_never_reports_an_observed_ack'), + }, + { + transcript: 'mutation-11-codex-capability.txt', + invariants: ['unavailable_queue_capability_falls_back_before_write'], + summary: + '`CodexQueueTarget::ensure_queue_capability` classified a missing `codex queue` as a committed error, so a Codex that cannot queue at all blocked the PTY fallback instead of refusing before any write.', + file: CODEX_QUEUE, + guards: [CODEX_QUEUE, RELAY_PTY_CODEX_SESSION], + find: ` DeliveryError::unavailable( + "installed Codex does not expose \`codex queue --thread --message\`", + )`, + replace: ` DeliveryError::committed( + "installed Codex does not expose \`codex queue --thread --message\`", + )`, + command: libTest('delivery::codex_queue::tests::unavailable_queue_capability_falls_back_before_write'), + }, + { + transcript: 'mutation-12-codex-committed.txt', + invariants: ['queue_process_failure_is_committed_and_does_not_fall_back'], + summary: + 'A started `codex queue` child that exited non-zero was classified `Unavailable`, so the seam fell back to the PTY after a write that may already have landed.', + file: CODEX_QUEUE, + guards: [CODEX_QUEUE, RELAY_PTY_CODEX_SESSION], + find: ` Err(DeliveryError::committed(format!( + "Codex queue exited with status {}{}",`, + replace: ` Err(DeliveryError::unavailable(format!( + "Codex queue exited with status {}{}",`, + command: libTest('delivery::codex_queue::tests::queue_process_failure_is_committed_and_does_not_fall_back'), + }, + { + transcript: 'mutation-13-codex-handover.txt', + invariants: ['successful_queue_send_is_handed_over_not_acked'], + summary: + '`CodexQueueBackend::send` fabricated an observed acknowledgement from a successful queue command instead of reporting only `HandedOver`.', + file: CODEX_QUEUE, + guards: [CODEX_QUEUE, RELAY_PTY_CODEX_SESSION], + find: ` target.queue_message(request, &queue_command).await?; + Ok(SendStatus::HandedOver(HandoverState::HandedOver))`, + replace: ` target.queue_message(request, &queue_command).await?; + Ok(SendStatus::Acked(ObservedAck::peer_ack("queued")))`, + command: libTest('delivery::codex_queue::tests::successful_queue_send_is_handed_over_not_acked'), + }, + { + transcript: 'mutation-15-codex-duplicate.txt', + invariants: ['a_repeated_send_never_queues_the_same_delivery_twice'], + summary: + 'The seam classified a recorded delivery id as `Fresh`, seen through the REAL codex route: the fake Codex records a second `codex queue` child for one delivery id.', + file: BACKEND, + guards: [BACKEND, CODEX_QUEUE, RELAY_PTY_CODEX_SESSION], + find: ' return Ok(SendOutcome::AlreadySent(receipt));', + replace: ' return Ok(SendOutcome::Fresh(receipt));', + command: libTest('delivery::codex_queue::tests::a_repeated_send_never_queues_the_same_delivery_twice'), + }, + { + transcript: 'mutation-16-codex-cancelled.txt', + invariants: ['a_cancelled_queue_send_is_not_retried_on_the_codex_route'], + summary: + 'The write-ahead in-doubt receipt was not recorded before awaiting the backend, so a cancelled `codex queue` left no memory and the next attempt wrote again.', + file: BACKEND, + guards: [BACKEND, CODEX_QUEUE, RELAY_PTY_CODEX_SESSION], + find: ` self.record_receipt(SendReceipt::new( + request.delivery_id.clone(), + route.clone(), + SendStatus::InDoubt, + ));`, + replace: ' let _ = &route;', + command: libTest('delivery::codex_queue::tests::a_cancelled_queue_send_is_not_retried_on_the_codex_route'), + }, + { + transcript: 'mutation-17-teardown-indoubt.txt', + invariants: ['releasing_an_agent_with_a_handed_over_native_delivery_dead_letters_it_in_doubt'], + summary: + 'Worker teardown stopped asking whether the delivery had reached a transport, so a handed-over native delivery was dead-lettered as freely redeliverable — the pre-repair behaviour.', + file: RUNTIME_DELIVERY, + guards: [RUNTIME_DELIVERY, RUNTIME_TESTS], + find: ' let Some(route) = handed_over_route_label(seam, pending) else {', + replace: ' let Some(route) = None:: else {', + command: libTest('runtime::tests::releasing_an_agent_with_a_handed_over_native_delivery_dead_letters_it_in_doubt'), + }, + { + transcript: 'mutation-18-teardown-sites.txt', + invariants: ['every_worker_teardown_site_disposes_through_the_seam_aware_path'], + summary: + 'One of the four teardown sites (Relaycast-driven agent release) went back to the seam-blind `emit_dropped_delivery_failures`.', + file: RUNTIME_RELAYCAST, + guards: [RUNTIME_RELAYCAST, RUNTIME_API, 'crates/broker/src/runtime/maintenance.rs', RUNTIME_TESTS], + find: ` let _ = dispose_pending_deliveries_for_teardown( + sdk_out_tx, + dead_letters, + delivery_seam, + node_delivery_probe, + &dropped, + "agent_released", + ) + .await;`, + replace: ` let _ = delivery_seam; + let _ = node_delivery_probe; + let _ = emit_dropped_delivery_failures( + sdk_out_tx, + dead_letters, + &dropped, + "agent_released", + ) + .await;`, + command: libTest('runtime::tests::every_worker_teardown_site_disposes_through_the_seam_aware_path'), + }, + { + transcript: 'mutation-19-restart-rehydrate.txt', + invariants: ['a_restarted_broker_does_not_queue_a_handed_over_codex_delivery_again'], + summary: + '`DeliverySeam::restore_handed_over` became a no-op, so a reloaded pending snapshot classified `Fresh` and the fake Codex recorded a SECOND `codex queue` write for a message already in Codex\'s durable queue.', + file: BACKEND, + guards: [BACKEND, RUNTIME_DELIVERY, RUNTIME_TESTS], + find: ` if self.was_sent(&delivery_id) { + return; + } + self.record_receipt(SendReceipt::new( + delivery_id, + route, + SendStatus::HandedOver(HandoverState::HandedOver), + ));`, + replace: ' let _ = (delivery_id, route);', + command: libTest('runtime::tests::a_restarted_broker_does_not_queue_a_handed_over_codex_delivery_again'), + }, + { + transcript: 'mutation-20-codex-settle-route.txt', + invariants: ['settlement_uses_the_recorded_thread_route_and_never_another_codex'], + summary: + 'Settlement resolved through the first offered backend instead of the recorded route, seen through the REAL codex route: it read a different Codex thread\'s rollout.', + file: BACKEND, + guards: [BACKEND, CODEX_QUEUE, RELAY_PTY_CODEX_SESSION], + find: ` let Some(backend) = backends + .iter_mut() + .find(|backend| backend.route_id() == route) + else { + return SettleOutcome::RouteUnavailable(route); + };`, + replace: ` let Some(backend) = backends.iter_mut().next() else { + return SettleOutcome::RouteUnavailable(route); + };`, + command: libTest('delivery::codex_queue::tests::settlement_uses_the_recorded_thread_route_and_never_another_codex'), + }, + { + transcript: 'mutation-21-selection-guard.txt', + invariants: ['only_a_codex_worker_with_a_known_thread_selects_the_codex_queue_route'], + summary: + 'The selection guard dropped its CLI check, so a claude / gemini / opencode worker with a session id became selectable for `codex queue`.', + file: CODEX_QUEUE, + guards: [CODEX_QUEUE], + find: ` let normalized = normalize_cli_name(&command).to_lowercase(); + if normalized != "codex" && normalized != "codex.exe" { + return None; + }`, + replace: ' let _ = normalize_cli_name(&command);', + command: libTest('delivery::codex_queue::tests::only_a_codex_worker_with_a_known_thread_selects_the_codex_queue_route'), + }, + { + transcript: 'mutation-22-unselectable-prewrite.txt', + invariants: ['an_unselectable_codex_backend_refuses_before_any_write'], + summary: + 'A backend with no target reported its refusal as a committed error, which would block the PTY fallback for every non-Codex worker.', + file: CODEX_QUEUE, + guards: [CODEX_QUEUE], + find: ` let target = self.target.as_ref().ok_or_else(|| { + DeliveryError::unavailable( + "Codex queue route requires a Codex worker with a known thread id", + ) + })?;`, + replace: ` let target = self.target.as_ref().ok_or_else(|| { + DeliveryError::committed( + "Codex queue route requires a Codex worker with a known thread id", + ) + })?;`, + command: libTest('delivery::codex_queue::tests::an_unselectable_codex_backend_refuses_before_any_write'), + }, + { + transcript: 'mutation-23-consumed-projection.txt', + invariants: ['a_consumed_user_item_is_observed_in_both_real_projections'], + summary: + 'The positive user-input matcher stopped recognising Codex\'s two real consumed projections, so a genuinely consumed message never acknowledged.', + file: CODEX_THREAD, + guards: [CODEX_THREAD], + find: `fn node_is_user_input(node: &Value) -> bool {`, + replace: `fn node_is_user_input(node: &Value) -> bool { + if true { + return false; + }`, + command: libTest('delivery::codex_thread::tests::a_consumed_user_item_is_observed_in_both_real_projections'), + }, + { + transcript: 'mutation-24-queued-not-consumed.txt', + invariants: ['a_queued_but_unconsumed_message_is_queued_not_consumed'], + summary: + 'A message sitting in Codex\'s own `queued_items` collapsed onto `Unknown`, erasing the difference between "durably delivered, unread" and "nothing is known".', + file: CODEX_THREAD, + guards: [CODEX_THREAD], + find: ' Ok(Some(source)) => CodexMarkerObservation::Queued { source },', + replace: ' Ok(Some(_source)) => CodexMarkerObservation::Unknown,', + command: libTest('delivery::codex_thread::tests::a_queued_but_unconsumed_message_is_queued_not_consumed'), + }, + { + transcript: 'mutation-25-quoted-marker.txt', + invariants: ['a_quoted_marker_in_a_non_user_record_is_not_an_acknowledgement'], + summary: + 'The matcher reverted to negative-only: any record quoting the marker that was not a named artifact acknowledged, including the synthetic shape no Codex emits.', + file: CODEX_THREAD, + guards: [CODEX_THREAD], + find: ` if node_is_user_input(node) { + return true; + }`, + replace: ' return true;', + command: libTest('delivery::codex_thread::tests::a_quoted_marker_in_a_non_user_record_is_not_an_acknowledgement'), + }, + { + transcript: 'mutation-26-native-parks.txt', + invariants: ['a_native_only_delivery_target_never_parks_an_inbound_message'], + summary: + 'A native-only delivery target parked again under manual flush, where the drain (`WorkerRegistry::deliver`) can never reach its route.', + file: RUNTIME_DELIVERY, + guards: [RUNTIME_DELIVERY, RUNTIME_TESTS], + find: ' let should_drain = native_only || state.should_drain_immediately();', + replace: ' let should_drain = state.should_drain_immediately();', + command: libTest('runtime::tests::a_native_only_delivery_target_never_parks_an_inbound_message'), + }, + { + transcript: 'mutation-27-manual-flush-refusal.txt', + invariants: ['manual_flush_is_refused_for_a_native_only_delivery_target'], + summary: + 'The delivery-mode setter stopped refusing manual flush for a native-only target, so the mode a message could be parked under became reachable again.', + file: RUNTIME_API, + guards: [RUNTIME_API, RUNTIME_TESTS], + find: ` if workers.is_native_only_delivery_target(&name) + && mode == InboundDeliveryMode::ManualFlush + {`, + replace: ' if false {', + command: libTest('runtime::tests::manual_flush_is_refused_for_a_native_only_delivery_target'), + }, +]; + +function arg(name, fallback = null) { + const index = process.argv.indexOf(name); + return index === -1 ? fallback : process.argv[index + 1]; +} + +function run(command) { + const [bin, ...args] = command; + const result = spawnSync(bin, args, { encoding: 'utf8' }); + return `${result.stdout ?? ''}${result.stderr ?? ''}`.trim(); +} + +function main() { + const art = arg('--art'); + if (!art) throw new Error('--art is required'); + const only = arg('--only'); + const evidence = path.join(art, 'evidence'); + const selected = only ? MUTATIONS.filter((entry) => entry.transcript.includes(only)) : MUTATIONS; + if (selected.length === 0) throw new Error(`no mutation matches --only ${only}`); + + const originals = new Map(); + const restoreAll = () => { + for (const [file, text] of originals) writeFileSync(file, text); + }; + process.on('SIGINT', () => { + restoreAll(); + process.exit(130); + }); + + const results = []; + try { + for (const entry of selected) { + const original = readFileSync(entry.file, 'utf8'); + originals.set(entry.file, original); + const occurrences = original.split(entry.find).length - 1; + if (occurrences !== 1) { + throw new Error( + `mutation ${entry.transcript}: anchor matched ${occurrences} times in ${entry.file}; ` + + 'the source moved — update the mutation, do not loosen it' + ); + } + writeFileSync(entry.file, original.replace(entry.find, entry.replace)); + let output; + try { + output = run(entry.command); + } finally { + writeFileSync(entry.file, original); + originals.delete(entry.file); + } + const bites = /FAILED|panicked|assertion .*failed/.test(output); + writeFileSync( + path.join(evidence, entry.transcript), + `$ ${entry.command.join(' ')}\n\n${output}\n` + ); + results.push({ ...entry, bites }); + process.stdout.write(`${bites ? 'BITES ' : 'NO-BITE '} ${entry.transcript}\n`); + } + } finally { + restoreAll(); + } + + const inert = results.filter((entry) => !entry.bites); + if (inert.length > 0) { + throw new Error( + `mutations that did not break their invariant: ${inert.map((entry) => entry.transcript).join(', ')}` + ); + } + + if (!only) { + const green = run([ + 'cargo', + 'test', + '-p', + 'agent-relay-broker', + '--features', + 'seam-probe', + '--test', + 'delivery_seam_invariants', + ]); + const greenLib = run(['cargo', 'test', '-p', 'agent-relay-broker', '--lib']); + writeFileSync( + path.join(evidence, 'mutation-restored-green.txt'), + `$ cargo test -p agent-relay-broker --features seam-probe --test delivery_seam_invariants\n\n${green}\n\n` + + `$ cargo test -p agent-relay-broker --lib\n\n${greenLib}\n` + ); + + const sources = {}; + for (const entry of MUTATIONS) { + for (const file of entry.guards) { + if (!existsSync(file)) throw new Error(`guarded source missing: ${file}`); + sources[file] = createHash('sha256').update(readFileSync(file)).digest('hex'); + } + } + writeFileSync( + path.join(evidence, 'mutation-proof.json'), + `${JSON.stringify( + { + schemaVersion: 1, + kind: 'native-delivery-mutation-proof', + recorder: 'scripts/migrate/mutation-proof.mjs', + invariants: MUTATIONS.flatMap((entry) => + entry.invariants.map((name) => ({ + name, + transcript: entry.transcript, + guards: entry.guards, + })) + ), + sources, + }, + null, + 2 + )}\n` + ); + writeFileSync(path.join(evidence, 'mutation-proof.md'), renderMarkdown(results)); + } +} + +function renderMarkdown(results) { + const sections = []; + for (const entry of results) { + const transcript = readFileSync( + path.join(arg('--art'), 'evidence', entry.transcript), + 'utf8' + ).trim(); + for (const invariant of entry.invariants) { + sections.push( + `### ${invariant}\n\n**Mutation:** ${entry.summary}\n\n` + + `**Mutated:** \`${entry.file}\`\n\n**Transcript:** \`${entry.transcript}\`\n\n` + + '```text\n' + + transcript + + '\n```\n' + ); + } + } + return `# Phase 1 Rust Mutation Proof + +Generated by \`scripts/migrate/mutation-proof.mjs\`, which applies each mutation +to product source, runs the invariant test it must break, restores the file +from the original bytes, and records the transcript verbatim. Nothing in this +file is hand-written, so it cannot describe a mutation that was never run. + +Every transcript below was recorded against the tree this file sits beside. +\`evidence/mutation-proof.json\` carries the sha256 of every guarded source at +recording time and \`native-delivery-gates.mjs seam-rules\` recomputes them, so +a later edit to guarded code invalidates the proof instead of silently +outliving it — the failure signoff finding F2 caught. + +One section per invariant, each carrying its own failing transcript. + +## Failing transcripts + +${sections.join('\n')} +## Restored + +Every mutation above was reverted by the recorder before the next one ran, and +the suites are green on the restored tree: \`mutation-restored-green.txt\`. +`; +} + +main(); diff --git a/scripts/migrate/native-delivery-gates.mjs b/scripts/migrate/native-delivery-gates.mjs index 230b623efd..1848d4fab6 100755 --- a/scripts/migrate/native-delivery-gates.mjs +++ b/scripts/migrate/native-delivery-gates.mjs @@ -47,7 +47,7 @@ const PLANNER = 'scripts/verify-features/targeted-pr-plan.mjs'; * PTY behaviour today; the whole point is that the same assertions pass with * the backend swapped, so every phase reruns all five. */ -const PARITY = { +export const PARITY = { 'parity-orch-to-worker': 'npx tsx tests/parity/orch-to-worker.ts', 'parity-multi-worker': 'npx tsx tests/parity/multi-worker.ts', 'parity-broadcast': 'npx tsx tests/parity/broadcast.ts', @@ -55,6 +55,198 @@ const PARITY = { 'parity-stability-soak': 'npx tsx tests/parity/stability-soak.ts', }; +/** + * The phase's unlaunched-session scenario, as a command. + * + * It lives here rather than in the flow for the same reason `PARITY` does: + * `contractCommands()` below compares recorded evidence against the command + * the contract names, and a command the contract does not know cannot be + * compared. The flow imports this instead of restating it. + */ +export function unlaunchedCommand(cli) { + return [ + 'npm run build:core', + 'cargo build -p agent-relay-broker --bin agent-relay-broker', + `npx vitest run --config tests/e2e/vitest.unlaunched.config.ts tests/e2e/unlaunched/unlaunched-${cli}-delivery.test.ts`, + ].join(' && '); +} + +/** + * Every evidence name whose command this contract owns, mapped to that exact + * command. + * + * Earned by codex-review-1 F2. `eval-codex` was tightened from `--harness=codex` + * to `--harness=codex --min-scenarios=1 --min-delivery-rate=1`, and the run + * carried forward an `evidence/eval-codex.json` recorded under the OLD, weaker + * command — a file whose own tail reads `sent=25% scenarios=1/8`, marked green. + * `require-green` and `accept` validated identity, run id and verdict and never + * asked what command produced the verdict, so a green recorded under a command + * the contract no longer names satisfied the contract. Tightening a gate then + * has no effect until someone notices the evidence is stale, which is the + * failure mode this campaign exists to prevent. + * + * Names the contract does NOT own (`rust-*`, `ts-typecheck`, `unit-tests`, the + * recorded structural gates) are absent from this map and are not + * command-checked; their commands live in the flow. + */ +export function contractCommands(config) { + const commands = {}; + for (const name of config.parity ?? []) { + if (PARITY[name]) commands[name] = PARITY[name]; + } + for (const [name, command] of Object.entries(config.evals ?? {})) commands[name] = command; + for (const [name, command] of Object.entries(config.e2e ?? {})) commands[name] = command; + if (Array.isArray(config.unlaunched)) { + for (const cli of config.unlaunched) { + commands[`unlaunched-${cli}-delivery`] = unlaunchedCommand(cli); + } + } + return commands; +} + +/** + * `null` when the evidence was produced by the command the contract names, or + * when the contract does not name one. A string describing the drift otherwise. + */ +function commandDrift(config, name, evidence) { + const expected = contractCommands(config)[name]; + if (expected === undefined) return null; + if (evidence.command === expected) return null; + return ( + `${name}: evidence was recorded under a command the contract no longer names\n` + + ` recorded: ${evidence.command ?? '(none)'}\n` + + ` contract: ${expected}` + ); +} + +/** + * The recorded structural gates: gates whose verdict is written to + * `evidence/.json` by `record` and read back later by `require-green` + * and `accept`, rather than decided in-line. + */ +const STRUCTURAL_GATE_ACTIONS = [ + 'edit-gate', + 'manifest-gate', + 'targeted-gate', + 'seam-rules', + 'unlaunched-gate', +]; + +/** + * Exactly which parts of the phase contract a structural gate reads. + * + * Earned by codex-review-2 F1. `commandDrift` closed the hole where evidence + * was recorded under a command the contract no longer names — but the + * structural gates take no contract text on their command line at all. Their + * command is `... seam-rules --phase 1 --artifact ... --run-id ...` whatever + * the contract says, so the command string is IDENTICAL before and after the + * contract tightens. When phase 1 grew the three Codex queue invariants + * (12 → 15), `evidence/seam-rules-final.json` still carried the green from the + * 12-invariant run, its own tail reading `invariants=12`, and `accept()` + * validated existence, run id and the green bit and never asked what contract + * the green covered. A tightened gate then has no effect until someone reads + * the tail by eye. + * + * So each structural gate stamps a digest of the contract inputs it actually + * consumed into its pass line, `record` parses that into `gateFacts.coverage`, + * and `require-green`/`accept` recompute the digest from the CURRENT contract + * and refuse a mismatch. Adding an invariant, a required source, a feature row + * or a wiring rule changes the digest, which makes every gate verdict recorded + * before that change fail closed instead of silently carrying forward. + * + * `null` for an action that reads no contract text, which is not checked. + */ +export function gateCoverage(config, action) { + switch (action) { + case 'edit-gate': + return { + scope: [...(config.scope ?? [])].sort(), + tsScope: [...(config.tsScope ?? [])].sort(), + requiredSources: [...(config.requiredSources ?? [])].sort(), + requiredArtifacts: [...(config.requiredArtifacts ?? [])].sort(), + allowed: [...allowedPaths(config)].sort(), + }; + // Both gates are driven by the same declared feature rows: manifest-gate + // checks they are registered with the required category/tier/location, and + // targeted-gate checks the selector picked them up. + case 'manifest-gate': + case 'targeted-gate': + return { + features: (config.features ?? []) + .map((feature) => [feature.id, feature.category, feature.verify_tier, feature.location].join('|')) + .sort(), + }; + case 'seam-rules': + return { + invariants: [...(config.invariants ?? [])].sort(), + invariantTestFiles: [...invariantTestFiles(config)].sort(), + wiring: (config.wiring ?? []) + .map((rule) => `${rule.symbol}|${rule.from ?? ''}|${rule.outside ?? ''}`) + .sort(), + forbidden: (config.forbidden ?? []) + .map((rule) => `${rule.where}|${rule.pattern}|${rule.unless}`) + .sort(), + untouched: [...(config.untouched ?? [])].sort(), + }; + case 'unlaunched-gate': + return { + unlaunched: Array.isArray(config.unlaunched) ? [...config.unlaunched].sort() : false, + }; + default: + return null; + } +} + +/** The stamp a structural gate writes into its pass line. `null` when unchecked. */ +export function coverageDigest(config, action) { + const coverage = gateCoverage(config, action); + if (coverage === null) return null; + return createHash('sha256').update(JSON.stringify(coverage)).digest('hex').slice(0, 16); +} + +/** + * `edit-gate-final` → `edit-gate`. `null` for a name this contract does not + * recognise as a structural gate, which is left unchecked rather than guessed + * at (`post-codex-review-1-edit` is a repair probe, not an acceptance gate). + */ +function structuralAction(name) { + const base = name.endsWith('-final') ? name.slice(0, -'-final'.length) : name; + return STRUCTURAL_GATE_ACTIONS.includes(base) ? base : null; +} + +/** + * `null` when the recorded verdict covers the contract as it stands now. + * A string describing the drift otherwise. + * + * Absent `gateFacts.coverage` is drift, not an exemption: evidence with no + * stamp was produced by a gate that did not know what contract it was proving, + * and that is precisely the stale file this check exists to reject. + */ +function coverageDrift(config, name, evidence) { + const action = structuralAction(name); + if (action === null) return null; + const expected = coverageDigest(config, action); + if (expected === null) return null; + const recorded = evidence.gateFacts?.coverage; + if (recorded === undefined) { + return ( + `${name}: evidence carries no contract-coverage stamp, so it predates the current gate — ` + + `re-record it (expected coverage ${expected})` + ); + } + if (recorded !== expected) { + const summary = Object.entries(gateCoverage(config, action)) + .map(([key, value]) => `${key}=${Array.isArray(value) ? value.length : value}`) + .join(' '); + return ( + `${name}: the recorded verdict covers a different contract than this phase declares\n` + + ` recorded coverage: ${recorded}\n` + + ` contract coverage: ${expected} (${summary})` + ); + } + return null; +} + /** * The seam contract tests that must exist, pass, and carry mutation evidence. * This includes the original four scripted-backend tests, every invariant @@ -71,13 +263,74 @@ const SEAM_INVARIANTS = [ 'settle_distinguishes_absence_from_an_unreachable_route', 'settle_reports_an_evicted_receipt_as_unknown_not_absent', 'an_acknowledgement_must_name_the_observation_behind_it', + 'unavailable_queue_capability_falls_back_before_write', + 'queue_process_failure_is_committed_and_does_not_fall_back', + 'successful_queue_send_is_handed_over_not_acked', 'real_pty_route_unknown_worker_is_pre_write_and_may_fall_back', 'real_pty_route_write_failure_after_commit_does_not_fall_back', 'real_pty_route_never_reports_an_observed_ack', ]; const INVARIANT_TEST_FILE = 'crates/broker/tests/delivery_seam_invariants.rs'; -const INVARIANT_TEST_FILES = [INVARIANT_TEST_FILE, 'crates/broker/src/delivery/pty.rs']; +const INVARIANT_TEST_FILES = [ + INVARIANT_TEST_FILE, + 'crates/broker/src/delivery/pty.rs', + 'crates/broker/src/delivery/codex_queue.rs', +]; + +/** + * Phase 1's own invariants, on top of the seam's. + * + * Phase 0's lesson 2 is that a scripted backend proves nothing about a real + * one, so each phase has to drive the four rules through ITS transport. The + * seam list covers rules 1 and 4 for the codex route; these cover rule 2's + * duplicate, cancellation, teardown and restart shapes, rule 3's + * settle-by-the-recorded-route, the selection guard the whole parity answer + * rests on, and the queued-versus-consumed acknowledgement boundary. + * + * Deliberately NOT appended to `SEAM_INVARIANTS`: phases 2-5 share that list + * and have no codex route to prove these against. + */ +const PHASE_1_NATIVE_ROUTE_INVARIANTS = [ + // Rule 2, on the real transport. + 'a_repeated_send_never_queues_the_same_delivery_twice', + 'a_cancelled_queue_send_is_not_retried_on_the_codex_route', + // Rule 2, across the two dispositions that outlive a send: worker teardown + // and broker restart. A native route survives both, so both can duplicate. + 'releasing_an_agent_with_a_handed_over_native_delivery_dead_letters_it_in_doubt', + 'every_worker_teardown_site_disposes_through_the_seam_aware_path', + 'a_restarted_broker_does_not_queue_a_handed_over_codex_delivery_again', + // Rule 3, on the real transport. + 'settlement_uses_the_recorded_thread_route_and_never_another_codex', + // The selection guard: only codex, only with a known thread. + 'only_a_codex_worker_with_a_known_thread_selects_the_codex_queue_route', + 'an_unselectable_codex_backend_refuses_before_any_write', + // Rule 4: what a codex acknowledgement is allowed to mean (decision D4). + 'a_consumed_user_item_is_observed_in_both_real_projections', + 'a_queued_but_unconsumed_message_is_queued_not_consumed', + 'a_quoted_marker_in_a_non_user_record_is_not_an_acknowledgement', + // Manual flush must not accumulate messages the flush path cannot deliver. + 'a_native_only_delivery_target_never_parks_an_inbound_message', + 'manual_flush_is_refused_for_a_native_only_delivery_target', +]; + +const PHASE_1_INVARIANT_TEST_FILES = [ + ...INVARIANT_TEST_FILES, + 'crates/broker/src/codex_thread.rs', + 'crates/broker/src/runtime/tests.rs', +]; + +/** + * Where a phase's invariant tests live. `invariantTestFile` (singular) is the + * legacy override for a phase with exactly one file; `invariantTestFiles` + * declares the set. Both feed the coverage digest, so widening the set is + * visible in every structural gate's pass line rather than silent. + */ +function invariantTestFiles(config) { + if (config.invariantTestFiles) return [...config.invariantTestFiles]; + if (config.invariantTestFile) return [config.invariantTestFile]; + return [...INVARIANT_TEST_FILES]; +} /** Phases, ordered as the doc orders them. Phase 6 is independent. */ export const PHASES = { @@ -154,35 +407,66 @@ export const PHASES = { 1: { slug: 'codex-queue', title: 'Codex native delivery over `codex queue`', + /** + * `wrap.rs` is the fifth widening, and it is the same shape as phase 0's: + * a lane that excludes an edit the phase's own repair requires makes the + * phase unsatisfiable. The PTY verification structs live in the in-scope + * `delivery_verification.rs`, but `wrap.rs` constructs them for the legacy + * wrap route. Leaving it out is not merely a stray: + * `commit-if-green` stages `git add -- `, so the commit would carry + * the new field without the caller that populates it and the committed + * tree would not compile. + */ scope: [ 'crates/broker/src/delivery/', 'crates/broker/src/codex_thread.rs', + 'crates/broker/src/broker/delivery_verification.rs', 'crates/broker/tests/', + 'crates/broker/src/worker.rs', + 'crates/broker/src/listen_api.rs', 'crates/broker/src/runtime/', 'crates/broker/src/pty_worker.rs', + 'crates/broker/src/wrap.rs', + 'crates/broker/Cargo.toml', + // The sixth widening, same shape as the five above. Worker teardown now + // dead-letters a delivery that already reached a native transport as IN + // DOUBT, and the withheld fleet ack that dies with it has to be recorded + // somewhere an operator can see rather than dropped with a log line. The + // probe owns every other `DeliverDisposition`, so the new + // `DroppedInDoubt` belongs beside them; leaving the file out would make + // the repair the signoff demanded unreachable. + 'crates/broker/src/node_delivery_probe.rs', + 'crates/relay-pty/src/codex_session.rs', ], - tsScope: ['tests/', '.agentworkforce/features/manifest.yaml', MATRIX], + tsScope: ['tests/', 'packages/cli/', '.agentworkforce/features/manifest.yaml', MATRIX], requiredSources: [ 'crates/broker/src/delivery/codex_queue.rs', 'crates/broker/src/codex_thread.rs', INVARIANT_TEST_FILE, ], - requiredArtifacts: ['decisions/D1-codex-thread-id.md'], + requiredArtifacts: [ + 'decisions/D1-codex-thread-id.md', + 'decisions/D2-codex-eval-floor.md', + 'decisions/D3-codex-steer-route.md', + 'decisions/D4-codex-read-receipt-standard.md', + ], features: [ { id: 'codex-queue-delivery', category: 'broker', - location: 'crates/broker/src/delivery/codex_queue.rs, crates/broker/src/codex_thread.rs', + location: + 'crates/broker/src/delivery/codex_queue.rs, crates/broker/src/codex_thread.rs, crates/broker/src/listen_api.rs, packages/cli/src/cli/agent-relay-mcp.ts', verify_tier: 4, }, ], wiring: [{ symbol: 'CodexQueueBackend', outside: 'crates/broker/src/delivery/codex_queue.rs' }], - invariants: SEAM_INVARIANTS, + invariants: [...SEAM_INVARIANTS, ...PHASE_1_NATIVE_ROUTE_INVARIANTS], + invariantTestFiles: PHASE_1_INVARIANT_TEST_FILES, parity: Object.keys(PARITY), rust: true, evals: { 'eval-codex': - 'npm run eval:build && cd tests/integration/broker && RELAY_INTEGRATION_REAL_CLI=1 node dist/evals/runner.js --harness=codex', + 'npm run eval:build && cd tests/integration/broker && RELAY_INTEGRATION_REAL_CLI=1 node dist/evals/runner.js --harness=codex --min-scenarios=1 --min-delivery-rate=1', }, e2e: {}, unlaunched: ['codex'], @@ -228,7 +512,7 @@ export const PHASES = { rust: true, evals: { 'eval-claude': - 'npm run eval:build && cd tests/integration/broker && RELAY_INTEGRATION_REAL_CLI=1 node dist/evals/runner.js --harness=claude', + 'npm run eval:build && cd tests/integration/broker && RELAY_INTEGRATION_REAL_CLI=1 node dist/evals/runner.js --harness=claude --min-scenarios=1 --min-delivery-rate=1', }, e2e: {}, unlaunched: ['claude'], @@ -259,7 +543,7 @@ export const PHASES = { rust: true, evals: { 'eval-acp-harnesses': - 'npm run eval:build && cd tests/integration/broker && RELAY_INTEGRATION_REAL_CLI=1 node dist/evals/runner.js --harness=grok,opencode,devin', + 'npm run eval:build && cd tests/integration/broker && RELAY_INTEGRATION_REAL_CLI=1 node dist/evals/runner.js --harness=grok,opencode,devin --min-scenarios=1 --min-delivery-rate=1', }, e2e: {}, unlaunched: false, @@ -598,6 +882,46 @@ function withinScope(file, scope) { return scope.some((entry) => (entry.endsWith('/') ? file.startsWith(entry) : file === entry)); } +/** + * Every path this phase is permitted to change: the declared product lane plus + * the campaign's own output. + * + * `edit-gate` rejects anything outside this set, and `commit-if-green` stages + * exactly what this set admits. Those two used to disagree — the gate allowed + * `package.json`, `CHANGELOG.md`, `.gitignore` and the `scripts/migrate/` + + * `flows/migrate/` harness, while the commit staged only `scope` + `tsScope`. + * So a phase could pass every gate and commit a tree missing the very files + * the gate had approved: the `test:e2e:unlaunched` script repointed at the new + * vitest config, the changelog entry CLAUDE.md requires, and the gate edits + * `accept` prints `HARNESS_MODIFIED` about precisely so a reviewer can read + * them in the diff. + */ +function allowedPaths(config) { + return [ + ...config.scope, + ...(config.tsScope ?? []), + '.workflow-artifacts/', + 'scripts/migrate/', + 'flows/migrate/', + 'flows/audit/', + // The harness adapter and the e2e wiring the phase needed: campaign + // tooling, not product, and so not manifest-routable either. + 'scripts/flows/', + 'package.json', + 'vitest.e2e.config.ts', + '.gitignore', + // Trail writes these as agents work; CLAUDE.md requires them tracked, so + // they are legitimate output of a run rather than scope creep. The bare + // `.trajectories/` path is the same tool's older location, which `trail + // compact` migrates out of mid-run. + '.agentworkforce/trajectories/', + '.trajectories/', + '.review-out/', + 'CHANGELOG.md', + 'docs/', + ]; +} + // ───────────────────────────── actions ───────────────────────────── /** @@ -696,6 +1020,10 @@ function contract() { phase: Number(phase), ...config, parityCommands: Object.fromEntries((config.parity ?? []).map((name) => [name, PARITY[name]])), + // The exact command each contract-owned evidence file must be recorded + // under. `require-green` and `accept` compare against this, so a recorder + // invoked by hand has the string to copy rather than to approximate. + contractCommands: contractCommands(config), seamRules: [ 'Fall back to another transport only on a strictly pre-write error.', 'Never re-send on doubt.', @@ -796,6 +1124,24 @@ async function record() { */ runId: option('--run-id', 'unknown'), command, + /** + * A stable digest of the command, so the seal and a reviewer can compare + * two evidence files without diffing shell strings by eye. The comparison + * gates use `command` itself; this is the audit handle. + */ + commandSha256: createHash('sha256').update(command).digest('hex'), + /** + * The `key=value` facts the wrapped gate stamped into its own pass line, + * chiefly `coverage` — the digest of the contract text that gate read. + * + * Structural gates take no contract text on their command line, so + * `commandSha256` is identical before and after the contract tightens and + * cannot tell a stale verdict from a current one. This can: `accept` and + * `require-green` recompute the digest from the contract and refuse a + * mismatch. `null` when the wrapped command is not a gate, or when it + * failed and therefore printed no pass line. + */ + gateFacts: parseGateFacts(output), exitCode, verdict, missingExpected: missing, @@ -813,8 +1159,28 @@ async function record() { process.stdout.write(`${output.slice(-4_000)}\n`); } +/** + * The `key=value` pairs on the LAST `GATE_PASSED` line of a recorded run. + * + * Last, not first: a recorder may wrap a `&&` chain of several gates, and it is + * the final verdict that the evidence file's exit code belongs to. Tokens that + * are not `key=value` (the gate's own action name, the parenthetical prose + * `targeted-gate` appends) are dropped rather than guessed at. + */ +function parseGateFacts(output) { + const lines = output.match(/^GATE_PASSED .*$/gm); + if (!lines || lines.length === 0) return null; + const facts = {}; + for (const token of lines[lines.length - 1].split(/\s+/).slice(1)) { + const match = /^([a-z][\w-]*)=(.*)$/.exec(token); + if (match) facts[match[1]] = match[2]; + } + return Object.keys(facts).length > 0 ? facts : null; +} + /** Read recorded evidence back. This is the only thing that says "green". */ function requireGreen() { + const { config } = phaseConfig(); const art = artifactRoot(); const names = list('--names'); if (names.length === 0) throw new Error('--names is required'); @@ -841,6 +1207,20 @@ function requireGreen() { problems.push(`${name}: stale evidence from run "${evidence.runId}" (current "${runId}")`); continue; } + const drift = commandDrift(config, name, evidence); + if (drift) { + problems.push(drift); + continue; + } + // Structural gates carry no contract text on their command line, so + // `commandDrift` cannot see a contract that tightened under a stable + // command. The coverage stamp can. Checked before the verdict, because the + // verdict is exactly what the older, narrower contract produced. + const coverage = coverageDrift(config, name, evidence); + if (coverage) { + problems.push(coverage); + continue; + } if (evidence.verdict !== 'green') { const detail = [ `exit=${evidence.exitCode}`, @@ -912,27 +1292,7 @@ function editGate() { } // Anything outside the declared lane is scope creep, and scope creep in a // delivery migration is how double-delivery ships. - const allowed = [ - ...config.scope, - ...(config.tsScope ?? []), - '.workflow-artifacts/', - 'scripts/migrate/', - 'flows/migrate/', - 'flows/audit/', - // The harness adapter and the e2e wiring the phase needed: campaign - // tooling, not product, and so not manifest-routable either. - 'scripts/flows/', - 'package.json', - 'vitest.e2e.config.ts', - '.gitignore', - // Trail writes these as agents work; CLAUDE.md requires them tracked, so - // they are legitimate output of a run rather than scope creep. - '.agentworkforce/trajectories/', - '.review-out/', - 'CHANGELOG.md', - 'docs/', - ]; - const strays = files.filter((file) => !withinScope(file, allowed)); + const strays = files.filter((file) => !withinScope(file, allowedPaths(config))); if (strays.length > 0) problems.push(`out-of-scope changes: ${strays.slice(0, 20).join(', ')}`); } @@ -941,7 +1301,10 @@ function editGate() { fail(`edit-gate phase=${phase} scope=${which}\n ${problems.join('\n ')}`); return; } - pass(`edit-gate phase=${phase} scope=${which} files=${inScope.length}`); + pass( + `edit-gate phase=${phase} scope=${which} files=${inScope.length} ` + + `coverage=${coverageDigest(config, 'edit-gate')}` + ); } /** @@ -1013,7 +1376,10 @@ function manifestGate() { fail(`manifest-gate phase=${phase}\n ${problems.join('\n ')}`); return; } - pass(`manifest-gate phase=${phase} features=${(config.features ?? []).length}`); + pass( + `manifest-gate phase=${phase} features=${(config.features ?? []).length} ` + + `coverage=${coverageDigest(config, 'manifest-gate')}` + ); } /** @@ -1066,6 +1432,10 @@ function targetedGate() { 'package.json', 'vitest.e2e.config.ts', '.gitignore', + // Trail records, for the same reason `edit-gate` allows them: run output, + // not a product runtime path, and so with no manifest row to earn. + '.agentworkforce/trajectories/', + '.trajectories/', ]; const unmatched = (plan.unmatchedRuntimeFiles ?? []).filter( (file) => !HARNESS.some((prefix) => file.startsWith(prefix)) @@ -1094,7 +1464,7 @@ function targetedGate() { } pass( `targeted-gate phase=${phase} mode=${plan.mode} features=${[...selected].join(',')} ` + - `unmapped=0 scenarios=${plan.scenarios.length}` + + `unmapped=0 scenarios=${plan.scenarios.length} coverage=${coverageDigest(config, 'targeted-gate')}` + (plan.mode === 'full-smoke' ? ' (full-smoke from the manifest self-check, which is expected)' : '') ); } @@ -1103,7 +1473,7 @@ function seamRules() { const { phase, config } = phaseConfig(); const art = artifactRoot(); const problems = []; - const testFiles = config.invariantTestFile ? [config.invariantTestFile] : INVARIANT_TEST_FILES; + const testFiles = invariantTestFiles(config); const existingTestFiles = testFiles.filter(existsSync); for (const testFile of testFiles) { if (!existsSync(testFile)) problems.push(`invariant test file missing: ${testFile}`); @@ -1221,12 +1591,111 @@ function seamRules() { } } } + problems.push(...mutationFreshnessProblems(config, art)); if (problems.length > 0) { fail(`seam-rules phase=${phase}\n ${problems.join('\n ')}`); return; } - pass(`seam-rules phase=${phase} invariants=${(config.invariants ?? []).length}`); + pass( + `seam-rules phase=${phase} invariants=${(config.invariants ?? []).length} ` + + `coverage=${coverageDigest(config, 'seam-rules')}` + ); +} + +/** + * A mutation transcript is only evidence about the tree it was recorded + * against. + * + * The prose check above asks whether each invariant has a transcript that + * FAILED. It cannot ask WHEN, so a transcript recorded before a later repair + * round rewrote the very code the invariant guards still satisfies it. That + * happened in phase 1: the codex-route transcripts panicked at + * `codex_queue.rs:465/503/546` while the shipped expectations sat ~76 lines + * lower, because the fix round after them changed queue capability resolution, + * the probe's argv and the PTY fallback predicate. Every deterministic gate + * passed anyway, which is exactly the failure mode `coverageDrift` already + * guards for the structural gates. + * + * So the proof has to carry a digest of what it proved. `mutation-proof.json` + * declares, per invariant, the source files that invariant guards and the + * sha256 each of those files had when the transcript was recorded. This gate + * recomputes them. A guarded file that has changed since invalidates its + * transcripts and the mutation must be re-run — a content digest, not a + * timestamp, so it survives a checkout and cannot be satisfied by `touch`. + */ +function mutationFreshnessProblems(config, art) { + const problems = []; + const manifestPath = path.join(art, 'evidence', 'mutation-proof.json'); + if (!existsSync(manifestPath)) { + return [ + 'evidence/mutation-proof.json missing: the mutation proof must declare, per invariant, ' + + 'which sources it guards and their sha256 at recording time', + ]; + } + let manifest; + try { + manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); + } catch (error) { + return [`evidence/mutation-proof.json is not valid JSON: ${error.message}`]; + } + const entries = new Map( + (manifest.invariants ?? []).map((entry) => [entry.name, entry]) + ); + const digests = manifest.sources ?? {}; + const actual = new Map(); + const digestOf = (file) => { + if (!actual.has(file)) { + actual.set( + file, + existsSync(file) + ? createHash('sha256').update(readFileSync(file)).digest('hex') + : null + ); + } + return actual.get(file); + }; + + for (const invariant of config.invariants ?? []) { + const entry = entries.get(invariant); + if (!entry) { + problems.push(`mutation-proof.json does not cover invariant: ${invariant}`); + continue; + } + const guards = entry.guards ?? []; + if (guards.length === 0) { + problems.push(`mutation-proof.json declares no guarded source for invariant: ${invariant}`); + } + const transcript = path.join(art, 'evidence', entry.transcript ?? ''); + if (!entry.transcript || !existsSync(transcript)) { + problems.push(`mutation transcript missing for invariant ${invariant}: ${entry.transcript}`); + } else if ( + !/FAILED|panicked|assertion .*failed/.test(readFileSync(transcript, 'utf8')) + ) { + problems.push( + `mutation transcript for ${invariant} records no failure: ${entry.transcript}` + ); + } + for (const file of guards) { + const recorded = digests[file]; + if (!recorded) { + problems.push( + `mutation-proof.json records no digest for ${file}, guarded by ${invariant}` + ); + continue; + } + const current = digestOf(file); + if (current === null) { + problems.push(`guarded source missing: ${file} (declared by ${invariant})`); + } else if (current !== recorded) { + problems.push( + `mutation evidence is stale: ${file} changed since the transcript for ${invariant} ` + + `was recorded (recorded ${recorded.slice(0, 12)}, now ${current.slice(0, 12)}) — re-run the mutation` + ); + } + } + } + return problems; } function walk(dir) { @@ -1303,7 +1772,10 @@ function unlaunchedGate() { } const checked = typecheck ? ` typechecked=${typecheck.files}` : ''; if (!config.unlaunched) { - pass(`unlaunched-gate phase=${phase} not-required${checked}`); + pass( + `unlaunched-gate phase=${phase} not-required${checked} ` + + `coverage=${coverageDigest(config, 'unlaunched-gate')}` + ); return; } const matrix = readJson(MATRIX); @@ -1327,14 +1799,20 @@ function unlaunchedGate() { problems.push(`${id} evidence is ${scenario.evidence}, must be integration`); if (!Array.isArray(scenario.command) || scenario.command.length === 0) problems.push(`${id} has no command`); - if (!(scenario.forbidOutput ?? []).includes('# SKIP')) - problems.push(`${id} must forbid "# SKIP" so a skipped test cannot read as a pass`); + const forbidden = new Set(scenario.forbidOutput ?? []); + for (const marker of ['# SKIP', 'skipped', 'SKIP', 'no tests']) { + if (!forbidden.has(marker)) + problems.push(`${id} must forbid "${marker}" so a skipped or empty test run cannot read as a pass`); + } } if (problems.length > 0) { fail(`unlaunched-gate phase=${phase}\n ${problems.join('\n ')}`); return; } - pass(`unlaunched-gate phase=${phase} clis=${config.unlaunched.join(',')}${checked}`); + pass( + `unlaunched-gate phase=${phase} clis=${config.unlaunched.join(',')}${checked} ` + + `coverage=${coverageDigest(config, 'unlaunched-gate')}` + ); } /** Hash every artifact so a reviewer reviews a fixed set, not a moving one. */ @@ -1447,6 +1925,7 @@ function accept() { ...(config.parity ?? []), ...Object.keys(config.evals ?? {}), ...Object.keys(config.e2e ?? {}), + ...(Array.isArray(config.unlaunched) ? config.unlaunched.map((cli) => `unlaunched-${cli}-delivery`) : []), ].filter((name) => (config.rust ? true : !name.startsWith('rust-'))); const runId = option('--run-id', 'unknown'); @@ -1459,6 +1938,19 @@ function accept() { const evidence = readJson(file); if (evidence.runId !== undefined && evidence.runId !== runId) problems.push(`${name}: stale evidence from run "${evidence.runId}" (current "${runId}")`); + // A green recorded under a command the contract no longer names is not + // evidence for this contract. Checked before the verdict, because the + // verdict is exactly the thing the weaker command produced. + const drift = commandDrift(config, name, evidence); + if (drift) { + problems.push(drift); + continue; + } + const coverage = coverageDrift(config, name, evidence); + if (coverage) { + problems.push(coverage); + continue; + } if (name === 'unit-tests') { // The workflow deliberately accepts the repo's declared, unreachable // baseline through `regression-gate`; acceptance must apply the same @@ -1473,15 +1965,35 @@ function accept() { } } - for (const action of ['edit-gate', 'manifest-gate', 'targeted-gate', 'seam-rules', 'unlaunched-gate']) { - const file = path.join(art, 'evidence', `${action}-final.json`); - if (!existsSync(file)) problems.push(`final gate never ran: ${action}`); - else { - const evidence = readJson(file); - if (evidence.runId !== undefined && evidence.runId !== runId) - problems.push(`${action}: stale final evidence from run "${evidence.runId}" (current "${runId}")`); - if (evidence.verdict !== 'green') problems.push(`final gate red: ${action}`); + /** + * The final structural gates, validated against the contract AS IT STANDS + * NOW rather than on the green bit alone. + * + * Earned by codex-review-2 F1. Acceptance used to check existence, run id + * and `verdict === 'green'`, none of which move when the contract tightens + * under an unchanged command line — so `seam-rules-final.json` carried a + * green whose own tail read `invariants=12` while the contract declared 15, + * and acceptance took it. `coverageDrift` recomputes the digest of the + * contract text each gate reads and rejects a verdict recorded against a + * different one; the missing `commandSha256` is the same staleness by an + * older shape, since `record` has stamped one on every file it writes since + * codex-review-1 F2. + */ + for (const action of STRUCTURAL_GATE_ACTIONS) { + const name = `${action}-final`; + const file = path.join(art, 'evidence', `${name}.json`); + if (!existsSync(file)) { + problems.push(`final gate never ran: ${action}`); + continue; } + const evidence = readJson(file); + if (evidence.runId !== undefined && evidence.runId !== runId) + problems.push(`${action}: stale final evidence from run "${evidence.runId}" (current "${runId}")`); + if (typeof evidence.commandSha256 !== 'string') + problems.push(`${action}: final evidence predates the command digest — re-record it`); + const coverage = coverageDrift(config, name, evidence); + if (coverage) problems.push(coverage); + if (evidence.verdict !== 'green') problems.push(`final gate red: ${action}`); } for (const provider of ['claude', 'codex']) { @@ -1549,8 +2061,21 @@ function commitIfGreen() { fail('refusing to commit on main'); return; } - const scope = [...config.scope, ...(config.tsScope ?? [])]; - execFileSync('git', ['add', '--', ...scope], { stdio: 'inherit' }); + /** + * Stage what `edit-gate` approved, not a narrower list. The declared lane + * goes in as path prefixes (they exist whether or not anything under them + * changed); everything else is named file by file from the change set, so no + * pathspec can fail to match. `.workflow-artifacts/` is evidence, not source, + * and stays out of the commit. + */ + const lane = [...config.scope, ...(config.tsScope ?? [])]; + const alsoAllowed = changedFiles().filter( + (file) => + !withinScope(file, lane) && + !file.startsWith('.workflow-artifacts/') && + withinScope(file, allowedPaths(config)) + ); + execFileSync('git', ['add', '--', ...lane, ...alsoAllowed], { stdio: 'inherit' }); const subject = `feat(delivery): phase ${phase} — ${config.title}`; execFileSync( 'git', diff --git a/tests/e2e/unlaunched/codex-session-host.ts b/tests/e2e/unlaunched/codex-session-host.ts new file mode 100644 index 0000000000..b1ab4323a1 --- /dev/null +++ b/tests/e2e/unlaunched/codex-session-host.ts @@ -0,0 +1,469 @@ +/** + * A **codex** session that relay did not launch. + * + * Phase 1 of `docs/native-delivery-migration.md` is the `codex queue` route, + * and the doc's "gate that does not exist yet" asks for a scenario that starts + * a bare `codex` OUTSIDE the broker and proves a relay message reaches it. The + * phase-0 host (`session-host.ts`) deliberately used `opencode serve` and said + * so: codex had no delivery route yet, and launching codex carelessly writes + * the operator's config. Phase 1 is the phase that owns codex, so this file is + * the codex half of the same idea. + * + * ## Why `codex app-server`, and not a TUI + * + * The scenario's constraint is "no wrap, no PTY" — relay must not own the + * session's terminal. `codex app-server` is a bare `codex` process speaking + * JSON-RPC over its own stdio: started by the test, parented to the test, with + * a real thread in codex's own store (a UUIDv7 `thread.id`, a rollout JSONL at + * `thread.path`, a row in `state_5.sqlite`) — exactly the thread handle + * `codex queue --thread ` addresses and `crates/broker/src/codex_thread.rs` + * settles from. No pseudo-terminal is allocated anywhere in this file. + * + * ## The config hazard, closed by construction + * + * The doc's testing-hazard section: launching codex in an untrusted directory, + * or answering its first-run prompts, writes the operator's config. Neither can + * happen here. `CODEX_HOME` points at a fresh temp directory, so every file + * codex touches — `config.toml`, `state_5.sqlite`, `sessions/`, `auth.json` — + * is inside that directory and is removed on `stop()`. `app-server` is + * non-interactive, so there is no first-run prompt to answer. Verified on + * codex-cli 0.144.5: after a full start/thread/stop cycle the operator's + * `~/.codex/config.toml` hash is unchanged. + * + * ## How the session registers itself + * + * The scenario requires the session to `set_workspace_key` + `register_agent`. + * Those are relay MCP tools, and in a real unlaunched setup codex is the MCP + * *client*: the operator configures `mcp_servers.agent-relay` and the session + * calls the tools. That is reproduced exactly — the relay MCP server is + * configured in the isolated `config.toml` and spawned by codex, not by this + * test — with one deliberate difference: the tool call is driven through the + * app-server's own `mcpServer/tool/call` method rather than by prompting a + * model to please call it. The call still originates from the codex process's + * MCP client; what is removed is the model's discretion, which is not the + * thing under test and which would make the gate nondeterministic and paid. + */ +import { spawn, execFileSync, type ChildProcessByStdio } from 'node:child_process'; +import type { Readable, Writable } from 'node:stream'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +/** Queue-capable Codex builds can ship with the desktop app before PATH updates. */ +const CODEX_FALLBACKS = [ + '/Applications/ChatGPT.app/Contents/Resources/codex', + '/opt/homebrew/bin/codex', + '/usr/local/bin/codex', +]; + +/** Enough for a cold `codex app-server` to complete `initialize`. */ +const STARTUP_TIMEOUT_MS = 60_000; +/** Any single JSON-RPC round trip. Thread creation is local and fast. */ +const RPC_TIMEOUT_MS = 60_000; + +type HostChild = ChildProcessByStdio; + +export interface RelayMcpServer { + /** Executable codex spawns for the relay MCP server (usually `node`). */ + command: string; + /** Arguments, e.g. `['packages/cli/dist/cli/index.js', 'mcp']`. */ + args: string[]; + /** Extra environment for that server process. Never logged. */ + env?: Record; +} + +export interface UnlaunchedCodexSession { + /** Codex's own thread id (UUIDv7) — the `--thread` argument. */ + threadId: string; + /** Rollout JSONL codex writes this thread to, or null if unreported. */ + rolloutPath: string | null; + /** PID of the `codex app-server` process. Its parent is this test. */ + pid: number; + /** Isolated `CODEX_HOME`; removed on stop. */ + codexHome: string; + /** Isolated working directory the thread was created in. */ + workdir: string; + /** Call a relay MCP tool through the codex session's own MCP client. */ + callRelayTool(tool: string, args: Record): Promise; + /** Every text fragment codex has recorded for this thread. */ + readThreadText(): Promise; + stop(): Promise; +} + +export interface RelayToolResult { + isError: boolean; + /** Flattened text content of the tool result. */ + text: string; + structuredContent: unknown; +} + +export interface CodexQueueCapability { + available: boolean; + /** Human-readable reason, safe to put in an assertion message. */ + reason: string; +} + +export function resolveCodexBinary(): string | null { + const configured = process.env.RELAY_UNLAUNCHED_CODEX_BIN?.trim(); + if (configured) return existsSync(configured) ? configured : null; + const candidates: string[] = []; + try { + const resolved = execFileSync('/bin/sh', ['-c', 'command -v codex'], { + encoding: 'utf8', + }).trim(); + if (resolved.length > 0) candidates.push(resolved); + } catch { + // A bundled app binary can still satisfy the gate. + } + candidates.push(...CODEX_FALLBACKS); + const existing = [...new Set(candidates)].filter((candidate) => existsSync(candidate)); + return existing.find((candidate) => probeCodexQueueCapability(candidate).available) ?? existing[0] ?? null; +} + +/** + * The same capability decision `CodexQueueTarget::ensure_queue_capability` + * makes in `crates/broker/src/delivery/codex_queue.rs`, reproduced here so the + * gate reports the route's own precondition rather than a downstream symptom. + * + * The flag check is not belt-and-braces. On codex-cli 0.144.5 an unknown + * subcommand is not an error: `codex queue --help` prints the top-level help + * and **exits 0**. A probe that trusted the exit status alone would report the + * route available on a codex that has no `queue` command at all, and the + * backend would then classify the real invocation's failure as committed — + * post-write, never retried, never falling back. Requiring `--thread` and + * `--message` in the help text is what keeps that failure pre-write. + */ +export function probeCodexQueueCapability(binary: string): CodexQueueCapability { + const codexHome = mkdtempSync(path.join(tmpdir(), 'relay-codex-probe-')); + try { + let stdout = ''; + let status = 0; + try { + stdout = execFileSync(binary, ['queue', '--help'], { + encoding: 'utf8', + env: { ...process.env, CODEX_HOME: codexHome }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch (error) { + const failure = error as { status?: number; stdout?: string; stderr?: string }; + status = failure.status ?? 1; + stdout = `${failure.stdout ?? ''}${failure.stderr ?? ''}`; + } + if (status !== 0) { + return { available: false, reason: `\`codex queue --help\` exited ${status}` }; + } + const hasThread = stdout.includes('--thread'); + const hasMessage = stdout.includes('--message'); + if (!hasThread || !hasMessage) { + return { + available: false, + reason: + '`codex queue --help` exited 0 but the help text does not document ' + + `${[!hasThread ? '--thread' : null, !hasMessage ? '--message' : null] + .filter(Boolean) + .join(' or ')}` + + ' — this codex has no `queue` subcommand and printed top-level help instead', + }; + } + return { available: true, reason: '`codex queue` documents --thread and --message' }; + } finally { + rmSync(codexHome, { recursive: true, force: true }); + } +} + +/** TOML string literal. Codex config values are parsed as TOML. */ +function tomlString(value: string): string { + return JSON.stringify(value); +} + +function relayMcpToml(server: RelayMcpServer): string { + const lines = [ + '[mcp_servers.agent-relay]', + `command = ${tomlString(server.command)}`, + `args = [${server.args.map(tomlString).join(', ')}]`, + ]; + const env = server.env ?? {}; + const entries = Object.entries(env); + if (entries.length > 0) { + lines.push('[mcp_servers.agent-relay.env]'); + for (const [key, value] of entries) lines.push(`${key} = ${tomlString(value)}`); + } + return `${lines.join('\n')}\n`; +} + +/** + * Start a bare codex and create one thread in it. + * + * Nothing about this is relay-owned: the process is a child of the test, its + * stdio is a pipe rather than a pty, and the thread is created through codex's + * own API before relay is told anything. + */ +export async function startUnlaunchedCodexSession(options: { + binary: string; + relayMcp: RelayMcpServer; + startupTimeoutMs?: number; +}): Promise { + const codexHome = mkdtempSync(path.join(tmpdir(), 'relay-unlaunched-codex-')); + const workdir = path.join(codexHome, 'work'); + const relayHome = path.join(codexHome, 'relay-home'); + mkdirSync(workdir, { recursive: true }); + mkdirSync(relayHome, { recursive: true }); + const isolatedRelayMcp: RelayMcpServer = { + ...options.relayMcp, + env: { + ...options.relayMcp.env, + // `optionsFromEnv()` also resumes the machine-global active workspace. + // An empty environment is therefore not enough to make this MCP session + // unconfigured; isolate both the workspace store and telemetry state. + AGENT_RELAY_HOME: relayHome, + AGENT_RELAY_DATA_DIR: relayHome, + AGENT_RELAY_PROJECT: workdir, + // Codex normally uses ~/.codex, which both the MCP process and broker can + // resolve independently. This gate intentionally relocates that store; + // carry the same location into the MCP child so native attach names the + // isolated thread store rather than the operator's default one. + CODEX_HOME: codexHome, + }, + }; + + // `approval_policy`/`sandbox_mode` are set so the session never blocks on an + // approval request; nothing in this gate runs a command in the session. + writeFileSync( + path.join(codexHome, 'config.toml'), + ['approval_policy = "never"', 'sandbox_mode = "read-only"', '', relayMcpToml(isolatedRelayMcp)].join( + '\n' + ), + 'utf8' + ); + + // This is deliberately an *unconfigured* Relay session. The parent test has + // a workspace key so it can create the hosted workspace and drive the + // control broker, but passing that key through Codex would make the MCP + // subprocess auto-register the default `orchestrator` before this session + // calls set_workspace_key/register_agent itself. Besides violating the + // scenario, a reused workspace can then make MCP startup exit on an existing + // orchestrator identity and close the initialize handshake. + const codexEnv: NodeJS.ProcessEnv = { ...process.env, CODEX_HOME: codexHome }; + for (const key of Object.keys(codexEnv)) { + if (key.startsWith('RELAY_') || key.startsWith('AGENT_RELAY_')) delete codexEnv[key]; + } + // The one interrupted priming turn below exists only to materialize Codex's + // rollout file. Never let it inherit a provider credential and accidentally + // become a paid model request. + delete codexEnv.OPENAI_API_KEY; + delete codexEnv.CODEX_API_KEY; + + const child = spawn(options.binary, ['app-server'], { + cwd: workdir, + env: codexEnv, + stdio: ['pipe', 'pipe', 'pipe'], + }) as HostChild; + + const pending = new Map) => void>(); + const stderrTail: string[] = []; + let buffered = ''; + let exited: { code: number | null; signal: NodeJS.Signals | null } | null = null; + let nextId = 1; + + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { + buffered += chunk; + for (;;) { + const newline = buffered.indexOf('\n'); + if (newline < 0) break; + const line = buffered.slice(0, newline).trim(); + buffered = buffered.slice(newline + 1); + if (line.length === 0) continue; + let message: Record; + try { + message = JSON.parse(line) as Record; + } catch { + continue; + } + const id = message.id; + if (typeof id === 'number' && pending.has(id)) { + pending.get(id)?.(message); + pending.delete(id); + } + } + }); + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk: string) => { + stderrTail.push(chunk); + if (stderrTail.length > 40) stderrTail.shift(); + }); + child.on('exit', (code, signal) => { + exited = { code, signal }; + for (const [id, resolve] of pending) { + resolve({ id, error: { message: `codex app-server exited (code=${code} signal=${signal})` } }); + } + pending.clear(); + }); + + const request = (method: string, params: unknown, timeoutMs = RPC_TIMEOUT_MS): Promise => { + if (exited) { + return Promise.reject( + new Error(`codex app-server is not running: ${JSON.stringify(exited)} ${stderrTail.join('')}`) + ); + } + const id = nextId++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + pending.delete(id); + reject(new Error(`codex app-server did not answer ${method} within ${timeoutMs}ms`)); + }, timeoutMs); + pending.set(id, (message) => { + clearTimeout(timer); + const error = message.error as { message?: string } | undefined; + if (error) { + reject(new Error(`codex app-server refused ${method}: ${error.message ?? JSON.stringify(error)}`)); + return; + } + resolve(message.result); + }); + child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id, method, params })}\n`); + }); + }; + + const cleanup = async (): Promise => { + if (!child.killed) child.kill('SIGTERM'); + await new Promise((resolve) => setTimeout(resolve, 250)); + if (child.exitCode === null) child.kill('SIGKILL'); + rmSync(codexHome, { recursive: true, force: true }); + }; + + let threadId: string; + let rolloutPath: string | null; + try { + await request( + 'initialize', + { + clientInfo: { name: 'relay-unlaunched-gate', version: '1.0.0' }, + capabilities: { experimentalApi: true }, + }, + options.startupTimeoutMs ?? STARTUP_TIMEOUT_MS + ); + child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', method: 'initialized', params: {} })}\n`); + const started = (await request('thread/start', { cwd: workdir })) as { + thread?: { id?: unknown; path?: unknown }; + }; + threadId = String(started.thread?.id ?? ''); + if (!threadId) throw new Error(`codex did not return a thread id: ${JSON.stringify(started)}`); + rolloutPath = typeof started.thread?.path === 'string' ? started.thread.path : null; + + // `thread/start` reserves the path but Codex does not create the rollout + // until the first turn begins. `codex queue` resolves its target through + // that rollout and otherwise returns "no rollout found". Real app/terminal + // threads have already crossed this boundary; reproduce it without a + // model call by starting and immediately interrupting a credential-less + // local turn. + const primed = (await request('turn/start', { + threadId, + input: [{ type: 'text', text: 'Initialize this local delivery test session.' }], + })) as { turn?: { id?: unknown } }; + const primingTurnId = String(primed.turn?.id ?? ''); + if (primingTurnId) { + try { + await request('turn/interrupt', { threadId, turnId: primingTurnId }, 10_000); + } catch { + // A missing credential can end the turn before the interrupt arrives; + // beginning it is sufficient to materialize the rollout. + } + } + if (rolloutPath) { + const deadline = Date.now() + 10_000; + while (!existsSync(rolloutPath) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + if (!existsSync(rolloutPath)) { + throw new Error(`codex did not materialize the thread rollout at ${rolloutPath}`); + } + } + } catch (error) { + await cleanup(); + throw error; + } + + const callRelayTool = async (tool: string, args: Record): Promise => { + const result = (await request('mcpServer/tool/call', { + server: 'agent-relay', + threadId, + tool, + arguments: args, + })) as { + content?: Array<{ type?: unknown; text?: unknown }>; + isError?: unknown; + structuredContent?: unknown; + }; + const text = (result.content ?? []) + .map((part) => (typeof part.text === 'string' ? part.text : '')) + .join('\n'); + return { + isError: result.isError === true, + text, + structuredContent: result.structuredContent, + }; + }; + + /** + * Read the thread back from codex's own records. + * + * `thread/items/list` is the supported view and is the only view returned + * when available. The rollout repeats one logical user item as both a raw + * response item and an `item_completed` event; combining the two projections + * would make one injection look like three. The rollout is therefore only a + * fallback when the supported item view is unavailable. + */ + const readThreadText = async (): Promise => { + const fragments: string[] = []; + try { + const listed = (await request('thread/items/list', { threadId })) as { data?: unknown[] }; + for (const item of listed.data ?? []) fragments.push(JSON.stringify(item)); + return fragments; + } catch { + // An app-server that has gone away can still leave an authoritative + // rollout for the gate to inspect. + } + if (rolloutPath && existsSync(rolloutPath)) { + for (const line of readFileSync(rolloutPath, 'utf8').split('\n')) { + if (line.trim().length > 0) fragments.push(line); + } + } + return fragments; + }; + + return { + threadId, + rolloutPath, + pid: child.pid as number, + codexHome, + workdir, + callRelayTool, + readThreadText, + stop: cleanup, + }; +} + +/** Read a process's parent pid. Used to prove relay did not launch the host. */ +export function parentPidOf(pid: number): number | null { + try { + const out = execFileSync('ps', ['-o', 'ppid=', '-p', String(pid)], { encoding: 'utf8' }); + const parsed = Number.parseInt(out.trim(), 10); + return Number.isFinite(parsed) ? parsed : null; + } catch { + return null; + } +} + +/** How many of `fragments` contain `needle`. */ +export function countOccurrences(fragments: string[], needle: string): number { + return fragments.reduce((total, fragment) => { + let count = 0; + let index = fragment.indexOf(needle); + while (index >= 0) { + count += 1; + index = fragment.indexOf(needle, index + needle.length); + } + return total + count; + }, 0); +} diff --git a/tests/e2e/unlaunched/unlaunched-codex-delivery.test.ts b/tests/e2e/unlaunched/unlaunched-codex-delivery.test.ts new file mode 100644 index 0000000000..68dbb1737f --- /dev/null +++ b/tests/e2e/unlaunched/unlaunched-codex-delivery.test.ts @@ -0,0 +1,486 @@ +/** + * Phase 1's exit gate: a relay message reaches a **codex** session relay did + * not launch, unprompted, exactly once. + * + * "No current scenario delivers into a session relay did NOT launch. That + * capability is the entire point of the migration and nothing tests it + * today. [...] Until that exists there is no proof of the thing being + * claimed." — docs/native-delivery-migration.md + * + * The phase-0 sibling (`unlaunched-delivery.test.ts`) proves the idea for the + * one route that existed then — an attached `opencode serve` — and says in its + * own header that it does NOT prove relay can reach a codex session, and that + * `unlaunched-codex-delivery` belongs to phase 1. This is that scenario. + * + * What it asserts, in order, each as its own failure: + * + * 1. `codex queue` exists on the installed codex and documents the flags the + * backend invokes. This is the route's own precondition; a phase-1 claim + * is worthless on a codex that cannot queue. + * 2. The session is not relay's. A bare `codex app-server` is started by this + * test, in an isolated CODEX_HOME, with no pty anywhere — and its parent + * pid is asserted to be this test process, not the broker. + * 3. The session registers ITSELF: `set_workspace_key` then `register_agent`, + * called through the codex process's own MCP client against the relay MCP + * server codex spawned. Relay is not told about the session by relay. + * 4. Relay accepts that already-running thread as a delivery target. + * 5. A message sent to the registered name lands in the codex thread's own + * records — read back from codex, not from relay's telemetry — exactly + * once, and stays exactly once after the retry window has passed. + * 6. No pty. The event kinds only a session relay OWNS can produce are + * asserted absent, with the phase-0 control proving those kinds still + * occur for a launched worker. + * + * Deliberately NOT skippable. `tests/relayflows/cleanroom/relay.matrix.json` + * forbids `# SKIP` in this scenario's output precisely so a missing + * prerequisite cannot read as a pass. Every prerequisite below fails. + * + * Config-safety: see `codex-session-host.ts`. CODEX_HOME is a temp directory + * for every codex this file starts, including the capability probe, so the + * operator's `~/.codex` is never read for state nor written. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { HarnessDriverClient, type BrokerEvent } from '@agent-relay/harness-driver'; +import { RelayCast } from '@relaycast/sdk'; + +import { + countOccurrences, + parentPidOf, + probeCodexQueueCapability, + resolveCodexBinary, + startUnlaunchedCodexSession, + type CodexQueueCapability, + type UnlaunchedCodexSession, +} from './codex-session-host.js'; + +/** Unique per run, so a stale thread or a reused workspace cannot satisfy it. */ +const RUN = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; +const AGENT_NAME = `unlaunched-codex-${RUN.slice(-6)}`; +const SENDER = 'Lead'; +/** The body token counted in the codex thread. Never a substring of anything else. */ +const MARKER = `relay-unlaunched-codex-${RUN}`; +/** Marker the codex-queue backend carries so it can settle (`codex_thread.rs`). */ +const ROUTE_MARKER_PREFIX = 'relay-delivery-id:'; +const DELIVERY_WAIT_MS = Number(process.env.RELAY_UNLAUNCHED_DELIVERY_WAIT_MS ?? 120_000); +const TERMINAL_WAIT_MS = Number(process.env.RELAY_UNLAUNCHED_TERMINAL_WAIT_MS ?? 120_000); + +/** + * Event kinds only a session relay OWNS can produce: `delivery_verified` comes + * from the pty/headless worker echo-checking a screen relay is reading, and + * `worker_stream` is that stream. `unlaunched-delivery.test.ts` carries the + * control that asserts both DO appear for a launched worker, so these absence + * assertions cannot quietly become vacuous if a kind is renamed. + */ +const PTY_OWNED_SESSION_KINDS = ['delivery_verified', 'worker_stream'] as const; + +function brokerBinary(): string { + if (process.env.AGENT_RELAY_BIN) return process.env.AGENT_RELAY_BIN; + const exe = process.platform === 'win32' ? 'agent-relay-broker.exe' : 'agent-relay-broker'; + for (const profile of ['release', 'debug']) { + const candidate = path.resolve(process.cwd(), 'target', profile, exe); + if (existsSync(candidate)) return candidate; + } + return exe; +} + +/** The relay MCP stdio server, as an unlaunched codex would be configured to spawn it. */ +function relayMcpEntrypoint(): string { + return ( + process.env.RELAY_UNLAUNCHED_CLI_ENTRYPOINT?.trim() || + path.resolve(process.cwd(), 'packages/cli/dist/cli/index.js') + ); +} + +let generatedWorkspaceKey: string | undefined; + +async function resolveWorkspaceKey(): Promise { + // Generic workspace variables belong to the developer's ambient Relay + // session and may name a legacy or non-canonical deployment. This clean-room + // gate must not silently inherit them: use the dedicated override or create + // a fresh workspace against the matching dedicated/default endpoint. + const configured = process.env.RELAY_UNLAUNCHED_WORKSPACE_KEY?.trim() || ''; + if (configured) return configured; + if (generatedWorkspaceKey) return generatedWorkspaceKey; + const baseUrl = process.env.RELAY_UNLAUNCHED_BASE_URL?.trim(); + const workspace = await RelayCast.createWorkspace(`unlaunched-codex-${RUN}`, { + ...(baseUrl ? { baseUrl } : {}), + }); + if (!workspace.apiKey) { + throw new Error('Relaycast workspace did not return an API key for the unlaunched Codex gate'); + } + generatedWorkspaceKey = workspace.apiKey; + return generatedWorkspaceKey; +} + +async function waitFor( + read: () => Promise, + timeoutMs: number, + describeFailure: string +): Promise { + const deadline = Date.now() + timeoutMs; + let lastError: unknown; + while (Date.now() < deadline) { + try { + const value = await read(); + if (value !== undefined) return value; + } catch (error) { + lastError = error; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error( + `${describeFailure} (waited ${timeoutMs}ms)${lastError ? `; last error: ${String(lastError)}` : ''}` + ); +} + +/** Distinct `relay-delivery-id:` values present in the thread's records. */ +function routeMarkerIds(fragments: string[]): string[] { + const ids = new Set(); + for (const fragment of fragments) { + for (const match of fragment.matchAll(/relay-delivery-id:([A-Za-z0-9_.:-]+)/g)) { + ids.add(match[1]); + } + } + return [...ids]; +} + +function markerRecordSummary(fragments: string[], marker: string): string { + return JSON.stringify( + fragments + .filter((fragment) => fragment.includes(marker)) + .map((fragment) => { + try { + const record = JSON.parse(fragment) as Record; + const payload = (record.payload ?? {}) as Record; + const item = (record.item ?? payload.item ?? {}) as Record; + return { + type: record.type, + payloadType: payload.type, + id: record.id ?? payload.id ?? item.id, + turnId: record.turnId ?? record.turn_id ?? payload.turnId ?? payload.turn_id, + itemType: item.type, + }; + } catch { + return { type: 'unparsed' }; + } + }) + ); +} + +/** Safe route diagnostics: deliberately excludes message bodies, tokens, and workspace keys. */ +function deliveryEventSummary(events: BrokerEvent[]): string { + return events + .map((event) => { + const detail = event as { + kind: string; + name?: unknown; + from?: unknown; + to?: unknown; + delivery_id?: unknown; + event_id?: unknown; + verification?: unknown; + reason?: unknown; + lastError?: unknown; + }; + return JSON.stringify({ + kind: detail.kind, + name: detail.name, + from: detail.from, + to: detail.to, + delivery_id: detail.delivery_id, + event_id: detail.event_id, + verification: detail.verification, + reason: detail.reason, + lastError: detail.lastError, + }); + }) + .join(', '); +} + +describe('delivery into a codex session relay did not launch', () => { + let codexBinary: string; + let capability: CodexQueueCapability; + let session: UnlaunchedCodexSession; + let client: HarnessDriverClient; + let key: string; + let brokerStateDir: string; + const events: BrokerEvent[] = []; + const brokerStderr: string[] = []; + const testPid = process.pid; + + beforeAll(async () => { + const binary = resolveCodexBinary(); + expect( + binary, + 'a real codex is required for the phase-1 gate; install it or set RELAY_UNLAUNCHED_CODEX_BIN' + ).toBeTruthy(); + codexBinary = binary as string; + + // Precondition 1. Checked before anything is started, because everything + // below is meaningless on a codex that cannot queue. + capability = probeCodexQueueCapability(codexBinary); + expect( + capability.available, + `the phase-1 route is unavailable on the installed codex: ${capability.reason}. ` + + '`codex queue --thread --message=` is the command this phase delivers over ' + + '(docs/native-delivery-migration.md, Phase 1); without it there is no route to gate.' + ).toBe(true); + + const entrypoint = relayMcpEntrypoint(); + expect( + existsSync(entrypoint), + `the relay MCP entrypoint ${entrypoint} is missing; run \`npm run build:core\` ` + + 'or set RELAY_UNLAUNCHED_CLI_ENTRYPOINT' + ).toBe(true); + + key = await resolveWorkspaceKey(); + expect( + key.length, + 'this gate needs a hosted workspace and two registered identities (verify_tier 4): ' + + 'set RELAY_UNLAUNCHED_WORKSPACE_KEY if automatic workspace creation is unavailable' + ).toBeGreaterThan(0); + + const relayBaseUrl = process.env.RELAY_UNLAUNCHED_BASE_URL?.trim(); + const brokerEnv = { ...process.env }; + for (const variable of [ + 'RELAY_API_KEY', + 'RELAY_WORKSPACE_KEY', + 'AGENT_RELAY_WORKSPACE_KEY', + 'RELAY_BASE_URL', + 'RELAYCAST_BASE_URL', + ]) { + delete brokerEnv[variable]; + } + brokerEnv.RELAY_API_KEY = key; + if (relayBaseUrl) brokerEnv.RELAYCAST_BASE_URL = relayBaseUrl; + + client = await HarnessDriverClient.spawn({ + binaryPath: brokerBinary(), + // A failed or interrupted hosted run may not reach `shutdown()`. Reusing + // the worktree basename would then collide with that run's still-owned + // broker identity and make a clean retry fail before the scenario starts. + brokerName: `unlaunched-broker-${RUN}`, + channels: ['general'], + binaryArgs: { + persist: true, + stateDir: (brokerStateDir = mkdtempSync(path.join(tmpdir(), 'relay-unlaunched-broker-'))), + }, + env: { ...brokerEnv, RUST_LOG: process.env.RUST_LOG ?? 'info' }, + onStderr: (line) => { + brokerStderr.push(line); + if (brokerStderr.length > 200) brokerStderr.shift(); + }, + }); + client.onEvent((event) => { + events.push(event); + }); + + const connection = JSON.parse(readFileSync(path.join(brokerStateDir, 'connection.json'), 'utf8')) as { + url?: unknown; + api_key?: unknown; + }; + expect(typeof connection.url).toBe('string'); + expect(typeof connection.api_key).toBe('string'); + + // Precondition 2. A bare codex, started here, over stdio, with no pty. + session = await startUnlaunchedCodexSession({ + binary: codexBinary, + relayMcp: { + command: process.execPath, + args: [entrypoint, 'mcp'], + // The MCP server gets no agent token: an unlaunched session has not been + // minted one. It mints its own in `register_agent`, below, which is the + // whole difference between this and a spawned worker. Broker credentials + // only authorize the local attach half of that same tool call. + env: { + AGENT_RELAY_TELEMETRY_DISABLED: '1', + RELAY_BROKER_URL: connection.url as string, + RELAY_BROKER_API_KEY: connection.api_key as string, + // `startUnlaunchedCodexSession` deliberately strips every RELAY_* + // variable from the bare Codex process so it cannot auto-register. + // Carry the selected Relaycast deployment back into the MCP child + // explicitly; otherwise a key for a non-default deployment is sent + // to production and `register_agent` reports "Invalid API key". + ...(relayBaseUrl ? { RELAY_BASE_URL: relayBaseUrl } : {}), + }, + }, + }); + }, 300_000); + + afterAll(async () => { + try { + await client?.release(AGENT_NAME, 'unlaunched codex gate complete'); + } catch { + // Release failures must not mask an assertion result. + } + try { + await client?.shutdown(); + } catch { + // Same. + } + await session?.stop(); + if (brokerStateDir && process.env.RELAY_KEEP_UNLAUNCHED_STATE !== '1') { + rmSync(brokerStateDir, { recursive: true, force: true }); + } + }, 120_000); + + it('the installed codex exposes the `codex queue` route this phase delivers over', () => { + expect(capability.available, capability.reason).toBe(true); + }); + + it('the codex session is not a broker child', () => { + const parent = parentPidOf(session.pid); + expect(parent, 'could not read the codex session pid lineage').not.toBeNull(); + expect( + parent, + `the codex session (pid ${session.pid}) must be a child of this test (pid ${testPid}); ` + + 'if relay launched it, the scenario proves nothing' + ).toBe(testPid); + }); + + it('the session registers itself with set_workspace_key then register_agent', async () => { + const keyed = await session.callRelayTool('set_workspace_key', { workspace_key: key }); + expect(keyed.isError, `set_workspace_key failed inside the codex session: ${keyed.text}`).toBe(false); + + const registered = await session.callRelayTool('register_agent', { + name: AGENT_NAME, + type: 'agent', + native_delivery: { + provider: 'codex', + thread_id: session.threadId, + }, + }); + expect(registered.isError, `register_agent failed inside the codex session: ${registered.text}`).toBe( + false + ); + // The registration is the session's own, so its own name must come back. + expect(registered.text).toContain(AGENT_NAME); + const result = registered.structuredContent as { + token?: unknown; + native_delivery?: { route?: unknown; owns_session?: unknown }; + } | null; + expect(typeof result?.token, 'register_agent did not return the session-owned agent token').toBe( + 'string' + ); + expect(result?.native_delivery?.route).toBe('codex-queue'); + expect(result?.native_delivery?.owns_session).toBe(false); + }, 120_000); + + it('the thread carries no relay conversation before relay sends one', async () => { + const fragments = await session.readThreadText(); + expect(countOccurrences(fragments, MARKER)).toBe(0); + expect(routeMarkerIds(fragments)).toEqual([]); + }); + + it('self-registration makes the already-running Codex thread addressable', async () => { + const listed = await waitFor( + async () => (await client.listAgents()).find((agent) => agent.name === AGENT_NAME), + 30_000, + `register_agent never made ${AGENT_NAME} addressable as a native Codex target` + ); + expect((listed as { runtime?: string }).runtime, 'a pty runtime means relay owns a terminal').not.toBe( + 'pty' + ); + + const inventory = await waitFor( + async () => { + const snapshot = await client.listFleetInventory(); + return snapshot.agents.find((agent) => agent.name === AGENT_NAME); + }, + 30_000, + `register_agent attached ${AGENT_NAME} locally but did not publish it in node inventory` + ); + expect(inventory.session_ref, 'the attached codex thread is absent from node inventory').toBe( + session.threadId + ); + }, 120_000); + + it('a relay message reaches the codex thread, unprompted and exactly once', async () => { + const result = await client.sendMessage({ to: AGENT_NAME, from: SENDER, text: MARKER, mode: 'steer' }); + expect(result.event_id, 'relay refused the send outright').not.toBe('unsupported_operation'); + + try { + await waitFor( + async () => { + const fragments = await session.readThreadText(); + return countOccurrences(fragments, MARKER) > 0 ? true : undefined; + }, + DELIVERY_WAIT_MS, + 'the message never reached the codex thread: relay could not deliver into a codex session it did not launch' + ); + } catch (error) { + const sendStatus = result as unknown as Record; + throw new Error( + `${String(error)}; send status: ${JSON.stringify({ + delivery_status: sendStatus.delivery_status, + recipient_live: sendStatus.recipient_live, + recipient_status: sendStatus.recipient_status, + local: sendStatus.local, + })}; broker route events: [${deliveryEventSummary(events)}]; broker stderr tail: ${brokerStderr.join( + '\n' + )}` + ); + } + + const fragments = await session.readThreadText(); + expect( + countOccurrences(fragments, MARKER), + 'the message body appears in the codex thread more than once: relay re-sent a delivery it had already handed over; ' + + markerRecordSummary(fragments, MARKER) + ).toBe(1); + const markerIds = routeMarkerIds(fragments); + expect(markerIds, `expected exactly one ${ROUTE_MARKER_PREFIX} marker for this send`).toHaveLength(1); + const deliveryId = markerIds[0]; + + let terminalDelivery: BrokerEvent; + try { + terminalDelivery = await waitFor( + async () => { + const terminal = events.find( + (event) => + (event.kind === 'message_delivery_confirmed' || event.kind === 'message_delivery_failed') && + (event as { name?: unknown }).name === AGENT_NAME && + (event as { delivery_id?: unknown }).delivery_id === deliveryId + ); + return terminal; + }, + TERMINAL_WAIT_MS, + 'relay never reported a terminal delivery outcome after codex queue handoff' + ); + } catch (error) { + throw new Error(`${String(error)}; broker route events: [${deliveryEventSummary(events)}]`); + } + expect( + terminalDelivery.kind, + `codex queue delivery reached the thread but relay reported failure: ${deliveryEventSummary(events)}` + ).toBe('message_delivery_confirmed'); + expect( + (terminalDelivery as { event_id?: unknown }).event_id, + 'confirmed delivery event did not name the Relaycast message id' + ).toBeTruthy(); + const settledFragments = await session.readThreadText(); + expect( + countOccurrences(settledFragments, MARKER), + 'the settled message was injected more than once; ' + markerRecordSummary(settledFragments, MARKER) + ).toBe(1); + expect(routeMarkerIds(settledFragments)).toEqual([deliveryId]); + }, 300_000); + + it('the delivery used the native route, not a PTY', async () => { + const kinds = new Set( + events.filter((event) => (event as { name?: unknown }).name === AGENT_NAME).map((event) => event.kind) + ); + for (const ownedSessionOnly of PTY_OWNED_SESSION_KINDS) { + expect( + [...kinds], + `${ownedSessionOnly} means relay owned this session's terminal or stdout` + ).not.toContain(ownedSessionOnly); + } + // And the process is still not relay's, after delivery as before it. + expect(parentPidOf(session.pid)).toBe(testPid); + }); +}); diff --git a/tests/e2e/vitest.unlaunched.config.ts b/tests/e2e/vitest.unlaunched.config.ts new file mode 100644 index 0000000000..8d5d261ba5 --- /dev/null +++ b/tests/e2e/vitest.unlaunched.config.ts @@ -0,0 +1,56 @@ +import path from 'node:path'; + +import { defineConfig } from 'vitest/config'; + +import e2eConfig from '../../vitest.e2e.config.js'; + +// The unlaunched-delivery suite: a message reaching a session relay did NOT +// launch. `docs/native-delivery-migration.md` calls this "the gate that does +// not exist yet", and it is the exit criterion for every native-delivery +// phase. +// +// It needs its own config because `vitest.e2e.config.ts` deliberately EXCLUDES +// `tests/e2e/unlaunched/**` — these scenarios need a real CLI on the box and a +// hosted workspace, so the default `npm run test:e2e` sweep must not pick them +// up. That exclude is correct and is kept. +// +// What was not correct: a vitest positional argument is a FILTER applied after +// include/exclude globbing, so it cannot resurrect an excluded file. Both +// `npm run test:e2e:unlaunched` and the `unlaunched-*-delivery` cleanroom +// scenarios pointed at `vitest.e2e.config.ts` with a path argument and +// therefore collected zero test files, exiting 1 with "No test files found" — +// a red that says nothing about delivery. This config inverts the include so +// the suite can actually run. +// +// It lives under `tests/e2e/` rather than at the repo root because the +// native-delivery phase lanes only admit `tests/**` on the TypeScript side, +// and one directory up from the suite rather than inside it because +// `unlaunched-gate` typechecks every `.ts` under `tests/e2e/unlaunched/` as +// suite source — a build config sitting there would drag `vitest.e2e.config.ts` +// into that check and fail it on a pre-existing, unrelated `minWorkers` type +// error. `root` is pinned back to the repo root so the inherited globs and the +// cleanroom scenario's positional filter stay repo-relative. +// +// Everything else (workspace aliases, single-threaded forks, no retry) is +// inherited by spread, so the two configs cannot drift. `mergeConfig` is +// deliberately NOT used: it concatenates arrays, so the inherited +// `include`/`exclude` would survive and this config would run the whole e2e +// sweep while still excluding the one suite it exists to run. +const repoRoot = path.resolve(__dirname, '../..'); + +export default defineConfig({ + ...e2eConfig, + root: repoRoot, + test: { + ...e2eConfig.test, + root: repoRoot, + include: ['tests/e2e/unlaunched/**/*.test.ts'], + exclude: [], + // A bare CLI has to start, register itself and be delivered into; the + // e2e default is not enough headroom for a cold start plus the retry + // window these scenarios must sit through before counting copies. + testTimeout: 300_000, + hookTimeout: 300_000, + teardownTimeout: 120_000, + }, +}); diff --git a/tests/fixtures/delivery-contract-evals.codex-queue.test.ts b/tests/fixtures/delivery-contract-evals.codex-queue.test.ts new file mode 100644 index 0000000000..b6a97c91e6 --- /dev/null +++ b/tests/fixtures/delivery-contract-evals.codex-queue.test.ts @@ -0,0 +1,314 @@ +/** + * Phase-1 delivery-contract regression coverage: the codex-queue route. + * + * `docs/native-delivery-migration.md` names four suites as the gate that "the + * delivery contract is unchanged" once a message stops arriving over the PTY: + * + * evals/suites/{delivery-modes,messaging,read-receipts,agent-directory} + * + * Phase 0's `delivery-contract-evals.test.ts` runs every compiled case in those + * four suites and adds the seam-rule cases the suites only state as prose. This + * file is the phase-1 layer on top: cases for the properties the *codex queue* + * route introduces, driven through the same executor and the same assertion + * functions `npm run evals` uses. + * + * Deliberately not re-asserted here, because phase 0 already gates it: that the + * four suites are populated, compiled, and pass. Re-running them would make a + * phase-1 failure report a phase-0 regression. + * + * Scope note, inherited and still true: these cases pin the delivery contract + * as the TypeScript SDK expresses it (`DeliveryRunner` over durable inbox + * state). They do not execute `crates/broker/src/delivery/codex_queue.rs`; that + * module's own behaviour is proven by its unit tests and by + * `crates/broker/tests/delivery_seam_invariants.rs`. What these cases catch is + * a native route that settles an inbox item differently from the PTY route. + * + * Every case below asserts something the *runner* produced, not something the + * case's own mock supplied. A check that reads back a fixture is a check that + * cannot fail. + */ + +import { assertHumanEvalExpected, validateHumanEvalCase } from '@agent-assistant/telemetry/evals'; +import { describe, expect, it } from 'vitest'; + +import { assertRelayExpected } from '../../scripts/evals/relay-checks.mjs'; +import { createRelayExecutor } from '../../scripts/evals/relay-executor.mjs'; + +interface EvalCase { + id: string; + suite?: string; + executor?: string; + input: { message?: string; operation?: unknown }; + expected?: Record; + mock?: Record; +} + +interface FailedCheck { + name: string; + message: string; +} + +const execute = createRelayExecutor(); + +/** Run one case exactly the way `scripts/evals/run-relay-evals.mjs` does. */ +async function failedChecks(testCase: EvalCase): Promise { + validateHumanEvalCase(testCase as any); + const actual = await execute(testCase as any, { providerMode: false, rootDir: process.cwd() }); + return [...assertHumanEvalExpected(testCase as any, actual), ...assertRelayExpected(testCase, actual)] + .filter((check: { passed: boolean }) => !check.passed) + .map(({ name, message }: FailedCheck) => ({ name, message })); +} + +/** + * The correlation marker `CodexThreadSession::body_with_marker` appends, because + * Codex assigns its own `client_id` to a queued message and relay therefore has + * to carry its delivery id inside the text + * (`crates/broker/src/codex_thread.rs`, and the Phase 1 section of the doc). + */ +const ROUTE_MARKER = 'relay-delivery-id:del_codex_1'; + +const CODEX_QUEUE_CASES: EvalCase[] = [ + { + /** + * `codex queue` is a QUEUE. It appends to a thread's durable queue; it + * cannot interrupt a turn the way typing into a terminal can. That is a + * real semantic difference between the PTY route and this one, and the + * risk is that steer quietly degrades into wait once the transport can no + * longer interrupt. + * + * This case pins the half the contract can still hold: the runner must + * carry steer intent to the adapter as `immediate`. The assertion reads + * `metadata.mode`, which the DEFAULT adapter builds from the runner's own + * `context.mode` (`relay-executor.mjs` `receiveMessage`) — there is no + * `result` fixture in the mock, so nothing here can be satisfied by a + * value the case supplied. + */ + id: 'delivery-modes.codex-queue-steer-still-reaches-the-route-as-immediate', + suite: 'delivery-modes', + executor: 'relay', + input: { + message: + 'A steer-mode delivery over a queue-shaped route still reaches the adapter as an immediate injection, not as an idle wait.', + operation: [{ op: 'deliver', as: 'codex-session', mode: 'steer', reason: 'mention' }], + }, + mock: { + agents: [{ name: 'codex-session', type: 'agent' }], + delivery: { target: 'codex-session', serverDeliveryState: true }, + }, + expected: { + ok: true, + contentMatches: ['"mode": "immediate"'], + // `next-message` is the wait mapping. Seeing it here would mean steer was + // downgraded to wait because the transport could not interrupt. + forbidPatterns: ['"mode": "next-message"'], + toolCallsInclude: ['deliver'], + maxToolCalls: 1, + }, + }, + { + /** + * Exactly once, at the layer the contract can see it. + * + * The migration doc calls double delivery "the worst failure mode, and the + * one agent-deck hit", and phase 1 is the first phase whose route can + * double-deliver in a way the PTY never could: `codex queue` writes into a + * durable queue, so a re-send after a send left in doubt lands a second + * copy that the session will actually read. + * + * "Twice" is counted where a duplicate would actually show up — inside the + * recipient's own conversation — not anywhere in the observed record. The + * record echoes each op's result, so the body legitimately appears once per + * op that returns it; forbidding two occurrences globally would fail on a + * correct run. The checks below are anchored past `list_dms:` instead, so a + * second `messageId` in the recipient's conversation is what fails, and + * `unreadCount` pins the same fact from the inbox side. + */ + id: 'messaging.codex-queue-body-lands-once-in-the-recipient-inbox', + suite: 'messaging', + executor: 'relay', + input: { + message: + 'A message delivered over a queue route appears once in the recipient inbox, with one unread item.', + operation: [ + { + op: 'send_dm', + as: 'Lead', + to: 'codex-session', + text: `queue once ${ROUTE_MARKER}`, + id: 'dm_codex_once_1', + }, + { op: 'check_inbox', as: 'codex-session' }, + { op: 'list_dms', as: 'codex-session' }, + ], + }, + mock: { + agents: [ + { name: 'Lead', type: 'human' }, + { name: 'codex-session', type: 'agent' }, + ], + }, + expected: { + ok: true, + messageExists: [{ kind: 'dm', from: 'Lead', text: `queue once ${ROUTE_MARKER}` }], + contentIncludes: [ROUTE_MARKER], + contentMatches: ['"unreadCount": 1'], + forbidPatterns: [ + // Two occurrences of the delivery id inside the recipient's own + // conversation. Anchored past `list_dms:` — the last op — so the + // per-op echoes of the body in the `send_dm` and `check_inbox` + // results, which a correct run always produces, are not counted. + `list_dms:[\\s\\S]*${ROUTE_MARKER}[\\s\\S]*${ROUTE_MARKER}`, + '"unreadCount": 2', + ], + toolCallsInclude: ['send_dm', 'check_inbox', 'list_dms'], + }, + }, + { + /** + * Seam rule 4, in the shape phase 1 gives it. + * + * The codex route settles by finding its marker in the thread's own session + * file (`observe_marker`). That is an observation about ARRIVAL. It is not + * a read receipt, and the contract must not let one become the other — a + * route that marked its own deliveries read would make every message look + * consumed the instant it was queued. + * + * The body deliberately carries the settle marker, so the case fails if + * anything treats that marker as evidence the recipient read the message. + */ + id: 'read-receipts.codex-queue-settle-marker-is-not-a-read-receipt', + suite: 'read-receipts', + executor: 'relay', + input: { + message: + 'A message whose body carries the route settle marker still has no readers until the recipient marks it read.', + operation: [ + { + op: 'send_dm', + as: 'Lead', + to: 'codex-session', + text: `settle check ${ROUTE_MARKER}`, + id: 'dm_codex_receipt_1', + }, + { op: 'get_readers', messageId: 'dm_codex_receipt_1' }, + { op: 'mark_read', as: 'codex-session', messageId: 'dm_codex_receipt_1' }, + { op: 'get_readers', messageId: 'dm_codex_receipt_1' }, + ], + }, + mock: { + agents: [ + { name: 'Lead', type: 'human' }, + { name: 'codex-session', type: 'agent' }, + ], + }, + expected: { + ok: true, + // Ordered: the first get_readers is empty and the reader only appears + // after mark_read. Anchored on the op trace so a later reader cannot + // satisfy the earlier check. + contentMatches: ['get_readers: \\[\\][\\s\\S]*mark_read:[\\s\\S]*"agentName": "codex-session"'], + eventEmitted: [{ type: 'messageRead', agentName: 'codex-session' }], + toolCallsInclude: ['send_dm', 'get_readers', 'mark_read'], + }, + }, + { + /** + * Decision D4 plus the thing a thread id can get wrong. + * + * `codex queue --thread ` addresses ONE thread. Relay resolves that + * id per agent (`target_for_worker`), so a mis-resolved id delivers a + * message into somebody else's session — the native-route equivalent of + * typing into the wrong terminal, and a confidentiality bug, not just a + * routing one. + * + * Two identities are registered (verify_tier 4): the unlaunched codex + * session, which registered itself rather than being spawned, and a second + * agent. A DM to the first must be visible to the first and to nobody else. + */ + id: 'agent-directory.codex-queue-delivery-reaches-only-the-addressed-session', + suite: 'agent-directory', + executor: 'relay', + input: { + message: + 'Two self-registered sessions are addressable; a direct message to one is not visible to the other.', + operation: [ + { op: 'register_agent', name: 'codex-session', type: 'agent', persona: 'Unlaunched' }, + { op: 'register_agent', name: 'other-session', type: 'agent', persona: 'Unlaunched' }, + { + op: 'send_dm', + as: 'Lead', + to: 'codex-session', + text: `addressed thread ${ROUTE_MARKER}`, + id: 'dm_codex_thread_1', + }, + { op: 'list_dms', as: 'codex-session' }, + { op: 'list_dms', as: 'other-session' }, + { op: 'list_agents', status: 'online' }, + ], + }, + mock: { agents: [{ name: 'Lead', type: 'human' }] }, + expected: { + ok: true, + agentPresence: [ + { name: 'codex-session', status: 'online' }, + { name: 'other-session', status: 'online' }, + ], + messageExists: [{ kind: 'dm', from: 'Lead', text: `addressed thread ${ROUTE_MARKER}` }], + // Bound to the registration result itself: being addressed later must not + // be what brings a self-registered session online. + contentMatches: [ + 'register_agent: \\{[^}]*"name": "codex-session"[^}]*"status": "online"', + 'register_agent: \\{[^}]*"name": "other-session"[^}]*"status": "online"', + ], + // The marker must not appear twice: once for the addressed session's + // list_dms, never again for the other session's. Anchored past the + // first `list_dms:` so the `send_dm` echo, which a correct run always + // produces, is not what fails it — a second marker at or after that + // point means the unaddressed session saw the message. + forbidPatterns: [`list_dms:[\\s\\S]*${ROUTE_MARKER}[\\s\\S]*${ROUTE_MARKER}`], + toolCallsInclude: ['register_agent', 'send_dm', 'list_dms', 'list_agents'], + // Reachability must not depend on the broker having spawned either one. + toolCallsExclude: ['add_agent'], + }, + }, +]; + +describe('phase-1 codex-queue route, at the delivery contract', () => { + for (const testCase of CODEX_QUEUE_CASES) { + it(testCase.id, async () => { + const failed = await failedChecks(testCase); + expect(failed, JSON.stringify(failed, null, 2)).toEqual([]); + }); + } + +}); + +/** + * Codex-route properties this harness cannot observe, recorded rather than + * faked. A check that passes no matter what the runner did is worse than no + * check, so none of these has a case above. + * + * - The capability probe. `ensure_queue_capability` runs `codex queue --help` + * as a separate child and classifies any failure as pre-write, which is what + * permits a PTY fallback without double-delivering. The executor's `deliver` + * op drives exactly one adapter with one outcome, so "route A refused + * pre-write, route B delivered, and the item was acked once" cannot be + * expressed. It is covered in Rust by + * `unavailable_queue_capability_falls_back_before_write`. + * `tests/e2e/unlaunched/unlaunched-codex-delivery.test.ts` reproduces the + * probe's decision against a real codex, where it is the gate's first + * precondition. + * - Hand-over versus acknowledgement. `CodexQueueBackend::send` always returns + * `HandedOver`, never `Acked`; only `settle` can produce an ack, and only + * from the thread's session file. `DeliveryRunner` does draw the + * distinction (`accepted` acks with `state: undefined`), but the executor's + * `inbox.ack` collapses it with `input.state ?? 'delivered'`, so an + * unobserved hand-over reads as delivered either way. Unchanged from phase 0. + * - Re-send on doubt. One `deliver` op creates one inbox item and drives the + * runner once; there is no second attempt to forbid. The exactly-once case + * above pins the adjacent, observable property (one copy in the recipient's + * inbox) instead. + * - The marker's own shape. `body_with_marker` is idempotent and appends an + * HTML comment; that is Rust-side string handling with no TypeScript + * counterpart, and is covered by `codex_thread::tests`. + */ diff --git a/tests/integration/broker/evals/runner.ts b/tests/integration/broker/evals/runner.ts index b008edb4d5..eee4b0432a 100644 --- a/tests/integration/broker/evals/runner.ts +++ b/tests/integration/broker/evals/runner.ts @@ -13,6 +13,9 @@ * --group=messaging|lifecycle|phrasing|auto-routing|lead-delegation|lead-quality|cross-cli-spawn|task-exit|all Scenario group (default: messaging) * --repeat=N Repeat each scenario N times (default: 1; use 10 for reliability) * --baseline=path.json Compare against a prior report; exit 1 on regression + * --require-all Exit 1 unless every requested scenario and harness ran and passed + * --min-scenarios=N Exit 1 when a harness passes fewer than N scenarios + * --min-delivery-rate=0..1 Exit 1 when a harness falls below this delivery rate * * Lifecycle eval quick-start (finds minimum onboarding for 10/10 reliability): * RELAY_INTEGRATION_REAL_CLI=1 node dist/evals/runner.js \ @@ -104,10 +107,19 @@ interface Flags { group: ScenarioGroup; repeat: number; baseline?: string; + requireAll: boolean; + minScenarios?: number; + minDeliveryRate?: number; } function parseFlags(argv: string[]): Flags { - const flags: Flags = { harnesses: DEFAULT_HARNESSES, tier: 'realistic', group: 'messaging', repeat: 1 }; + const flags: Flags = { + harnesses: DEFAULT_HARNESSES, + tier: 'realistic', + group: 'messaging', + repeat: 1, + requireAll: false, + }; for (const arg of argv) { const [key, value] = arg.replace(/^--/, '').split('='); if (key === 'harness' && value) flags.harnesses = value.split(',').map((s) => s.trim()); @@ -130,6 +142,14 @@ function parseFlags(argv: string[]): Flags { flags.group = value as ScenarioGroup; else if (key === 'repeat' && value) flags.repeat = Math.max(1, Number(value) || 1); else if (key === 'baseline' && value) flags.baseline = value; + else if (key === 'require-all') flags.requireAll = true; + else if (key === 'min-scenarios' && value) { + const parsed = Number(value); + if (Number.isInteger(parsed) && parsed >= 0) flags.minScenarios = parsed; + } else if (key === 'min-delivery-rate' && value) { + const parsed = Number(value); + if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 1) flags.minDeliveryRate = parsed; + } } return flags; } @@ -219,6 +239,7 @@ async function runOnce( const harness = new BrokerHarness({ channels: scenario.channels, env }); await harness.start(); try { + await waitForNodeDelivery(harness, `${spec.cli} ${scenario.id}`); // Pass model in context so scenarios can forward it to spawnAgent (e.g. claude:haiku). return await scenario.run({ harness, cli: spec.cli, model: spec.model, suffix: uniqueSuffix(), sleep }); } finally { @@ -226,6 +247,25 @@ async function runOnce( } } +async function waitForNodeDelivery(harness: BrokerHarness, label: string): Promise { + const deadline = Date.now() + 45_000; + let last: unknown; + while (Date.now() < deadline) { + const status = await harness.client.getStatus(); + const nodeDelivery = (status as { node_delivery?: { connected?: unknown }; node_connected?: unknown }) + .node_delivery; + if (status.node_connected === true || nodeDelivery?.connected === true) { + return; + } + last = { + node_connected: status.node_connected, + node_delivery: nodeDelivery, + }; + await sleep(500); + } + throw new Error(`node delivery did not become ready before ${label}; last status ${JSON.stringify(last)}`); +} + async function runHarness( spec: { cli: string; model?: string }, scenarios: EvalScenario[], @@ -383,11 +423,15 @@ async function main(): Promise { const allReports: Array<{ harness: string; report: EvalReport }> = []; let anyRegression = false; + let anySkippedHarness = false; + let anyScenarioFailure = false; + let anyFloorFailure = false; for (const harnessSpec of flags.harnesses) { const spec = parseHarnessSpec(harnessSpec); if (!isCliAvailable(spec.cli)) { console.log(`\n[${harnessSpec}] skipped — CLI not found on PATH`); + anySkippedHarness = true; continue; } const label = spec.model ? `${spec.cli}:${spec.model.split('/').pop()}` : spec.cli; @@ -398,9 +442,32 @@ async function main(): Promise { matrix.harnesses[label] = report.metrics; allReports.push({ harness: label, report }); printMetrics(label, report.metrics); + if (report.metrics.scenariosTotal === 0 || report.metrics.scenariosPassed < report.metrics.scenariosTotal) { + anyScenarioFailure = true; + console.error( + ` scenarios failed: ${report.metrics.scenariosPassed}/${report.metrics.scenariosTotal} passed` + ); + } + if ( + flags.minScenarios !== undefined && + report.metrics.scenariosPassed < flags.minScenarios + ) { + anyFloorFailure = true; + console.error( + ` scenario floor missed: ${report.metrics.scenariosPassed} < ${flags.minScenarios}` + ); + } + if ( + flags.minDeliveryRate !== undefined && + report.metrics.deliverySuccessRate < flags.minDeliveryRate + ) { + anyFloorFailure = true; + console.error( + ` delivery floor missed: ${report.metrics.deliverySuccessRate} < ${flags.minDeliveryRate}` + ); + } console.log(` report → ${file}`); console.log(` html → ${htmlFile}`); - if (flags.baseline) { try { const deltas = compareReports(readReport(flags.baseline), report); @@ -428,7 +495,20 @@ async function main(): Promise { printLifecycleMatrix(allReports); } - process.exit(anyRegression ? 1 : 0); + if (anySkippedHarness) { + console.error('\nOne or more requested harnesses were skipped.'); + } + if (allReports.length === 0) { + console.error('\nNo requested harness ran.'); + } + + const strictFailure = + flags.requireAll && (anySkippedHarness || anyScenarioFailure || allReports.length === 0); + const floorWasRequested = + flags.minScenarios !== undefined || flags.minDeliveryRate !== undefined; + const floorFailure = + floorWasRequested && (anySkippedHarness || anyFloorFailure || allReports.length === 0); + process.exit(anyRegression || strictFailure || floorFailure ? 1 : 0); } main().catch((err) => { diff --git a/tests/parity/broadcast.ts b/tests/parity/broadcast.ts index 808b39743d..3c0d568bc7 100644 --- a/tests/parity/broadcast.ts +++ b/tests/parity/broadcast.ts @@ -17,6 +17,7 @@ import { const AGENT_COUNT = 3; const TIMEOUT_MS = 15_000; +const ECHO_CLI = "sh -c 'stty -echo; cat'"; async function main(): Promise { console.log('=== Parity Test: Broadcast Messages ===\n'); @@ -37,7 +38,7 @@ async function main(): Promise { const name = randomName(`bcast-${i}`); await client.spawnPty({ name, - cli: 'cat', + cli: ECHO_CLI, channels: ['general'], }); agents.push(name); diff --git a/tests/parity/continuity-handoff.ts b/tests/parity/continuity-handoff.ts index 01f8d877c9..8de16a0c8b 100644 --- a/tests/parity/continuity-handoff.ts +++ b/tests/parity/continuity-handoff.ts @@ -17,6 +17,7 @@ import { } from '../benchmarks/harness.js'; const DELIVERY_TIMEOUT_MS = 15_000; +const ECHO_CLI = "sh -c 'stty -echo; cat'"; async function main(): Promise { console.log('=== Parity Test: Continuity Handoff (Spawn/Release Cycle) ===\n'); @@ -34,7 +35,7 @@ async function main(): Promise { console.log('1. First spawn cycle...'); await client.spawnPty({ name: agentName, - cli: 'cat', + cli: ECHO_CLI, channels: ['general'], }); console.log(` Spawned: ${agentName}`); @@ -83,7 +84,7 @@ async function main(): Promise { console.log('3. Re-spawning with same name...'); await client.spawnPty({ name: agentName, - cli: 'cat', + cli: ECHO_CLI, channels: ['general'], }); console.log(` Re-spawned: ${agentName}`); diff --git a/tests/parity/multi-worker.ts b/tests/parity/multi-worker.ts index 0c0a80e592..c8c1da2dc6 100644 --- a/tests/parity/multi-worker.ts +++ b/tests/parity/multi-worker.ts @@ -16,6 +16,7 @@ import { } from '../benchmarks/harness.js'; const WORKER_COUNT = 3; +const ECHO_CLI = "sh -c 'stty -echo; cat'"; async function main(): Promise { console.log('=== Parity Test: Multiple Workers Communication ===\n'); @@ -35,7 +36,7 @@ async function main(): Promise { const name = randomName(`mw-${i}`); await client.spawnPty({ name, - cli: 'cat', + cli: ECHO_CLI, channels: ['general'], }); workers.push(name); diff --git a/tests/parity/orch-to-worker.ts b/tests/parity/orch-to-worker.ts index 9d8d53eb4f..1775b41c5a 100644 --- a/tests/parity/orch-to-worker.ts +++ b/tests/parity/orch-to-worker.ts @@ -16,6 +16,8 @@ import { randomName, } from '../benchmarks/harness.js'; +const ECHO_CLI = "sh -c 'stty -echo; cat'"; + async function main(): Promise { console.log('=== Parity Test: Orchestrator → Worker ===\n'); @@ -32,7 +34,7 @@ async function main(): Promise { console.log('1. Spawning worker...'); await client.spawnPty({ name: workerName, - cli: 'cat', + cli: ECHO_CLI, channels: ['general'], }); console.log(` Worker: ${workerName}\n`); diff --git a/tests/parity/stability-soak.ts b/tests/parity/stability-soak.ts index 7fdd550a2c..8d2baf6ab0 100644 --- a/tests/parity/stability-soak.ts +++ b/tests/parity/stability-soak.ts @@ -19,6 +19,7 @@ import { const QUICK = process.argv.includes('--quick'); const DURATION_MS = QUICK ? 15_000 : 60_000; // 15s quick, 60s full const INTERVAL_MS = 200; // 5 msgs/sec +const ECHO_CLI = "sh -c 'stty -echo; cat'"; async function main(): Promise { const expectedMsgs = Math.floor(DURATION_MS / INTERVAL_MS); @@ -44,7 +45,7 @@ async function main(): Promise { console.log('1. Spawning worker...'); await client.spawnPty({ name: workerName, - cli: 'cat', + cli: ECHO_CLI, channels: ['general'], }); await new Promise((r) => setTimeout(r, 500)); diff --git a/tests/relayflows/cleanroom/relay.matrix.json b/tests/relayflows/cleanroom/relay.matrix.json index 379aa1cf29..6ae9b47005 100644 --- a/tests/relayflows/cleanroom/relay.matrix.json +++ b/tests/relayflows/cleanroom/relay.matrix.json @@ -197,6 +197,28 @@ "evidence": "integration", "coversCategories": ["broker", "agent-management", "local-agents"] }, + { + "id": "unlaunched-codex-delivery", + "title": "A relay message reaches a bare codex session relay did not launch, exactly once", + "command": [ + "npx", + "vitest", + "run", + "--config", + "tests/e2e/vitest.unlaunched.config.ts", + "tests/e2e/unlaunched/unlaunched-codex-delivery.test.ts" + ], + "timeoutSeconds": 900, + "prSetup": ["build-core", "build-broker"], + "environment": { + "AGENT_RELAY_BIN": "{{brokerBinary}}" + }, + "forbidOutput": ["# SKIP", "skipped", "SKIP", "no tests"], + "evidence": "integration", + "coversCategories": ["broker"], + "coversFeatures": ["codex-queue-delivery"], + "issues": [1812, 1832] + }, { "id": "spawn-release-soak", "title": "Repeated spawn, task completion, release and same-name reclaim", From e5c95e38512add44ca9432f5b672b4ef9d86c9ff Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 22 Sep 2026 22:31:47 +0000 Subject: [PATCH 2/3] style: auto-format with Prettier --- flows/migrate/native-delivery.spec.ts | 14 ++++-- packages/cli/src/cli/agent-relay-mcp.ts | 3 +- scripts/migrate/mutation-proof.mjs | 47 +++++++++++-------- scripts/migrate/native-delivery-gates.mjs | 20 ++------ ...elivery-contract-evals.codex-queue.test.ts | 1 - tests/integration/broker/evals/runner.ts | 25 ++++------ 6 files changed, 53 insertions(+), 57 deletions(-) diff --git a/flows/migrate/native-delivery.spec.ts b/flows/migrate/native-delivery.spec.ts index b135dad59d..b2e30082ec 100644 --- a/flows/migrate/native-delivery.spec.ts +++ b/flows/migrate/native-delivery.spec.ts @@ -655,7 +655,10 @@ if (CONFIG.rust) { record('rust-fmt', `${CARGO} fmt --all -- --check`), record('rust-clippy', `${CARGO} clippy --all-targets -- -D warnings`), record('rust-build', `${CARGO} build --release --bin agent-relay-broker`), - record('invariant-tests', `${CARGO} test -p agent-relay-broker --features seam-probe --test delivery_seam_invariants`), + record( + 'invariant-tests', + `${CARGO} test -p agent-relay-broker --features seam-probe --test delivery_seam_invariants` + ), ].join('\n'), ['repair-rust'], 5_400_000 @@ -764,9 +767,7 @@ const nativeNames = [...Object.keys(CONFIG.evals ?? {}), ...Object.keys(CONFIG.e const nativeCommands = { ...(CONFIG.evals ?? {}), ...(CONFIG.e2e ?? {}) }; const nativeRecord = (name: string): string => record(name, nativeCommands[name]!, { - ...(Object.hasOwn(CONFIG.evals ?? {}, name) - ? { expect: ['delivery=100%', 'scenarios='] } - : {}), + ...(Object.hasOwn(CONFIG.evals ?? {}, name) ? { expect: ['delivery=100%', 'scenarios='] } : {}), forbid: ['# SKIP', 'skipped'], }); if (nativeNames.length > 0) { @@ -974,7 +975,10 @@ det( record('rust-fmt', `${CARGO} fmt --all -- --check`), record('rust-clippy', `${CARGO} clippy --all-targets -- -D warnings`), record('rust-build', `${CARGO} build --release --bin agent-relay-broker`), - record('invariant-tests', `${CARGO} test -p agent-relay-broker --features seam-probe --test delivery_seam_invariants`), + record( + 'invariant-tests', + `${CARGO} test -p agent-relay-broker --features seam-probe --test delivery_seam_invariants` + ), ] : []), record('ts-typecheck', 'npm run typecheck'), diff --git a/packages/cli/src/cli/agent-relay-mcp.ts b/packages/cli/src/cli/agent-relay-mcp.ts index 4e18b3fd88..fb9e454dce 100644 --- a/packages/cli/src/cli/agent-relay-mcp.ts +++ b/packages/cli/src/cli/agent-relay-mcp.ts @@ -924,7 +924,8 @@ async function verifyMetadataLanded( } const record = agents.find((agent) => (agent as { name?: string } | null)?.name === name) as - { metadata?: Record } | undefined; + | { metadata?: Record } + | undefined; if (!record) { return { diff --git a/scripts/migrate/mutation-proof.mjs b/scripts/migrate/mutation-proof.mjs index c18fbfb901..9e6e4f5414 100644 --- a/scripts/migrate/mutation-proof.mjs +++ b/scripts/migrate/mutation-proof.mjs @@ -165,7 +165,8 @@ const MUTATIONS = [ { transcript: 'mutation-06-unreachable.txt', invariants: ['settle_distinguishes_absence_from_an_unreachable_route'], - summary: 'An unreachable recorded route reported `NoReceipt` — positive evidence of absence — instead of `RouteUnavailable`.', + summary: + 'An unreachable recorded route reported `NoReceipt` — positive evidence of absence — instead of `RouteUnavailable`.', file: BACKEND, guards: [BACKEND, SEAM_TEST_FILE], find: ' return SettleOutcome::RouteUnavailable(route);', @@ -272,7 +273,9 @@ const MUTATIONS = [ "Codex queue exited with status {}{}",`, replace: ` Err(DeliveryError::unavailable(format!( "Codex queue exited with status {}{}",`, - command: libTest('delivery::codex_queue::tests::queue_process_failure_is_committed_and_does_not_fall_back'), + command: libTest( + 'delivery::codex_queue::tests::queue_process_failure_is_committed_and_does_not_fall_back' + ), }, { transcript: 'mutation-13-codex-handover.txt', @@ -311,7 +314,9 @@ const MUTATIONS = [ SendStatus::InDoubt, ));`, replace: ' let _ = &route;', - command: libTest('delivery::codex_queue::tests::a_cancelled_queue_send_is_not_retried_on_the_codex_route'), + command: libTest( + 'delivery::codex_queue::tests::a_cancelled_queue_send_is_not_retried_on_the_codex_route' + ), }, { transcript: 'mutation-17-teardown-indoubt.txt', @@ -322,7 +327,9 @@ const MUTATIONS = [ guards: [RUNTIME_DELIVERY, RUNTIME_TESTS], find: ' let Some(route) = handed_over_route_label(seam, pending) else {', replace: ' let Some(route) = None:: else {', - command: libTest('runtime::tests::releasing_an_agent_with_a_handed_over_native_delivery_dead_letters_it_in_doubt'), + command: libTest( + 'runtime::tests::releasing_an_agent_with_a_handed_over_native_delivery_dead_letters_it_in_doubt' + ), }, { transcript: 'mutation-18-teardown-sites.txt', @@ -355,7 +362,7 @@ const MUTATIONS = [ transcript: 'mutation-19-restart-rehydrate.txt', invariants: ['a_restarted_broker_does_not_queue_a_handed_over_codex_delivery_again'], summary: - '`DeliverySeam::restore_handed_over` became a no-op, so a reloaded pending snapshot classified `Fresh` and the fake Codex recorded a SECOND `codex queue` write for a message already in Codex\'s durable queue.', + "`DeliverySeam::restore_handed_over` became a no-op, so a reloaded pending snapshot classified `Fresh` and the fake Codex recorded a SECOND `codex queue` write for a message already in Codex's durable queue.", file: BACKEND, guards: [BACKEND, RUNTIME_DELIVERY, RUNTIME_TESTS], find: ` if self.was_sent(&delivery_id) { @@ -373,7 +380,7 @@ const MUTATIONS = [ transcript: 'mutation-20-codex-settle-route.txt', invariants: ['settlement_uses_the_recorded_thread_route_and_never_another_codex'], summary: - 'Settlement resolved through the first offered backend instead of the recorded route, seen through the REAL codex route: it read a different Codex thread\'s rollout.', + "Settlement resolved through the first offered backend instead of the recorded route, seen through the REAL codex route: it read a different Codex thread's rollout.", file: BACKEND, guards: [BACKEND, CODEX_QUEUE, RELAY_PTY_CODEX_SESSION], find: ` let Some(backend) = backends @@ -385,7 +392,9 @@ const MUTATIONS = [ replace: ` let Some(backend) = backends.iter_mut().next() else { return SettleOutcome::RouteUnavailable(route); };`, - command: libTest('delivery::codex_queue::tests::settlement_uses_the_recorded_thread_route_and_never_another_codex'), + command: libTest( + 'delivery::codex_queue::tests::settlement_uses_the_recorded_thread_route_and_never_another_codex' + ), }, { transcript: 'mutation-21-selection-guard.txt', @@ -399,7 +408,9 @@ const MUTATIONS = [ return None; }`, replace: ' let _ = normalize_cli_name(&command);', - command: libTest('delivery::codex_queue::tests::only_a_codex_worker_with_a_known_thread_selects_the_codex_queue_route'), + command: libTest( + 'delivery::codex_queue::tests::only_a_codex_worker_with_a_known_thread_selects_the_codex_queue_route' + ), }, { transcript: 'mutation-22-unselectable-prewrite.txt', @@ -424,7 +435,7 @@ const MUTATIONS = [ transcript: 'mutation-23-consumed-projection.txt', invariants: ['a_consumed_user_item_is_observed_in_both_real_projections'], summary: - 'The positive user-input matcher stopped recognising Codex\'s two real consumed projections, so a genuinely consumed message never acknowledged.', + "The positive user-input matcher stopped recognising Codex's two real consumed projections, so a genuinely consumed message never acknowledged.", file: CODEX_THREAD, guards: [CODEX_THREAD], find: `fn node_is_user_input(node: &Value) -> bool {`, @@ -432,7 +443,9 @@ const MUTATIONS = [ if true { return false; }`, - command: libTest('delivery::codex_thread::tests::a_consumed_user_item_is_observed_in_both_real_projections'), + command: libTest( + 'delivery::codex_thread::tests::a_consumed_user_item_is_observed_in_both_real_projections' + ), }, { transcript: 'mutation-24-queued-not-consumed.txt', @@ -456,7 +469,9 @@ const MUTATIONS = [ return true; }`, replace: ' return true;', - command: libTest('delivery::codex_thread::tests::a_quoted_marker_in_a_non_user_record_is_not_an_acknowledgement'), + command: libTest( + 'delivery::codex_thread::tests::a_quoted_marker_in_a_non_user_record_is_not_an_acknowledgement' + ), }, { transcript: 'mutation-26-native-parks.txt', @@ -533,10 +548,7 @@ function main() { originals.delete(entry.file); } const bites = /FAILED|panicked|assertion .*failed/.test(output); - writeFileSync( - path.join(evidence, entry.transcript), - `$ ${entry.command.join(' ')}\n\n${output}\n` - ); + writeFileSync(path.join(evidence, entry.transcript), `$ ${entry.command.join(' ')}\n\n${output}\n`); results.push({ ...entry, bites }); process.stdout.write(`${bites ? 'BITES ' : 'NO-BITE '} ${entry.transcript}\n`); } @@ -603,10 +615,7 @@ function main() { function renderMarkdown(results) { const sections = []; for (const entry of results) { - const transcript = readFileSync( - path.join(arg('--art'), 'evidence', entry.transcript), - 'utf8' - ).trim(); + const transcript = readFileSync(path.join(arg('--art'), 'evidence', entry.transcript), 'utf8').trim(); for (const invariant of entry.invariants) { sections.push( `### ${invariant}\n\n**Mutation:** ${entry.summary}\n\n` + diff --git a/scripts/migrate/native-delivery-gates.mjs b/scripts/migrate/native-delivery-gates.mjs index 1848d4fab6..4297ab9cf0 100755 --- a/scripts/migrate/native-delivery-gates.mjs +++ b/scripts/migrate/native-delivery-gates.mjs @@ -1639,18 +1639,14 @@ function mutationFreshnessProblems(config, art) { } catch (error) { return [`evidence/mutation-proof.json is not valid JSON: ${error.message}`]; } - const entries = new Map( - (manifest.invariants ?? []).map((entry) => [entry.name, entry]) - ); + const entries = new Map((manifest.invariants ?? []).map((entry) => [entry.name, entry])); const digests = manifest.sources ?? {}; const actual = new Map(); const digestOf = (file) => { if (!actual.has(file)) { actual.set( file, - existsSync(file) - ? createHash('sha256').update(readFileSync(file)).digest('hex') - : null + existsSync(file) ? createHash('sha256').update(readFileSync(file)).digest('hex') : null ); } return actual.get(file); @@ -1669,19 +1665,13 @@ function mutationFreshnessProblems(config, art) { const transcript = path.join(art, 'evidence', entry.transcript ?? ''); if (!entry.transcript || !existsSync(transcript)) { problems.push(`mutation transcript missing for invariant ${invariant}: ${entry.transcript}`); - } else if ( - !/FAILED|panicked|assertion .*failed/.test(readFileSync(transcript, 'utf8')) - ) { - problems.push( - `mutation transcript for ${invariant} records no failure: ${entry.transcript}` - ); + } else if (!/FAILED|panicked|assertion .*failed/.test(readFileSync(transcript, 'utf8'))) { + problems.push(`mutation transcript for ${invariant} records no failure: ${entry.transcript}`); } for (const file of guards) { const recorded = digests[file]; if (!recorded) { - problems.push( - `mutation-proof.json records no digest for ${file}, guarded by ${invariant}` - ); + problems.push(`mutation-proof.json records no digest for ${file}, guarded by ${invariant}`); continue; } const current = digestOf(file); diff --git a/tests/fixtures/delivery-contract-evals.codex-queue.test.ts b/tests/fixtures/delivery-contract-evals.codex-queue.test.ts index b6a97c91e6..5196d7eb22 100644 --- a/tests/fixtures/delivery-contract-evals.codex-queue.test.ts +++ b/tests/fixtures/delivery-contract-evals.codex-queue.test.ts @@ -280,7 +280,6 @@ describe('phase-1 codex-queue route, at the delivery contract', () => { expect(failed, JSON.stringify(failed, null, 2)).toEqual([]); }); } - }); /** diff --git a/tests/integration/broker/evals/runner.ts b/tests/integration/broker/evals/runner.ts index eee4b0432a..57f0440083 100644 --- a/tests/integration/broker/evals/runner.ts +++ b/tests/integration/broker/evals/runner.ts @@ -442,25 +442,20 @@ async function main(): Promise { matrix.harnesses[label] = report.metrics; allReports.push({ harness: label, report }); printMetrics(label, report.metrics); - if (report.metrics.scenariosTotal === 0 || report.metrics.scenariosPassed < report.metrics.scenariosTotal) { + if ( + report.metrics.scenariosTotal === 0 || + report.metrics.scenariosPassed < report.metrics.scenariosTotal + ) { anyScenarioFailure = true; console.error( ` scenarios failed: ${report.metrics.scenariosPassed}/${report.metrics.scenariosTotal} passed` ); } - if ( - flags.minScenarios !== undefined && - report.metrics.scenariosPassed < flags.minScenarios - ) { + if (flags.minScenarios !== undefined && report.metrics.scenariosPassed < flags.minScenarios) { anyFloorFailure = true; - console.error( - ` scenario floor missed: ${report.metrics.scenariosPassed} < ${flags.minScenarios}` - ); + console.error(` scenario floor missed: ${report.metrics.scenariosPassed} < ${flags.minScenarios}`); } - if ( - flags.minDeliveryRate !== undefined && - report.metrics.deliverySuccessRate < flags.minDeliveryRate - ) { + if (flags.minDeliveryRate !== undefined && report.metrics.deliverySuccessRate < flags.minDeliveryRate) { anyFloorFailure = true; console.error( ` delivery floor missed: ${report.metrics.deliverySuccessRate} < ${flags.minDeliveryRate}` @@ -504,10 +499,8 @@ async function main(): Promise { const strictFailure = flags.requireAll && (anySkippedHarness || anyScenarioFailure || allReports.length === 0); - const floorWasRequested = - flags.minScenarios !== undefined || flags.minDeliveryRate !== undefined; - const floorFailure = - floorWasRequested && (anySkippedHarness || anyFloorFailure || allReports.length === 0); + const floorWasRequested = flags.minScenarios !== undefined || flags.minDeliveryRate !== undefined; + const floorFailure = floorWasRequested && (anySkippedHarness || anyFloorFailure || allReports.length === 0); process.exit(anyRegression || strictFailure || floorFailure ? 1 : 0); } From 982322a37ff61d91dd5fbe37d38b47ae9ef787f9 Mon Sep 17 00:00:00 2001 From: Miya Date: Wed, 23 Sep 2026 00:32:13 +0200 Subject: [PATCH 3/3] docs(trajectory): record Phase 1 stacked draft --- .../active/traj_nd5a2dd4waas/trajectory.json | 31 +++++++++++++++++-- .agentworkforce/trajectories/index.json | 5 --- 2 files changed, 29 insertions(+), 7 deletions(-) delete mode 100644 .agentworkforce/trajectories/index.json diff --git a/.agentworkforce/trajectories/active/traj_nd5a2dd4waas/trajectory.json b/.agentworkforce/trajectories/active/traj_nd5a2dd4waas/trajectory.json index 19e44b00f5..129148b200 100644 --- a/.agentworkforce/trajectories/active/traj_nd5a2dd4waas/trajectory.json +++ b/.agentworkforce/trajectories/active/traj_nd5a2dd4waas/trajectory.json @@ -6,8 +6,35 @@ }, "status": "active", "startedAt": "2026-09-21T06:13:08.229Z", - "agents": [], - "chapters": [], + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-09-22T22:32:13.924Z" + } + ], + "chapters": [ + { + "id": "chap_60kdytoq4uet", + "title": "Work", + "agentName": "default", + "startedAt": "2026-09-22T22:32:13.924Z", + "events": [ + { + "ts": 1790116333925, + "type": "decision", + "content": "Published Phase 1 as stacked draft PR #1846 on Phase 0 PR #1825: Published Phase 1 as stacked draft PR #1846 on Phase 0 PR #1825", + "raw": { + "question": "Published Phase 1 as stacked draft PR #1846 on Phase 0 PR #1825", + "chosen": "Published Phase 1 as stacked draft PR #1846 on Phase 0 PR #1825", + "alternatives": [], + "reasoning": "The user requested the work be pushed now. A separate draft branch preserves Phase 0 review scope, exposes the green deterministic evidence, and keeps the pending dual-provider signoff explicit without pushing to main." + }, + "significance": "high" + } + ] + } + ], "commits": [], "filesChanged": [], "projectId": "AgentWorkforce/relay", diff --git a/.agentworkforce/trajectories/index.json b/.agentworkforce/trajectories/index.json deleted file mode 100644 index e736ceed3b..0000000000 --- a/.agentworkforce/trajectories/index.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "version": 1, - "lastUpdated": "2026-07-16T20:05:37.749Z", - "trajectories": {} -}