From 6bdec7b2026d94b01a77156a7964e4a8bf4bfcfd Mon Sep 17 00:00:00 2001 From: KJGBot Date: Thu, 8 Oct 2026 16:52:34 -0700 Subject: [PATCH 01/24] fix(broker): verify PTY turn submission (#1891) * fix(broker): verify PTY turn submission * style: auto-format with Prettier * docs: clarify PTY human ownership * fix(broker): serialize PTY acceptance recovery * fix: harden PTY acceptance recovery * fix: dedupe delivery stuck transitions * fix: distinguish drafts from PTY activity * fix(broker): harden delivery acceptance edges * fix(broker): close harness recovery edge cases * fix(broker): remove dead wrap lifetime reset * fix(broker): normalize Gemini composer borders * fix(broker): confirm Codex composer receipt across reflow * fix(broker): recognize native Codex idle placeholder after recovery --------- Co-authored-by: kjgbot Co-authored-by: github-actions[bot] Co-authored-by: Miya Co-authored-by: Khaliq --- CHANGELOG.md | 1 + .../src/broker/delivery_verification.rs | 725 ++++++++++++- crates/broker/src/protocol.rs | 82 +- crates/broker/src/pty_worker.rs | 995 ++++++++++++++---- crates/broker/src/runtime/delivery.rs | 19 +- crates/broker/src/runtime/fleet.rs | 9 +- crates/broker/src/runtime/tests.rs | 156 ++- crates/broker/src/runtime/worker_events.rs | 176 +++- crates/broker/src/wrap.rs | 519 +++++---- crates/relay-pty/src/detection.rs | 73 +- crates/relay-pty/src/snapshot.rs | 74 ++ docs/harnesses/devin.md | 7 +- docs/harnesses/pty-delivery.md | 59 ++ .../contracts/fixtures/event-fixtures.json | 29 + .../harness-driver/src/lifecycle-hooks.ts | 2 + packages/harness-driver/src/protocol.ts | 49 +- packages/sdk-py/src/agent_relay/protocol.py | 3 +- .../AgentRelayBrokerSDK/BrokerTypes.swift | 10 +- .../case.json | 21 + .../real-codex.mjs | 436 ++++++++ .../run.mjs | 320 ++++++ 21 files changed, 3256 insertions(+), 509 deletions(-) create mode 100644 docs/harnesses/pty-delivery.md create mode 100644 tests/relayflows/cases/1891-codex-parked-composer-recovery/case.json create mode 100644 tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs create mode 100644 tests/relayflows/cases/1891-codex-parked-composer-recovery/run.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 21f3330a17..aebbe86cd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Broker fleet control keeps a healthy node WebSocket open when Relaycast returns retryable `d1_pressure` for `node.register` or `inventory.sync`, retrying the same frame with bounded exponential backoff while the existing registration and application-liveness deadlines remain fail-closed. +- Broker-managed PTY messages now remain pending until the harness accepts them; failed delivery blocks further sends, with up to two submit-only retries (three total attempts) instead of silently parked input. Codex recovery uses `End`, a distinct carriage return, and a guarded line-feed fallback without replaying the body; human input takes ownership and cancels recovery. Upgrade and restart brokers and PTY agents for the planned 13.2.0 rollout. - `--state-dir` on `node agent` commands, `node status`, and `node down` also accepts a fleet node directory whose broker state lives in `state/`. - An explicit `--state-dir` on `node agent` commands is no longer overridden by `RELAY_BROKER_URL` / `RELAY_BROKER_API_KEY`. - "No broker connection" errors name the path searched and whether it was the project default or `--state-dir`, instead of implying a running broker is down. diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 93da568903..55c8aa8553 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -8,8 +8,10 @@ use serde_json::{json, Value}; use crate::{ ids::{DeliveryId, EventId, MessageTarget, RequestId, WorkspaceAlias, WorkspaceId}, + readiness::{cli_prompt_ready, GridReadinessSnapshot}, + snapshot::Snapshot, util::ansi::strip_ansi, - worker::detection::ActivityDetector, + worker::detection::{is_codex_busy_status_line, ActivityDetector}, }; pub(crate) const ACTIVITY_WINDOW: Duration = Duration::from_secs(5); @@ -26,6 +28,7 @@ pub(crate) enum DeliveryOutcome { /// Neither speeds up nor backs off the throttle, but breaks the /// consecutive-success streak so unverified deliveries never drive /// the delay down. + #[allow(dead_code)] Unverified, Failed, } @@ -92,14 +95,16 @@ pub(crate) struct PendingActivity { pub detector: ActivityDetector, } -/// Maximum number of injection attempts before accepting delivery via timeout. -/// Set to 1 to avoid re-injecting the same message when echo detection fails - -/// duplicate injections cause agents to process messages multiple times, -/// multiplying Relaycast API calls and triggering rate limits. -pub(crate) const MAX_VERIFICATION_ATTEMPTS: usize = 1; +/// Maximum number of submit-key attempts for one already-written body. +/// +/// A retry never writes the body again: it only nudges a composer that is +/// still proven to contain this delivery. This preserves at-most-once turn +/// creation while giving paste-aware TUIs a bounded recovery path. +pub(crate) const MAX_VERIFICATION_ATTEMPTS: usize = 3; /// Time window to wait for echo verification before accepting delivery. pub(crate) const VERIFICATION_WINDOW: std::time::Duration = std::time::Duration::from_secs(5); +pub(crate) const MAX_ACCEPTANCE_LIFETIME: std::time::Duration = std::time::Duration::from_secs(30); /// A pending delivery waiting for echo verification in PTY output. #[derive(Debug)] @@ -113,14 +118,307 @@ pub(crate) struct PendingVerification { /// PTY reader and this event loop. pub output_boundary: u64, pub injected_at: std::time::Instant, + pub verification_started_at: std::time::Instant, pub attempts: usize, pub max_attempts: usize, pub request_id: Option, + #[allow(dead_code)] pub workspace_id: Option, + #[allow(dead_code)] pub workspace_alias: Option, + #[allow(dead_code)] pub from: String, + #[allow(dead_code)] pub body: String, + #[allow(dead_code)] pub target: MessageTarget, + /// The formatted body appeared in post-submission PTY output. This proves + /// terminal echo only; it does not prove the harness accepted a turn. + pub echo_seen: bool, + /// Post-submission output retained until a harness activity marker proves + /// the turn started. Kept separately from `VerificationOutput` so matching + /// remains stable after the global tail trims. + pub activity_buffer: String, + pub detector: ActivityDetector, +} + +impl PendingVerification { + pub(crate) fn observe_raw_process_receipt(&mut self, output: &VerificationOutput, cli: &str) { + if self.echo_seen || !is_cat_process(cli) || self.body.is_empty() { + return; + } + let observed = strip_ansi(&output.since(self.output_boundary)); + if observed.contains(&self.body) || observed.replace("\r\n", "\n").contains(&self.body) { + self.echo_seen = true; + } + } + + /// A native TUI may wrap or repaint the injected text, so its raw PTY + /// output need not contain the formatted body as one contiguous string. + /// The injection gate starts from an empty composer and delivery stays + /// single-flight. Seeing this delivery's tail in the *current* composer + /// therefore proves the editor received it. Keep that fact when a later + /// submit clears the composer. + pub(crate) fn observe_visible_composer(&mut self, snapshot: &Snapshot, cli: &str) { + if self.echo_seen { + return; + } + let tail = expected_tail(&self.expected_echo); + if !tail.is_empty() + && current_composer(snapshot, cli) + .is_some_and(|composer| compact_render(&composer).contains(&tail)) + { + self.echo_seen = true; + self.activity_buffer.clear(); + } + } + + pub(crate) fn observe(&mut self, output: &VerificationOutput, text: &str) { + let echo_now = pending_verification_echo_seen(output, self); + if !self.echo_seen && echo_now { + // Ignore activity that preceded the editor echo. A message may be + // injected while a previous turn is still rendering its own busy + // marker; that marker cannot prove this delivery was accepted. + self.echo_seen = true; + self.activity_buffer.clear(); + let clean = strip_ansi(&output.since(self.output_boundary)); + if let Some(index) = clean.rfind(&self.expected_echo) { + self.activity_buffer + .push_str(&clean[index + self.expected_echo.len()..]); + } + } else if self.echo_seen { + self.activity_buffer.push_str(text); + } + if self.activity_buffer.len() > ACTIVITY_BUFFER_MAX_BYTES { + let start = crate::util::ansi::floor_char_boundary( + &self.activity_buffer, + self.activity_buffer.len() - ACTIVITY_BUFFER_KEEP_BYTES, + ); + self.activity_buffer = self.activity_buffer[start..].to_string(); + } + } + + pub(crate) fn accepted_activity(&self) -> Option { + if !self.echo_seen || !self.detector.has_explicit_patterns() { + return None; + } + self.detector + .detect_activity(&self.activity_buffer, &self.expected_echo) + } + + pub(crate) fn acceptance_expired(&self) -> bool { + self.verification_started_at.elapsed() >= MAX_ACCEPTANCE_LIFETIME + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum HarnessAcceptance { + Accepted(String), + Parked, + Inconclusive, +} + +fn compact_render(text: &str) -> String { + text.chars() + .filter(|character| !character.is_whitespace()) + .collect() +} + +fn is_cat_process(cli: &str) -> bool { + cli.rsplit(['/', '\\']) + .next() + .is_some_and(|name| matches!(name.to_ascii_lowercase().as_str(), "cat" | "cat.exe")) +} + +fn expected_tail(expected: &str) -> String { + let compact = compact_render(expected); + let keep = compact.len().min(96); + let start = crate::util::ansi::floor_char_boundary(&compact, compact.len() - keep); + compact[start..].to_string() +} + +fn gemini_composer_row(line: &str) -> &str { + let trimmed = line.trim_start(); + let Some(bordered) = trimmed.strip_prefix('│') else { + return trimmed; + }; + let bordered = bordered.trim_start(); + bordered + .trim_end() + .strip_suffix('│') + .map(str::trim_end) + .unwrap_or(bordered) +} + +fn current_composer(snapshot: &Snapshot, cli: &str) -> Option { + let plain = snapshot.to_plain_through_cursor(); + let lines: Vec<_> = plain.lines().collect(); + let end = snapshot.cursor.0.checked_sub(1)? as usize; + if end >= lines.len() { + return None; + } + let lower = cli.to_ascii_lowercase(); + if lower.contains("gemini") { + if lines + .get(end) + .is_some_and(|line| cursor_row_is_activity(cli, line)) + { + return None; + } + // Gemini renders its live input with a `> ` prefix (inside a `│` + // border on terminals that cannot use the background-colour frame). + // Anchor to that boundary instead of treating arbitrary transcript + // rows near the cursor as the composer. Keeping every row through the + // cursor also preserves long, wrapped drafts beyond four rows. + let start = lines.iter().take(end + 1).rposition(|line| { + let input = gemini_composer_row(line); + input == ">" || input.starts_with("> ") + })?; + return Some( + lines[start..=end] + .iter() + .map(|line| gemini_composer_row(line)) + .collect::>() + .join("\n"), + ); + } + let is_prompt = |line: &&str| { + let trimmed = line.trim_start(); + if lower.contains("codex") { + trimmed == "›" + || trimmed.starts_with("› ") + || trimmed == "codex>" + || trimmed.starts_with("codex> ") + } else if lower.contains("claude") { + trimmed == "❯" + || trimmed.starts_with("❯ ") + || trimmed == ">" + || trimmed.starts_with("> ") + } else if crate::readiness::is_devin_cli(cli) { + trimmed == "❭" || trimmed.starts_with("❭ ") + } else { + trimmed == ">" + || trimmed.starts_with("> ") + || trimmed == "$" + || trimmed.starts_with("$ ") + || trimmed == ">>>" + || trimmed.starts_with(">>> ") + || trimmed == "›" + || trimmed.starts_with("› ") + || trimmed == "❯" + || trimmed.starts_with("❯ ") + } + }; + let start = lines.iter().take(end + 1).rposition(is_prompt)?; + let composer_lines = &lines[start..=end]; + if composer_lines.len() > 1 + && composer_lines + .last() + .is_some_and(|line| cursor_row_is_activity(cli, line)) + { + // The latest prompt-looking line is part of the submitted transcript; + // a distinctive harness status row owns the cursor. Do not use the + // broader ActivityDetector patterns here because ordinary multiline + // draft text can contain Tool:, Write(, or shell prompts. + return None; + } + Some(composer_lines.join("\n")) +} + +fn cursor_row_is_activity(cli: &str, line: &str) -> bool { + let lower_cli = cli.to_ascii_lowercase(); + let trimmed = line.trim_start(); + let lower_line = trimmed.to_ascii_lowercase(); + if lower_cli.contains("codex") { + is_codex_busy_status_line(line) + } else if lower_cli.contains("claude") { + lower_line.contains("esc to interrupt") + || matches!(trimmed.chars().next(), Some('⠋' | '⠙' | '⠹')) + } else if crate::readiness::is_devin_cli(cli) { + trimmed.starts_with("Thinking ·") || trimmed.starts_with("Guide Devin while it works") + } else if lower_cli.contains("gemini") { + trimmed.starts_with("Generating") + || trimmed.starts_with("Action:") + || trimmed.starts_with("Executing") + } else { + false + } +} + +fn codex_busy(screen: &str) -> bool { + screen.lines().any(is_codex_busy_status_line) +} + +fn composer_is_idle(snapshot: &Snapshot, cli: &str) -> bool { + let screen = snapshot.to_plain(); + if cli.to_ascii_lowercase().contains("codex") && codex_busy(&screen) { + return false; + } + let codex_placeholder_after_cursor = cli.to_ascii_lowercase().contains("codex") + && snapshot.cursor.1 == 3 + && screen + .lines() + .nth(snapshot.cursor.0.saturating_sub(1) as usize) + .is_some_and(|line| line.trim_start() == "› Ask Codex to do anything"); + cli_prompt_ready( + cli, + GridReadinessSnapshot { + screen: &screen, + cursor: Some(snapshot.cursor), + }, + ) && (!snapshot.has_visible_text_at_or_after_cursor() || codex_placeholder_after_cursor) + && current_composer(snapshot, cli) + .map(|composer| { + let trimmed = composer.trim(); + matches!(trimmed, "›" | "codex>" | "❯" | "❭" | ">" | "$" | ">>>") + || trimmed.contains("Ask Codex to do anything") + || trimmed.contains("Type your message or @path/to/file") + || trimmed + .contains("Ask Devin to build features, fix bugs, or work on your code") + }) + .unwrap_or(false) +} + +/// Distinguish terminal echo from actual harness acceptance. +/// +/// Activity is definitive acceptance. A body is considered parked only when +/// its tail is still in the live composer at the cursor. Once echo was seen, +/// a proven empty composer is also acceptance. Every other state is +/// inconclusive and must fail or wait — never blindly press a key. +pub(crate) fn assess_harness_acceptance( + cli: &str, + verification: &PendingVerification, + snapshot: &Snapshot, +) -> HarnessAcceptance { + let tail = expected_tail(&verification.expected_echo); + let parked = !tail.is_empty() + && current_composer(snapshot, cli) + .map(|composer| compact_render(&composer).contains(&tail)) + .unwrap_or(false); + if parked { + return HarnessAcceptance::Parked; + } + if let Some(pattern) = verification.accepted_activity() { + return HarnessAcceptance::Accepted(format!("activity:{pattern}")); + } + if verification.echo_seen && composer_is_idle(snapshot, cli) { + return HarnessAcceptance::Accepted("composer_cleared".to_string()); + } + // `cat` is also a supported local PTY recipient. It has no composer or + // turn-start marker; its PTY echo confirms transport receipt. Keep native + // agent TUIs on the stronger gate. + if verification.echo_seen && is_cat_process(cli) { + return HarnessAcceptance::Accepted("process_echo".to_string()); + } + if verification.echo_seen && !verification.detector.has_explicit_patterns() { + if let Some(pattern) = verification + .detector + .detect_activity(&verification.activity_buffer, &verification.expected_echo) + { + return HarnessAcceptance::Accepted(format!("activity:{pattern}")); + } + } + HarnessAcceptance::Inconclusive } #[derive(Debug)] @@ -214,26 +512,14 @@ pub(crate) fn pending_verification_echo_seen( verification: &PendingVerification, ) -> bool { let observed = output.since(verification.output_boundary); - check_echo_in_output(&observed, &verification.expected_echo) -} - -/// Return a verification whose echo arrived before the PTY write ack was -/// processed, otherwise queue it for later output-driven verification. -/// -/// Both wrap-mode and fleet PTY workers use this policy. Keeping the decision -/// here ensures the two delivery paths cannot drift on the echo-before-ack -/// race while still letting each caller perform its own confirmation effects. -pub(crate) fn queue_or_take_confirmed_verification( - verification: PendingVerification, - output: &VerificationOutput, - pending_verifications: &mut std::collections::VecDeque, -) -> Option { - if pending_verification_echo_seen(output, &verification) { - Some(verification) - } else { - pending_verifications.push_back(verification); - None + if check_echo_in_output(&observed, &verification.expected_echo) { + return true; } + // Native TUIs can wrap and indent every line of a long delivery. The + // receive-time boundary still excludes older turns; compare this body's + // compact tail when the full byte-for-byte echo is unavailable. + let tail = expected_tail(&verification.expected_echo); + !tail.is_empty() && compact_render(&strip_ansi(&observed)).contains(&tail) } /// Start activity detection with every post-submission byte already observed. @@ -243,6 +529,7 @@ pub(crate) fn queue_or_take_confirmed_verification( /// acknowledgement later confirms the delivery, seeding this buffer prevents /// the already-observed activity from being lost merely because no more output /// follows. +#[cfg(test)] pub(crate) fn pending_activity_from_confirmed_output( verification: &PendingVerification, output: &VerificationOutput, @@ -262,6 +549,7 @@ pub(crate) fn pending_activity_from_confirmed_output( /// state for later output. The returned activity carries the matched pattern /// so each runtime can emit its own protocol event or log without duplicating /// this race-handling policy. +#[cfg(test)] pub(crate) fn queue_or_take_detected_activity( verification: &PendingVerification, output: &VerificationOutput, @@ -283,7 +571,7 @@ pub(crate) fn queue_or_take_detected_activity( /// Check if the expected echo string appears in PTY output (after stripping ANSI). pub(crate) fn check_echo_in_output(output: &str, expected: &str) -> bool { let clean = strip_ansi(output); - clean.contains(expected) + clean.contains(expected) || clean.replace("\r\n", "\n").contains(expected) } pub(crate) fn current_timestamp_ms() -> u64 { @@ -325,6 +613,375 @@ pub(crate) fn delivery_injected_event_payload( mod tests { use super::*; + fn codex_verification(expected: &str) -> PendingVerification { + PendingVerification { + delivery_id: "delivery-acceptance".into(), + event_id: "event-acceptance".into(), + expected_echo: expected.to_string(), + output_boundary: 0, + injected_at: Instant::now(), + verification_started_at: Instant::now(), + attempts: 1, + max_attempts: MAX_VERIFICATION_ATTEMPTS, + request_id: None, + workspace_id: None, + workspace_alias: None, + from: "Lead".to_string(), + body: "fix idle injection".to_string(), + target: "Worker".into(), + echo_seen: true, + activity_buffer: String::new(), + detector: ActivityDetector::for_cli("codex"), + } + } + + #[cfg(unix)] + async fn codex_snapshot(screen: &str) -> (crate::pty::PtySession, Snapshot) { + let encoded = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + screen.as_bytes(), + ); + let script = format!( + "stty raw -echo; printf '\\033[2J\\033[H'; printf '%s' '{encoded}' | base64 -d; sleep 3" + ); + let (pty, mut rx) = + crate::pty::PtySession::spawn("/bin/sh", &["-c".into(), script], 24, 120).unwrap(); + let expected_tail = screen + .split('\x1b') + .next() + .and_then(|visible| visible.chars().last()) + .unwrap_or('›'); + for _ in 0..100 { + let _ = tokio::time::timeout(Duration::from_millis(20), rx.recv()).await; + if pty.screen_text().contains(expected_tail) { + break; + } + } + tokio::time::sleep(Duration::from_millis(30)).await; + let snapshot = Snapshot::capture(&pty); + (pty, snapshot) + } + + #[cfg(unix)] + #[tokio::test] + async fn editor_echo_is_parked_until_codex_accepts_the_turn() { + let expected = "Relay message from Lead [evt]: fix idle injection"; + let (pty, snapshot) = codex_snapshot(&format!("› {expected}")).await; + let verification = codex_verification(expected); + + assert_eq!( + assess_harness_acceptance("codex", &verification, &snapshot), + HarnessAcceptance::Parked, + "visible composer text must never be mistaken for acceptance" + ); + pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn visible_wrapped_composer_tail_survives_codex_submit() { + let expected = "Relay message from Lead [evt]: Reply with exactly WRAPPED_CODEX_ACK"; + let (parked_pty, parked) = codex_snapshot( + "› Relay message from Lead [evt]: Reply with exactly\n WRAPPED_CODEX_ACK", + ) + .await; + let mut verification = codex_verification(expected); + verification.echo_seen = false; + verification.observe_visible_composer(&parked, "codex"); + assert!(verification.echo_seen); + assert_eq!( + assess_harness_acceptance("codex", &verification, &parked), + HarnessAcceptance::Parked, + ); + parked_pty.shutdown().unwrap(); + + let (idle_pty, idle) = codex_snapshot("› Ask Codex to do anything").await; + assert_eq!( + assess_harness_acceptance("codex", &verification, &idle), + HarnessAcceptance::Accepted("composer_cleared".to_string()), + ); + idle_pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn cat_process_echo_confirms_local_pty_receipt() { + let (pty, snapshot) = codex_snapshot("LOCAL_WORK_PROOF").await; + let mut verification = codex_verification("LOCAL_WORK_PROOF"); + verification.body = "LOCAL_WORK_PROOF".to_string(); + verification.echo_seen = false; + assert_eq!( + assess_harness_acceptance("cat", &verification, &snapshot), + HarnessAcceptance::Inconclusive, + ); + let mut output = VerificationOutput::default(); + output.push_str("raw PTY echo: LOCAL_WORK_PROOF\r\n"); + verification.observe_raw_process_receipt(&output, "/bin/cat"); + assert!(verification.echo_seen); + assert_eq!( + assess_harness_acceptance("cat", &verification, &snapshot), + HarnessAcceptance::Accepted("process_echo".to_string()), + ); + pty.shutdown().unwrap(); + } + + #[test] + fn pty_crlf_echo_matches_formatted_message() { + assert!(check_echo_in_output( + "Relay message from Lead:\r\n LOCAL_WORK_PROOF", + "Relay message from Lead:\n LOCAL_WORK_PROOF", + )); + } + + #[cfg(unix)] + #[tokio::test] + async fn codex_activity_and_cleared_composer_prove_acceptance() { + let expected = "Relay message from Lead [evt]: fix idle injection"; + let (busy_pty, busy_snapshot) = + codex_snapshot(&format!("› {expected}\nWorking (1s • esc to interrupt)")).await; + let mut active = codex_verification(expected); + active + .activity_buffer + .push_str("Working (1s • esc to interrupt)"); + assert!(matches!( + assess_harness_acceptance("codex", &active, &busy_snapshot), + HarnessAcceptance::Accepted(ref evidence) if evidence.starts_with("activity:") + )); + busy_pty.shutdown().unwrap(); + + let (idle_pty, idle_snapshot) = codex_snapshot("› Ask Codex to do anything").await; + let cleared = codex_verification(expected); + assert_eq!( + assess_harness_acceptance("codex", &cleared, &idle_snapshot), + HarnessAcceptance::Accepted("composer_cleared".to_string()) + ); + idle_pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn codex_idle_placeholder_to_right_of_cursor_proves_cleared_composer() { + let expected = "Relay message from Lead [evt]: fix idle injection"; + let (pty, snapshot) = codex_snapshot("› Ask Codex to do anything\x1b[1;3H").await; + assert_eq!(snapshot.cursor, (1, 3)); + assert!(snapshot.has_visible_text_at_or_after_cursor()); + let verification = codex_verification(expected); + assert_eq!( + assess_harness_acceptance("codex", &verification, &snapshot), + HarnessAcceptance::Accepted("composer_cleared".to_string()) + ); + pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn generic_redraw_cannot_confirm_a_body_still_parked() { + let expected = "Relay message from Lead [evt]: fix idle injection"; + let (pty, snapshot) = codex_snapshot(&format!("› {expected}")).await; + let mut verification = codex_verification(expected); + verification.detector = ActivityDetector::for_cli("muse"); + verification + .activity_buffer + .push_str("composer repaint after echo"); + + assert_eq!( + assess_harness_acceptance("muse", &verification, &snapshot), + HarnessAcceptance::Parked, + "generic output must not outrank a visibly parked composer" + ); + pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn previous_turn_busy_redraw_cannot_confirm_new_parked_body() { + let expected = "Relay message from Lead [evt]: queued while busy"; + let screen = format!("› previous request\nWorking (4s • esc to interrupt)\n› {expected}"); + let (pty, snapshot) = codex_snapshot(&screen).await; + let mut verification = codex_verification(expected); + verification + .activity_buffer + .push_str("Working (5s • esc to interrupt)"); + + assert_eq!( + assess_harness_acceptance("codex", &verification, &snapshot), + HarnessAcceptance::Parked, + "a continuing marker from the previous turn cannot outrank the new live composer" + ); + pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn multiline_draft_activity_words_remain_parked() { + let expected = "Relay message from Lead [evt]: call Write(test)"; + let screen = format!("❯ {expected}\nTool: still part of the draft"); + let (pty, snapshot) = codex_snapshot(&screen).await; + let mut verification = codex_verification(expected); + verification.detector = ActivityDetector::for_cli("claude"); + verification + .activity_buffer + .push_str("Tool: still part of the draft"); + + assert_eq!( + assess_harness_acceptance("claude", &verification, &snapshot), + HarnessAcceptance::Parked + ); + pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn claude_interrupt_status_is_activity_not_a_parked_transcript() { + let expected = "Relay message from Lead [evt]: fix busy Claude"; + let screen = format!("❯ {expected}\nResponding… esc to interrupt"); + let (pty, snapshot) = codex_snapshot(&screen).await; + let mut verification = codex_verification(expected); + verification.detector = ActivityDetector::for_cli("claude"); + verification + .activity_buffer + .push_str("Responding… esc to interrupt"); + + assert!(matches!( + assess_harness_acceptance("claude", &verification, &snapshot), + HarnessAcceptance::Accepted(ref evidence) if evidence == "activity:esc to interrupt" + )); + pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn gemini_composer_is_prompt_anchored_and_keeps_long_drafts() { + let expected = "a".repeat(120); + let wrapped = expected + .as_bytes() + .chunks(20) + .map(|chunk| std::str::from_utf8(chunk).unwrap()) + .collect::>() + .join("\n"); + let (parked_pty, parked_snapshot) = codex_snapshot(&format!("> {wrapped}")).await; + let mut parked = codex_verification(&expected); + parked.detector = ActivityDetector::for_cli("gemini"); + assert_eq!( + assess_harness_acceptance("gemini", &parked, &parked_snapshot), + HarnessAcceptance::Parked, + "Gemini drafts longer than four rows retain the complete recovery tail" + ); + parked_pty.shutdown().unwrap(); + + let bordered = expected + .as_bytes() + .chunks(20) + .enumerate() + .map(|(index, chunk)| { + let prefix = if index == 0 { "> " } else { " " }; + format!("│ {prefix}{} │", std::str::from_utf8(chunk).unwrap()) + }) + .collect::>() + .join("\n"); + let (bordered_pty, bordered_snapshot) = codex_snapshot(&bordered).await; + let mut bordered_verification = codex_verification(&expected); + bordered_verification.detector = ActivityDetector::for_cli("gemini"); + assert_eq!( + assess_harness_acceptance("gemini", &bordered_verification, &bordered_snapshot), + HarnessAcceptance::Parked, + "Gemini border glyphs must not split a wrapped recovery tail" + ); + bordered_pty.shutdown().unwrap(); + + let (active_pty, active_snapshot) = + codex_snapshot(&format!("{expected}\nGenerating response")).await; + let mut active = codex_verification(&expected); + active.detector = ActivityDetector::for_cli("gemini"); + active.activity_buffer.push_str("Generating response"); + assert!(matches!( + assess_harness_acceptance("gemini", &active, &active_snapshot), + HarnessAcceptance::Accepted(ref evidence) if evidence == "activity:Generating" + )); + active_pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn text_right_of_cursor_cannot_look_like_a_cleared_composer() { + let expected = "Relay message from Lead [evt]: hidden draft"; + let screen = format!("› {expected}\x1b[1;3H"); + let (pty, snapshot) = codex_snapshot(&screen).await; + let verification = codex_verification(expected); + + assert_eq!( + assess_harness_acceptance("codex", &verification, &snapshot), + HarnessAcceptance::Inconclusive, + "right-side draft text prevents composer-cleared acceptance while recovery stays fail-closed" + ); + pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn generic_post_echo_output_confirms_after_composer_disappears() { + let expected = "Relay message from Lead [evt]: fix idle injection"; + let (pty, snapshot) = codex_snapshot("Processing accepted turn").await; + let mut verification = codex_verification(expected); + verification.detector = ActivityDetector::for_cli("muse"); + verification + .activity_buffer + .push_str("Processing accepted turn"); + + assert_eq!( + assess_harness_acceptance("muse", &verification, &snapshot), + HarnessAcceptance::Accepted("activity:any_output".to_string()) + ); + pty.shutdown().unwrap(); + } + + #[test] + fn split_echo_preserves_same_chunk_acceptance_activity() { + let expected = "Relay message from Lead [evt]: split echo"; + let mut verification = codex_verification(expected); + verification.echo_seen = false; + let mut output = VerificationOutput::default(); + output.push_output(1, b"Relay message from Lead [evt]: split "); + verification.observe(&output, "Relay message from Lead [evt]: split "); + assert!(!verification.echo_seen); + + output.push_output(2, "echo\nWorking (1s • esc to interrupt)".as_bytes()); + verification.observe(&output, "echo\nWorking (1s • esc to interrupt)"); + + assert!(verification.echo_seen); + assert_eq!( + verification.accepted_activity(), + Some("Working+esc to interrupt".to_string()) + ); + } + + #[test] + fn wrapped_tui_echo_is_scoped_to_delivery_output_boundary() { + let expected = "Relay message from Lead [evt]: Reply with exactly WRAPPED_CODEX_ACK"; + let mut verification = codex_verification(expected); + verification.echo_seen = false; + verification.output_boundary = 1; + let mut output = VerificationOutput::default(); + output.push_output( + 1, + b"Relay message from Lead [evt]: Reply with exactly WRAPPED_CODEX_ACK", + ); + assert!(!pending_verification_echo_seen(&output, &verification)); + output.push_output( + 2, + b"Relay message from Lead [evt]: Reply with exactly\r\n WRAPPED_CODEX_ACK", + ); + assert!(pending_verification_echo_seen(&output, &verification)); + } + + #[test] + fn acceptance_lifetime_is_not_reset_by_retry_windows() { + let mut verification = codex_verification("bounded delivery"); + verification.verification_started_at = Instant::now() - MAX_ACCEPTANCE_LIFETIME; + verification.injected_at = Instant::now(); + assert!(verification.acceptance_expired()); + } + #[test] fn check_echo_clean_text() { let output = "some preamble\nRelay message from Alice [evt_1]: hello world\nmore output"; @@ -383,6 +1040,7 @@ mod tests { expected_echo: expected.to_string(), output_boundary, injected_at: Instant::now(), + verification_started_at: Instant::now(), attempts: 1, max_attempts: 1, request_id: None, @@ -391,6 +1049,9 @@ mod tests { from: "Alice".to_string(), body: "same body".to_string(), target: "Worker".into(), + echo_seen: false, + activity_buffer: String::new(), + detector: ActivityDetector::for_cli("codex"), }; assert!(!pending_verification_echo_seen(&output, &verification)); @@ -416,6 +1077,7 @@ mod tests { expected_echo: expected.to_string(), output_boundary, injected_at: Instant::now(), + verification_started_at: Instant::now(), attempts: 1, max_attempts: 1, request_id: None, @@ -424,6 +1086,9 @@ mod tests { from: "Alice".to_string(), body: "same body".to_string(), target: "Worker".into(), + echo_seen: false, + activity_buffer: String::new(), + detector: ActivityDetector::for_cli("codex"), }; output.push_output(2, expected.as_bytes()); @@ -470,6 +1135,7 @@ mod tests { expected_echo: expected.to_string(), output_boundary, injected_at: Instant::now(), + verification_started_at: Instant::now(), attempts: 1, max_attempts: 1, request_id: None, @@ -478,6 +1144,9 @@ mod tests { from: "Lead".to_string(), body: "review this".to_string(), target: "Worker".into(), + echo_seen: false, + activity_buffer: String::new(), + detector: ActivityDetector::for_cli("claude"), }; let mut pending_activities = std::collections::VecDeque::new(); @@ -507,6 +1176,7 @@ mod tests { expected_echo: expected.to_string(), output_boundary, injected_at: Instant::now(), + verification_started_at: Instant::now(), attempts: 1, max_attempts: 1, request_id: None, @@ -515,6 +1185,9 @@ mod tests { from: "Lead".to_string(), body: "review this".to_string(), target: "Worker".into(), + echo_seen: false, + activity_buffer: String::new(), + detector: ActivityDetector::for_cli("claude"), }; let mut pending_activities = std::collections::VecDeque::new(); diff --git a/crates/broker/src/protocol.rs b/crates/broker/src/protocol.rs index 1f95ac4cba..3938e003ae 100644 --- a/crates/broker/src/protocol.rs +++ b/crates/broker/src/protocol.rs @@ -411,16 +411,36 @@ pub enum BrokerEvent { count: usize, reason: String, }, + DeliveryUnconfirmed { + name: WorkerName, + delivery_id: DeliveryId, + event_id: EventId, + reason: String, + attempts: u32, + max_attempts: u32, + }, + DeliveryResubmitted { + name: WorkerName, + delivery_id: DeliveryId, + event_id: EventId, + attempt: u32, + strategy: String, + }, DeliveryVerified { name: WorkerName, delivery_id: DeliveryId, event_id: EventId, - /// "echo" when confirmed in PTY output, "timeout_fallback" when the - /// delivery was acked without echo verification. + /// "harness_acceptance" when activity or a cleared composer proves + /// that the PTY harness accepted the turn. Older workers may report + /// legacy values such as "echo" or "timeout_fallback". #[serde(default, skip_serializing_if = "Option::is_none")] verification: Option, #[serde(default, skip_serializing_if = "Option::is_none")] reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + evidence: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + attempts: Option, }, DeliveryFailed { name: WorkerName, @@ -633,6 +653,27 @@ pub enum WorkerToBroker { DeliveryVerified { delivery_id: DeliveryId, event_id: EventId, + #[serde(default, skip_serializing_if = "Option::is_none")] + verification: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + evidence: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + attempts: Option, + }, + DeliveryUnconfirmed { + delivery_id: DeliveryId, + event_id: EventId, + reason: String, + attempts: u32, + max_attempts: u32, + }, + DeliveryResubmitted { + delivery_id: DeliveryId, + event_id: EventId, + attempt: u32, + strategy: String, }, DeliveryFailed { delivery_id: DeliveryId, @@ -817,12 +858,45 @@ mod tests { let msg = WorkerToBroker::DeliveryVerified { delivery_id: "del_v1".into(), event_id: "evt_v1".into(), + verification: Some("harness_acceptance".into()), + reason: Some("accepted by live harness".into()), + evidence: Some("activity:Working".into()), + attempts: Some(2), }; let encoded = serde_json::to_string(&msg).unwrap(); let decoded: WorkerToBroker = serde_json::from_str(&encoded).unwrap(); assert_eq!(decoded, msg); } + #[test] + fn delivery_recovery_observability_round_trips() { + let unconfirmed = WorkerToBroker::DeliveryUnconfirmed { + delivery_id: "del_u1".into(), + event_id: "evt_u1".into(), + reason: "body remains parked".into(), + attempts: 1, + max_attempts: 3, + }; + let encoded = serde_json::to_string(&unconfirmed).unwrap(); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + unconfirmed + ); + + let resubmitted = BrokerEvent::DeliveryResubmitted { + name: "Worker1".into(), + delivery_id: "del_u1".into(), + event_id: "evt_u1".into(), + attempt: 2, + strategy: "submit_key_only".into(), + }; + let encoded = serde_json::to_string(&resubmitted).unwrap(); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + resubmitted + ); + } + #[test] fn worker_to_broker_delivery_failed_round_trip() { let msg = WorkerToBroker::DeliveryFailed { @@ -843,6 +917,8 @@ mod tests { event_id: "evt_v2".into(), verification: None, reason: None, + evidence: None, + attempts: None, }; let encoded = serde_json::to_string(&event).unwrap(); assert!( @@ -861,6 +937,8 @@ mod tests { event_id: "evt_v3".into(), verification: Some("timeout_fallback".into()), reason: Some("echo not detected within 5s window".into()), + evidence: None, + attempts: None, }; let encoded = serde_json::to_string(&event).unwrap(); let decoded: BrokerEvent = serde_json::from_str(&encoded).unwrap(); diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index 6ee5866da7..79c5897412 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -28,11 +28,10 @@ use tokio::{ use crate::broker::{ continuity::parse_continuity_command, delivery_verification::{ - current_timestamp_ms, delivery_injected_event_payload, delivery_queued_event_payload, - pending_verification_echo_seen, queue_or_take_confirmed_verification, - queue_or_take_detected_activity, DeliveryOutcome, PendingActivity, PendingVerification, - ThrottleState, VerificationOutput, ACTIVITY_BUFFER_KEEP_BYTES, ACTIVITY_BUFFER_MAX_BYTES, - ACTIVITY_WINDOW, VERIFICATION_WINDOW, + assess_harness_acceptance, current_timestamp_ms, delivery_injected_event_payload, + delivery_queued_event_payload, DeliveryOutcome, HarnessAcceptance, PendingActivity, + PendingVerification, ThrottleState, VerificationOutput, ACTIVITY_BUFFER_KEEP_BYTES, + ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW, MAX_VERIFICATION_ATTEMPTS, VERIFICATION_WINDOW, }, injection_format::{format_injection_for_worker_with_workspace, McpReminderThrottle}, }; @@ -46,9 +45,10 @@ use crate::util::terminal::{ detect_claude_trust_prompt, detect_codex_trust_prompt, detect_muse_device_auth_prompt, }; use crate::util::utf8_stream::Utf8StreamDecoder; -use crate::worker::detection::ActivityDetector; +use crate::worker::detection::{is_codex_busy_status_line, ActivityDetector}; use crate::wrap::{ - submit_injection_body, warn_on_auto_response_write, PtyAutoState, AUTO_SUGGESTION_BLOCK_TIMEOUT, + submit_injection_body, submit_injection_recovery, warn_on_auto_response_write, PtyAutoState, + AUTO_SUGGESTION_BLOCK_TIMEOUT, }; use base64::Engine; @@ -183,6 +183,10 @@ struct ActiveInjection { /// delivery and must never turn into an allowance the next queued relay /// message can spend. targeted_hold_exemption: bool, + /// Human PTY input generation when this automated write started. A later + /// generation transfers composer ownership to the human and cancels + /// automatic verification/recovery. + human_input_generation: u64, } /// Result of an in-flight `write_pty` awaiting the PTY drainer: the request id @@ -199,6 +203,35 @@ type PtyWriteAck = ( /// because each `async move` block is a distinct anonymous type. type PtyWriteAckFuture = Pin + Send>>; +type RecoveryWriteAck = ( + PendingVerification, + u64, + u64, + Result, tokio::sync::oneshot::error::RecvError>, +); +type RecoveryWriteAckFuture = Pin + Send>>; + +fn queue_post_acceptance_activity( + verification: &PendingVerification, + detector: Option<&ActivityDetector>, + pending_activities: &mut VecDeque, +) { + let Some(detector) = detector else { + return; + }; + pending_activities.push_back(PendingActivity { + delivery_id: verification.delivery_id.clone(), + event_id: verification.event_id.clone(), + expected_echo: verification.expected_echo.clone(), + verified_at: Instant::now(), + // `observe` clears this buffer when the delivery echo first appears, + // so it cannot carry a busy marker from the previous turn into this + // delivery's post-acceptance activity probe. + output_buffer: verification.activity_buffer.clone(), + detector: detector.clone(), + }); +} + fn cli_basename(command: &str) -> &str { command .rsplit(['/', '\\']) @@ -299,16 +332,6 @@ fn compact_render(text: &str) -> String { text.chars().filter(|c| !c.is_whitespace()).collect() } -// Codex's actual busy status line always pairs "working" with the -// "esc to interrupt" hint on the same row (e.g. "Working (12s • esc to -// interrupt)"). Requiring both, rather than the hint phrase alone, keeps -// this from matching ordinary task text that happens to mention "esc to -// interrupt" without being the real indicator (relay#1782 review, round 3). -fn is_codex_busy_status_line(line: &str) -> bool { - let lower = line.to_ascii_lowercase(); - lower.contains("working") && lower.contains("esc to interrupt") -} - // Only the current composer, never transcript/history. A scrolling composer // keeps its prompt on the first visible row; the cursor bounds its last row. // @@ -707,11 +730,12 @@ fn injection_pop_allowed( fn next_injection_index( pending: &VecDeque, active_injection_present: bool, + pending_acceptance: bool, interactive_hold: bool, hold_exempt_remaining: usize, hold_exempt_event_ids: &HashSet, ) -> Option { - if active_injection_present { + if active_injection_present || pending_acceptance { return None; } if injection_pop_allowed( @@ -726,8 +750,8 @@ fn next_injection_index( .position(|entry| hold_exempt_event_ids.contains(entry.delivery.event_id.as_str())) } -/// Return an interrupted injection's hold exemption so a requeued frame is -/// retried through the hold instead of freezing behind it. A targeted exemption +/// Return a requeued injection's hold exemption so the same frame is retried +/// through the hold instead of freezing behind it. A targeted exemption /// goes back to the event-id set — it must stay bound to that one delivery — /// while a blanket `flush_injections` allowance goes back to the counter. fn restore_hold_exemption( @@ -751,8 +775,9 @@ fn restore_hold_exemption( fn injected_output_command_detection_allowed( active_injection_present: bool, pending_verifications_empty: bool, + pending_recovery_writes_empty: bool, ) -> bool { - !active_injection_present && pending_verifications_empty + !active_injection_present && pending_verifications_empty && pending_recovery_writes_empty } /// What to do with an in-flight injection once its combined Body+Enter write @@ -1062,6 +1087,12 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { // predictive-echo rollback). Draining lives in its own select arm so the // loop keeps forwarding PTY output and handling input while writes settle. let mut pending_pty_writes: FuturesUnordered = FuturesUnordered::new(); + // Submit-key recovery is tracked separately from initial body writes. The + // body is never replayed; each future owns the verification record until + // the drainer proves the recovery key reached the child. + let mut pending_recovery_writes: FuturesUnordered = + FuturesUnordered::new(); + let mut human_input_generation = 0u64; let mut startup_output = String::new(); let mut startup_total_bytes = 0usize; let mut init_request_id: Option = None; @@ -1388,7 +1419,29 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { // child process sees the keystrokes. match frame.payload.get("data").and_then(Value::as_str) { Some(data) => { + human_input_generation = human_input_generation.saturating_add(1); if let Some(cancel) = &initial_injection_cancel { cancel.store(true, Ordering::Relaxed); } + // Once a human writes into the PTY we can + // no longer distinguish the broker body + // from their partial composer text. Give + // the human ownership and terminate every + // pending automatic verification without + // pressing a recovery key. + let mut cancelled_verification = false; + while let Some(pv) = pending_verifications.pop_front() { + cancelled_verification = true; + let delivery_id = pv.delivery_id.clone(); + let _ = send_frame(&out_tx, "delivery_failed", None, json!({ + "delivery_id": delivery_id, + "event_id": pv.event_id, + "reason": "human PTY input took ownership before harness acceptance", + "attempts": pv.attempts, + })).await; + pending_worker_delivery_ids.remove(&delivery_id); + } + if cancelled_verification { + throttle.record(DeliveryOutcome::Failed); + } // Non-blocking submission: never parks the // select loop even if the drainer is wedged // behind a child that stopped reading stdin. @@ -1715,6 +1768,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { if injected_output_command_detection_allowed( active_injection.is_some(), pending_verifications.is_empty(), + pending_recovery_writes.is_empty(), ) && clean_text.lines().any(|line| line.trim() == "/exit") { @@ -1768,6 +1822,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { if injected_output_command_detection_allowed( active_injection.is_some(), pending_verifications.is_empty(), + pending_recovery_writes.is_empty(), ) { continuity_buffer.push_str(&clean_text); if continuity_buffer.len() > CONTINUITY_BUFFER_MAX { @@ -1809,23 +1864,38 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { } } - // Check pending verifications against new output + // Terminal echo only proves that the editor received the + // body. Confirm delivery only after the harness starts a + // turn (activity marker) or the echoed body has left a + // proven idle composer. let mut verified_indices = Vec::new(); - for (i, pv) in pending_verifications.iter().enumerate() { - if pending_verification_echo_seen(&echo_buffer, pv) { - verified_indices.push(i); + if !pending_verifications.is_empty() { + let snapshot = Snapshot::capture(&pty); + for (i, pv) in pending_verifications.iter_mut().enumerate() { + pv.observe(&echo_buffer, &clean_text); + pv.observe_raw_process_receipt(&echo_buffer, &resolved_cli); + pv.observe_visible_composer(&snapshot, &resolved_cli); + if let HarnessAcceptance::Accepted(evidence) = + assess_harness_acceptance(&resolved_cli, pv, &snapshot) + { + verified_indices.push((i, evidence)); + } } } - // Remove verified entries in reverse order to preserve indices - for &i in verified_indices.iter().rev() { + // Remove verified entries in reverse order to preserve indices. + for (i, evidence) in verified_indices.into_iter().rev() { let pv = pending_verifications.remove(i).unwrap(); let delivery_id = pv.delivery_id.clone(); let event_id = pv.event_id.clone(); tracing::debug!( delivery_id = %delivery_id, attempts = pv.attempts, - "delivery echo verified" + evidence = %evidence, + "delivery accepted by harness" ); + let activity_pattern = evidence + .strip_prefix("activity:") + .map(str::to_string); let _ = send_frame( &out_tx, "delivery_ack", @@ -1843,20 +1913,31 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { json!({ "delivery_id": delivery_id, "event_id": event_id, - "verification": "echo" + "verification": "harness_acceptance", + "evidence": evidence, + "attempts": pv.attempts, }), ) .await; throttle.record(DeliveryOutcome::Success); - if let Some(detector) = activity_detector.as_ref() { - pending_activities.push_back(PendingActivity { - delivery_id: delivery_id.clone(), - event_id: event_id.clone(), - expected_echo: pv.expected_echo, - verified_at: Instant::now(), - output_buffer: String::new(), - detector: detector.clone(), - }); + if let Some(pattern) = activity_pattern { + let _ = send_frame( + &out_tx, + "delivery_active", + None, + json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "pattern": pattern, + }), + ) + .await; + } else { + queue_post_acceptance_activity( + &pv, + activity_detector.as_ref(), + &mut pending_activities, + ); } pending_worker_delivery_ids.remove(&delivery_id); completed_worker_deliveries.insert(delivery_id, event_id); @@ -2004,6 +2085,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { if let Some(index) = next_injection_index( &pending_worker_injections, active_injection.is_some(), + !pending_verifications.is_empty() || !pending_recovery_writes.is_empty(), pty_auto.interactive_hold, hold_exempt_injections, &hold_exempt_event_ids, @@ -2026,6 +2108,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { output_boundary: None, hold_exempt, targeted_hold_exemption, + human_input_generation, }); } } @@ -2044,6 +2127,21 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { && (!pty_auto.interactive_hold || active_injection.as_ref().is_some_and(|inj| inj.hold_exempt)) => { let mut inj = active_injection.take().expect("active injection present under guard"); + if inj.human_input_generation != human_input_generation { + let delivery_id = inj.pending.delivery.delivery_id.clone(); + let _ = send_frame(&out_tx, "delivery_failed", None, json!({ + "delivery_id": delivery_id, + "event_id": inj.pending.delivery.event_id, + "reason": "human PTY input took ownership before body injection", + })).await; + // This delivery is terminally failed rather than + // requeued, so its one-shot flush exemption is consumed. + // Restoring it would let the next queued delivery bypass + // the human's interactive hold. + pending_worker_delivery_ids.remove(&delivery_id); + throttle.record(DeliveryOutcome::Failed); + continue; + } match inj.stage { InjectionStage::Escape => { if matches!(inj.pending.delivery.injection_mode, MessageInjectionMode::Steer) { @@ -2249,6 +2347,17 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { })).await; continue; } + if inj.human_input_generation != human_input_generation { + let delivery_id = inj.pending.delivery.delivery_id.clone(); + let _ = send_frame(&out_tx, "delivery_failed", None, json!({ + "delivery_id": delivery_id, + "event_id": inj.pending.delivery.event_id, + "reason": "human PTY input took ownership during injection", + })).await; + pending_worker_delivery_ids.remove(&delivery_id); + throttle.record(DeliveryOutcome::Failed); + continue; + } match injection_ack_outcome(inj.stage, confirmed) { InjectionAckOutcome::Finalize => { // Body+Enter confirmed on the child: only now is @@ -2267,11 +2376,12 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { .await; pty_auto.note_completed_injection(&resolved_cli); - // Queue echo verification against the confirmed body, - // or confirm immediately when the echo raced ahead of - // this ack while Claude's delayed Enter was pending. + // Seed harness verification with any output that + // raced ahead of this ack. Echo alone is not an ack: + // the live screen must show activity or a cleared + // composer before broker custody is released. let injection = inj.injection_text.take().unwrap_or_default(); - let verification = PendingVerification { + let mut verification = PendingVerification { delivery_id: inj.pending.delivery.delivery_id.clone(), event_id: inj.pending.delivery.event_id.clone(), expected_echo: injection, @@ -2279,27 +2389,41 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { .output_boundary .unwrap_or_else(|| echo_buffer.boundary()), injected_at: Instant::now(), + verification_started_at: Instant::now(), attempts: 1, - max_attempts: 1, + max_attempts: MAX_VERIFICATION_ATTEMPTS, request_id: inj.pending.request_id, workspace_id: inj.pending.delivery.workspace_id.clone(), workspace_alias: inj.pending.delivery.workspace_alias.clone(), from: inj.pending.delivery.from, body: inj.pending.delivery.body, target: inj.pending.delivery.target, + echo_seen: false, + activity_buffer: String::new(), + detector: ActivityDetector::for_cli(&resolved_cli), }; - if let Some(pv) = queue_or_take_confirmed_verification( - verification, - &echo_buffer, - &mut pending_verifications, - ) { + let raced_output = echo_buffer + .since(verification.output_boundary) + .into_owned(); + verification.observe(&echo_buffer, &raced_output); + let acceptance = assess_harness_acceptance( + &resolved_cli, + &verification, + &Snapshot::capture(&pty), + ); + if let HarnessAcceptance::Accepted(evidence) = acceptance { + let pv = verification; let delivery_id = pv.delivery_id.clone(); let event_id = pv.event_id.clone(); tracing::debug!( delivery_id = %delivery_id, attempts = pv.attempts, - "delivery echo verified before write ack was processed" + evidence = %evidence, + "delivery accepted before write ack was processed" ); + let activity_pattern = evidence + .strip_prefix("activity:") + .map(str::to_string); let _ = send_frame( &out_tx, "delivery_ack", @@ -2317,40 +2441,36 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { json!({ "delivery_id": delivery_id, "event_id": event_id, - "verification": "echo" + "verification": "harness_acceptance", + "evidence": evidence, + "attempts": pv.attempts, }), ) .await; throttle.record(DeliveryOutcome::Success); - if let Some(detector) = activity_detector.as_ref() { - if let Some((activity, pattern)) = queue_or_take_detected_activity( - &pv, - &echo_buffer, - detector, + if let Some(pattern) = activity_pattern { + let _ = send_frame( + &out_tx, + "delivery_active", + None, + json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "pattern": pattern, + }), + ) + .await; + } else { + queue_post_acceptance_activity( + &pv, + activity_detector.as_ref(), &mut pending_activities, - ) { - tracing::debug!( - target: "agent_relay::worker::pty", - delivery_id = %activity.delivery_id, - event_id = %activity.event_id, - pattern = %pattern, - "delivery activity detected before write ack was processed" - ); - let _ = send_frame( - &out_tx, - "delivery_active", - None, - json!({ - "delivery_id": activity.delivery_id, - "event_id": activity.event_id, - "pattern": pattern, - }), - ) - .await; - } + ); } pending_worker_delivery_ids.remove(&delivery_id); completed_worker_deliveries.insert(delivery_id, event_id); + } else { + pending_verifications.push_back(verification); } // active_injection remains None: injection complete. } @@ -2382,6 +2502,116 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { } } + // Settle submit-key-only recovery writes without blocking PTY + // output or human input. A failed recovery is terminal because the + // body may already be in the composer and must never be replayed. + Some((mut pv, output_boundary, submitted_generation, ack)) = pending_recovery_writes.next(), + if !pending_recovery_writes.is_empty() => { + let delivery_id = pv.delivery_id.clone(); + let event_id = pv.event_id.clone(); + if submitted_generation != human_input_generation { + let _ = send_frame(&out_tx, "delivery_failed", None, json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "reason": "human PTY input took ownership during submit recovery", + "attempts": pv.attempts, + })).await; + throttle.record(DeliveryOutcome::Failed); + pending_worker_delivery_ids.remove(&delivery_id); + continue; + } + match ack { + Ok(Ok(())) => { + pv.output_boundary = output_boundary; + pv.injected_at = Instant::now(); + pv.activity_buffer.clear(); + let raced_output = echo_buffer.since(output_boundary).into_owned(); + pv.observe(&echo_buffer, &raced_output); + pv.observe_raw_process_receipt(&echo_buffer, &resolved_cli); + let snapshot = Snapshot::capture(&pty); + pv.observe_visible_composer(&snapshot, &resolved_cli); + match assess_harness_acceptance( + &resolved_cli, + &pv, + &snapshot, + ) { + HarnessAcceptance::Accepted(evidence) => { + let activity_pattern = evidence + .strip_prefix("activity:") + .map(str::to_string); + let _ = send_frame( + &out_tx, + "delivery_ack", + pv.request_id.clone(), + json!({ "delivery_id": delivery_id, "event_id": event_id }), + ) + .await; + let _ = send_frame( + &out_tx, + "delivery_verified", + None, + json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "verification": "harness_acceptance", + "evidence": evidence, + "attempts": pv.attempts, + }), + ) + .await; + if let Some(pattern) = activity_pattern { + let _ = send_frame( + &out_tx, + "delivery_active", + None, + json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "pattern": pattern, + }), + ) + .await; + } else { + queue_post_acceptance_activity( + &pv, + activity_detector.as_ref(), + &mut pending_activities, + ); + } + throttle.record(DeliveryOutcome::Success); + pending_worker_delivery_ids.remove(&delivery_id); + completed_worker_deliveries.insert(delivery_id, event_id); + } + HarnessAcceptance::Parked | HarnessAcceptance::Inconclusive => { + pending_verifications.push_back(pv); + } + } + } + Ok(Err(error)) => { + let reason = format!("submit recovery write failed: {error}"); + let _ = send_frame(&out_tx, "delivery_failed", None, json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "reason": reason, + "attempts": pv.attempts, + })).await; + throttle.record(DeliveryOutcome::Failed); + pending_worker_delivery_ids.remove(&delivery_id); + } + Err(_) => { + let reason = "submit recovery drainer exited before acknowledging the key"; + let _ = send_frame(&out_tx, "delivery_failed", None, json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "reason": reason, + "attempts": pv.attempts, + })).await; + throttle.record(DeliveryOutcome::Failed); + pending_worker_delivery_ids.remove(&delivery_id); + } + } + } + // --- Verification tick: check for timed-out verifications --- _ = verification_tick.tick() => { let gate = startup_gate( @@ -2405,43 +2635,157 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { let mut i = 0; while i < pending_verifications.len() { if pending_verifications[i].injected_at.elapsed() >= verification_window { - let pv = pending_verifications.remove(i).unwrap(); + let mut pv = pending_verifications.remove(i).unwrap(); let delivery_id = pv.delivery_id.clone(); let event_id = pv.event_id.clone(); - // Do not re-inject on verification timeout. Re-injection can duplicate - // already-delivered messages when terminal echo parsing is noisy. - tracing::info!( - delivery_id = %delivery_id, - attempts = pv.attempts, - "delivery echo not detected within verification window; acknowledging via timeout fallback (unverified)" - ); - let _ = send_frame( - &out_tx, - "delivery_ack", - pv.request_id.clone(), - json!({ - "delivery_id": delivery_id, - "event_id": event_id - }), - ) - .await; - let _ = send_frame( - &out_tx, - "delivery_verified", - pv.request_id.clone(), - json!({ - "delivery_id": delivery_id, - "event_id": event_id, - "verification": "timeout_fallback", - "reason": format!("echo not detected within {}s window", verification_window.as_secs()) - }), - ) - .await; - // Timeout-fallback acks are not verified deliveries: - // keep them out of the throttle's success signal. - throttle.record(DeliveryOutcome::Unverified); - pending_worker_delivery_ids.remove(&delivery_id); - completed_worker_deliveries.insert(delivery_id, event_id); + pv.observe_raw_process_receipt(&echo_buffer, &resolved_cli); + let snapshot = Snapshot::capture(&pty); + pv.observe_visible_composer(&snapshot, &resolved_cli); + match assess_harness_acceptance( + &resolved_cli, + &pv, + &snapshot, + ) { + HarnessAcceptance::Accepted(evidence) => { + let activity_pattern = evidence + .strip_prefix("activity:") + .map(str::to_string); + let _ = send_frame( + &out_tx, + "delivery_ack", + pv.request_id.clone(), + json!({ "delivery_id": delivery_id, "event_id": event_id }), + ) + .await; + let _ = send_frame( + &out_tx, + "delivery_verified", + None, + json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "verification": "harness_acceptance", + "evidence": evidence, + "attempts": pv.attempts, + }), + ) + .await; + if let Some(pattern) = activity_pattern { + let _ = send_frame( + &out_tx, + "delivery_active", + None, + json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "pattern": pattern, + }), + ) + .await; + } else { + queue_post_acceptance_activity( + &pv, + activity_detector.as_ref(), + &mut pending_activities, + ); + } + throttle.record(DeliveryOutcome::Success); + pending_worker_delivery_ids.remove(&delivery_id); + completed_worker_deliveries.insert(delivery_id, event_id); + } + HarnessAcceptance::Parked + if pv.attempts < pv.max_attempts + && !pv.acceptance_expired() + && !pty_auto.interactive_hold + && pending_pty_writes.is_empty() + && active_injection.is_none() + && pending_recovery_writes.is_empty() => + { + let _ = send_frame( + &out_tx, + "delivery_unconfirmed", + None, + json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "reason": "body remains parked in the live composer", + "attempts": pv.attempts, + "max_attempts": pv.max_attempts, + }), + ) + .await; + match submit_injection_recovery(&pty, &resolved_cli, pv.attempts) { + Ok((ack_rx, output_boundary)) => { + pv.attempts += 1; + let _ = send_frame( + &out_tx, + "delivery_resubmitted", + None, + json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "attempt": pv.attempts, + "strategy": "submit_key_only", + }), + ) + .await; + pending_recovery_writes.push(Box::pin(async move { + (pv, output_boundary, human_input_generation, ack_rx.await) + })); + } + Err(error) => { + let reason = format!("failed to queue submit recovery: {error}"); + let _ = send_frame(&out_tx, "delivery_failed", None, json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "reason": reason, + "attempts": pv.attempts, + })).await; + throttle.record(DeliveryOutcome::Failed); + pending_worker_delivery_ids.remove(&delivery_id); + } + } + } + HarnessAcceptance::Parked + if !pv.acceptance_expired() + && (pty_auto.interactive_hold + || !pending_pty_writes.is_empty() + || active_injection.is_some() + || !pending_recovery_writes.is_empty()) => + { + // A human or another writer owns the input FIFO. + // Keep the delivery pending and retry the check + // later without spending its submit-key budget. + if pty_auto.interactive_hold { + // Human drive pauses the total acceptance + // clock. Automated-writer contention stays + // wall-clock bounded. + pv.verification_started_at = Instant::now(); + } + pv.injected_at = Instant::now(); + pending_verifications.push_back(pv); + } + state @ (HarnessAcceptance::Parked + | HarnessAcceptance::Inconclusive) => { + let reason = match state { + HarnessAcceptance::Parked => { + "body remained parked after bounded submit-key recovery" + } + HarnessAcceptance::Inconclusive => { + "harness acceptance could not be proven" + } + HarnessAcceptance::Accepted(_) => unreachable!(), + }; + let _ = send_frame(&out_tx, "delivery_failed", None, json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "reason": reason, + "attempts": pv.attempts, + })).await; + throttle.record(DeliveryOutcome::Failed); + pending_worker_delivery_ids.remove(&delivery_id); + } + } } else { i += 1; } @@ -2469,13 +2813,20 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { // --- Auto-enter for stuck agents --- _ = auto_enter_interval.tick() => { - if active_injection.is_none() { pty_auto.try_auto_enter(&pty); } + if active_injection.is_none() + && pending_verifications.is_empty() + && pending_recovery_writes.is_empty() + { + pty_auto.try_auto_enter(&pty); + } // Idle detection: emit agent_idle once when silence exceeds threshold. // Granularity depends on auto_enter_interval tick rate (2s). if pending_worker_injections.is_empty() && pending_verifications.is_empty() && pending_activities.is_empty() + && pending_recovery_writes.is_empty() + && active_injection.is_none() { if let Some(threshold) = idle_threshold { if let Some(idle_secs) = pty_auto.check_idle_transition(threshold) { @@ -2639,6 +2990,37 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { mod tests { use super::*; + #[test] + fn post_acceptance_activity_is_seeded_only_from_post_echo_output() { + let verification = PendingVerification { + delivery_id: "delivery-current".into(), + event_id: "event-current".into(), + expected_echo: "current request".into(), + output_boundary: 0, + injected_at: Instant::now(), + verification_started_at: Instant::now(), + attempts: 1, + max_attempts: MAX_VERIFICATION_ATTEMPTS, + request_id: None, + workspace_id: None, + workspace_alias: None, + from: "lead".into(), + body: "current request".into(), + target: "worker".into(), + echo_seen: true, + activity_buffer: "post-echo repaint".into(), + detector: ActivityDetector::for_cli("codex"), + }; + let mut pending = VecDeque::new(); + + queue_post_acceptance_activity(&verification, Some(&verification.detector), &mut pending); + + assert_eq!( + pending.pop_front().unwrap().output_buffer, + "post-echo repaint" + ); + } + #[test] fn completed_delivery_replay_retains_the_original_event_identity() { let mut completed = CompletedWorkerDeliveries::default(); @@ -2757,7 +3139,7 @@ mod tests { } assert!(pty.screen_text().contains("Ask Codex")); let before = pty.consumed_offset(); - let ack = pty.submit_write(body[..768].as_bytes().to_vec()).unwrap(); + let ack = pty.submit_write(body.as_bytes()[..768].to_vec()).unwrap(); let cancelled = AtomicBool::new(cancel); let started = Instant::now(); let result = @@ -2768,7 +3150,7 @@ mod tests { let max_writer_submits = 1 + INITIAL_SUBMIT_RETRIES; if cancel { assert!(result.is_err()); - assert_eq!(actual, body[..768].as_bytes()); + assert_eq!(actual, &body.as_bytes()[..768]); } else if submits <= max_writer_submits { result.unwrap(); assert!(started.elapsed() >= CODEX_STARTUP_SETTLE * 3); @@ -2836,7 +3218,7 @@ mod tests { } assert!(pty.screen_text().contains(&body[..768])); let before = pty.consumed_offset(); - let ack = pty.submit_write(body[..768].as_bytes().to_vec()).unwrap(); + let ack = pty.submit_write(body.as_bytes()[..768].to_vec()).unwrap(); let result = write_initial_codex( &pty, &body, @@ -2852,7 +3234,7 @@ mod tests { .unwrap_err() .to_string() .contains("render gate timed out")); - assert_eq!(std::fs::read(log).unwrap(), body[..768].as_bytes()); + assert_eq!(std::fs::read(log).unwrap(), &body.as_bytes()[..768]); } // codex_composer_ready (startup) already accepted both native `›` and @@ -3041,7 +3423,7 @@ mod tests { } let before = pty.consumed_offset(); let ack = pty - .submit_write(body[..chunk_end(&body, 0)].as_bytes().to_vec()) + .submit_write(body.as_bytes()[..chunk_end(&body, 0)].to_vec()) .unwrap(); let result = write_initial_codex( &pty, @@ -3070,6 +3452,287 @@ mod tests { ); } + #[cfg(unix)] + async fn submit_live_codex_probe( + pty: &PtySession, + rx: &mut tokio::sync::mpsc::Receiver, + output: &mut VerificationOutput, + body: &str, + response: &str, + ) { + let (ack_rx, output_boundary) = + submit_injection_body(pty, "codex", body.as_bytes().to_vec(), Duration::ZERO) + .expect("queue live Codex injection"); + tokio::time::timeout(Duration::from_secs(5), ack_rx) + .await + .expect("initial injection ack timed out") + .expect("PTY drainer exited") + .expect("initial injection write failed"); + let mut verification = PendingVerification { + delivery_id: "live-codex-delivery".into(), + event_id: "live-codex-event".into(), + expected_echo: body.to_string(), + output_boundary, + injected_at: Instant::now(), + verification_started_at: Instant::now(), + attempts: 1, + max_attempts: MAX_VERIFICATION_ATTEMPTS, + request_id: None, + workspace_id: None, + workspace_alias: None, + from: "live-probe".into(), + body: body.into(), + target: "codex".into(), + echo_seen: false, + activity_buffer: String::new(), + detector: ActivityDetector::for_cli("codex"), + }; + let started = Instant::now(); + let mut accepted = false; + let mut transcript = String::new(); + let mut attempt_started = Instant::now(); + while started.elapsed() < Duration::from_secs(45) { + if let Ok(Some(chunk)) = + tokio::time::timeout(Duration::from_millis(250), rx.recv()).await + { + output.push_output(chunk.sequence(), chunk.as_bytes()); + let text = String::from_utf8_lossy(chunk.as_bytes()); + transcript.push_str(&text); + verification.observe(output, &text); + } + match assess_harness_acceptance("codex", &verification, &Snapshot::capture(pty)) { + HarnessAcceptance::Accepted(_) => accepted = true, + HarnessAcceptance::Parked + if attempt_started.elapsed() >= VERIFICATION_WINDOW + && verification.attempts < verification.max_attempts => + { + let (recovery_ack, boundary) = + submit_injection_recovery(pty, "codex", verification.attempts) + .expect("queue submit-key recovery"); + tokio::time::timeout(Duration::from_secs(5), recovery_ack) + .await + .expect("submit-key recovery ack timed out") + .expect("PTY drainer exited during recovery") + .expect("submit-key recovery failed"); + verification.attempts += 1; + verification.output_boundary = boundary; + verification.activity_buffer.clear(); + attempt_started = Instant::now(); + } + _ => {} + } + if accepted && strip_ansi(&transcript).contains(response) { + return; + } + } + panic!( + "Codex did not accept/respond after {} submit attempts; screen={:?}", + verification.attempts, + pty.screen_text() + ); + } + + /// Real Codex regression: one delivery enters an already-idle composer; + /// the second is held while the first is busy and released only after the + /// prompt becomes idle again. Both must advance the harness, not merely + /// echo into its editor. Ignored in ordinary CI because it needs a local + /// authenticated Codex installation. + #[cfg(unix)] + #[tokio::test] + #[ignore = "requires authenticated Codex; native idle and queued-to-idle probe"] + async fn live_codex_idle_and_queued_to_idle_delivery() { + let (pty, mut rx) = PtySession::spawn( + "codex", + &[ + "--no-alt-screen".into(), + "-a".into(), + "never".into(), + "-s".into(), + "read-only".into(), + ], + 24, + 120, + ) + .unwrap(); + let mut output = VerificationOutput::default(); + let ready_deadline = Instant::now() + Duration::from_secs(25); + while Instant::now() < ready_deadline && !pty.screen_text().contains("Ask Codex") { + if let Ok(Some(chunk)) = + tokio::time::timeout(Duration::from_millis(250), rx.recv()).await + { + output.push_output(chunk.sequence(), chunk.as_bytes()); + } + } + assert!( + pty.screen_text().contains("Ask Codex"), + "Codex did not become idle: {:?}", + pty.screen_text() + ); + + submit_live_codex_probe( + &pty, + &mut rx, + &mut output, + "Reply only with IDLE_INJECTION_OK. Do not use tools.", + "IDLE_INJECTION_OK", + ) + .await; + + // Model a delivery queued during the preceding turn: do not write it + // until Codex has redrawn its idle composer. + let idle_deadline = Instant::now() + Duration::from_secs(25); + while Instant::now() < idle_deadline && !pty.screen_text().contains("Ask Codex") { + if let Ok(Some(chunk)) = + tokio::time::timeout(Duration::from_millis(250), rx.recv()).await + { + output.push_output(chunk.sequence(), chunk.as_bytes()); + } + } + assert!(pty.screen_text().contains("Ask Codex")); + submit_live_codex_probe( + &pty, + &mut rx, + &mut output, + "Reply only with QUEUED_TO_IDLE_OK. Do not use tools.", + "QUEUED_TO_IDLE_OK", + ) + .await; + pty.shutdown().unwrap(); + } + + /// Reproduces Codex's multiline/paste-burst parking with the real TUI: + /// body+CR in one PTY write remains in the composer. Recovery then sends + /// only End+CR and, if still parked, LF, stopping as soon as the harness + /// shows acceptance so a turn can never be submitted twice. + #[cfg(unix)] + #[tokio::test] + #[ignore = "requires authenticated Codex; native parked-composer probe"] + async fn live_codex_burst_parks_then_submit_only_recovery_accepts() { + let (pty, mut rx) = PtySession::spawn( + "codex", + &[ + "--no-alt-screen".into(), + "-a".into(), + "never".into(), + "-s".into(), + "read-only".into(), + ], + 24, + 120, + ) + .unwrap(); + let mut output = VerificationOutput::default(); + let ready_deadline = Instant::now() + Duration::from_secs(25); + while Instant::now() < ready_deadline && !pty.screen_text().contains("Ask Codex") { + if let Ok(Some(chunk)) = + tokio::time::timeout(Duration::from_millis(250), rx.recv()).await + { + output.push_output(chunk.sequence(), chunk.as_bytes()); + } + } + assert!(pty.screen_text().contains("Ask Codex")); + + let body = "Reply only with BURST_RECOVERY_OK. Do not use tools."; + let mut burst = body.as_bytes().to_vec(); + burst.push(b'\r'); + let (ack_rx, output_boundary) = pty + .submit_write_paced_with_output_boundary(burst, Duration::ZERO) + .unwrap(); + ack_rx.await.unwrap().unwrap(); + let mut verification = PendingVerification { + delivery_id: "live-codex-parked".into(), + event_id: "live-codex-parked-event".into(), + expected_echo: body.to_string(), + output_boundary, + injected_at: Instant::now(), + verification_started_at: Instant::now(), + attempts: 1, + max_attempts: MAX_VERIFICATION_ATTEMPTS, + request_id: None, + workspace_id: None, + workspace_alias: None, + from: "live-probe".into(), + body: body.into(), + target: "codex".into(), + echo_seen: false, + activity_buffer: String::new(), + detector: ActivityDetector::for_cli("codex"), + }; + let mut transcript = String::new(); + let parked_deadline = Instant::now() + Duration::from_secs(3); + while Instant::now() < parked_deadline { + if let Ok(Some(chunk)) = + tokio::time::timeout(Duration::from_millis(200), rx.recv()).await + { + output.push_output(chunk.sequence(), chunk.as_bytes()); + let text = String::from_utf8_lossy(chunk.as_bytes()); + transcript.push_str(&text); + verification.observe(&output, &text); + } + } + assert_eq!( + assess_harness_acceptance("codex", &verification, &Snapshot::capture(&pty)), + HarnessAcceptance::Parked, + "same-burst body+CR should reproduce the Codex parked composer" + ); + + let mut accepted_after = None; + for completed_attempts in [1, 2] { + let (recovery_ack, boundary) = + submit_injection_recovery(&pty, "codex", completed_attempts).unwrap(); + recovery_ack.await.unwrap().unwrap(); + verification.attempts += 1; + verification.output_boundary = boundary; + verification.activity_buffer.clear(); + let recovery_deadline = Instant::now() + Duration::from_secs(4); + while Instant::now() < recovery_deadline { + if let Ok(Some(chunk)) = + tokio::time::timeout(Duration::from_millis(200), rx.recv()).await + { + output.push_output(chunk.sequence(), chunk.as_bytes()); + let text = String::from_utf8_lossy(chunk.as_bytes()); + transcript.push_str(&text); + verification.observe(&output, &text); + } + if matches!( + assess_harness_acceptance("codex", &verification, &Snapshot::capture(&pty)), + HarnessAcceptance::Accepted(_) + ) { + accepted_after = Some(completed_attempts + 1); + break; + } + } + if accepted_after.is_some() { + break; + } + } + assert!( + accepted_after.is_some(), + "submit-only recovery never advanced Codex; screen={:?}", + pty.screen_text() + ); + eprintln!( + "real Codex accepted the parked turn after submit attempt {}", + accepted_after.unwrap() + ); + let response_deadline = Instant::now() + Duration::from_secs(20); + while Instant::now() < response_deadline + && !strip_ansi(&transcript).contains("BURST_RECOVERY_OK") + { + if let Ok(Some(chunk)) = + tokio::time::timeout(Duration::from_millis(250), rx.recv()).await + { + transcript.push_str(&String::from_utf8_lossy(chunk.as_bytes())); + } + } + assert!( + strip_ansi(&transcript).contains("BURST_RECOVERY_OK"), + "Codex accepted the turn but did not answer: {:?}", + pty.screen_text() + ); + pty.shutdown().unwrap(); + } + #[test] fn codex_defaults_to_bulk_injection() { assert_eq!(resolve_inject_rate("codex", None), Duration::ZERO); @@ -3478,12 +4141,19 @@ mod tests { #[test] fn injected_output_commands_are_suppressed_through_injection_lifecycle() { - // (active_injection_present, pending_verifications_empty): detection is - // allowed only when no injection is active and no verification is pending. - assert!(injected_output_command_detection_allowed(false, true)); - assert!(!injected_output_command_detection_allowed(true, true)); - assert!(!injected_output_command_detection_allowed(false, false)); - assert!(!injected_output_command_detection_allowed(true, false)); + // A recovery owns the verification record while its write is in + // flight. Command-like text in a Codex redraw is still an echo then. + assert!(injected_output_command_detection_allowed(false, true, true)); + assert!(!injected_output_command_detection_allowed(true, true, true)); + assert!(!injected_output_command_detection_allowed( + false, false, true + )); + assert!(!injected_output_command_detection_allowed( + true, false, true + )); + assert!(!injected_output_command_detection_allowed( + false, true, false + )); } #[test] @@ -3529,32 +4199,37 @@ mod tests { let targeted = HashSet::from(["init_task".to_string()]); assert_eq!( - next_injection_index(&pending, false, true, 0, &targeted), + next_injection_index(&pending, false, false, true, 0, &targeted), Some(1) ); // Nothing targeted: the whole queue stays parked under the hold. assert_eq!( - next_injection_index(&pending, false, true, 0, &HashSet::new()), + next_injection_index(&pending, false, false, true, 0, &HashSet::new()), None ); // A blanket flush still drains from the front, in order. assert_eq!( - next_injection_index(&pending, false, true, 2, &HashSet::new()), + next_injection_index(&pending, false, false, true, 2, &HashSet::new()), Some(0) ); // Unheld, the target is irrelevant — normal FIFO order applies. assert_eq!( - next_injection_index(&pending, false, false, 0, &targeted), + next_injection_index(&pending, false, false, false, 0, &targeted), Some(0) ); // An injection already in flight is never preempted, targeted or not. assert_eq!( - next_injection_index(&pending, true, true, 0, &targeted), + next_injection_index(&pending, true, false, true, 0, &targeted), + None + ); + // A body awaiting harness acceptance owns the one live composer. + assert_eq!( + next_injection_index(&pending, false, true, false, 0, &targeted), None ); // An empty queue has nothing to start. assert_eq!( - next_injection_index(&VecDeque::new(), false, false, 0, &targeted), + next_injection_index(&VecDeque::new(), false, false, false, 0, &targeted), None ); } @@ -3574,6 +4249,7 @@ mod tests { output_boundary: None, hold_exempt: true, targeted_hold_exemption: true, + human_input_generation: 0, }; restore_hold_exemption(&targeted, &mut count, &mut event_ids); assert_eq!(count, 0); @@ -3613,6 +4289,7 @@ mod tests { output_boundary: None, hold_exempt: true, targeted_hold_exemption: true, + human_input_generation: 0, }; // What the blanket branch does to an already-exempt in-flight injection. injection.hold_exempt = true; @@ -3660,68 +4337,6 @@ mod tests { ); } - #[test] - fn echo_arriving_before_worker_write_ack_is_confirmed_immediately() { - let injection = "From: Lead\nReview this change"; - let mut output = VerificationOutput::default(); - output.push_output(1, b"older output\n"); - // Sequence 2 was assigned by the reader before the write was queued, - // but this worker has not consumed that matching chunk yet. - let output_boundary = 2; - let verification = || PendingVerification { - delivery_id: "del_echo_before_ack".into(), - event_id: "evt_echo_before_ack".into(), - expected_echo: injection.to_string(), - output_boundary, - injected_at: Instant::now(), - attempts: 1, - max_attempts: 1, - request_id: None, - workspace_id: None, - workspace_alias: None, - from: "Lead".to_string(), - body: "Review this change".to_string(), - target: "Worker".into(), - }; - let mut pending_verifications = VecDeque::new(); - - output.push_output(2, format!("stale composer echo: {injection}").as_bytes()); - - let stale = queue_or_take_confirmed_verification( - verification(), - &output, - &mut pending_verifications, - ); - assert!(stale.is_none(), "a retained pre-submission echo is stale"); - pending_verifications.clear(); - - output.push_output( - 3, - format!("\nnew composer echo: {injection}\nTool: Write(review.md)").as_bytes(), - ); - let confirmed = queue_or_take_confirmed_verification( - verification(), - &output, - &mut pending_verifications, - ) - .expect("the buffered echo must be confirmed"); - - assert!( - pending_verifications.is_empty(), - "an already-observed echo must not be queued to time out" - ); - let mut pending_activities = VecDeque::new(); - let (_, pattern) = queue_or_take_detected_activity( - &confirmed, - &output, - &ActivityDetector::for_cli("claude"), - &mut pending_activities, - ) - .expect("activity buffered before the ack must be detected immediately"); - assert_eq!(pattern, "Tool:"); - assert!(pending_activities.is_empty()); - } - #[test] fn detects_context_budget_percentage() { assert_eq!(detect_context_budget_pct("Context 6% left"), Some(6)); diff --git a/crates/broker/src/runtime/delivery.rs b/crates/broker/src/runtime/delivery.rs index 9f4630c339..c7f52fc364 100644 --- a/crates/broker/src/runtime/delivery.rs +++ b/crates/broker/src/runtime/delivery.rs @@ -13,13 +13,13 @@ pub(crate) struct PendingDelivery { pub(super) queued_at_ms: u64, pub(super) last_error: Option, /// Fleet (engine-facing) `delivery_ack` withheld until the worker confirms - /// this specific PTY injection landed — echo-verified, or its bounded - /// timeout fallback — rather than acked the instant the write is merely - /// handed to the worker. See relay#1310. + /// this specific PTY injection was accepted by the harness, rather than + /// acked the instant the write is merely handed to the worker. See + /// relay#1310. /// /// Lives on the `PendingDelivery` itself, not a second map keyed by /// `DeliveryId`, so it cannot outlive the delivery it belongs to: every - /// path that disposes of a `PendingDelivery` (echo confirmation, dead + /// path that disposes of a `PendingDelivery` (acceptance confirmation, dead /// letter, worker teardown) disposes of its withheld ack with it, by /// construction, instead of needing a matching removal remembered at /// every one of those call sites. See relay#1543. @@ -1065,6 +1065,17 @@ pub(crate) fn delivery_ack_timeout( std::cmp::max(retry_interval, minimum) } +/// A worker recovery progress event starts another bounded verification window +/// after its submit key reaches the PTY. Keep the broker from replaying the +/// delivery while that write and verification are in flight. +pub(crate) fn delivery_recovery_ack_timeout( + injection_mode: &MessageInjectionMode, + retry_interval: Duration, +) -> Duration { + delivery_ack_timeout(injection_mode, retry_interval) + .saturating_add(crate::broker::delivery_verification::VERIFICATION_WINDOW) +} + pub(crate) async fn emit_delivery_attempt_outcome( sdk_out_tx: &mpsc::Sender>, dead_letters: &mut DeadLetterStore, diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index a946318223..92d335fe9a 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -199,8 +199,8 @@ enum FleetDeliverySurfaceOutcome { /// or an unrecognized payload type), so there is nothing to verify. Acknowledge, /// A PTY injection was handed to the worker. The engine ack is withheld - /// until the worker confirms it landed (echo-verified, or the bounded - /// timeout fallback) — see relay#1310. The withheld ack was already + /// until the worker confirms harness acceptance — see relay#1310. The + /// withheld ack was already /// registered on the corresponding `PendingDelivery` at insertion time /// (see relay#1543), so there is nothing left to carry here. AcknowledgeAfterEcho, @@ -1940,9 +1940,8 @@ pub(super) async fn enqueue_delivery_ack( /// confirmation of a specific PTY injection (relay#1310: the ack must not /// fire before the worker confirms the write landed). Called with the /// `delivery_id`/`event_id` from the worker's own internal `delivery_ack` -/// event (`worker_events.rs`), which pty_worker.rs sends only after echo -/// verification succeeds or its bounded timeout fallback fires — never at -/// write-enqueue time. +/// event (`worker_events.rs`), which pty_worker.rs sends only after activity or +/// a cleared composer proves harness acceptance — never at write-enqueue time. /// /// Returns the `(agent, up_to_seq)` to send to the engine once resolved, or /// `None` when there is nothing withheld for `delivery_id` (already diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index 173bf1e089..b1c4baf680 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -847,6 +847,8 @@ fn delivery_lifecycle_worker_event( "delivery_id": delivery_id, "event_id": event_id, "reason": "test terminal disposition", + "attempts": 1, + "max_attempts": 3, }, }), } @@ -2928,7 +2930,8 @@ async fn a_worker_confirmed_ack_becomes_visible_on_the_node_delivery_endpoint() ..withheld_ack_for(delivery_id.as_str()) }; - // The state right after an injection: received, ack withheld pending echo. + // The state right after an injection: received, ack withheld pending + // harness acceptance. fixture .runtime .fleet_delivery_book @@ -3172,9 +3175,8 @@ async fn every_terminal_disposition_drops_its_withheld_fleet_ack() { cleanup_worker_registry(fixture.runtime.workers).await; } -// relay#1310 MUST-NOT-FIRE: once the worker confirms the injection landed -// (echo-verified, or its bounded timeout fallback — pty_worker.rs sends the -// same internal `delivery_ack` event either way), the engine ack must still +// relay#1310 MUST-NOT-FIRE: once the worker confirms harness acceptance, the +// engine ack must still // fire, with the delivery's own (agent, up_to_seq) — i.e. the happy path is // unchanged, just correctly gated on confirmation instead of write-enqueue. // Exercises the full wiring: a real handoff through @@ -3228,7 +3230,7 @@ async fn worker_confirmation_ack_diagnostics(closed: bool) { .expect("the delivery must be tracked pending the worker's confirmation") .withheld_fleet_ack .is_some(), - "a successful handoff must still withhold the ack pending echo confirmation" + "a successful handoff must still withhold the ack pending harness acceptance" ); fixture @@ -4332,6 +4334,150 @@ fn worker_reported_delivery_failures_use_the_dead_letter_path() { ); } +#[tokio::test] +async fn unconfirmed_pty_delivery_is_visible_as_blocked_on_send() { + let worker_name = "worker-a"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let delivery_id = DeliveryId::new("del_blocked_on_send"); + let pending = make_pending_delivery(delivery_id.as_str(), worker_name); + let event_id = pending.delivery.event_id.clone(); + let mut fixture = + worker_event_runtime_fixture(registry, HashMap::from([(delivery_id.clone(), pending)])); + + fixture + .runtime + .handle_worker_event(delivery_lifecycle_worker_event( + worker_name, + generation, + "delivery_unconfirmed", + delivery_id.as_str(), + event_id.as_str(), + )) + .await; + + assert_eq!( + fixture.runtime.workers.workers[worker_name].state, + AgentWorkState::BlockedOnSend + ); + let mut kinds = Vec::new(); + while let Ok(frame) = fixture._sdk_out_rx.try_recv() { + if let Some(kind) = frame.payload.get("kind").and_then(Value::as_str) { + if kind == "delivery_unconfirmed" { + assert_eq!(frame.payload["attempts"], 1); + assert_eq!(frame.payload["max_attempts"], 3); + } + kinds.push(kind.to_string()); + } + } + assert!(kinds.iter().any(|kind| kind == "delivery_unconfirmed")); + assert!(kinds.iter().any(|kind| kind == "agent_blocked_on_send")); + cleanup_worker_registry(fixture.runtime.workers).await; +} + +#[tokio::test] +async fn prior_delivery_activity_does_not_clear_blocked_on_send() { + let worker_name = "worker-a"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let delivery_id = DeliveryId::new("del_blocked_with_prior_activity"); + let pending = make_pending_delivery(delivery_id.as_str(), worker_name); + let event_id = pending.delivery.event_id.clone(); + let mut fixture = + worker_event_runtime_fixture(registry, HashMap::from([(delivery_id.clone(), pending)])); + + fixture + .runtime + .handle_worker_event(delivery_lifecycle_worker_event( + worker_name, + generation, + "delivery_unconfirmed", + delivery_id.as_str(), + event_id.as_str(), + )) + .await; + fixture + .runtime + .handle_worker_event(delivery_lifecycle_worker_event( + worker_name, + generation, + "delivery_active", + "del_prior_delivery", + "event_prior_delivery", + )) + .await; + + assert_eq!( + fixture.runtime.workers.workers[worker_name].state, + AgentWorkState::BlockedOnSend, + "late activity from an earlier accepted delivery must not clear recovery state" + ); + cleanup_worker_registry(fixture.runtime.workers).await; +} + +#[tokio::test] +async fn stale_recovery_progress_does_not_block_the_worker() { + let worker_name = "worker-a"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let mut fixture = worker_event_runtime_fixture(registry, HashMap::new()); + + fixture + .runtime + .handle_worker_event(delivery_lifecycle_worker_event( + worker_name, + generation, + "delivery_unconfirmed", + "del_already_complete", + "evt_already_complete", + )) + .await; + + assert_eq!( + fixture.runtime.workers.workers[worker_name].state, + AgentWorkState::Working + ); + cleanup_worker_registry(fixture.runtime.workers).await; +} + +#[tokio::test] +async fn submit_recovery_progress_defers_broker_delivery_retry() { + let worker_name = "worker-a"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let delivery_id = DeliveryId::new("del_recovery_progress"); + let mut pending = make_pending_delivery(delivery_id.as_str(), worker_name); + pending.delivery.injection_mode = MessageInjectionMode::Steer; + let event_id = pending.delivery.event_id.clone(); + let mut fixture = + worker_event_runtime_fixture(registry, HashMap::from([(delivery_id.clone(), pending)])); + let before = Instant::now(); + + fixture + .runtime + .handle_worker_event(delivery_lifecycle_worker_event( + worker_name, + generation, + "delivery_unconfirmed", + delivery_id.as_str(), + event_id.as_str(), + )) + .await; + + let deadline = fixture.runtime.pending_deliveries[&delivery_id].next_retry_at; + assert!( + deadline + .checked_duration_since(before) + .is_some_and(|delay| { + delay + >= crate::broker::delivery_verification::VERIFICATION_WINDOW + + crate::broker::delivery_verification::VERIFICATION_WINDOW + }), + "broker retry must wait through the in-flight submit write and verification window" + ); + cleanup_worker_registry(fixture.runtime.workers).await; +} + #[test] fn contract_broadcast_whitelist_fixture_requires_filtering_to_required_kinds() { let event_fixture: Value = serde_json::from_str(include_str!( diff --git a/crates/broker/src/runtime/worker_events.rs b/crates/broker/src/runtime/worker_events.rs index 2134304d15..21983feb99 100644 --- a/crates/broker/src/runtime/worker_events.rs +++ b/crates/broker/src/runtime/worker_events.rs @@ -859,6 +859,82 @@ impl BrokerRuntime { ) .await; } + } else if msg_type == "delivery_unconfirmed" + || msg_type == "delivery_resubmitted" + { + if let Some(payload) = value.get("payload") { + let delivery_id = payload + .get("delivery_id") + .and_then(Value::as_str) + .unwrap_or(""); + let event_id = payload + .get("event_id") + .and_then(Value::as_str) + .unwrap_or(""); + let matching_pending = pending_deliveries + .get_mut(delivery_id) + .filter(|pending| pending.delivery.event_id.as_str() == event_id) + .map(|pending| { + pending.next_retry_at = Instant::now() + + delivery_recovery_ack_timeout( + &pending.delivery.injection_mode, + delivery_retry_interval, + ); + }) + .is_some(); + if matching_pending { + if let Some(handle) = workers.workers.get_mut(&name) { + handle.last_activity_at = Instant::now(); + handle.state = AgentWorkState::BlockedOnSend; + } + if msg_type == "delivery_unconfirmed" { + let pending_delivery_count = pending_deliveries + .values() + .filter(|pending| pending.worker_name == name) + .count(); + let _ = send_broker_event( + sdk_out_tx, + BrokerEvent::AgentBlockedOnSend { + name: name.clone(), + blocked_secs: 0, + pending_delivery_count, + }, + ) + .await; + publish_agent_state_transition( + ws_control_tx, + &name, + "stuck", + Some("blocked_on_send"), + ) + .await; + } + } + tracing::warn!( + target = "agent_relay::broker", + worker = %name, + delivery_id = %delivery_id, + event_id = %event_id, + attempts = ?payload.get("attempts").or_else(|| payload.get("attempt")), + kind = msg_type, + "delivery is not yet accepted by the harness" + ); + let _ = send_event( + sdk_out_tx, + json!({ + "kind": msg_type, + "name": name, + "delivery_id": payload.get("delivery_id"), + "event_id": payload.get("event_id"), + "reason": payload.get("reason"), + "attempts": payload.get("attempts"), + "attempt": payload.get("attempt"), + "max_attempts": payload.get("max_attempts"), + "strategy": payload.get("strategy"), + }), + ) + .await; + } } else if msg_type == "delivery_verified" { if let Some(payload) = value.get("payload") { let delivery_id = payload @@ -869,32 +945,21 @@ impl BrokerRuntime { .get("event_id") .and_then(Value::as_str) .unwrap_or(""); - // "echo" when the injection was confirmed in PTY - // output; "timeout_fallback" when the worker acked - // without ever seeing the echo. + // Modern PTY workers use "harness_acceptance": + // editor echo alone is never delivery confirmation. let verification = payload .get("verification") .and_then(Value::as_str) - .unwrap_or("echo"); + .unwrap_or("worker_confirmation"); let reason = payload.get("reason").and_then(Value::as_str); - if verification == "timeout_fallback" { - tracing::info!( - target = "agent_relay::broker", - worker = %name, - delivery_id = %delivery_id, - event_id = %event_id, - reason = reason.unwrap_or(""), - "delivery acked via timeout fallback — echo never verified" - ); - } else { - tracing::debug!( - target = "agent_relay::broker", - worker = %name, - delivery_id = %delivery_id, - event_id = %event_id, - "delivery verified by echo detection" - ); - } + tracing::debug!( + target = "agent_relay::broker", + worker = %name, + delivery_id = %delivery_id, + event_id = %event_id, + verification = %verification, + "delivery acceptance verified" + ); let pending_for_confirmation = clear_pending_delivery_if_event_matches( pending_deliveries, delivery_id, @@ -908,6 +973,8 @@ impl BrokerRuntime { "delivery_id": delivery_id, "event_id": event_id, "verification": verification, + "evidence": payload.get("evidence"), + "attempts": payload.get("attempts"), }); if let (Some(reason), Some(map)) = (reason, verified_event.as_object_mut()) @@ -937,7 +1004,9 @@ impl BrokerRuntime { if let Some(payload) = value.get("payload") { if let Some(handle) = workers.workers.get_mut(&name) { handle.last_activity_at = Instant::now(); - handle.state = AgentWorkState::Working; + if handle.state != AgentWorkState::BlockedOnSend { + handle.state = AgentWorkState::Working; + } } let _ = send_event( sdk_out_tx, @@ -971,7 +1040,7 @@ impl BrokerRuntime { delivery_id = %delivery_id, event_id = %event_id, reason = %reason, - "delivery failed — echo not detected" + "delivery failed — harness acceptance not confirmed" ); let pending_for_failure = clear_pending_delivery_if_event_matches( pending_deliveries, @@ -997,8 +1066,28 @@ impl BrokerRuntime { if let Some(pending) = pending_for_failure { if let Some(handle) = workers.workers.get_mut(&name) { handle.last_activity_at = Instant::now(); - handle.state = AgentWorkState::Working; + handle.state = AgentWorkState::BlockedOnSend; } + let pending_delivery_count = pending_deliveries + .values() + .filter(|candidate| candidate.worker_name == name) + .count(); + let _ = send_broker_event( + sdk_out_tx, + BrokerEvent::AgentBlockedOnSend { + name: name.clone(), + blocked_secs: 0, + pending_delivery_count, + }, + ) + .await; + publish_agent_state_transition( + ws_control_tx, + &name, + "stuck", + Some("blocked_on_send"), + ) + .await; let _ = emit_dropped_delivery_failures( sdk_out_tx, dead_letters, @@ -1351,7 +1440,9 @@ impl BrokerRuntime { .is_some_and(|handle| handle.spec.runtime == AgentRuntime::Pty); if let Some(handle) = workers.workers.get_mut(&name) { handle.last_activity_at = Instant::now(); - handle.state = AgentWorkState::Working; + if handle.state != AgentWorkState::BlockedOnSend { + handle.state = AgentWorkState::Working; + } } if is_pty { publish_pty_busy(pty_observability, hosted_agent_event_tx, &name); @@ -1600,13 +1691,20 @@ impl BrokerRuntime { .unwrap_or(0); let since = chrono::Utc::now() - chrono::Duration::seconds(idle_secs as i64); - if let Some(handle) = workers.workers.get_mut(&name) { - handle.state = AgentWorkState::Idle; - } - if workers + let remains_blocked = workers .workers .get(&name) - .is_some_and(|handle| handle.spec.runtime == AgentRuntime::Pty) + .is_some_and(|handle| handle.state == AgentWorkState::BlockedOnSend); + if !remains_blocked { + if let Some(handle) = workers.workers.get_mut(&name) { + handle.state = AgentWorkState::Idle; + } + } + if !remains_blocked + && workers + .workers + .get(&name) + .is_some_and(|handle| handle.spec.runtime == AgentRuntime::Pty) { publish_pty_idle(pty_observability, hosted_agent_event_tx, &name); } @@ -1621,13 +1719,15 @@ impl BrokerRuntime { }), ) .await; - publish_agent_state_transition( - ws_control_tx, - &name, - "idle", - Some("idle_threshold"), - ) - .await; + if !remains_blocked { + publish_agent_state_transition( + ws_control_tx, + &name, + "idle", + Some("idle_threshold"), + ) + .await; + } } else if msg_type == "agent_blocked_on_send" { let blocked_secs = value .get("payload") diff --git a/crates/broker/src/wrap.rs b/crates/broker/src/wrap.rs index c38b63e2cb..98ee1fa56f 100644 --- a/crates/broker/src/wrap.rs +++ b/crates/broker/src/wrap.rs @@ -23,10 +23,9 @@ use tokio::{sync::mpsc, time::MissedTickBehavior}; use crate::broker::{ delivery_verification::{ - pending_verification_echo_seen, queue_or_take_confirmed_verification, - queue_or_take_detected_activity, DeliveryOutcome, PendingActivity, PendingVerification, - ThrottleState, VerificationOutput, ACTIVITY_BUFFER_KEEP_BYTES, ACTIVITY_BUFFER_MAX_BYTES, - ACTIVITY_WINDOW, MAX_VERIFICATION_ATTEMPTS, VERIFICATION_WINDOW, + assess_harness_acceptance, DeliveryOutcome, HarnessAcceptance, PendingActivity, + PendingVerification, ThrottleState, VerificationOutput, ACTIVITY_BUFFER_KEEP_BYTES, + ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW, MAX_VERIFICATION_ATTEMPTS, VERIFICATION_WINDOW, }, injection_format::{format_injection_for_worker_with_workspace, McpReminderThrottle}, }; @@ -36,6 +35,7 @@ use crate::runtime::{ extract_mcp_message_ids, get_terminal_size, terminal_cols, terminal_rows, RelaySession, RelaySessionOptions, RelayWorkspace, }; +use crate::snapshot::Snapshot; use crate::spawner::{spawn_env_vars, with_commit_attestation_env, Spawner}; use crate::util::{ ansi::{floor_char_boundary, strip_ansi, AnsiStripper}, @@ -116,6 +116,37 @@ pub(crate) fn injection_submit_followup_delay(cli: &str) -> Option { paste_submit_harness(cli).then_some(PASTE_INJECTION_SUBMIT_DELAY) } +/// Retry only the submit gesture for a body proven to remain in the live +/// composer. Never write the body twice: an accepted first write whose activity +/// marker was missed would otherwise create a duplicate turn. +pub(crate) fn submit_injection_recovery( + pty: &PtySession, + resolved_cli: &str, + completed_attempts: usize, +) -> anyhow::Result<(tokio::sync::oneshot::Receiver>, u64)> { + let lower = resolved_cli.to_ascii_lowercase(); + if lower.contains("codex") && completed_attempts == 1 { + // Field recovery for Codex 0.150/0.160: a plain Enter left an idle + // multiline composer parked, while End followed by a distinct CR and + // later LF submitted it. First reproduce the non-destructive + // End+Enter half; the final bounded attempt below sends LF only if the + // same delivery is still visibly parked. + return pty.submit_write_paced_with_followup_and_output_boundary( + b"\x1b[F".to_vec(), + Duration::ZERO, + Duration::from_secs(1), + b"\r".to_vec(), + ); + } + + let key = if completed_attempts >= 2 { + b'\n' + } else { + b'\r' + }; + pty.submit_write_paced_with_output_boundary(vec![key], Duration::ZERO) +} + /// Warn (without retrying) when a one-shot auto-response keystroke can't be /// enqueued to the PTY write drainer. These prompts — MCP approval, trust /// dialogs, model-picker dismissals, auto-enter nudges, escape dismissals — are @@ -209,12 +240,12 @@ enum PendingWrapWrite { injection: String, output_boundary: u64, include_reminder: bool, + human_input_generation: u64, }, Retry { verification: PendingVerification, - injection: String, output_boundary: u64, - include_reminder: bool, + human_input_generation: u64, }, } @@ -244,8 +275,8 @@ fn wrap_write_ack_error(ack: WrapWriteAck) -> Option { /// Keep wrap injections single-flight until the PTY drainer confirms the /// previous write. Besides preserving delivery order, this ensures an MCP /// reminder is recorded before the next delivery decides whether to include it. -fn wrap_injection_timer_allowed(has_pending_write_ack: bool) -> bool { - !has_pending_write_ack +fn wrap_injection_timer_allowed(has_pending_write_ack: bool, has_pending_acceptance: bool) -> bool { + !has_pending_write_ack && !has_pending_acceptance } // Readiness deferrals must not spend the bounded delivery retry budget. @@ -266,40 +297,34 @@ fn prepare_wrap_retry( /// raced ahead of the ack select arm. Returns `true` when the echo was already /// present and the delivery was confirmed immediately. fn queue_or_confirm_wrap_verification( - verification: PendingVerification, + mut verification: PendingVerification, output: &VerificationOutput, - activity_detector: Option<&ActivityDetector>, + pty: &PtySession, + resolved_cli: &str, throttle: &mut ThrottleState, pending_verifications: &mut VecDeque, - pending_activities: &mut VecDeque, ) -> bool { - let Some(verification) = - queue_or_take_confirmed_verification(verification, output, pending_verifications) - else { - return false; - }; - - tracing::debug!( - event_id = %verification.event_id, - delivery_id = %verification.delivery_id, - attempts = verification.attempts, - "wrap: delivery echo verified before write ack was processed" - ); - throttle.record(DeliveryOutcome::Success); - if let Some(detector) = activity_detector { - if let Some((activity, pattern)) = - queue_or_take_detected_activity(&verification, output, detector, pending_activities) - { - tracing::info!( - target = "agent_relay::worker::wrap", - delivery_id = %activity.delivery_id, - event_id = %activity.event_id, - pattern = %pattern, - "delivery became active before write ack was processed" - ); - } + let raced_output = output.since(verification.output_boundary).into_owned(); + verification.observe(output, &raced_output); + verification.observe_raw_process_receipt(output, resolved_cli); + verification.observe_visible_composer(&Snapshot::capture(pty), resolved_cli); + if let HarnessAcceptance::Accepted(evidence) = + assess_harness_acceptance(resolved_cli, &verification, &Snapshot::capture(pty)) + { + tracing::info!( + target = "agent_relay::worker::wrap", + delivery_id = %verification.delivery_id, + event_id = %verification.event_id, + attempts = verification.attempts, + evidence = %evidence, + "wrap: delivery accepted by harness before write ack was processed" + ); + throttle.record(DeliveryOutcome::Success); + true + } else { + pending_verifications.push_back(verification); + false } - true } // Shared PTY auto-response state used by run_wrap and run_pty_worker. @@ -1540,6 +1565,7 @@ pub(crate) async fn run_wrap( // this without limit — the oldest chunk is dropped with a warning if the cap // is hit (at which point the child is not consuming input anyway). let mut stdin_pending: VecDeque> = VecDeque::new(); + let mut human_input_generation = 0u64; let mut stdin_retry_deadline: Option = None; const STDIN_RETRY_INTERVAL: Duration = Duration::from_millis(4); @@ -1555,6 +1581,19 @@ pub(crate) async fn run_wrap( // buffered and the retry-deadline arm below flushes it, so // keystrokes are never dropped or reordered under back-pressure. Some(data) = stdin_rx.recv() => { + human_input_generation = human_input_generation.saturating_add(1); + if !pending_verifications.is_empty() { + for verification in pending_verifications.drain(..) { + tracing::warn!( + target = "agent_relay::worker::wrap", + delivery_id = %verification.delivery_id, + event_id = %verification.event_id, + attempts = verification.attempts, + "wrap: human input took ownership before harness acceptance; automatic recovery cancelled" + ); + } + throttle.record(DeliveryOutcome::Failed); + } let backlogged = buffer_and_drain_stdin( &mut stdin_pending, data, @@ -1658,31 +1697,32 @@ pub(crate) async fn run_wrap( pty_auto.handle_claude_trust(&text, &pty).await; } - // Check pending verifications against new output - let mut verified_indices = Vec::new(); - for (i, pv) in pending_verifications.iter().enumerate() { - if pending_verification_echo_seen(&echo_buffer, pv) { - verified_indices.push(i); + // Echo is editor receipt, not turn acceptance. Keep the + // delivery pending until activity starts or the body is + // gone from a proven idle composer. + if !pending_verifications.is_empty() { + let snapshot = Snapshot::capture(&pty); + let mut verified_indices = Vec::new(); + for (i, pv) in pending_verifications.iter_mut().enumerate() { + pv.observe(&echo_buffer, &clean_text); + pv.observe_raw_process_receipt(&echo_buffer, &resolved_cli); + pv.observe_visible_composer(&snapshot, &resolved_cli); + if let HarnessAcceptance::Accepted(evidence) = + assess_harness_acceptance(&resolved_cli, pv, &snapshot) + { + verified_indices.push((i, evidence)); + } } - } - for &i in verified_indices.iter().rev() { - let pv = pending_verifications.remove(i).unwrap(); - tracing::debug!( - event_id = %pv.event_id, - delivery_id = %pv.delivery_id, - attempts = pv.attempts, - "wrap: delivery echo verified" - ); - throttle.record(DeliveryOutcome::Success); - if let Some(detector) = activity_detector.as_ref() { - pending_activities.push_back(PendingActivity { - delivery_id: pv.delivery_id, - event_id: pv.event_id, - expected_echo: pv.expected_echo, - verified_at: Instant::now(), - output_buffer: String::new(), - detector: detector.clone(), - }); + for (i, evidence) in verified_indices.into_iter().rev() { + let pv = pending_verifications.remove(i).unwrap(); + tracing::info!( + event_id = %pv.event_id, + delivery_id = %pv.delivery_id, + attempts = pv.attempts, + evidence = %evidence, + "wrap: delivery accepted by harness" + ); + throttle.record(DeliveryOutcome::Success); } } @@ -2094,7 +2134,10 @@ pub(crate) async fn run_wrap( } _ = pending_injection_interval.tick(), - if wrap_injection_timer_allowed(!pending_wrap_writes.is_empty()) => { + if wrap_injection_timer_allowed( + !pending_wrap_writes.is_empty(), + !pending_verifications.is_empty(), + ) => { // Give backlogged human keystrokes priority onto the PTY FIFO // over a new automated injection — see the auto-responder gate // above for why. @@ -2153,6 +2196,7 @@ pub(crate) async fn run_wrap( injection, output_boundary, include_reminder, + human_input_generation, }; pending_wrap_writes.push(Box::pin(async move { ( @@ -2197,7 +2241,17 @@ pub(crate) async fn run_wrap( injection, output_boundary, include_reminder, + human_input_generation: submitted_generation, } => { + if submitted_generation != human_input_generation { + tracing::warn!( + event_id = %pending.event_id, + "wrap: human input followed the injection write; automatic verification cancelled" + ); + throttle.record(DeliveryOutcome::Failed); + continue; + } + if let Some(error) = error { tracing::warn!( event_id = %pending.event_id, @@ -2226,6 +2280,7 @@ pub(crate) async fn run_wrap( expected_echo: injection, output_boundary, injected_at: Instant::now(), + verification_started_at: Instant::now(), attempts: 1, max_attempts: MAX_VERIFICATION_ATTEMPTS, request_id: None, @@ -2234,58 +2289,60 @@ pub(crate) async fn run_wrap( from: pending.from, body: pending.body, target: pending.target, + echo_seen: false, + activity_buffer: String::new(), + detector: ActivityDetector::for_cli(&resolved_cli), }; queue_or_confirm_wrap_verification( verification, &echo_buffer, - activity_detector.as_ref(), + &pty, + &resolved_cli, &mut throttle, &mut pending_verifications, - &mut pending_activities, ); } PendingWrapWrite::Retry { mut verification, - injection, output_boundary, - include_reminder, + human_input_generation: submitted_generation, } => { - if let Some(error) = error { + if submitted_generation != human_input_generation { tracing::warn!( event_id = %verification.event_id, - error = %error, - "wrap: retry PTY injection was not confirmed; re-queuing delivery" + "wrap: human input followed submit-key recovery; further automatic recovery cancelled" ); - pending_wrap_injections.push_front(PendingWrapInjection { - from: verification.from, - event_id: verification.event_id, - workspace_id: verification.workspace_id, - workspace_alias: verification.workspace_alias, - body: verification.body, - target: verification.target, - queued_at: Instant::now(), - }); + throttle.record(DeliveryOutcome::Failed); continue; } - if include_reminder { - mcp_reminder_throttle.note_sent(Instant::now()); + if let Some(error) = error { + tracing::error!( + event_id = %verification.event_id, + error = %error, + "wrap: submit-key recovery was not confirmed; body left untouched" + ); + throttle.record(DeliveryOutcome::Failed); + continue; } + + tracing::debug!( delivery_id = %verification.delivery_id, event_id = %verification.event_id, - "wrap: delivery re-injection confirmed (retry)" + attempt = verification.attempts, + "wrap: submit-key recovery confirmed" ); - verification.expected_echo = injection; verification.output_boundary = output_boundary; verification.injected_at = Instant::now(); + verification.activity_buffer.clear(); queue_or_confirm_wrap_verification( verification, &echo_buffer, - activity_detector.as_ref(), + &pty, + &resolved_cli, &mut throttle, &mut pending_verifications, - &mut pending_activities, ); } } @@ -2293,20 +2350,105 @@ pub(crate) async fn run_wrap( // Verification tick: check for timed-out wrap verifications _ = verification_tick.tick() => { - let mut retry_queue: Vec = Vec::new(); let mut i = 0; while i < pending_verifications.len() { if pending_verifications[i].injected_at.elapsed() >= VERIFICATION_WINDOW { - let pv = pending_verifications.remove(i).unwrap(); - if pv.attempts < pv.max_attempts { - retry_queue.push(pv); - } else { - tracing::warn!( - event_id = %pv.event_id, - attempts = pv.attempts, - "wrap: delivery verification failed after max retries" - ); - throttle.record(DeliveryOutcome::Failed); + let mut pv = pending_verifications.remove(i).unwrap(); + pv.observe_raw_process_receipt(&echo_buffer, &resolved_cli); + let snapshot = Snapshot::capture(&pty); + pv.observe_visible_composer(&snapshot, &resolved_cli); + match assess_harness_acceptance( + &resolved_cli, + &pv, + &snapshot, + ) { + HarnessAcceptance::Accepted(evidence) => { + tracing::info!( + target = "agent_relay::worker::wrap", + event_id = %pv.event_id, + attempts = pv.attempts, + evidence = %evidence, + "wrap: delivery accepted by harness" + ); + throttle.record(DeliveryOutcome::Success); + } + HarnessAcceptance::Parked + if pv.attempts < pv.max_attempts + && !pv.acceptance_expired() + && stdin_pending.is_empty() + && pending_wrap_writes.is_empty() + && crate::devin::can_inject(&resolved_cli, &pty) => + { + let completed_attempts = pv.attempts; + prepare_wrap_retry(&mut pv, true, Instant::now()); + tracing::warn!( + target = "agent_relay::worker::wrap", + event_id = %pv.event_id, + attempt = pv.attempts, + max = pv.max_attempts, + strategy = "submit_key_only", + "wrap: body remains parked; retrying only the submit gesture" + ); + match submit_injection_recovery( + &pty, + &resolved_cli, + completed_attempts, + ) { + Ok((ack_rx, output_boundary)) => { + let pending_write = PendingWrapWrite::Retry { + verification: pv, + output_boundary, + human_input_generation, + }; + pending_wrap_writes.push(Box::pin(async move { + ( + pending_write, + await_wrap_write_ack( + ack_rx, + WRAP_WRITE_ACK_TIMEOUT, + ) + .await, + ) + })); + } + Err(error) => { + tracing::error!( + event_id = %pv.event_id, + error = %error, + "wrap: failed to queue submit-key recovery; body left untouched" + ); + throttle.record(DeliveryOutcome::Failed); + } + } + } + HarnessAcceptance::Parked + if !pv.acceptance_expired() + && (!stdin_pending.is_empty() + || !pending_wrap_writes.is_empty()) => + { + // An in-flight writer takes priority. Human + // input already drains these verifications in + // the stdin arm, so this deferral never resets + // the total acceptance lifetime. + pv.injected_at = Instant::now(); + pending_verifications.push_back(pv); + } + HarnessAcceptance::Parked => { + tracing::error!( + event_id = %pv.event_id, + attempts = pv.attempts, + "wrap: body remained parked after bounded submit-key recovery" + ); + throttle.record(DeliveryOutcome::Failed); + } + HarnessAcceptance::Inconclusive => { + tracing::error!( + event_id = %pv.event_id, + attempts = pv.attempts, + "wrap: harness acceptance could not be proven; body left untouched" + ); + throttle.record(DeliveryOutcome::Failed); + } } } else { i += 1; @@ -2324,77 +2466,16 @@ pub(crate) async fn run_wrap( } } - // Re-inject retries - for mut pv in retry_queue { - tokio::time::sleep(throttle.delay()).await; - if !prepare_wrap_retry(&mut pv, crate::devin::can_inject(&resolved_cli, &pty), Instant::now()) { - pending_verifications.push_back(pv); - continue; - } - tracing::warn!( - event_id = %pv.event_id, - attempt = pv.attempts, - max = pv.max_attempts, - "wrap: echo verification timeout, retrying injection" - ); - // Retries consult the throttle like first injections: the - // failed attempt usually already echoed the full block, so - // a fresh one within the cooldown is redundant. - let include_reminder = !skip_prompt - && mcp_reminder_throttle.should_include(Instant::now()); - let injection = format_injection_for_worker_with_workspace( - &pv.from, - &pv.event_id, - &pv.body, - &pv.target, - include_reminder, - true, - None, - pv.workspace_id.as_deref(), - pv.workspace_alias.as_deref(), - ); - let bytes = crate::devin::injection_bytes(&resolved_cli, &injection); - let write = submit_injection_body(&pty, &resolved_cli, bytes, Duration::ZERO); - match write { - Ok((ack_rx, output_boundary)) => { - let pending_write = PendingWrapWrite::Retry { - verification: pv, - injection, - output_boundary, - include_reminder, - }; - pending_wrap_writes.push(Box::pin(async move { - ( - pending_write, - await_wrap_write_ack(ack_rx, WRAP_WRITE_ACK_TIMEOUT).await, - ) - })); - } - Err(error) => { - tracing::warn!( - event_id = %pv.event_id, - error = %error, - "wrap: retry PTY injection write failed; re-queuing delivery" - ); - pending_wrap_injections.push_front(PendingWrapInjection { - from: pv.from, - event_id: pv.event_id, - workspace_id: pv.workspace_id, - workspace_alias: pv.workspace_alias, - body: pv.body, - target: pv.target, - queued_at: Instant::now(), - }); - } - } - } } // Auto-enter for stuck agents. Gated on the same backlog check as // the other automation arms above — a stuck-agent nudge must not // jump ahead of keystrokes the human already typed. _ = auto_enter_interval.tick() => { - if stdin_pending.is_empty() { + if stdin_pending.is_empty() + && pending_verifications.is_empty() + && pending_wrap_writes.is_empty() + { pty_auto.try_auto_enter(&pty); } } @@ -2460,12 +2541,10 @@ pub(crate) async fn run_wrap( mod tests { use super::{ await_wrap_write_ack, buffer_and_drain_stdin, drain_stdin_buffer, - injection_submit_followup_delay, queue_or_confirm_wrap_verification, submit_injection_body, + injection_submit_followup_delay, submit_injection_body, submit_injection_recovery, wrap_injection_timer_allowed, wrap_write_ack_error, STDIN_PENDING_MAX_CHUNKS, }; - use crate::broker::delivery_verification::{ - PendingVerification, ThrottleState, VerificationOutput, MAX_VERIFICATION_ATTEMPTS, - }; + use crate::broker::delivery_verification::{PendingVerification, MAX_VERIFICATION_ATTEMPTS}; use crate::ids::{DeliveryId, EventId, MessageTarget}; use crate::pty::PtySession; use crate::worker::detection::ActivityDetector; @@ -2537,6 +2616,43 @@ mod tests { } } + #[cfg(unix)] + #[tokio::test] + async fn codex_recovery_writes_only_end_cr_then_lf() { + let dir = tempfile::tempdir().unwrap(); + let log = dir.path().join("recovery-bytes"); + let script = format!( + "stty raw -echo; dd bs=1 count=5 of='{}' 2>/dev/null; sleep 1", + log.display() + ); + let (pty, mut rx) = PtySession::spawn("/bin/sh", &["-c".into(), script], 24, 80).unwrap(); + let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); + tokio::time::sleep(Duration::from_millis(100)).await; + + let (first_ack, _) = submit_injection_recovery(&pty, "codex", 1).unwrap(); + tokio::time::timeout(Duration::from_secs(3), first_ack) + .await + .expect("End+CR recovery timed out") + .expect("drainer exited") + .expect("End+CR recovery failed"); + let (second_ack, _) = submit_injection_recovery(&pty, "codex", 2).unwrap(); + tokio::time::timeout(Duration::from_secs(3), second_ack) + .await + .expect("LF recovery timed out") + .expect("drainer exited") + .expect("LF recovery failed"); + + for _ in 0..50 { + if std::fs::read(&log).is_ok_and(|bytes| bytes.len() == 5) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!(std::fs::read(&log).unwrap(), b"\x1b[F\r\n"); + pty.shutdown().unwrap(); + drain.abort(); + } + /// Fake Muse-like composer: raw stdin, then one verdict line. A submit key /// that shares its read with multiline body bytes was swallowed into the /// paste (PARKED — the reported manual-Enter bug); a submit key arriving @@ -2672,9 +2788,10 @@ sys.stdout.flush()"#; } #[test] - fn wrap_injection_timer_waits_for_the_previous_write_ack() { - assert!(wrap_injection_timer_allowed(false)); - assert!(!wrap_injection_timer_allowed(true)); + fn wrap_injection_timer_waits_for_write_ack_and_harness_acceptance() { + assert!(wrap_injection_timer_allowed(false, false)); + assert!(!wrap_injection_timer_allowed(true, false)); + assert!(!wrap_injection_timer_allowed(false, true)); } #[test] @@ -2685,6 +2802,7 @@ sys.stdout.flush()"#; expected_echo: String::new(), output_boundary: 0, injected_at: Instant::now(), + verification_started_at: Instant::now(), attempts: 1, max_attempts: MAX_VERIFICATION_ATTEMPTS, request_id: None, @@ -2693,6 +2811,9 @@ sys.stdout.flush()"#; from: "sender".into(), body: "task".into(), target: MessageTarget::new("devin"), + echo_seen: false, + activity_buffer: String::new(), + detector: ActivityDetector::for_cli("devin"), }; for _ in 0..20 { let now = verification.injected_at + super::VERIFICATION_WINDOW; @@ -2708,70 +2829,6 @@ sys.stdout.flush()"#; assert_eq!(verification.attempts, 2); } - #[test] - fn echo_arriving_before_write_ack_is_confirmed_immediately() { - let injection = "multiline task\nwith a delayed submit"; - let mut output = VerificationOutput::default(); - output.push_output(1, b"older output\n"); - // Sequence 2 was assigned by the PTY reader before this wrap write - // was queued, even though wrap has not consumed the chunk yet. - let output_boundary = 2; - let verification = || PendingVerification { - delivery_id: DeliveryId::new("delivery-before-ack"), - event_id: EventId::new("event-before-ack"), - expected_echo: injection.to_string(), - output_boundary, - injected_at: Instant::now(), - attempts: 1, - max_attempts: MAX_VERIFICATION_ATTEMPTS, - request_id: None, - workspace_id: None, - workspace_alias: None, - from: "reviewer".to_string(), - body: "review this".to_string(), - target: MessageTarget::new("claude-worker"), - }; - let mut throttle = ThrottleState::default(); - let mut pending_verifications = VecDeque::new(); - let mut pending_activities = VecDeque::new(); - - output.push_output(2, format!("stale composer echo: {injection}").as_bytes()); - - let stale = queue_or_confirm_wrap_verification( - verification(), - &output, - None, - &mut throttle, - &mut pending_verifications, - &mut pending_activities, - ); - assert!(!stale, "a retained pre-submission echo is stale"); - pending_verifications.clear(); - - output.push_output( - 3, - format!("\nnew composer echo: {injection}\nTool: Write(review.md)").as_bytes(), - ); - let confirmed = queue_or_confirm_wrap_verification( - verification(), - &output, - Some(&ActivityDetector::for_cli("claude")), - &mut throttle, - &mut pending_verifications, - &mut pending_activities, - ); - - assert!(confirmed, "the buffered echo must not wait for new output"); - assert!( - pending_verifications.is_empty(), - "an already-observed echo must not be queued to time out" - ); - assert!( - pending_activities.is_empty(), - "already-buffered activity must be consumed immediately" - ); - } - #[test] fn stdin_drains_in_order_when_queue_accepts() { let mut pending: VecDeque> = VecDeque::new(); diff --git a/crates/relay-pty/src/detection.rs b/crates/relay-pty/src/detection.rs index 5e83c8c581..35eb9e5c94 100644 --- a/crates/relay-pty/src/detection.rs +++ b/crates/relay-pty/src/detection.rs @@ -1,26 +1,66 @@ use crate::ansi::strip_ansi; +/// Whether a rendered row is Codex's actual busy indicator. +/// +/// Requiring both fragments on the same row avoids treating ordinary task +/// text that mentions the interrupt hint as harness activity. +pub fn is_codex_busy_status_line(line: &str) -> bool { + let lower = line.to_ascii_lowercase(); + lower.contains("working") && lower.contains("esc to interrupt") +} + #[derive(Debug, Clone)] pub struct ActivityDetector { patterns: Vec<&'static str>, + codex_working_pair: bool, } impl ActivityDetector { pub fn for_cli(cli: &str) -> Self { let lower = cli.to_lowercase(); + let codex_working_pair = lower.contains("codex"); let patterns = if lower.contains("claude") { - vec!["⠋", "⠙", "⠹", "Tool:", "Read(", "Write(", "Edit("] + vec![ + "⠋", + "⠙", + "⠹", + "Tool:", + "Read(", + "Write(", + "Edit(", + "esc to interrupt", + ] } else if lower.contains("codex") { + // Current Codex renders `Working (… • esc to interrupt)` after a + // composer submission. Older builds used `Thinking...`/`Running:`. + // Keep both generations: this detector is acceptance evidence, so + // missing the current marker leaves a body parked while an echo is + // incorrectly treated as delivery success. vec!["Thinking...", "Running:", "$ ", "function_call"] } else if crate::readiness::is_devin_cli(cli) { vec!["Thinking ·", "Guide Devin while it works"] } else if lower.contains("gemini") { vec!["Generating", "Action:", "Executing"] + } else if lower.contains("opencode") { + // OpenCode's ordinary transcript frequently contains the generic + // words "tool", "Thinking", and "Working". Until a distinctive + // busy marker is established, rely on composer clearing instead. + Vec::new() } else { Vec::new() }; - Self { patterns } + Self { + patterns, + codex_working_pair, + } + } + + /// Whether this detector has harness-specific evidence that a turn began. + /// Generic output is useful for activity telemetry, but a TUI repaint is + /// not strong enough to acknowledge a broker delivery. + pub fn has_explicit_patterns(&self) -> bool { + !self.patterns.is_empty() } pub fn detect_activity(&self, output: &str, expected_echo: &str) -> Option { @@ -31,6 +71,10 @@ impl ActivityDetector { clean_output.replace(expected_echo, "") }; + if self.codex_working_pair && relevant_output.lines().any(is_codex_busy_status_line) { + return Some("Working+esc to interrupt".to_string()); + } + if self.patterns.is_empty() { if relevant_output.trim().is_empty() { None @@ -81,6 +125,20 @@ mod tests { #[test] fn detect_activity_for_codex_patterns() { let detector = ActivityDetector::for_cli("codex"); + assert_eq!( + detector.detect_activity( + "Working appears in the parked task body", + "Relay message from Alice [evt_1]: hello" + ), + None + ); + assert_eq!( + detector.detect_activity( + "Working (2s • esc to interrupt)", + "Relay message from Alice [evt_1]: hello" + ), + Some("Working+esc to interrupt".to_string()) + ); assert_eq!( detector.detect_activity( "Thinking... running tool", @@ -88,6 +146,10 @@ mod tests { ), Some("Thinking...".to_string()) ); + assert!(is_codex_busy_status_line("Working (2s • esc to interrupt)")); + assert!(!is_codex_busy_status_line( + "task text mentions esc to interrupt" + )); } #[test] @@ -114,6 +176,13 @@ mod tests { ); } + #[test] + fn only_known_harnesses_have_explicit_acceptance_patterns() { + assert!(ActivityDetector::for_cli("codex").has_explicit_patterns()); + assert!(ActivityDetector::for_cli("claude").has_explicit_patterns()); + assert!(!ActivityDetector::for_cli("muse").has_explicit_patterns()); + } + #[test] fn detect_activity_defaults_to_any_output() { let detector = ActivityDetector::for_cli("mystery-cli"); diff --git a/crates/relay-pty/src/snapshot.rs b/crates/relay-pty/src/snapshot.rs index b66e4960c0..a00428674c 100644 --- a/crates/relay-pty/src/snapshot.rs +++ b/crates/relay-pty/src/snapshot.rs @@ -71,6 +71,11 @@ pub struct Snapshot { pub rows: u16, pub cols: u16, pub cursor: (u16, u16), + /// The cursor cell contains the final character of an autowrapped line. + /// Alacritty keeps the cursor on that occupied cell until the next input + /// character performs the wrap, so cursor-bounded rendering must include + /// it rather than treating it as the insertion point. + input_needs_wrap: bool, cells: Vec>, /// Terminal mode flags captured at the same locked point as the grid. /// `to_ansi` re-emits the relevant ones so an attaching client's terminal @@ -112,6 +117,7 @@ impl Snapshot { // visible viewport. let cursor_row = (cursor_point.line.0.max(0) as u16).saturating_add(1); let cursor_col = (cursor_point.column.0 as u16).saturating_add(1); + let input_needs_wrap = grid.cursor.input_needs_wrap; let mut cells = Vec::with_capacity(rows as usize); for row_index in 0..(rows as usize) { @@ -127,6 +133,7 @@ impl Snapshot { rows, cols, cursor: (cursor_row, cursor_col), + input_needs_wrap, cells, // Copy the mode flags out under the same lock as the cells so the // captured modes match the captured grid exactly. @@ -152,6 +159,49 @@ impl Snapshot { out } + /// Plain text from the top-left of the viewport through the cell directly + /// before the cursor. Rows below the cursor and cells to its right are + /// excluded so composer detection cannot mistake stale right-side cells + /// for text the user is currently editing. + pub fn to_plain_through_cursor(&self) -> String { + let cursor_row = self.cursor.0.saturating_sub(1) as usize; + let cursor_col = self.cursor.1.saturating_sub(1) as usize; + let mut out = String::with_capacity( + (cursor_row.saturating_add(1)) * ((self.cols as usize).saturating_add(1)), + ); + for (row_index, row) in self.cells.iter().enumerate().take(cursor_row + 1) { + let limit = if row_index == cursor_row { + cursor_col + .saturating_add(usize::from(self.input_needs_wrap)) + .min(row.len()) + } else { + row.len() + }; + let row_start = out.len(); + for cell in row.iter().take(limit) { + out.push(cell.c); + } + while out.len() > row_start && out.ends_with(' ') { + out.pop(); + } + out.push('\n'); + } + out + } + + /// Whether the cursor row contains visible cells at or to the right of + /// the cursor. A cursor-bounded composer can look like a bare prompt while + /// a draft remains in cells to its right, so callers must not classify it + /// as empty in that state. + pub fn has_visible_text_at_or_after_cursor(&self) -> bool { + let cursor_row = self.cursor.0.saturating_sub(1) as usize; + let cursor_col = (self.cursor.1.saturating_sub(1) as usize) + .saturating_add(usize::from(self.input_needs_wrap)); + self.cells + .get(cursor_row) + .is_some_and(|row| row.iter().skip(cursor_col).any(|cell| cell.c != ' ')) + } + /// ANSI bytes that redraw the captured grid on a fresh terminal. /// /// Layout: @@ -583,6 +633,30 @@ mod tests { assert_eq!(lines[2], " hello", "row 3 should be {:?}", lines[2]); } + #[test] + fn plain_through_cursor_excludes_stale_right_and_lower_cells() { + let term = parse_into(4, 20, &[b"abcSTALE\r\nLOWER\x1b[1;4H"]); + let snap = Snapshot::from_term(&term); + assert_eq!(snap.cursor, (1, 4)); + assert_eq!(snap.to_plain_through_cursor(), "abc\n"); + assert!(snap.has_visible_text_at_or_after_cursor()); + + let term = parse_into(4, 20, &[b"abc"]); + let snap = Snapshot::from_term(&term); + assert!(!snap.has_visible_text_at_or_after_cursor()); + } + + #[test] + fn plain_through_cursor_includes_occupied_cell_while_wrap_is_pending() { + let term = parse_into(2, 5, &[b"abcde"]); + let snap = Snapshot::from_term(&term); + + assert_eq!(snap.cursor, (1, 5)); + assert!(snap.input_needs_wrap); + assert_eq!(snap.to_plain_through_cursor(), "abcde\n"); + assert!(!snap.has_visible_text_at_or_after_cursor()); + } + #[test] fn ansi_emits_clear_and_home_prefix() { let term = parse_into(2, 5, &[b"hi"]); diff --git a/docs/harnesses/devin.md b/docs/harnesses/devin.md index 505baa63c6..6d59a6c764 100644 --- a/docs/harnesses/devin.md +++ b/docs/harnesses/devin.md @@ -30,8 +30,11 @@ an approval or other dialog occupies the composer. Both initial tasks and follow-up messages use bracketed paste, then a separate Enter after 250 ms. In the installed CLI, a paste and Enter in one terminal -write left the prompt in the composer; a later Enter submitted it. Relay does -not send repeated recovery Enters that might accidentally approve a tool. +write left the prompt in the composer; a later Enter submitted it. Relay +fleet PTY workers retry submit only while the compacted message tail remains +visibly parked, and never while an approval dialog or human input owns the +composer. Interactive `relay wrap` sessions fail an unconfirmed parked write +instead of pressing a key when the Devin readiness gate is no longer idle. ## MCP configuration diff --git a/docs/harnesses/pty-delivery.md b/docs/harnesses/pty-delivery.md new file mode 100644 index 0000000000..ee1b217fd8 --- /dev/null +++ b/docs/harnesses/pty-delivery.md @@ -0,0 +1,59 @@ +# PTY delivery acceptance + +Relay keeps custody of a PTY delivery until the harness accepts a turn. Seeing +the formatted message in terminal output is only editor echo: it does not prove +that the composer submitted it. + +## Submit strategies + +| Harness | Initial body | Submit gesture | Bounded recovery | +| ------------------------------------------------ | ------------------------------ | -------------------------- | ---------------------------------------------------------------------------- | +| Codex | bulk editor input | distinct `CR` after 250 ms | `End`, then distinct `CR`; `LF` only if the same body remains visibly parked | +| Claude and Muse | paced or configured body input | distinct `CR` after 250 ms | distinct `CR`, then guarded `LF` | +| Devin | bracketed paste | distinct `CR` after 250 ms | distinct `CR`, then guarded `LF` | +| OpenCode, Gemini, Cursor and other PTY harnesses | paced typed input | trailing `CR` | distinct `CR`, then guarded `LF` | + +A recovery never writes the message body again. Relay retries a submit key only +when a whitespace-stripped suffix of at most 96 UTF-8 bytes from an ordinary +post-startup message is still present in the live composer at the cursor. It +stops immediately when an activity marker appears or an echoed body has left a +proven idle composer. An inconclusive screen is reported as failure instead of +receiving a blind keypress. The separate broker-spawned Codex `init_` path +checks a suffix of up to 128 UTF-8 bytes before stripping whitespace. + +Human drive sessions and queued human PTY writes take priority. An interactive +hold pauses recovery without consuming its retry budget. Actual human input +transfers composer ownership to the operator, cancels automatic recovery, and +reports the broker delivery as failed rather than submitting mixed input. + +## Codex mechanism + +Broker-spawned `init_` tasks use Codex's existing render-gated, chunked initial +write path before the same separate submit gesture and acceptance checks. The +table's bulk-input description applies to ordinary post-startup deliveries. + +Reproduced with authenticated Codex CLI 0.160.0 in a real PTY: a message body +and `CR` delivered in the same terminal write remained in Codex's multiline +composer. Terminal output contained the complete body, which made the previous +echo verifier acknowledge a turn that had never started. Moving the cursor to +the end and sending `CR` as a later write advanced the harness and produced the +expected response. The final `LF` exists for the older observed field state and +is sent only after the body is still proven to be parked. + +## Events and status + +PTY workers emit `delivery_unconfirmed` before recovery, +`delivery_resubmitted` after a submit-only retry is queued, +`delivery_verified` with `verification: harness_acceptance` after acceptance, +and `delivery_failed` after a write error, inconclusive screen, or exhausted +budget. Unconfirmed and retry events put the worker in `blocked_on_send`; agent +and fleet listings already expose that state and retain the pending message +count while recovery is in flight. A terminal failure moves the delivery to the +dead-letter path and removes it from the pending count. + +## Rollout + +The change is planned for broker 13.2.0. Upgrade and restart brokers and their +PTY agents. Existing processes keep the old verification and recovery behavior +until restarted. Native Codex app-server/IPC delivery is a separate transport +and is not affected by this PTY-specific bug. diff --git a/packages/contracts/fixtures/event-fixtures.json b/packages/contracts/fixtures/event-fixtures.json index 5d5ee0a7ea..50cd101738 100644 --- a/packages/contracts/fixtures/event-fixtures.json +++ b/packages/contracts/fixtures/event-fixtures.json @@ -11,6 +11,8 @@ "agent_permanently_dead", "agent_context_low", "agent_blocked_on_send", + "delivery_unconfirmed", + "delivery_resubmitted", "delivery_verified", "delivery_failed", "message_delivery_confirmed", @@ -188,6 +190,33 @@ "blockedSecs": 120, "pendingDeliveryCount": 1 } + }, + { + "kind": "delivery_unconfirmed", + "eventId": "evt_0017", + "seq": 117, + "timestamp": "2026-02-23T00:00:16.000Z", + "payload": { + "name": "WorkerA", + "deliveryId": "del_5", + "eventId": "relay_evt_5", + "reason": "body remains parked in the live composer", + "attempts": 1, + "maxAttempts": 3 + } + }, + { + "kind": "delivery_resubmitted", + "eventId": "evt_0018", + "seq": 118, + "timestamp": "2026-02-23T00:00:17.000Z", + "payload": { + "name": "WorkerA", + "deliveryId": "del_5", + "eventId": "relay_evt_5", + "attempt": 2, + "strategy": "submit_key_only" + } } ], "wave0_timeout_terminal_semantics": { diff --git a/packages/harness-driver/src/lifecycle-hooks.ts b/packages/harness-driver/src/lifecycle-hooks.ts index e7ade85c98..ff07a7a6bd 100644 --- a/packages/harness-driver/src/lifecycle-hooks.ts +++ b/packages/harness-driver/src/lifecycle-hooks.ts @@ -165,6 +165,8 @@ export interface DriverAgentResult { export type DriverAgentActivityReason = | 'delivery_queued' | 'delivery_injected' + | 'delivery_unconfirmed' + | 'delivery_resubmitted' | 'delivery_active' | 'delivery_ack' | 'delivery_read_ack' diff --git a/packages/harness-driver/src/protocol.ts b/packages/harness-driver/src/protocol.ts index 1e524c9c75..e7d8984cf6 100644 --- a/packages/harness-driver/src/protocol.ts +++ b/packages/harness-driver/src/protocol.ts @@ -613,9 +613,28 @@ export type BrokerEvent = name: string; delivery_id: string; event_id: string; - /** 'echo' when confirmed in PTY output, 'timeout_fallback' when acked unverified. */ + /** 'harness_acceptance' for current PTY workers; legacy workers may report 'echo' or 'timeout_fallback'. */ verification?: string; reason?: string; + evidence?: string | null; + attempts?: number | null; + } + | { + kind: 'delivery_unconfirmed'; + name: string; + delivery_id: string; + event_id: string; + reason: string; + attempts: number; + max_attempts: number; + } + | { + kind: 'delivery_resubmitted'; + name: string; + delivery_id: string; + event_id: string; + attempt: number; + strategy: string; } | { kind: 'delivery_failed'; @@ -838,7 +857,33 @@ export type WorkerToBroker = } | { type: 'delivery_verified'; - payload: { delivery_id: string; event_id: string; verification?: string; reason?: string }; + payload: { + delivery_id: string; + event_id: string; + verification?: string; + reason?: string; + evidence?: string; + attempts?: number; + }; + } + | { + type: 'delivery_unconfirmed'; + payload: { + delivery_id: string; + event_id: string; + reason: string; + attempts: number; + max_attempts: number; + }; + } + | { + type: 'delivery_resubmitted'; + payload: { + delivery_id: string; + event_id: string; + attempt: number; + strategy: string; + }; } | { type: 'delivery_failed'; diff --git a/packages/sdk-py/src/agent_relay/protocol.py b/packages/sdk-py/src/agent_relay/protocol.py index e0f263e50c..d4be128f80 100644 --- a/packages/sdk-py/src/agent_relay/protocol.py +++ b/packages/sdk-py/src/agent_relay/protocol.py @@ -14,7 +14,8 @@ # BrokerEvent is a dict with a 'kind' field discriminator. # Event kinds: agent_spawned, agent_released, agent_exit, agent_exited, # relay_inbound, worker_stream, worker_ready, worker_error, -# delivery_queued, delivery_injected, delivery_verified, delivery_failed, +# delivery_queued, delivery_injected, delivery_unconfirmed, +# delivery_resubmitted, delivery_verified, delivery_failed, # delivery_active, delivery_ack, delivery_retry, delivery_dropped, # relaycast_published, relaycast_publish_failed, acl_denied, # agent_idle, agent_restarting, agent_restarted, agent_permanently_dead diff --git a/packages/sdk-swift/Sources/AgentRelayBrokerSDK/BrokerTypes.swift b/packages/sdk-swift/Sources/AgentRelayBrokerSDK/BrokerTypes.swift index 59c8d0b314..b49085ebea 100644 --- a/packages/sdk-swift/Sources/AgentRelayBrokerSDK/BrokerTypes.swift +++ b/packages/sdk-swift/Sources/AgentRelayBrokerSDK/BrokerTypes.swift @@ -245,6 +245,8 @@ public enum BrokerEvent: Sendable { case deliveryDropped(DeliveryDroppedEvent) case deliveryQueued(DeliveryStateEvent) case deliveryInjected(DeliveryStateEvent) + case deliveryUnconfirmed(DeliveryUnconfirmedEvent) + case deliveryResubmitted(DeliveryResubmittedEvent) case deliveryVerified(DeliveryStateEvent) case deliveryFailed(DeliveryFailedEvent) case deliveryActive(DeliveryStateEvent) @@ -277,6 +279,8 @@ extension BrokerEvent: Codable { case "delivery_dropped": self = .deliveryDropped(try DeliveryDroppedEvent(from: decoder)) case "delivery_queued": self = .deliveryQueued(try DeliveryStateEvent(from: decoder)) case "delivery_injected": self = .deliveryInjected(try DeliveryStateEvent(from: decoder)) + case "delivery_unconfirmed": self = .deliveryUnconfirmed(try DeliveryUnconfirmedEvent(from: decoder)) + case "delivery_resubmitted": self = .deliveryResubmitted(try DeliveryResubmittedEvent(from: decoder)) case "delivery_verified": self = .deliveryVerified(try DeliveryStateEvent(from: decoder)) case "delivery_failed": self = .deliveryFailed(try DeliveryFailedEvent(from: decoder)) case "delivery_active": self = .deliveryActive(try DeliveryStateEvent(from: decoder)) @@ -308,6 +312,8 @@ extension BrokerEvent: Codable { case .deliveryDropped(let value): try value.encode(to: encoder) case .deliveryQueued(let value): try value.encode(to: encoder) case .deliveryInjected(let value): try value.encode(to: encoder) + case .deliveryUnconfirmed(let value): try value.encode(to: encoder) + case .deliveryResubmitted(let value): try value.encode(to: encoder) case .deliveryVerified(let value): try value.encode(to: encoder) case .deliveryFailed(let value): try value.encode(to: encoder) case .deliveryActive(let value): try value.encode(to: encoder) @@ -409,7 +415,9 @@ public struct RelayInboundEvent: Codable, Sendable { public var kind: String = " public struct WorkerStreamEvent: Codable, Sendable { public var kind: String = "worker_stream"; public var name: String; public var stream: String; public var chunk: String } public struct DeliveryRetryEvent: Codable, Sendable { public var kind: String = "delivery_retry"; public var name: String; public var deliveryId: String; public var eventId: String; public var attempts: Int; enum CodingKeys: String, CodingKey { case kind, name, attempts; case deliveryId = "delivery_id"; case eventId = "event_id" } } public struct DeliveryDroppedEvent: Codable, Sendable { public var kind: String = "delivery_dropped"; public var name: String; public var count: Int; public var reason: String } -public struct DeliveryStateEvent: Codable, Sendable { public var kind: String; public var name: String; public var deliveryId: String; public var eventId: String; enum CodingKeys: String, CodingKey { case kind, name; case deliveryId = "delivery_id"; case eventId = "event_id" } } +public struct DeliveryStateEvent: Codable, Sendable { public var kind: String; public var name: String; public var deliveryId: String; public var eventId: String; public var verification: String?; public var reason: String?; public var evidence: String?; public var attempts: Int?; enum CodingKeys: String, CodingKey { case kind, name, verification, reason, evidence, attempts; case deliveryId = "delivery_id"; case eventId = "event_id" } } +public struct DeliveryUnconfirmedEvent: Codable, Sendable { public var kind: String = "delivery_unconfirmed"; public var name: String; public var deliveryId: String; public var eventId: String; public var reason: String; public var attempts: Int; public var maxAttempts: Int; enum CodingKeys: String, CodingKey { case kind, name, reason, attempts; case deliveryId = "delivery_id"; case eventId = "event_id"; case maxAttempts = "max_attempts" } } +public struct DeliveryResubmittedEvent: Codable, Sendable { public var kind: String = "delivery_resubmitted"; public var name: String; public var deliveryId: String; public var eventId: String; public var attempt: Int; public var strategy: String; enum CodingKeys: String, CodingKey { case kind, name, attempt, strategy; case deliveryId = "delivery_id"; case eventId = "event_id" } } public struct DeliveryFailedEvent: Codable, Sendable { public var kind: String = "delivery_failed"; public var name: String; public var deliveryId: String; public var eventId: String; public var reason: String; enum CodingKeys: String, CodingKey { case kind, name, reason; case deliveryId = "delivery_id"; case eventId = "event_id" } } public struct WorkerReadyEvent: Codable, Sendable { public var kind: String = "worker_ready"; public var name: String; public var runtime: AgentRuntime; public var provider: HeadlessProvider?; public var cli: String?; public var model: String?; public var sessionId: String?; enum CodingKeys: String, CodingKey { case kind, name, runtime, provider, cli, model; case sessionId = "session_id" } } public struct WorkerErrorEvent: Codable, Sendable { public var kind: String = "worker_error"; public var name: String; public var code: String; public var message: String } diff --git a/tests/relayflows/cases/1891-codex-parked-composer-recovery/case.json b/tests/relayflows/cases/1891-codex-parked-composer-recovery/case.json new file mode 100644 index 0000000000..b3b0097262 --- /dev/null +++ b/tests/relayflows/cases/1891-codex-parked-composer-recovery/case.json @@ -0,0 +1,21 @@ +{ + "version": 1, + "id": "1891-codex-parked-composer-recovery", + "kind": "bugfix", + "title": "Recover a Relay delivery that Codex leaves in its composer", + "requirements": ["broker-linux-x64"], + "runner": { + "command": ["node", "tests/relayflows/cases/1891-codex-parked-composer-recovery/run.mjs"] + }, + "timeoutSeconds": 300, + "expected": { + "base": { + "outcome": "bug", + "signature": "codex_delivery_left_in_composer" + }, + "head": { + "outcome": "fixed", + "signature": "codex_parked_delivery_recovers_once" + } + } +} diff --git a/tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs b/tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs new file mode 100644 index 0000000000..ad72a56c51 --- /dev/null +++ b/tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs @@ -0,0 +1,436 @@ +#!/usr/bin/env node + +/** + * Provider-backed acceptance check for relay#1891. + * + * This deliberately is not the RelayFlow CI runner: it uses an authenticated + * interactive Codex CLI and therefore spends provider capacity. Run it from + * a disposable working directory after building the broker under test: + * + * node tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs \ + * --broker target/release/agent-relay-broker + * + * It proves the exact user-facing path. A Relay delivery arrives after Codex + * has rendered its initial composer. The broker must observe actual Codex turn + * activity, may issue bounded submit-only recovery, and must never inject the + * body a second time. The response marker is checked only after turn activity, + * so the instruction's editor echo cannot satisfy the test. + */ + +import { constants as fsConstants } from 'node:fs'; +import { access, chmod, copyFile, mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { spawn } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import process from 'node:process'; +import { createInterface } from 'node:readline'; + +const CASE_ID = '1891-codex-parked-composer-recovery'; +const RESPONSE_MARKER = 'RELAY_1891_REAL_CODEX_ACK'; +const BODY = [ + 'Relay verification task.', + `Reply with exactly ${RESPONSE_MARKER}.`, + 'Do not use tools, edit files, or ask a follow-up question.', +].join('\n'); +const brokerPath = path.resolve(option('--broker', process.env.RELAY_REAL_CODEX_BROKER_BINARY ?? '')); +const resultPath = option('--result', ''); +const timeoutSeconds = Number(option('--timeout-seconds', '180')); +const settleSeconds = Number(option('--settle-seconds', '60')); + +if (!brokerPath || brokerPath === path.resolve('.')) { + throw new Error('Pass --broker or RELAY_REAL_CODEX_BROKER_BINARY.'); +} +if (!Number.isSafeInteger(timeoutSeconds) || timeoutSeconds < 20 || timeoutSeconds > 600) { + throw new Error('--timeout-seconds must be an integer from 20 to 600.'); +} +if (!Number.isSafeInteger(settleSeconds) || settleSeconds < 10 || settleSeconds > 120) { + throw new Error('--settle-seconds must be an integer from 10 to 120.'); +} +await assertExecutable(brokerPath, 'broker'); +await requireCommand('codex'); + +const probeDir = await mkdtemp(path.join(tmpdir(), 'relayflow-real-codex-1891-')); +const cwd = path.join(probeDir, 'workspace'); +const codexHome = path.join(probeDir, 'codex-home'); +const agentName = `relayflow-real-codex-1891-${process.pid}`; +let worker; +let stderr = ''; + +try { + await mkdir(cwd, { recursive: true }); + await mkdir(codexHome, { mode: 0o700 }); + // Codex must have separate SQLite state from the live sessions on this + // machine. Copy only the login into a private disposable home. + const authSource = path.join(process.env.CODEX_HOME ?? path.join(process.env.HOME, '.codex'), 'auth.json'); + const authCopy = path.join(codexHome, 'auth.json'); + await copyFile(authSource, authCopy); + await chmod(authCopy, 0o600); + worker = spawn( + brokerPath, + [ + 'pty', + '--agent-name', + agentName, + 'codex', + '--', + '--config', + 'check_for_update_on_startup=false', + '--model', + 'gpt-6-sol', + '--config', + 'model_reasoning_effort="low"', + '--sandbox', + 'workspace-write', + '--no-daemon', + ], + { + cwd, + env: { + ...process.env, + CODEX_HOME: codexHome, + // The disposable cwd prevents the check from modifying a repository. + RELAY_INJECT_RATE_MS: '0', + }, + stdio: ['pipe', 'pipe', 'pipe'], + } + ); + worker.stderr.on('data', (chunk) => { + stderr = `${stderr}${chunk}`.slice(-8_000); + }); + const frames = createFrameQueue(worker, () => stderr); + + sendFrame(worker, { v: 2, type: 'init_worker', payload: { agent: { name: agentName } } }); + await frames.waitFor( + (frame) => frame.type === 'worker_stream' && typeof frame.payload?.chunk === 'string', + 45_000, + 'initial interactive Codex output' + ); + const dismissedUpdateDialogs = await settleCodexComposer(worker, frames, settleSeconds * 1_000); + if (frames.count('worker_ready') === 0) { + await frames.waitFor((frame) => frame.type === 'worker_ready', 30_000, 'broker worker readiness'); + } + + sendFrame(worker, { + v: 2, + type: 'deliver_relay', + request_id: 'relayflow-real-codex-delivery', + payload: { + delivery_id: 'delivery_relayflow_real_codex_1891', + event_id: 'event_relayflow_real_codex_1891', + from: 'relayflow-verifier', + target: agentName, + body: BODY, + priority: 2, + injection_mode: 'wait', + }, + }); + + // Capture a post-paste grid before the recovery deadline. This becomes + // bounded local evidence if a future Codex rendering change prevents the + // parked-composer recognizer from arming submit-only recovery. + await new Promise((resolve) => setTimeout(resolve, 2_000)); + sendFrame(worker, { + v: 2, + type: 'snapshot_pty', + request_id: 'real-codex-parked-snapshot', + payload: { format: 'plain' }, + }); + const parkedSnapshot = await frames.waitFor( + (frame) => frame.type === 'snapshot_response' && frame.request_id === 'real-codex-parked-snapshot', + 10_000, + 'post-paste Codex snapshot' + ); + + const observation = await observeOneCodexTurn( + worker, + frames, + parkedSnapshot.payload, + timeoutSeconds * 1_000 + ); + if (observation.deliveryInjections !== 1) { + throw new Error(`Expected one body injection, observed ${observation.deliveryInjections}.`); + } + if ( + observation.recoveries.length > 2 || + observation.recoveries.some((strategy) => strategy !== 'submit_key_only') + ) { + throw new Error(`Unexpected recovery strategy: ${JSON.stringify(observation.recoveries)}.`); + } + if (!observation.verified) throw new Error('Broker never verified harness acceptance.'); + await waitForVisibleAnswer(worker, frames, 120_000); + + const result = { + version: 1, + caseId: CASE_ID, + outcome: 'passed', + signature: + observation.recoveries.length > 0 + ? 'real_codex_parked_delivery_recovers_once' + : 'real_codex_delivery_completes_once', + deliveryInjections: observation.deliveryInjections, + submitOnlyRecoveries: observation.recoveries.length, + dismissedUpdateDialogs, + }; + if (resultPath) { + await mkdir(path.dirname(path.resolve(resultPath)), { recursive: true }); + await writeFile(resultPath, `${JSON.stringify(result)}\n`, 'utf8'); + } + console.log(`REAL_CODEX_1891_PASS ${JSON.stringify(result)}`); +} finally { + if (worker?.exitCode === null) { + sendFrame(worker, { v: 2, type: 'shutdown_worker', payload: { reason: 'real Codex proof complete' } }); + await Promise.race([ + new Promise((resolve) => worker.once('exit', resolve)), + new Promise((resolve) => setTimeout(resolve, 5_000)), + ]); + if (worker.exitCode === null) worker.kill('SIGKILL'); + } + await rm(probeDir, { recursive: true, force: true }); +} + +async function settleCodexComposer(worker, frames, settleMs) { + const deadline = Date.now() + settleMs; + let cleanSince; + let dismissed = 0; + let acceptedDisposableFolder = false; + let snapshotIndex = 0; + let lastScreen = ''; + while (true) { + const snapshot = await snapshotPty(worker, frames, `real-codex-settle-${snapshotIndex++}`); + const screen = String(snapshot.payload?.screen ?? ''); + lastScreen = screen; + if (screen.includes('Trust this folder?') && screen.includes('Trust and continue')) { + // The cwd was created by this test and contains no project files. + await writePty(worker, frames, 'real-codex-trust-disposable-folder', '\r'); + acceptedDisposableFolder = true; + cleanSince = undefined; + await sleep(750); + continue; + } + if (hasCodexUpdateDialog(screen)) { + // Escape closes this specific client-side update prompt. This runs before + // Relay delivery, so it cannot cancel delivery verification or simulate a + // user submitting the Relay body. + await writePty(worker, frames, `real-codex-dismiss-update-${dismissed}`, '\u001b'); + dismissed += 1; + cleanSince = undefined; + await sleep(750); + continue; + } + if (isEmptyCodexComposer(screen)) { + cleanSince ??= Date.now(); + } else { + cleanSince = undefined; + } + const remaining = deadline - Date.now(); + if (remaining <= 0) { + if (cleanSince && Date.now() - cleanSince >= 5_000) return dismissed; + throw new Error( + `Codex did not remain at an empty composer for five seconds: ${JSON.stringify({ + dismissedUpdateDialogs: dismissed, + acceptedDisposableFolder, + screen: lastScreen.slice(-4_000), + })}` + ); + } + await sleep(Math.min(1_000, remaining)); + } +} + +function hasCodexUpdateDialog(screen) { + return /Update available/i.test(screen) && /Skip(?: until next version)?/i.test(screen); +} + +function isEmptyCodexComposer(screen) { + return ( + /Ask Codex to do anything/i.test(screen) && + !screen.includes('Trust this folder?') && + !hasCodexUpdateDialog(screen) && + !/(?:Working \(|esc to interrupt|thinking|processing)/i.test(screen) + ); +} + +async function snapshotPty(worker, frames, requestId) { + sendFrame(worker, { + v: 2, + type: 'snapshot_pty', + request_id: requestId, + payload: { format: 'plain' }, + }); + return frames.waitFor( + (frame) => frame.type === 'snapshot_response' && frame.request_id === requestId, + 10_000, + `PTY snapshot ${requestId}` + ); +} + +async function writePty(worker, frames, requestId, data) { + sendFrame(worker, { + v: 2, + type: 'write_pty', + request_id: requestId, + payload: { data }, + }); + const response = await frames.waitFor( + (frame) => frame.type === 'write_pty_response' && frame.request_id === requestId, + 10_000, + `PTY write ${requestId}` + ); + if (response.payload?.error) throw new Error(`PTY write failed: ${JSON.stringify(response.payload.error)}`); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function observeOneCodexTurn(worker, frames, parkedSnapshot, timeoutMs) { + let deliveryInjections = frames.count('delivery_injected'); + const recoveries = []; + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const frame = await frames.next(Math.min(1_000, deadline - Date.now())); + if (!frame) continue; + if (frame.type === 'delivery_injected') deliveryInjections = frames.count('delivery_injected'); + if (frame.type === 'delivery_resubmitted') recoveries.push(frame.payload?.strategy); + if (frame.type === 'delivery_verified') { + return { deliveryInjections, recoveries, verified: true }; + } + if (frame.type === 'delivery_failed') { + const snapshot = await snapshotPty(worker, frames, 'real-codex-rejected-snapshot'); + throw new Error( + `Broker rejected the real Codex delivery: ${JSON.stringify({ + reason: frame.payload?.reason, + deliveryInjections, + recoveries, + cursor: snapshot.payload?.cursor, + screen: String(snapshot.payload?.screen ?? '').slice(-4_000), + })}` + ); + } + } + const finalSnapshot = await snapshotPty(worker, frames, 'real-codex-final-snapshot'); + throw new Error( + `Timed out awaiting one accepted Codex turn: ${JSON.stringify({ + deliveryInjections, + recoveries, + parkedSnapshot: { + cursor: parkedSnapshot?.cursor, + screen: String(parkedSnapshot?.screen ?? '').slice(-4_000), + }, + finalScreen: String(finalSnapshot.payload?.screen ?? '').slice(-4_000), + frames: frames.debugSummary(), + })}` + ); +} + +async function waitForVisibleAnswer(worker, frames, timeoutMs) { + const deadline = Date.now() + timeoutMs; + let lastScreen = ''; + let count = 0; + while (Date.now() < deadline) { + const snapshot = await snapshotPty(worker, frames, `real-codex-answer-${count++}`); + lastScreen = String(snapshot.payload?.screen ?? ''); + const answerLines = lastScreen.split('\n').filter((line) => line.trim() === `• ${RESPONSE_MARKER}`); + if (answerLines.length === 1 && isEmptyCodexComposer(lastScreen)) return; + if (answerLines.length > 1) break; + await sleep(1_000); + } + throw new Error( + `Codex did not finish exactly one visible answer: ${JSON.stringify({ + answerCount: lastScreen.split('\n').filter((line) => line.trim() === `• ${RESPONSE_MARKER}`).length, + screen: lastScreen.slice(-4_000), + })}` + ); +} + +function createFrameQueue(child, getStderr) { + const queue = []; + const recent = []; + const counts = new Map(); + const waiters = new Set(); + let parseError; + let exitError; + const lines = createInterface({ input: child.stdout }); + lines.on('line', (line) => { + try { + const frame = JSON.parse(line); + queue.push(frame); + counts.set(frame.type, (counts.get(frame.type) ?? 0) + 1); + recent.push({ type: frame.type, requestId: frame.request_id }); + if (recent.length > 30) recent.shift(); + } catch (error) { + parseError = new Error(`Broker emitted invalid JSON: ${error.message}; line=${line.slice(0, 2_000)}`); + } + for (const notify of waiters) notify(); + }); + child.once('exit', (code, signal) => { + exitError = new Error( + `Broker exited before proof completed (${signal ?? code ?? 'unknown'}): ${JSON.stringify({ stderr: getStderr(), counts: Object.fromEntries(counts), recent })}` + ); + for (const notify of waiters) notify(); + }); + const awaitChange = (timeoutMs) => + new Promise((resolve) => { + const timer = setTimeout(() => { + waiters.delete(notify); + resolve(); + }, timeoutMs); + const notify = () => { + clearTimeout(timer); + waiters.delete(notify); + resolve(); + }; + waiters.add(notify); + }); + return { + count(type) { + return counts.get(type) ?? 0; + }, + debugSummary() { + return { counts: Object.fromEntries(counts), recent }; + }, + async next(timeoutMs) { + if (parseError) throw parseError; + if (exitError) throw exitError; + if (queue.length > 0) return queue.shift(); + await awaitChange(timeoutMs); + if (parseError) throw parseError; + if (exitError) throw exitError; + return queue.shift(); + }, + async waitFor(predicate, timeoutMs, label) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const frame = await this.next(Math.min(500, deadline - Date.now())); + if (frame && predicate(frame)) return frame; + } + throw new Error(`Timed out waiting for ${label}: ${getStderr()}`); + }, + }; +} + +function sendFrame(child, frame) { + if (child?.stdin?.writable) child.stdin.write(`${JSON.stringify(frame)}\n`); +} + +function option(name, fallback) { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : fallback; +} + +async function assertExecutable(candidate, name) { + try { + await access(candidate, fsConstants.X_OK); + } catch { + throw new Error(`${name} must be an executable file: ${candidate}`); + } +} + +async function requireCommand(command) { + const result = await new Promise((resolve) => { + const child = spawn('sh', ['-lc', `command -v ${command}`], { stdio: ['ignore', 'pipe', 'ignore'] }); + child.once('error', () => resolve(false)); + child.once('exit', (code) => resolve(code === 0)); + }); + if (!result) throw new Error(`${command} must be on PATH for the provider-backed proof.`); +} diff --git a/tests/relayflows/cases/1891-codex-parked-composer-recovery/run.mjs b/tests/relayflows/cases/1891-codex-parked-composer-recovery/run.mjs new file mode 100644 index 0000000000..8b9cb01071 --- /dev/null +++ b/tests/relayflows/cases/1891-codex-parked-composer-recovery/run.mjs @@ -0,0 +1,320 @@ +#!/usr/bin/env node + +import { constants as fsConstants } from 'node:fs'; +import { access, chmod, mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { spawn, execFileSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import process from 'node:process'; +import { createInterface } from 'node:readline'; +import { fileURLToPath } from 'node:url'; + +const CASE_ID = '1891-codex-parked-composer-recovery'; +const BODY = 'RELAY_1891_PARKED_COMPOSER_SENTINEL\nComplete the requested task exactly once.'; +const targetDir = requiredDirectory('RELAY_PR_PROOF_TARGET_DIR'); +const harnessDir = requiredDirectory('RELAY_PR_PROOF_HARNESS_DIR'); +const binaryPath = await requiredExecutable('RELAY_PR_PROOF_BROKER_BINARY'); +const resultPath = requiredValue('RELAY_PR_PROOF_RESULT_PATH'); +const arm = requiredValue('RELAY_PR_PROOF_ARM'); + +if (arm !== 'base' && arm !== 'head') throw new Error(`Unexpected proof arm ${JSON.stringify(arm)}.`); +const expectedSha = + arm === 'base' ? process.env.RELAY_PR_PROOF_BASE_SHA : process.env.RELAY_PR_PROOF_HEAD_SHA; +if (!expectedSha) throw new Error(`Missing expected ${arm} SHA.`); +const targetSha = execFileSync('git', ['-C', targetDir, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); +if (targetSha !== expectedSha) + throw new Error(`Target checkout ${targetSha} does not match ${arm} SHA ${expectedSha}.`); +if (!isWithin(harnessDir, fileURLToPath(import.meta.url))) { + throw new Error('The RelayFlow runner must execute from the exact-head harness checkout.'); +} + +const probeDir = await mkdtemp(path.join(tmpdir(), 'relayflow-1891-')); +const binDir = path.join(probeDir, 'bin'); +const fakeCodexPath = path.join(binDir, 'codex'); +const fakeCodexSource = String.raw`#!/usr/bin/env node +const sentinel = ${JSON.stringify(BODY)}; +process.stdin.setRawMode?.(true); +process.stdin.resume(); +// The placeholder is transcript text; Codex's actual idle composer is an +// empty prompt with its cursor immediately after "› ". That cursor placement +// is what the broker's startup gate must see before it emits worker_ready. +process.stdout.write('Welcome to Codex\r\nAsk Codex to do anything\r\n› '); + +let composer = ''; +let parked = false; +let sawEnd = false; +let submitted = false; +let copies = 0; +let escape = []; + +function park() { + parked = true; + // Reflow the envelope like the native Codex TUI. The raw PTY output cannot + // match the original formatted body byte-for-byte, while the exact compact + // tail remains visible in the live composer. + process.stdout.write('\r\n' + composer.replaceAll('\n', '\r\n ') + '\r\nCOMPOSER_PARKED\r\n› ' + composer.replace(/\s+/g, '').slice(-96)); +} + +function submit() { + submitted = true; + composer = ''; + process.stdout.write('\x1b[2J\x1b[HWorking (1s · esc to interrupt)\r\nTASK_STARTED copies=' + copies + '\r\n'); +} + +function receiveByte(byte) { + if (escape.length > 0) { + escape.push(byte); + if (escape.length === 3) { + if (escape[0] === 0x1b && escape[1] === 0x5b && escape[2] === 0x46) sawEnd = true; + escape = []; + } + return; + } + if (byte === 0x1b) { + escape = [byte]; + return; + } + if (byte === 13) { + if (!parked && composer.includes(sentinel)) { + park(); + } else if (parked && sawEnd && !submitted) { + submit(); + } + return; + } + // A Relay envelope is multiline. Codex keeps pasted LFs in the composer; + // retaining them lets the fixture recognize the exact delivery body rather + // than a flattened approximation. + if (byte === 10) { + composer += '\n'; + return; + } + composer += String.fromCharCode(byte); + if (composer.endsWith(sentinel)) copies += 1; +} + +process.stdin.on('data', (chunk) => { + for (const byte of chunk) receiveByte(byte); +}); + +setTimeout(() => { + if (!submitted) process.stdout.write('\r\nTASK_PARKED_TIMEOUT copies=' + copies + '\r\n'); +}, 12_000).unref(); +`; + +let worker; +let stderr = ''; +try { + await mkdir(binDir, { recursive: true }); + await writeFile(fakeCodexPath, fakeCodexSource, { encoding: 'utf8', mode: 0o700 }); + await chmod(fakeCodexPath, 0o700); + worker = spawn(binaryPath, ['pty', '--agent-name', 'relayflow-codex-1891', 'codex'], { + cwd: targetDir, + env: { + ...Object.fromEntries(Object.entries(process.env).filter(([name]) => !/^(?:LD_|DYLD_)/.test(name))), + PATH: `${binDir}:${process.env.PATH ?? '/usr/local/bin:/usr/bin:/bin'}`, + RELAY_INJECT_RATE_MS: '0', + }, + stdio: ['pipe', 'pipe', 'pipe'], + }); + worker.stderr.on('data', (chunk) => { + stderr = `${stderr}${chunk}`.slice(-8_000); + }); + const frames = createFrameQueue(worker, () => stderr); + sendFrame(worker, { v: 2, type: 'init_worker', payload: { agent: { name: 'relayflow-codex-1891' } } }); + await frames.waitFor((frame) => frame.type === 'worker_ready', 15_000, 'worker readiness'); + sendFrame(worker, { + v: 2, + type: 'deliver_relay', + request_id: 'relayflow-delivery', + payload: { + delivery_id: 'delivery_relayflow_1891', + event_id: 'event_relayflow_1891', + from: 'broker', + target: 'relayflow-codex-1891', + body: BODY, + priority: 2, + injection_mode: 'wait', + }, + }); + await new Promise((resolve) => setTimeout(resolve, 1_000)); + sendFrame(worker, { + v: 2, + type: 'snapshot_pty', + request_id: 'parked-snapshot', + payload: { format: 'plain' }, + }); + const parkedSnapshot = await frames.waitFor( + (frame) => frame.type === 'snapshot_response' && frame.request_id === 'parked-snapshot', + 5_000, + 'parked Codex snapshot' + ); + const taskObserver = createTaskObserver(); + let observation; + await frames.waitFor( + (frame) => { + observation = taskObserver.observe(frame); + return observation !== undefined; + }, + 20_000, + 'parked or submitted task marker' + ); + let outcome; + let signature; + let details; + if (arm === 'base' && observation.marker === 'TASK_PARKED_TIMEOUT' && observation.copies === 1) { + outcome = 'bug'; + signature = 'codex_delivery_left_in_composer'; + details = `The delivery body reached the Codex composer ${observation.copies} time without a turn; the baseline never sent the bounded submit-only recovery.`; + } else if (arm === 'head' && observation.marker === 'TASK_STARTED' && observation.copies === 1) { + outcome = 'fixed'; + signature = 'codex_parked_delivery_recovers_once'; + details = + 'The body remained parked, then the broker sent End plus a distinct submit key; Codex accepted one turn without replaying the body.'; + } else { + throw new Error( + `Unexpected parked-composer outcome ${JSON.stringify({ arm, observation, parkedSnapshot: parkedSnapshot.payload, frames: frames.debugSummary() })}.` + ); + } + if (arm === 'head') { + const settlement = await frames.waitFor( + (frame) => frame.type === 'delivery_verified' || frame.type === 'delivery_failed', + 35_000, + 'broker acceptance of the submitted Codex turn' + ); + if (settlement.type !== 'delivery_verified') { + throw new Error( + `Codex started the turn but broker rejected delivery: ${JSON.stringify(settlement.payload)}.` + ); + } + if (frames.count('delivery_injected') !== 1 || frames.count('delivery_resubmitted') !== 1) { + throw new Error( + `Expected one body write and one submit-only recovery: ${JSON.stringify(frames.debugSummary())}.` + ); + } + } + await mkdir(path.dirname(resultPath), { recursive: true }); + await writeFile( + resultPath, + `${JSON.stringify({ version: 1, caseId: CASE_ID, arm, outcome, signature, details })}\n`, + 'utf8' + ); +} finally { + if (worker?.exitCode === null) { + sendFrame(worker, { v: 2, type: 'shutdown_worker', payload: { reason: 'proof complete' } }); + await Promise.race([ + new Promise((resolve) => worker.once('exit', resolve)), + new Promise((resolve) => setTimeout(resolve, 2_000)), + ]); + if (worker.exitCode === null) worker.kill('SIGKILL'); + } + await rm(probeDir, { recursive: true, force: true }); +} + +function createTaskObserver() { + let output = ''; + const seen = new WeakSet(); + return { + observe(frame) { + if (frame?.type !== 'worker_stream' || typeof frame.payload?.chunk !== 'string') return undefined; + // waitFor can revisit queued frames on each poll. Each stream chunk must + // contribute to the split-marker buffer exactly once. + if (seen.has(frame)) return undefined; + seen.add(frame); + // The broker may split terminal output across stream frames. Keep a + // bounded tail so a marker is recognized even when it crosses a frame. + output = `${output}${frame.payload.chunk}`.slice(-8_000); + const match = /\b(TASK_(?:STARTED|PARKED_TIMEOUT)) copies=(\d+)\b/.exec(output); + return match ? { marker: match[1], copies: Number(match[2]) } : undefined; + }, + }; +} + +function createFrameQueue(child, getStderr) { + const queue = []; + const recent = []; + const counts = new Map(); + const waiters = new Set(); + let parseError; + let exitError; + const lines = createInterface({ input: child.stdout }); + lines.on('line', (line) => { + try { + const frame = JSON.parse(line); + queue.push(frame); + counts.set(frame.type, (counts.get(frame.type) ?? 0) + 1); + recent.push({ type: frame.type, requestId: frame.request_id }); + if (recent.length > 20) recent.shift(); + } catch (error) { + parseError = new Error(`Broker emitted invalid JSON: ${error.message}; line=${line.slice(0, 2_000)}`); + } + for (const notify of waiters) notify(); + }); + child.once('exit', (code, signal) => { + exitError = new Error( + `Broker exited before proof completed (${signal ?? code ?? 'unknown'}): ${getStderr()}` + ); + for (const notify of waiters) notify(); + }); + return { + count(type) { + return counts.get(type) ?? 0; + }, + debugSummary() { + return { counts: Object.fromEntries(counts), recent }; + }, + async waitFor(predicate, timeoutMs, label) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (parseError) throw parseError; + if (exitError) throw exitError; + const index = queue.findIndex(predicate); + if (index >= 0) return queue.splice(index, 1)[0]; + await new Promise((resolve) => { + const timer = setTimeout( + () => { + waiters.delete(notify); + resolve(); + }, + Math.min(100, Math.max(1, deadline - Date.now())) + ); + const notify = () => { + clearTimeout(timer); + waiters.delete(notify); + resolve(); + }; + waiters.add(notify); + }); + } + throw new Error(`Timed out waiting for ${label}: ${getStderr()}`); + }, + }; +} + +function sendFrame(child, frame) { + if (child?.stdin?.writable) child.stdin.write(`${JSON.stringify(frame)}\n`); +} +function requiredValue(name) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`Missing required environment variable ${name}.`); + return value; +} +function requiredDirectory(name) { + return path.resolve(requiredValue(name)); +} +async function requiredExecutable(name) { + const candidate = path.resolve(requiredValue(name)); + try { + await access(candidate, fsConstants.R_OK | fsConstants.X_OK); + } catch { + throw new Error(`${name} must name a readable executable file.`); + } + return candidate; +} +function isWithin(directory, candidate) { + const relative = path.relative(directory, candidate); + return ( + relative === '' || + (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) + ); +} From de91fc873a81c568f79a9801b30fbd68263d65e4 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 8 Oct 2026 17:28:15 -0700 Subject: [PATCH 02/24] =?UTF-8?q?style:=20prettier-format=20merge-train=20?= =?UTF-8?q?files=20(unblocks=20trunk=20=E2=86=92=20main=20lint)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/codegen-models.yml | 3 +-- .github/workflows/security.yml | 2 -- AGENTS.md | 15 ++++++++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/codegen-models.yml b/.github/workflows/codegen-models.yml index 72f21434cb..8568f7f10a 100644 --- a/.github/workflows/codegen-models.yml +++ b/.github/workflows/codegen-models.yml @@ -4,8 +4,7 @@ on: push: paths: - 'packages/utils/cli-registry.yaml' - branches: - [main] + branches: [main] pull_request: paths: - 'packages/utils/cli-registry.yaml' diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 89d55e1c7c..903a6b9074 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -32,7 +32,6 @@ jobs: ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main')) && (needs.changes.outputs.node_changed == 'true') runs-on: ubuntu-latest - steps: - name: Checkout code uses: actions/checkout@v4 @@ -147,7 +146,6 @@ jobs: ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main')) && (needs.changes.outputs.node_changed == 'true') runs-on: ubuntu-latest - steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/AGENTS.md b/AGENTS.md index b9bbd8cac9..9459ebd043 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -315,6 +315,7 @@ Future agents can query past trajectories to learn from your decisions. + ## Merging: `trunk` + the `mergeable` label CI suites do **not** run automatically on feature branches. They run only on @@ -326,21 +327,24 @@ run on any branch. (Repos whose default branch is not agent batches ready PRs into `trunk`, gets that one PR green, and merges it. **When you open a PR** + 1. Branch from `trunk` and open the PR with **base `trunk`**, not `main`. A PR into `main` from any other branch fails the `Trunk guard` check. 2. No CI runs on your PR, so verify locally before calling it ready: run the typecheck, tests and lint this repo uses, and list the exact commands and results in the PR body. -**When the PR is ready** -3. Add the label **`mergeable`** once all of these are true: - - The change is complete and the local checks above pass. - - Review feedback (human and bot) is addressed or answered. - - It is not a draft and does not depend on an unmerged PR. +**When the PR is ready** 3. Add the label **`mergeable`** once all of these are true: + +- The change is complete and the local checks above pass. +- Review feedback (human and bot) is addressed or answered. +- It is not a draft and does not depend on an unmerged PR. + 4. Remove `mergeable` if the PR stops being ready (new work, a failing check, a blocking question). The label is read live from GitHub on every sweep. **What you must not do** + - Do not merge your own PR, and never merge into or push to `trunk` or `main` directly. - Do not re-enable CI for feature branches or edit the `trunk` gates in @@ -355,4 +359,5 @@ to rebase on `trunk`; do so and keep the label. > Interim: the sweep worker is not deployed yet. Until it is, a human or a > designated agent performs the merge-agent steps manually. Labelling is unchanged. + From 65bf077b985f8a3df4985fbf2db344fa84b20998 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Thu, 8 Oct 2026 18:26:43 -0700 Subject: [PATCH 03/24] ci: route Linux CI jobs through vars.CI_LINUX_RUNNER Rust tests/clippy/fmt/cross-compile, Node test/coverage/lint, build-bun.sh tests and publish Build & Version use vars.CI_LINUX_RUNNER with an ubuntu-latest fallback; matrix jobs switch only their ubuntu-latest entry, so check names are unchanged. macOS/Windows jobs are untouched. The PR-proof Landlock/devpts/no_new_privs tests self-skip on kernels that cannot host the sandbox (including StarSling's 6.1 kernel), so a new sandbox-proof job pinned to ubuntu-latest runs that file and fails if any of its tests is skipped. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/publish.yml | 2 +- .github/workflows/rust-ci.yml | 8 +++--- .github/workflows/test-build.yml | 4 +-- .github/workflows/test.yml | 49 ++++++++++++++++++++++++++++++-- 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4fa5768d65..ac4ce1e51c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -334,7 +334,7 @@ jobs: # Build all packages once, version them, and upload build: name: Build & Version - runs-on: ubuntu-latest + runs-on: ${{ vars.CI_LINUX_RUNNER || 'ubuntu-latest' }} outputs: new_version: ${{ steps.bump.outputs.new_version }} is_prerelease: ${{ steps.bump.outputs.is_prerelease }} diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 21eb358cf3..c2f97271f9 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -26,7 +26,7 @@ jobs: if: >- ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main')) && (needs.changes.outputs.rust_changed == 'true') - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os == 'ubuntu-latest' && (vars.CI_LINUX_RUNNER || 'ubuntu-latest') || matrix.os }} strategy: fail-fast: false matrix: @@ -70,7 +70,7 @@ jobs: if: >- ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main')) && (needs.changes.outputs.rust_changed == 'true') - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os == 'ubuntu-latest' && (vars.CI_LINUX_RUNNER || 'ubuntu-latest') || matrix.os }} strategy: fail-fast: false matrix: @@ -106,7 +106,7 @@ jobs: if: >- ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main')) && (needs.changes.outputs.rust_changed == 'true') - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os == 'ubuntu-latest' && (vars.CI_LINUX_RUNNER || 'ubuntu-latest') || matrix.os }} strategy: fail-fast: false matrix: @@ -127,7 +127,7 @@ jobs: if: >- ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main')) && (needs.changes.outputs.rust_changed == 'true') - runs-on: ubuntu-latest + runs-on: ${{ vars.CI_LINUX_RUNNER || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 4a15760e00..4ba527e587 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -23,7 +23,7 @@ jobs: test-build-script: if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main') name: Test build-bun.sh - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os == 'ubuntu-latest' && (vars.CI_LINUX_RUNNER || 'ubuntu-latest') || matrix.os }} strategy: fail-fast: false matrix: @@ -111,7 +111,7 @@ jobs: test-build-edge-cases: if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main') name: Test edge cases - runs-on: ubuntu-latest + runs-on: ${{ vars.CI_LINUX_RUNNER || 'ubuntu-latest' }} steps: - name: Checkout code diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 785c27a46b..1aaf102c9a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: if: >- ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main')) && (needs.changes.outputs.node_changed == 'true') - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os == 'ubuntu-latest' && (vars.CI_LINUX_RUNNER || 'ubuntu-latest') || matrix.os }} strategy: matrix: os: [ubuntu-latest, macos-latest] @@ -61,6 +61,49 @@ jobs: - name: Run tests run: npm test + # The PR-proof Landlock / devpts / no_new_privs sandbox tests skip themselves + # (it.skipIf(!HAS_TRUSTED_LANDLOCK_RUNTIME)) on kernels that cannot host the + # sandbox, which includes the StarSling runners that `test` above may run on + # via vars.CI_LINUX_RUNNER. This job stays pinned to GitHub's ubuntu-latest + # (never the runner variable) and fails if any test in the file is skipped, + # so those tests can never silently stop running. + sandbox-proof: + name: PR-proof sandbox tests (Landlock, ubuntu-latest) + needs: changes + if: >- + ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main')) && + (needs.changes.outputs.node_changed == 'true') + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22.14.0' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Ensure rollup optional dependencies are installed + run: npm install --no-save rollup || true + + - name: Run PR-proof sandbox tests + run: npx vitest run tests/fixtures/pr-proof-contract.test.ts --reporter=default --reporter=json --outputFile.json=pr-proof-results.json + + - name: Require every PR-proof test to have run + run: | + node -e ' + const r = require("./pr-proof-results.json"); + const all = r.testResults.flatMap((f) => f.assertionResults); + const notRun = all.filter((t) => t.status !== "passed" && t.status !== "failed"); + console.log("pr-proof-contract: " + all.length + " tests, " + notRun.length + " not run"); + for (const t of notRun) console.log("::error::not run (" + t.status + "): " + t.fullName); + if (all.length === 0 || notRun.length > 0) process.exit(1); + ' + windows-credential-acl: name: Windows credential ACL validation needs: changes @@ -100,7 +143,7 @@ jobs: if: >- ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main')) && (needs.changes.outputs.node_changed == 'true') - runs-on: ubuntu-latest + runs-on: ${{ vars.CI_LINUX_RUNNER || 'ubuntu-latest' }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -130,7 +173,7 @@ jobs: if: >- ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main')) && (needs.changes.outputs.node_changed == 'true') - runs-on: ubuntu-latest + runs-on: ${{ vars.CI_LINUX_RUNNER || 'ubuntu-latest' }} steps: - name: Checkout repository uses: actions/checkout@v4 From 4faa95ab803f698bb06381ce7f523e75d0f3640a Mon Sep 17 00:00:00 2001 From: kjgbot Date: Thu, 8 Oct 2026 19:30:10 -0700 Subject: [PATCH 04/24] ci: install Rust toolchain in jobs that build the broker on StarSling StarSling Linux runners preinstall rustc 1.92.0; GitHub's ubuntu-24.04 image ships 1.99.0. The broker uses atomic try_update (#1907), which 1.92 rejects as unstable (E0658), so test, Coverage (upload) and Test build-bun.sh failed on trunk -> main #1939 after #1946 routed them to vars.CI_LINUX_RUNNER. Rust CI jobs already install stable via dtolnay/rust-toolchain and passed on the same runners. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/test-build.yml | 6 ++++++ .github/workflows/test.yml | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 4ba527e587..43b8e7cb97 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -33,6 +33,12 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + # StarSling Linux runners (vars.CI_LINUX_RUNNER) preinstall rustc 1.92, older + # than GitHub's ubuntu-24.04 image (1.99); 1.92 cannot compile the + # broker (stable-only atomic try_update). Install the toolchain explicitly. + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable + - name: Setup Bun uses: oven-sh/setup-bun@v2 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1aaf102c9a..27c1ecf2d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,6 +40,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + # StarSling Linux runners (vars.CI_LINUX_RUNNER) preinstall rustc 1.92, older + # than GitHub's ubuntu-24.04 image (1.99); 1.92 cannot compile the + # broker (stable-only atomic try_update). Install the toolchain explicitly. + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable + - name: Setup Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: @@ -148,6 +154,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + # StarSling Linux runners (vars.CI_LINUX_RUNNER) preinstall rustc 1.92, older + # than GitHub's ubuntu-24.04 image (1.99); 1.92 cannot compile the + # broker (stable-only atomic try_update). Install the toolchain explicitly. + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable + - name: Setup Node.js uses: actions/setup-node@v4 with: From e1287f91b47142d6d54d5bc87a08b62e4e3a180a Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Fri, 9 Oct 2026 08:50:17 -0700 Subject: [PATCH 05/24] fix(broker): address #1939 must-fix PTY delivery review findings (WIP) - M1: latch a terminally failed, already-written delivery: fence its id against replay and hold injection until the composer releases the draft - M2: generic any_output cannot confirm a draft touching the cursor; wrapped-echo compact-tail matches keep same-read activity - M3: delivery_failed leaves BlockedOnSend only while work remains - M4: recovery writes are cancellable; human input withholds queued keys - M5: delivery_verified rejects legacy echo/timeout_fallback PTY frames - M6: empty or unadmitted write_pty does not take composer ownership Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- .../src/broker/delivery_verification.rs | 117 ++++++++- crates/broker/src/pty_worker.rs | 240 ++++++++++++++---- crates/broker/src/runtime/tests.rs | 141 ++++++++++ crates/broker/src/runtime/worker_events.rs | 92 +++++-- crates/broker/src/wrap.rs | 39 ++- crates/relay-pty/src/pty.rs | 159 +++++++++++- crates/relay-pty/src/snapshot.rs | 26 ++ 7 files changed, 735 insertions(+), 79 deletions(-) diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 55c8aa8553..9c3917801a 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -185,6 +185,12 @@ impl PendingVerification { if let Some(index) = clean.rfind(&self.expected_echo) { self.activity_buffer .push_str(&clean[index + self.expected_echo.len()..]); + } else if let Some(end) = + end_of_compact_match(&clean, &expected_tail(&self.expected_echo)) + { + // A wrapped echo matched only by its compact tail still + // carries any same-read turn marker after that tail. + self.activity_buffer.push_str(&clean[end..]); } } else if self.echo_seen { self.activity_buffer.push_str(text); @@ -224,6 +230,45 @@ fn compact_render(text: &str) -> String { .collect() } +/// Byte offset in `text` just past the last occurrence of `compact_needle` +/// when whitespace in `text` is ignored, matching [`compact_render`]. +fn end_of_compact_match(text: &str, compact_needle: &str) -> Option { + if compact_needle.is_empty() { + return None; + } + let mut compact = String::with_capacity(text.len()); + let mut ends = Vec::with_capacity(text.len()); + for (index, character) in text.char_indices() { + if !character.is_whitespace() { + compact.push(character); + ends.push(index + character.len_utf8()); + } + } + let start = compact.rfind(compact_needle)?; + // `ends` is indexed by compact character, so count characters (not bytes) + // through the final matched one. + let matched_chars = compact[..start + compact_needle.len()].chars().count(); + ends.get(matched_chars - 1).copied() +} + +/// Whether the delivery's compact tail sits immediately at the cursor: +/// ending just before it, starting at it, or straddling it. Any of these +/// means the editor may still hold the draft, regardless of which prompt +/// glyph (or body line that looks like one) the composer parser anchored on. +fn tail_touches_cursor(snapshot: &Snapshot, tail: &str) -> bool { + if tail.is_empty() { + return false; + } + let before = compact_render(&snapshot.to_plain_through_cursor()); + let after = compact_render(&snapshot.to_plain_from_cursor()); + if before.ends_with(tail) || after.starts_with(tail) { + return true; + } + tail.char_indices() + .skip(1) + .any(|(split, _)| before.ends_with(&tail[..split]) && after.starts_with(&tail[split..])) +} + fn is_cat_process(cli: &str) -> bool { cli.rsplit(['/', '\\']) .next() @@ -379,6 +424,22 @@ fn composer_is_idle(snapshot: &Snapshot, cli: &str) -> bool { .unwrap_or(false) } +/// Whether a terminally failed delivery's body has left the live composer. +/// +/// A failed body can remain typed in the editor. Injecting the next delivery +/// on top of it would append to that draft or submit both as one turn, so the +/// worker holds injection until the composer is proven idle, or the failed +/// tail is neither in the parsed composer nor touching the cursor. +pub(crate) fn failed_draft_released(cli: &str, expected_echo: &str, snapshot: &Snapshot) -> bool { + let tail = expected_tail(expected_echo); + if tail.is_empty() || composer_is_idle(snapshot, cli) { + return true; + } + let in_composer = current_composer(snapshot, cli) + .is_some_and(|composer| compact_render(&composer).contains(&tail)); + !in_composer && !tail_touches_cursor(snapshot, &tail) +} + /// Distinguish terminal echo from actual harness acceptance. /// /// Activity is definitive acceptance. A body is considered parked only when @@ -410,7 +471,14 @@ pub(crate) fn assess_harness_acceptance( if verification.echo_seen && is_cat_process(cli) { return HarnessAcceptance::Accepted("process_echo".to_string()); } - if verification.echo_seen && !verification.detector.has_explicit_patterns() { + // Harnesses without a turn-start marker fall back to post-echo output, but + // a repaint of a still-unsent draft is output too. Only accept it once the + // body has left the cursor: a tail ending at, starting at, or straddling + // the cursor stays inconclusive. + if verification.echo_seen + && !verification.detector.has_explicit_patterns() + && !tail_touches_cursor(snapshot, &tail) + { if let Some(pattern) = verification .detector .detect_activity(&verification.activity_buffer, &verification.expected_echo) @@ -935,6 +1003,53 @@ mod tests { pty.shutdown().unwrap(); } + #[cfg(unix)] + #[tokio::test] + async fn generic_repaint_cannot_confirm_a_draft_straddling_the_cursor() { + let expected = "Relay message from Lead [evt]: fix idle injection"; + // The cursor sits inside the draft, so the parser's through-cursor + // composer cannot see the tail, yet the body is still unsent. + let (pty, snapshot) = codex_snapshot(&format!("› {expected}\x1b[1;20H")).await; + let mut verification = codex_verification(expected); + verification.detector = ActivityDetector::for_cli("muse"); + verification + .activity_buffer + .push_str("composer repaint after echo"); + + assert_eq!( + assess_harness_acceptance("muse", &verification, &snapshot), + HarnessAcceptance::Inconclusive, + "generic output must not confirm a body that still sits at the cursor" + ); + pty.shutdown().unwrap(); + } + + #[test] + fn wrapped_echo_preserves_same_chunk_acceptance_activity() { + let expected = "Relay message from Lead [evt]: Reply with exactly WRAPPED_CODEX_ACK"; + let mut verification = codex_verification(expected); + verification.echo_seen = false; + let mut output = VerificationOutput::default(); + let chunk = "Relay message from Lead [evt]: Reply with exactly\r\n WRAPPED_CODEX_ACK\r\nWorking (1s • esc to interrupt)"; + output.push_str(chunk); + verification.observe(&output, chunk); + + assert!(verification.echo_seen); + assert_eq!( + verification.accepted_activity(), + Some("Working+esc to interrupt".to_string()), + "a turn marker in the same read as a wrapped echo must be retained" + ); + } + + #[test] + fn compact_match_end_maps_back_to_raw_offsets() { + let text = "a b\r\n cé d tail"; + let end = end_of_compact_match(text, "abcéd").unwrap(); + assert_eq!(&text[end..], " tail"); + assert_eq!(end_of_compact_match(text, "zzz"), None); + } + #[test] fn split_echo_preserves_same_chunk_acceptance_activity() { let expected = "Relay message from Lead [evt]: split echo"; diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index 79c5897412..173dd573e0 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -29,7 +29,7 @@ use crate::broker::{ continuity::parse_continuity_command, delivery_verification::{ assess_harness_acceptance, current_timestamp_ms, delivery_injected_event_payload, - delivery_queued_event_payload, DeliveryOutcome, HarnessAcceptance, PendingActivity, + delivery_queued_event_payload, failed_draft_released, DeliveryOutcome, HarnessAcceptance, PendingActivity, PendingVerification, ThrottleState, VerificationOutput, ACTIVITY_BUFFER_KEEP_BYTES, ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW, MAX_VERIFICATION_ATTEMPTS, VERIFICATION_WINDOW, }, @@ -47,7 +47,8 @@ use crate::util::terminal::{ use crate::util::utf8_stream::Utf8StreamDecoder; use crate::worker::detection::{is_codex_busy_status_line, ActivityDetector}; use crate::wrap::{ - submit_injection_body, submit_injection_recovery, warn_on_auto_response_write, PtyAutoState, + cancel_recovery_writes, new_recovery_cancel, submit_injection_body, submit_injection_recovery, + warn_on_auto_response_write, PtyAutoState, AUTO_SUGGESTION_BLOCK_TIMEOUT, }; use base64::Engine; @@ -100,6 +101,35 @@ impl CompletedWorkerDeliveries { } } +/// A delivery whose body was written to the composer but never proven +/// accepted. Its text may still be in the editor, so no further delivery is +/// injected until the composer is observed without it; otherwise the next body +/// would append to the failed draft or submit both as one turn. +#[derive(Debug, Clone)] +struct FailedComposerLatch { + delivery_id: DeliveryId, + event_id: EventId, + expected_echo: String, +} + +/// Record a terminal failure for a delivery whose body may remain in the +/// composer: fence its id against replay for the PTY's lifetime and latch +/// injection until the composer releases the draft. +fn latch_failed_written_delivery( + failed_written_deliveries: &mut CompletedWorkerDeliveries, + failed_composer_latch: &mut Option, + delivery_id: &DeliveryId, + event_id: &EventId, + expected_echo: &str, +) { + failed_written_deliveries.insert(delivery_id.clone(), event_id.clone()); + *failed_composer_latch = Some(FailedComposerLatch { + delivery_id: delivery_id.clone(), + event_id: event_id.clone(), + expected_echo: expected_echo.to_string(), + }); +} + /// Default per-atom gap for escape-aware paced injection, in milliseconds. /// Overridable at runtime via `RELAY_INJECT_RATE_MS`; `0` disables pacing and /// restores the single bulk write. Kept small because a full injection @@ -1047,6 +1077,10 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { let mut pending_worker_injections: VecDeque = VecDeque::new(); let mut pending_worker_delivery_ids: HashSet = HashSet::new(); let mut completed_worker_deliveries = CompletedWorkerDeliveries::default(); + // Deliveries written to this PTY that then failed terminally. A replay of + // one of these ids is refused rather than pasted over the failed draft. + let mut failed_written_deliveries = CompletedWorkerDeliveries::default(); + let mut failed_composer_latch: Option = None; // The injection currently being written across paced stages, if any. Only // one injection is in flight at a time; the pending-injection interval arm // starts the next one once this returns to `None`. @@ -1093,6 +1127,9 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { let mut pending_recovery_writes: FuturesUnordered = FuturesUnordered::new(); let mut human_input_generation = 0u64; + // Cancellation flags for queued submit-key recoveries. Human input sets + // them so a delayed recovery key cannot submit the human's draft. + let mut recovery_cancels: Vec> = Vec::new(); let mut startup_output = String::new(); let mut startup_total_bytes = 0usize; let mut init_request_id: Option = None; @@ -1353,6 +1390,18 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { "retryable": false, })).await; } + } else if failed_written_deliveries.get(&delivery.delivery_id).is_some() { + tracing::warn!( + delivery_id = %delivery.delivery_id, + event_id = %delivery.event_id, + "refusing to re-inject a delivery that already failed in this composer" + ); + let _ = send_frame(&out_tx, "delivery_failed", None, json!({ + "delivery_id": delivery.delivery_id, + "event_id": delivery.event_id, + "reason": "delivery was already written to this PTY and failed; refusing to inject it again", + "attempts": 0, + })).await; } else if pending_worker_delivery_ids.insert(delivery.delivery_id.clone()) { let _ = send_frame( &out_tx, @@ -1419,35 +1468,54 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { // child process sees the keystrokes. match frame.payload.get("data").and_then(Value::as_str) { Some(data) => { - human_input_generation = human_input_generation.saturating_add(1); - if let Some(cancel) = &initial_injection_cancel { cancel.store(true, Ordering::Relaxed); } - // Once a human writes into the PTY we can - // no longer distinguish the broker body - // from their partial composer text. Give - // the human ownership and terminate every - // pending automatic verification without - // pressing a recovery key. - let mut cancelled_verification = false; - while let Some(pv) = pending_verifications.pop_front() { - cancelled_verification = true; - let delivery_id = pv.delivery_id.clone(); - let _ = send_frame(&out_tx, "delivery_failed", None, json!({ - "delivery_id": delivery_id, - "event_id": pv.event_id, - "reason": "human PTY input took ownership before harness acceptance", - "attempts": pv.attempts, - })).await; - pending_worker_delivery_ids.remove(&delivery_id); - } - if cancelled_verification { - throttle.record(DeliveryOutcome::Failed); - } // Non-blocking submission: never parks the // select loop even if the drainer is wedged // behind a child that stopped reading stdin. let bytes = data.as_bytes().to_vec(); let byte_len = bytes.len(); - match pty.submit_write(bytes) { + let admitted = pty.submit_write(bytes); + // An empty frame changes nothing in the + // composer, and a rejected write never + // reached it: neither takes ownership. + if byte_len > 0 && admitted.is_ok() { + // Withdraw queued recovery keys first so + // a delayed submit cannot send the + // human's draft. + cancel_recovery_writes(&mut recovery_cancels); + human_input_generation = human_input_generation.saturating_add(1); + if let Some(cancel) = &initial_injection_cancel { cancel.store(true, Ordering::Relaxed); } + // Once a human writes into the PTY we can + // no longer distinguish the broker body + // from their partial composer text. Give + // the human ownership and terminate every + // pending automatic verification without + // pressing a recovery key. Only an admitted + // write transfers ownership: a rejected one + // never reached the composer. + let mut cancelled_verification = false; + while let Some(pv) = pending_verifications.pop_front() { + cancelled_verification = true; + let delivery_id = pv.delivery_id.clone(); + let _ = send_frame(&out_tx, "delivery_failed", None, json!({ + "delivery_id": delivery_id, + "event_id": pv.event_id, + "reason": "human PTY input took ownership before harness acceptance", + "attempts": pv.attempts, + })).await; + pending_worker_delivery_ids.remove(&delivery_id); + latch_failed_written_delivery( + &mut failed_written_deliveries, + &mut failed_composer_latch, + &delivery_id, + &pv.event_id, + &pv.expected_echo, + ); + } + if cancelled_verification { + throttle.record(DeliveryOutcome::Failed); + } + } + match admitted { Ok(ack_rx) => { // Defer the ack until the drainer confirms the // write landed (or failed). The broker holds the @@ -2085,7 +2153,9 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { if let Some(index) = next_injection_index( &pending_worker_injections, active_injection.is_some(), - !pending_verifications.is_empty() || !pending_recovery_writes.is_empty(), + !pending_verifications.is_empty() + || !pending_recovery_writes.is_empty() + || failed_composer_latch.is_some(), pty_auto.interactive_hold, hold_exempt_injections, &hold_exempt_event_ids, @@ -2355,6 +2425,13 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { "reason": "human PTY input took ownership during injection", })).await; pending_worker_delivery_ids.remove(&delivery_id); + latch_failed_written_delivery( + &mut failed_written_deliveries, + &mut failed_composer_latch, + &delivery_id, + &inj.pending.delivery.event_id, + inj.injection_text.as_deref().unwrap_or_default(), + ); throttle.record(DeliveryOutcome::Failed); continue; } @@ -2509,17 +2586,10 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { if !pending_recovery_writes.is_empty() => { let delivery_id = pv.delivery_id.clone(); let event_id = pv.event_id.clone(); - if submitted_generation != human_input_generation { - let _ = send_frame(&out_tx, "delivery_failed", None, json!({ - "delivery_id": delivery_id, - "event_id": event_id, - "reason": "human PTY input took ownership during submit recovery", - "attempts": pv.attempts, - })).await; - throttle.record(DeliveryOutcome::Failed); - pending_worker_delivery_ids.remove(&delivery_id); - continue; - } + // Human input cancels any part of the recovery that had not yet + // reached the child. A key that did reach it may still have + // started the turn, so assess acceptance before failing. + let human_took_over = submitted_generation != human_input_generation; match ack { Ok(Ok(())) => { pv.output_boundary = output_boundary; @@ -2582,13 +2652,38 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { pending_worker_delivery_ids.remove(&delivery_id); completed_worker_deliveries.insert(delivery_id, event_id); } - HarnessAcceptance::Parked | HarnessAcceptance::Inconclusive => { + HarnessAcceptance::Parked | HarnessAcceptance::Inconclusive + if !human_took_over => + { pending_verifications.push_back(pv); } + HarnessAcceptance::Parked | HarnessAcceptance::Inconclusive => { + let _ = send_frame(&out_tx, "delivery_failed", None, json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "reason": "human PTY input took ownership during submit recovery", + "attempts": pv.attempts, + })).await; + throttle.record(DeliveryOutcome::Failed); + pending_worker_delivery_ids.remove(&delivery_id); + latch_failed_written_delivery( + &mut failed_written_deliveries, + &mut failed_composer_latch, + &delivery_id, + &event_id, + &pv.expected_echo, + ); + } } } Ok(Err(error)) => { - let reason = format!("submit recovery write failed: {error}"); + let reason = if human_took_over { + format!( + "human PTY input took ownership during submit recovery; queued submit key withheld: {error}" + ) + } else { + format!("submit recovery write failed: {error}") + }; let _ = send_frame(&out_tx, "delivery_failed", None, json!({ "delivery_id": delivery_id, "event_id": event_id, @@ -2597,6 +2692,13 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { })).await; throttle.record(DeliveryOutcome::Failed); pending_worker_delivery_ids.remove(&delivery_id); + latch_failed_written_delivery( + &mut failed_written_deliveries, + &mut failed_composer_latch, + &delivery_id, + &event_id, + &pv.expected_echo, + ); } Err(_) => { let reason = "submit recovery drainer exited before acknowledging the key"; @@ -2608,6 +2710,13 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { })).await; throttle.record(DeliveryOutcome::Failed); pending_worker_delivery_ids.remove(&delivery_id); + latch_failed_written_delivery( + &mut failed_written_deliveries, + &mut failed_composer_latch, + &delivery_id, + &event_id, + &pv.expected_echo, + ); } } } @@ -2714,7 +2823,12 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { }), ) .await; - match submit_injection_recovery(&pty, &resolved_cli, pv.attempts) { + match submit_injection_recovery( + &pty, + &resolved_cli, + pv.attempts, + new_recovery_cancel(&mut recovery_cancels), + ) { Ok((ack_rx, output_boundary)) => { pv.attempts += 1; let _ = send_frame( @@ -2743,6 +2857,13 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { })).await; throttle.record(DeliveryOutcome::Failed); pending_worker_delivery_ids.remove(&delivery_id); + latch_failed_written_delivery( + &mut failed_written_deliveries, + &mut failed_composer_latch, + &delivery_id, + &event_id, + &pv.expected_echo, + ); } } } @@ -2783,6 +2904,17 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { "attempts": pv.attempts, })).await; throttle.record(DeliveryOutcome::Failed); + // Latch before removing the pending id: the body + // may still be in the composer, so neither a + // replay nor the next queued delivery may be + // typed on top of it. + latch_failed_written_delivery( + &mut failed_written_deliveries, + &mut failed_composer_latch, + &delivery_id, + &event_id, + &pv.expected_echo, + ); pending_worker_delivery_ids.remove(&delivery_id); } } @@ -2791,6 +2923,28 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { } } + // Release the failed-draft latch once the composer no longer + // holds that body (the harness took it late, or a human + // submitted or cleared it). Until then queued deliveries wait. + if let Some(latch) = failed_composer_latch.as_ref() { + if pending_verifications.is_empty() + && pending_recovery_writes.is_empty() + && active_injection.is_none() + && failed_draft_released( + &resolved_cli, + &latch.expected_echo, + &Snapshot::capture(&pty), + ) + { + tracing::info!( + delivery_id = %latch.delivery_id, + event_id = %latch.event_id, + "failed delivery left the composer; resuming injection" + ); + failed_composer_latch = None; + } + } + if activity_detector.is_some() { let mut i = 0; while i < pending_activities.len() { @@ -3507,7 +3661,7 @@ mod tests { && verification.attempts < verification.max_attempts => { let (recovery_ack, boundary) = - submit_injection_recovery(pty, "codex", verification.attempts) + submit_injection_recovery(pty, "codex", verification.attempts, Default::default()) .expect("queue submit-key recovery"); tokio::time::timeout(Duration::from_secs(5), recovery_ack) .await @@ -3679,7 +3833,7 @@ mod tests { let mut accepted_after = None; for completed_attempts in [1, 2] { let (recovery_ack, boundary) = - submit_injection_recovery(&pty, "codex", completed_attempts).unwrap(); + submit_injection_recovery(&pty, "codex", completed_attempts, Default::default()).unwrap(); recovery_ack.await.unwrap().unwrap(); verification.attempts += 1; verification.output_boundary = boundary; diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index b1c4baf680..5ee6155686 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -4440,6 +4440,147 @@ async fn stale_recovery_progress_does_not_block_the_worker() { cleanup_worker_registry(fixture.runtime.workers).await; } +fn delivery_verified_worker_event( + name: &str, + generation: Uuid, + delivery_id: &str, + event_id: &str, + verification: Option<&str>, +) -> WorkerEvent { + let mut payload = json!({ + "delivery_id": delivery_id, + "event_id": event_id, + }); + if let Some(verification) = verification { + payload["verification"] = json!(verification); + } + WorkerEvent::Message { + name: WorkerName::from(name), + generation, + value: json!({ "type": "delivery_verified", "payload": payload }), + } +} + +#[tokio::test] +async fn terminal_failure_without_remaining_work_does_not_block_the_worker() { + let worker_name = "worker-a"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let delivery_id = DeliveryId::new("del_terminal_failure_only"); + let pending = make_pending_delivery(delivery_id.as_str(), worker_name); + let event_id = pending.delivery.event_id.clone(); + let mut fixture = + worker_event_runtime_fixture(registry, HashMap::from([(delivery_id.clone(), pending)])); + + fixture + .runtime + .handle_worker_event(delivery_lifecycle_worker_event( + worker_name, + generation, + "delivery_failed", + delivery_id.as_str(), + event_id.as_str(), + )) + .await; + + assert!(!fixture.runtime.pending_deliveries.contains_key(&delivery_id)); + assert_eq!( + fixture.runtime.workers.workers[worker_name].state, + AgentWorkState::Working, + "a dead-lettered delivery with no remaining work must not pin blocked_on_send" + ); + let mut kinds = Vec::new(); + while let Ok(frame) = fixture._sdk_out_rx.try_recv() { + if let Some(kind) = frame.payload.get("kind").and_then(Value::as_str) { + kinds.push(kind.to_string()); + } + } + assert!(kinds.iter().any(|kind| kind == "delivery_failed")); + assert!(!kinds.iter().any(|kind| kind == "agent_blocked_on_send")); + cleanup_worker_registry(fixture.runtime.workers).await; +} + +#[tokio::test] +async fn terminal_failure_with_remaining_work_stays_blocked_on_send() { + let worker_name = "worker-a"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let failed_id = DeliveryId::new("del_terminal_failure_first"); + let waiting_id = DeliveryId::new("del_terminal_failure_waiting"); + let failed = make_pending_delivery(failed_id.as_str(), worker_name); + let waiting = make_pending_delivery(waiting_id.as_str(), worker_name); + let event_id = failed.delivery.event_id.clone(); + let mut fixture = worker_event_runtime_fixture( + registry, + HashMap::from([(failed_id.clone(), failed), (waiting_id.clone(), waiting)]), + ); + + fixture + .runtime + .handle_worker_event(delivery_lifecycle_worker_event( + worker_name, + generation, + "delivery_failed", + failed_id.as_str(), + event_id.as_str(), + )) + .await; + + assert_eq!( + fixture.runtime.workers.workers[worker_name].state, + AgentWorkState::BlockedOnSend + ); + cleanup_worker_registry(fixture.runtime.workers).await; +} + +#[tokio::test] +async fn pty_delivery_verified_requires_harness_acceptance() { + let worker_name = "worker-a"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let delivery_id = DeliveryId::new("del_legacy_verification"); + let pending = make_pending_delivery(delivery_id.as_str(), worker_name); + let event_id = pending.delivery.event_id.clone(); + let mut fixture = + worker_event_runtime_fixture(registry, HashMap::from([(delivery_id.clone(), pending)])); + + for legacy in [Some("echo"), Some("timeout_fallback"), None] { + fixture + .runtime + .handle_worker_event(delivery_verified_worker_event( + worker_name, + generation, + delivery_id.as_str(), + event_id.as_str(), + legacy, + )) + .await; + assert!( + fixture.runtime.pending_deliveries.contains_key(&delivery_id), + "PTY verification {legacy:?} must not clear delivery custody" + ); + } + while let Ok(frame) = fixture._sdk_out_rx.try_recv() { + assert_ne!( + frame.payload.get("kind").and_then(Value::as_str), + Some("message_delivery_confirmed") + ); + } + + fixture + .runtime + .handle_worker_event(delivery_verified_worker_event( + worker_name, + generation, + delivery_id.as_str(), + event_id.as_str(), + Some("harness_acceptance"), + )) + .await; + assert!(!fixture.runtime.pending_deliveries.contains_key(&delivery_id)); + cleanup_worker_registry(fixture.runtime.workers).await; +} + #[tokio::test] async fn submit_recovery_progress_defers_broker_delivery_retry() { let worker_name = "worker-a"; diff --git a/crates/broker/src/runtime/worker_events.rs b/crates/broker/src/runtime/worker_events.rs index 21983feb99..1003793834 100644 --- a/crates/broker/src/runtime/worker_events.rs +++ b/crates/broker/src/runtime/worker_events.rs @@ -183,6 +183,22 @@ fn record_started_harness_pid( true } +/// Validate a worker's `delivery_verified` frame before it clears custody. +/// +/// PTY workers must name harness acceptance (or a replay of an already +/// completed delivery). Legacy `echo` and `timeout_fallback` frames prove only +/// that bytes reached the terminal, so they never confirm a delivery. Headless +/// workers report process success without a verification field. +fn accepted_delivery_verification(verification: Option<&str>, is_pty: bool) -> Option<&str> { + match verification { + Some("harness_acceptance" | "completed_replay") => verification, + Some("echo" | "timeout_fallback") => None, + Some(_) | None if is_pty => None, + Some(other) => Some(other), + None => Some("worker_confirmation"), + } +} + fn worker_event_is_current(current_generation: Option, event_generation: Uuid) -> bool { current_generation == Some(event_generation) } @@ -945,12 +961,24 @@ impl BrokerRuntime { .get("event_id") .and_then(Value::as_str) .unwrap_or(""); - // Modern PTY workers use "harness_acceptance": - // editor echo alone is never delivery confirmation. - let verification = payload - .get("verification") - .and_then(Value::as_str) - .unwrap_or("worker_confirmation"); + let is_pty = workers + .workers + .get(&name) + .is_some_and(|handle| handle.spec.runtime == AgentRuntime::Pty); + let Some(verification) = accepted_delivery_verification( + payload.get("verification").and_then(Value::as_str), + is_pty, + ) else { + tracing::warn!( + target = "agent_relay::broker", + worker = %name, + delivery_id = %delivery_id, + event_id = %event_id, + verification = ?payload.get("verification"), + "ignoring delivery_verified without harness-acceptance evidence" + ); + return; + }; let reason = payload.get("reason").and_then(Value::as_str); tracing::debug!( target = "agent_relay::broker", @@ -1064,30 +1092,42 @@ impl BrokerRuntime { ) .await; if let Some(pending) = pending_for_failure { - if let Some(handle) = workers.workers.get_mut(&name) { - handle.last_activity_at = Instant::now(); - handle.state = AgentWorkState::BlockedOnSend; - } let pending_delivery_count = pending_deliveries .values() .filter(|candidate| candidate.worker_name == name) .count(); - let _ = send_broker_event( - sdk_out_tx, - BrokerEvent::AgentBlockedOnSend { - name: name.clone(), - blocked_secs: 0, - pending_delivery_count, - }, - ) - .await; - publish_agent_state_transition( - ws_control_tx, - &name, - "stuck", - Some("blocked_on_send"), - ) - .await; + // The failed delivery is already dead-lettered. + // Stay blocked only while another delivery for + // this worker still awaits acceptance; otherwise + // later output and idle events must be able to + // move the worker on. + let still_blocked = pending_delivery_count > 0; + if let Some(handle) = workers.workers.get_mut(&name) { + handle.last_activity_at = Instant::now(); + handle.state = if still_blocked { + AgentWorkState::BlockedOnSend + } else { + AgentWorkState::Working + }; + } + if still_blocked { + let _ = send_broker_event( + sdk_out_tx, + BrokerEvent::AgentBlockedOnSend { + name: name.clone(), + blocked_secs: 0, + pending_delivery_count, + }, + ) + .await; + publish_agent_state_transition( + ws_control_tx, + &name, + "stuck", + Some("blocked_on_send"), + ) + .await; + } let _ = emit_dropped_delivery_failures( sdk_out_tx, dead_letters, diff --git a/crates/broker/src/wrap.rs b/crates/broker/src/wrap.rs index 98ee1fa56f..67a8584a3e 100644 --- a/crates/broker/src/wrap.rs +++ b/crates/broker/src/wrap.rs @@ -1,6 +1,10 @@ use std::collections::VecDeque; use std::future::Future; use std::pin::Pin; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; use std::time::{Duration, Instant}; use crate::{ @@ -119,10 +123,14 @@ pub(crate) fn injection_submit_followup_delay(cli: &str) -> Option { /// Retry only the submit gesture for a body proven to remain in the live /// composer. Never write the body twice: an accepted first write whose activity /// marker was missed would otherwise create a duplicate turn. +/// +/// `cancel` lets human input withdraw the gesture until it reaches the child: +/// the drainer skips a not-yet-started key and withholds Codex's delayed CR. pub(crate) fn submit_injection_recovery( pty: &PtySession, resolved_cli: &str, completed_attempts: usize, + cancel: Arc, ) -> anyhow::Result<(tokio::sync::oneshot::Receiver>, u64)> { let lower = resolved_cli.to_ascii_lowercase(); if lower.contains("codex") && completed_attempts == 1 { @@ -131,11 +139,10 @@ pub(crate) fn submit_injection_recovery( // later LF submitted it. First reproduce the non-destructive // End+Enter half; the final bounded attempt below sends LF only if the // same delivery is still visibly parked. - return pty.submit_write_paced_with_followup_and_output_boundary( + return pty.submit_cancellable_write_with_output_boundary( b"\x1b[F".to_vec(), - Duration::ZERO, - Duration::from_secs(1), - b"\r".to_vec(), + Some((Duration::from_secs(1), b"\r".to_vec())), + cancel, ); } @@ -144,7 +151,22 @@ pub(crate) fn submit_injection_recovery( } else { b'\r' }; - pty.submit_write_paced_with_output_boundary(vec![key], Duration::ZERO) + pty.submit_cancellable_write_with_output_boundary(vec![key], None, cancel) +} + +/// Cancel every queued submit-key recovery because a human now owns the input. +pub(crate) fn cancel_recovery_writes(cancels: &mut Vec>) { + for cancel in cancels.drain(..) { + cancel.store(true, Ordering::Release); + } +} + +/// Start a cancellable recovery, pruning flags whose writes already settled. +pub(crate) fn new_recovery_cancel(cancels: &mut Vec>) -> Arc { + cancels.retain(|cancel| Arc::strong_count(cancel) > 1); + let cancel = Arc::new(AtomicBool::new(false)); + cancels.push(cancel.clone()); + cancel } /// Warn (without retrying) when a one-shot auto-response keystroke can't be @@ -1566,6 +1588,7 @@ pub(crate) async fn run_wrap( // is hit (at which point the child is not consuming input anyway). let mut stdin_pending: VecDeque> = VecDeque::new(); let mut human_input_generation = 0u64; + let mut recovery_cancels: Vec> = Vec::new(); let mut stdin_retry_deadline: Option = None; const STDIN_RETRY_INTERVAL: Duration = Duration::from_millis(4); @@ -1582,6 +1605,7 @@ pub(crate) async fn run_wrap( // keystrokes are never dropped or reordered under back-pressure. Some(data) = stdin_rx.recv() => { human_input_generation = human_input_generation.saturating_add(1); + cancel_recovery_writes(&mut recovery_cancels); if !pending_verifications.is_empty() { for verification in pending_verifications.drain(..) { tracing::warn!( @@ -2393,6 +2417,7 @@ pub(crate) async fn run_wrap( &pty, &resolved_cli, completed_attempts, + new_recovery_cancel(&mut recovery_cancels), ) { Ok((ack_rx, output_boundary)) => { let pending_write = PendingWrapWrite::Retry { @@ -2629,13 +2654,13 @@ mod tests { let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); tokio::time::sleep(Duration::from_millis(100)).await; - let (first_ack, _) = submit_injection_recovery(&pty, "codex", 1).unwrap(); + let (first_ack, _) = submit_injection_recovery(&pty, "codex", 1, Default::default()).unwrap(); tokio::time::timeout(Duration::from_secs(3), first_ack) .await .expect("End+CR recovery timed out") .expect("drainer exited") .expect("End+CR recovery failed"); - let (second_ack, _) = submit_injection_recovery(&pty, "codex", 2).unwrap(); + let (second_ack, _) = submit_injection_recovery(&pty, "codex", 2, Default::default()).unwrap(); tokio::time::timeout(Duration::from_secs(3), second_ack) .await .expect("LF recovery timed out") diff --git a/crates/relay-pty/src/pty.rs b/crates/relay-pty/src/pty.rs index 36e521604d..8a76beb0d3 100644 --- a/crates/relay-pty/src/pty.rs +++ b/crates/relay-pty/src/pty.rs @@ -132,10 +132,23 @@ enum WriteMsg { /// Relay still must prevent a terminal reply or human keystroke from /// splicing into that boundary. followup: Option, + /// Optional cancellation shared with the submitter. The drainer checks + /// it before the primary bytes and again after the follow-up delay, so + /// a later owner of the input (a human typing) can stop a queued submit + /// key from reaching the child. A cancelled write acks + /// [`io::ErrorKind::Interrupted`] and leaves the drainer running. + cancel: Option>, ack: oneshot::Sender>, }, } +fn write_cancelled() -> io::Error { + io::Error::new( + io::ErrorKind::Interrupted, + "queued PTY write cancelled before it reached the child", + ) +} + struct FollowupWrite { delay: Duration, bytes: Vec, @@ -656,12 +669,22 @@ fn drain_write_queue( bytes, pace, followup, + cancel, ack, } => { if bytes.is_empty() && followup.as_ref().is_none_or(|part| part.bytes.is_empty()) { let _ = ack.send(Ok(())); continue; } + let is_cancelled = || { + cancel + .as_ref() + .is_some_and(|flag| flag.load(Ordering::Acquire)) + }; + if is_cancelled() { + let _ = ack.send(Err(write_cancelled())); + continue; + } if matches!(bytes.as_slice(), b"\x1b[B" | b"\x1b[A" | b"\r" | b"\n") { tracing::debug!(target: "relay_pty::startup_input", control = ?bytes, followup_length = followup.as_ref().map(|part| part.bytes.len()), @@ -682,6 +705,9 @@ fn drain_write_queue( return Ok(()); } thread::sleep(part.delay); + if is_cancelled() { + return Err(write_cancelled()); + } writer.write_all(&part.bytes).and_then(|_| writer.flush()) }); match result { @@ -692,6 +718,10 @@ fn drain_write_queue( no_pid_alive_checks.store(0, Ordering::Relaxed); let _ = ack.send(Ok(())); } + Err(err) if err.kind() == io::ErrorKind::Interrupted && is_cancelled() => { + // Only the follow-up was withheld; the PTY is healthy. + let _ = ack.send(Err(err)); + } Err(err) => { let _ = ack.send(Err(err)); break; @@ -712,12 +742,14 @@ fn enqueue_user_write( bytes: Vec, pace: Duration, followup: Option, + cancel: Option>, ) -> Result>> { let (ack_tx, ack_rx) = oneshot::channel::>(); match write_tx.try_send(WriteMsg::UserInput { bytes, pace, followup, + cancel, ack: ack_tx, }) { Ok(()) => Ok(ack_rx), @@ -933,6 +965,7 @@ impl PtySession { bytes: Vec, pace: Duration, followup: Option, + cancel: Option>, ) -> Result<(oneshot::Receiver>, u64)> { #[cfg(unix)] let _order_guard = { @@ -974,6 +1007,7 @@ impl PtySession { bytes, pace, followup, + cancel, )?; Ok((ack, boundary)) } @@ -1221,6 +1255,7 @@ impl PtySession { bytes, Duration::ZERO, None, + None, ) } @@ -1247,7 +1282,14 @@ impl PtySession { bytes: Vec, pace: Duration, ) -> Result>> { - enqueue_user_write(&self.write_tx, &self.no_pid_alive_checks, bytes, pace, None) + enqueue_user_write( + &self.write_tx, + &self.no_pid_alive_checks, + bytes, + pace, + None, + None, + ) } /// Queue a paced input atomically with the receive-time output watermark. @@ -1261,7 +1303,7 @@ impl PtySession { bytes: Vec, pace: Duration, ) -> Result<(oneshot::Receiver>, u64)> { - self.enqueue_write_with_output_boundary(bytes, pace, None) + self.enqueue_write_with_output_boundary(bytes, pace, None, None) } /// Submit a paced primary payload followed by a delayed second write as @@ -1288,6 +1330,7 @@ impl PtySession { delay: followup_delay, bytes: followup_bytes, }), + None, ) } @@ -1307,6 +1350,26 @@ impl PtySession { delay: followup_delay, bytes: followup_bytes, }), + None, + ) + } + + /// Queue a submit-key write, with an optional delayed follow-up, that the + /// caller can cancel until each part reaches the child. Setting `cancel` + /// before the drainer starts the entry skips it entirely; setting it during + /// the follow-up delay withholds the follow-up. Either way the ack resolves + /// to [`io::ErrorKind::Interrupted`]. + pub fn submit_cancellable_write_with_output_boundary( + &self, + bytes: Vec, + followup: Option<(Duration, Vec)>, + cancel: Arc, + ) -> Result<(oneshot::Receiver>, u64)> { + self.enqueue_write_with_output_boundary( + bytes, + Duration::ZERO, + followup.map(|(delay, bytes)| FollowupWrite { delay, bytes }), + Some(cancel), ) } @@ -2421,6 +2484,7 @@ mod tests { bytes: b"injection-body".to_vec(), pace: Duration::ZERO, followup: None, + cancel: None, ack: ack_tx_1, }) .unwrap(); @@ -2430,6 +2494,7 @@ mod tests { bytes: b"\r".to_vec(), pace: Duration::ZERO, followup: None, + cancel: None, ack: ack_tx_2, }) .unwrap(); @@ -2505,6 +2570,7 @@ mod tests { bytes: b"should-fail\n".to_vec(), pace: Duration::ZERO, followup: None, + cancel: None, ack: ack_tx, }) .expect("queue accepts user input"); @@ -2544,6 +2610,7 @@ mod tests { bytes: b"flush-should-fail\n".to_vec(), pace: Duration::ZERO, followup: None, + cancel: None, ack: ack_tx, }) .expect("queue accepts user input"); @@ -2961,6 +3028,7 @@ mod tests { bytes: b"go\x1b[A\r".to_vec(), pace: StdDuration::from_millis(1), followup: None, + cancel: None, ack: ack_tx, }) .expect("queue accepts paced user input"); @@ -3017,6 +3085,7 @@ mod tests { delay: StdDuration::from_millis(5), bytes: b"\r".to_vec(), }), + cancel: None, ack: ack_tx, }) .expect("queue accepts the compound write"); @@ -3083,6 +3152,7 @@ mod tests { delay: StdDuration::ZERO, bytes: b"\r".to_vec(), }), + cancel: None, ack: ack_tx, }) .expect("queue accepts the compound write"); @@ -3100,6 +3170,91 @@ mod tests { drainer.join().expect("drainer thread joins cleanly"); } + /// A human taking over the composer during a delayed submit recovery must + /// stop the queued follow-up key, not merely relabel its ack afterwards. + #[tokio::test] + async fn cancelled_followup_is_withheld_and_drainer_keeps_running() { + use std::sync::Mutex as StdMutex; + + struct RecordingWriter { + chunks: Arc>>>, + } + impl std::io::Write for RecordingWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.chunks.lock().unwrap().push(buf.to_vec()); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let chunks = Arc::new(StdMutex::new(Vec::new())); + let (tx, rx) = std_mpsc::sync_channel::(WRITE_QUEUE_DEPTH); + let writer = RecordingWriter { + chunks: chunks.clone(), + }; + let drainer = std::thread::spawn(move || { + super::drain_write_queue(writer, rx, Arc::new(AtomicU32::new(0))) + }); + + let cancel = Arc::new(AtomicBool::new(false)); + let (ack_tx, ack_rx) = oneshot::channel::>(); + tx.send(WriteMsg::UserInput { + bytes: b"\x1b[F".to_vec(), + pace: StdDuration::ZERO, + followup: Some(FollowupWrite { + delay: StdDuration::from_millis(200), + bytes: b"\r".to_vec(), + }), + cancel: Some(cancel.clone()), + ack: ack_tx, + }) + .expect("queue accepts the recovery write"); + tokio::time::sleep(Duration::from_millis(50)).await; + cancel.store(true, Ordering::Release); + + let ack = tokio::time::timeout(Duration::from_secs(1), ack_rx) + .await + .expect("drainer resolves the cancelled write") + .expect("ack sender not dropped"); + assert_eq!( + ack.expect_err("cancelled follow-up must not ack success").kind(), + std::io::ErrorKind::Interrupted + ); + + let skipped = Arc::new(AtomicBool::new(true)); + let (skip_tx, skip_rx) = oneshot::channel::>(); + tx.send(WriteMsg::UserInput { + bytes: b"\r".to_vec(), + pace: StdDuration::ZERO, + followup: None, + cancel: Some(skipped), + ack: skip_tx, + }) + .expect("queue accepts a pre-cancelled write"); + assert!(skip_rx.await.unwrap().is_err()); + + let (human_tx, human_rx) = oneshot::channel::>(); + tx.send(WriteMsg::UserInput { + bytes: b"human".to_vec(), + pace: StdDuration::ZERO, + followup: None, + cancel: None, + ack: human_tx, + }) + .expect("queue accepts the human write"); + assert!(human_rx.await.unwrap().is_ok(), "drainer survives cancellation"); + + drop(tx); + drainer.join().expect("drainer thread joins cleanly"); + assert_eq!( + chunks.lock().unwrap().as_slice(), + [b"\x1b[F".to_vec(), b"human".to_vec()], + "neither the cancelled follow-up CR nor the pre-cancelled CR reaches the child" + ); + } + /// Regression: mise/asdf/rtx expose provider CLIs (codex, claude, gemini) /// as symlinked shims that all point at the version-manager binary. Those /// shims dispatch on `argv[0]` (the shim's basename), so canonicalising diff --git a/crates/relay-pty/src/snapshot.rs b/crates/relay-pty/src/snapshot.rs index a00428674c..5d288bce8e 100644 --- a/crates/relay-pty/src/snapshot.rs +++ b/crates/relay-pty/src/snapshot.rs @@ -189,6 +189,32 @@ impl Snapshot { out } + /// Plain text from the cursor cell through the bottom of the viewport: the + /// complement of [`Self::to_plain_through_cursor`]. Together they let a + /// caller ask whether specific text sits immediately around the cursor. + pub fn to_plain_from_cursor(&self) -> String { + let cursor_row = self.cursor.0.saturating_sub(1) as usize; + let cursor_col = (self.cursor.1.saturating_sub(1) as usize) + .saturating_add(usize::from(self.input_needs_wrap)); + let mut out = String::new(); + for (row_index, row) in self.cells.iter().enumerate().skip(cursor_row) { + let skip = if row_index == cursor_row { + cursor_col.min(row.len()) + } else { + 0 + }; + let row_start = out.len(); + for cell in row.iter().skip(skip) { + out.push(cell.c); + } + while out.len() > row_start && out.ends_with(' ') { + out.pop(); + } + out.push('\n'); + } + out + } + /// Whether the cursor row contains visible cells at or to the right of /// the cursor. A cursor-bounded composer can look like a bare prompt while /// a draft remains in cells to its right, so callers must not classify it From b8fb6681c71fd6ce49770c88917325ea0d7f98bb Mon Sep 17 00:00:00 2001 From: Miya Date: Fri, 9 Oct 2026 18:33:12 +0200 Subject: [PATCH 06/24] fix: address #1939 P2/P3 review items (changelog, docs, real-codex guards) - CHANGELOG: move the PTY delivery entry from the released 13.1.1 section to [Unreleased - Minor] / Fixed, impact-first, without the release-only sentence. 6bdec7b2 is not an ancestor of v13.2.0 or main. - AGENTS.md: restore "When the PR is ready" as its own line so items 3 and 4 render as one ordered list with nested bullets. - fleet.rs: the enqueue_delivery_ack comment names process echo as acceptance evidence alongside activity and a cleared composer. - real-codex.mjs: `--broker` without a value falls back to the friendly error instead of path.resolve(undefined); a directory is rejected by the executable-file guard. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 2a0590dc-a82a-4174-9eff-c23d59571061 --- AGENTS.md | 10 +++++----- CHANGELOG.md | 7 +++++-- crates/broker/src/runtime/fleet.rs | 5 +++-- .../1891-codex-parked-composer-recovery/real-codex.mjs | 6 ++++-- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9459ebd043..294c2e182a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -334,12 +334,12 @@ agent batches ready PRs into `trunk`, gets that one PR green, and merges it. typecheck, tests and lint this repo uses, and list the exact commands and results in the PR body. -**When the PR is ready** 3. Add the label **`mergeable`** once all of these are true: - -- The change is complete and the local checks above pass. -- Review feedback (human and bot) is addressed or answered. -- It is not a draft and does not depend on an unmerged PR. +**When the PR is ready** +3. Add the label **`mergeable`** once all of these are true: + - The change is complete and the local checks above pass. + - Review feedback (human and bot) is addressed or answered. + - It is not a draft and does not depend on an unmerged PR. 4. Remove `mergeable` if the PR stops being ready (new work, a failing check, a blocking question). The label is read live from GitHub on every sweep. diff --git a/CHANGELOG.md b/CHANGELOG.md index aebbe86cd1..676e57e343 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Minor] + +### Fixed + +- Broker-managed PTY messages now stay pending until the harness accepts them instead of being acknowledged while still parked in the composer. A delivery that is never accepted blocks further sends to that agent, gets up to two submit-only retries without resending the body, and is then reported as failed; typing into the agent's terminal takes over and cancels the retries. ## [13.2.0] - 2026-10-08 @@ -77,7 +81,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Broker fleet control keeps a healthy node WebSocket open when Relaycast returns retryable `d1_pressure` for `node.register` or `inventory.sync`, retrying the same frame with bounded exponential backoff while the existing registration and application-liveness deadlines remain fail-closed. -- Broker-managed PTY messages now remain pending until the harness accepts them; failed delivery blocks further sends, with up to two submit-only retries (three total attempts) instead of silently parked input. Codex recovery uses `End`, a distinct carriage return, and a guarded line-feed fallback without replaying the body; human input takes ownership and cancels recovery. Upgrade and restart brokers and PTY agents for the planned 13.2.0 rollout. - `--state-dir` on `node agent` commands, `node status`, and `node down` also accepts a fleet node directory whose broker state lives in `state/`. - An explicit `--state-dir` on `node agent` commands is no longer overridden by `RELAY_BROKER_URL` / `RELAY_BROKER_API_KEY`. - "No broker connection" errors name the path searched and whether it was the project default or `--state-dir`, instead of implying a running broker is down. diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index 92d335fe9a..418390dabc 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -1940,8 +1940,9 @@ pub(super) async fn enqueue_delivery_ack( /// confirmation of a specific PTY injection (relay#1310: the ack must not /// fire before the worker confirms the write landed). Called with the /// `delivery_id`/`event_id` from the worker's own internal `delivery_ack` -/// event (`worker_events.rs`), which pty_worker.rs sends only after activity or -/// a cleared composer proves harness acceptance — never at write-enqueue time. +/// event (`worker_events.rs`), which pty_worker.rs sends only after activity, +/// a cleared composer, or process echo proves harness acceptance — never at +/// write-enqueue time. /// /// Returns the `(agent, up_to_seq)` to send to the engine once resolved, or /// `None` when there is nothing withheld for `delivery_id` (already diff --git a/tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs b/tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs index ad72a56c51..dcac3761fc 100644 --- a/tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs +++ b/tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs @@ -18,7 +18,7 @@ */ import { constants as fsConstants } from 'node:fs'; -import { access, chmod, copyFile, mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { access, chmod, copyFile, mkdtemp, mkdir, rm, stat, writeFile } from 'node:fs/promises'; import { spawn } from 'node:child_process'; import { tmpdir } from 'node:os'; import path from 'node:path'; @@ -415,11 +415,13 @@ function sendFrame(child, frame) { function option(name, fallback) { const index = process.argv.indexOf(name); - return index >= 0 ? process.argv[index + 1] : fallback; + const value = index >= 0 ? process.argv[index + 1] : undefined; + return value === undefined || value.startsWith('--') ? fallback : value; } async function assertExecutable(candidate, name) { try { + if (!(await stat(candidate)).isFile()) throw new Error('not a file'); await access(candidate, fsConstants.X_OK); } catch { throw new Error(`${name} must be an executable file: ${candidate}`); From ca627bf7d03b032d72a7e77131ecea52bc22a31e Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Fri, 9 Oct 2026 09:57:40 -0700 Subject: [PATCH 07/24] test(broker): M1/M6/OpenCode regression tests; fmt Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- .../src/broker/delivery_verification.rs | 32 ++++++ crates/broker/src/pty_worker.rs | 101 +++++++++++++++--- crates/broker/src/runtime/tests.rs | 15 ++- crates/broker/src/wrap.rs | 6 +- crates/relay-pty/src/pty.rs | 14 ++- 5 files changed, 146 insertions(+), 22 deletions(-) diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 9c3917801a..982d47073b 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -1024,6 +1024,38 @@ mod tests { pty.shutdown().unwrap(); } + #[cfg(unix)] + #[tokio::test] + async fn opencode_output_cannot_confirm_a_draft_at_the_cursor() { + let expected = "Relay message from Lead [evt]: opencode draft"; + let (pty, snapshot) = codex_snapshot(&format!("┃ {expected}")).await; + let mut verification = codex_verification(expected); + verification.detector = ActivityDetector::for_cli("opencode"); + verification + .activity_buffer + .push_str("Thinking about the tool"); + + assert_eq!( + assess_harness_acceptance("opencode", &verification, &snapshot), + HarnessAcceptance::Inconclusive, + "OpenCode transcript words must not confirm a typed-but-unsent body" + ); + pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn failed_draft_latch_holds_until_the_composer_releases_it() { + let expected = "Relay message from Lead [evt]: parked after recovery"; + let (parked_pty, parked) = codex_snapshot(&format!("› {expected}")).await; + assert!(!failed_draft_released("codex", expected, &parked)); + parked_pty.shutdown().unwrap(); + + let (idle_pty, idle) = codex_snapshot("› Ask Codex to do anything").await; + assert!(failed_draft_released("codex", expected, &idle)); + idle_pty.shutdown().unwrap(); + } + #[test] fn wrapped_echo_preserves_same_chunk_acceptance_activity() { let expected = "Relay message from Lead [evt]: Reply with exactly WRAPPED_CODEX_ACK"; diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index 173dd573e0..40b1d4341d 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -29,9 +29,10 @@ use crate::broker::{ continuity::parse_continuity_command, delivery_verification::{ assess_harness_acceptance, current_timestamp_ms, delivery_injected_event_payload, - delivery_queued_event_payload, failed_draft_released, DeliveryOutcome, HarnessAcceptance, PendingActivity, - PendingVerification, ThrottleState, VerificationOutput, ACTIVITY_BUFFER_KEEP_BYTES, - ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW, MAX_VERIFICATION_ATTEMPTS, VERIFICATION_WINDOW, + delivery_queued_event_payload, failed_draft_released, DeliveryOutcome, HarnessAcceptance, + PendingActivity, PendingVerification, ThrottleState, VerificationOutput, + ACTIVITY_BUFFER_KEEP_BYTES, ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW, + MAX_VERIFICATION_ATTEMPTS, VERIFICATION_WINDOW, }, injection_format::{format_injection_for_worker_with_workspace, McpReminderThrottle}, }; @@ -48,8 +49,7 @@ use crate::util::utf8_stream::Utf8StreamDecoder; use crate::worker::detection::{is_codex_busy_status_line, ActivityDetector}; use crate::wrap::{ cancel_recovery_writes, new_recovery_cancel, submit_injection_body, submit_injection_recovery, - warn_on_auto_response_write, PtyAutoState, - AUTO_SUGGESTION_BLOCK_TIMEOUT, + warn_on_auto_response_write, PtyAutoState, AUTO_SUGGESTION_BLOCK_TIMEOUT, }; use base64::Engine; @@ -130,6 +130,25 @@ fn latch_failed_written_delivery( }); } +/// Whether the composer is owned by an unresolved delivery: one awaiting +/// acceptance, one whose submit-key recovery is in flight, or a failed body that +/// may still be typed there. The next injection must wait in every case. +fn composer_owned_by_delivery( + pending_verifications_empty: bool, + pending_recovery_writes_empty: bool, + failed_composer_latch: Option<&FailedComposerLatch>, +) -> bool { + !pending_verifications_empty + || !pending_recovery_writes_empty + || failed_composer_latch.is_some() +} + +/// Only a non-empty `write_pty` that the drainer admitted can change the +/// composer, so only that write takes ownership away from relay deliveries. +fn human_write_takes_ownership(byte_len: usize, admitted: bool) -> bool { + byte_len > 0 && admitted +} + /// Default per-atom gap for escape-aware paced injection, in milliseconds. /// Overridable at runtime via `RELAY_INJECT_RATE_MS`; `0` disables pacing and /// restores the single bulk write. Kept small because a full injection @@ -1477,7 +1496,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { // An empty frame changes nothing in the // composer, and a rejected write never // reached it: neither takes ownership. - if byte_len > 0 && admitted.is_ok() { + if human_write_takes_ownership(byte_len, admitted.is_ok()) { // Withdraw queued recovery keys first so // a delayed submit cannot send the // human's draft. @@ -2153,9 +2172,11 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { if let Some(index) = next_injection_index( &pending_worker_injections, active_injection.is_some(), - !pending_verifications.is_empty() - || !pending_recovery_writes.is_empty() - || failed_composer_latch.is_some(), + composer_owned_by_delivery( + pending_verifications.is_empty(), + pending_recovery_writes.is_empty(), + failed_composer_latch.as_ref(), + ), pty_auto.interactive_hold, hold_exempt_injections, &hold_exempt_event_ids, @@ -3660,9 +3681,13 @@ mod tests { if attempt_started.elapsed() >= VERIFICATION_WINDOW && verification.attempts < verification.max_attempts => { - let (recovery_ack, boundary) = - submit_injection_recovery(pty, "codex", verification.attempts, Default::default()) - .expect("queue submit-key recovery"); + let (recovery_ack, boundary) = submit_injection_recovery( + pty, + "codex", + verification.attempts, + Default::default(), + ) + .expect("queue submit-key recovery"); tokio::time::timeout(Duration::from_secs(5), recovery_ack) .await .expect("submit-key recovery ack timed out") @@ -3833,7 +3858,8 @@ mod tests { let mut accepted_after = None; for completed_attempts in [1, 2] { let (recovery_ack, boundary) = - submit_injection_recovery(&pty, "codex", completed_attempts, Default::default()).unwrap(); + submit_injection_recovery(&pty, "codex", completed_attempts, Default::default()) + .unwrap(); recovery_ack.await.unwrap().unwrap(); verification.attempts += 1; verification.output_boundary = boundary; @@ -4388,6 +4414,55 @@ mod tests { ); } + #[test] + fn failed_written_delivery_latches_injection_and_fences_replay() { + let mut failed = CompletedWorkerDeliveries::default(); + let mut latch = None; + let delivery_id = DeliveryId::from("del_parked_failure"); + let event_id = EventId::from("evt_parked_failure"); + assert!(!composer_owned_by_delivery(true, true, latch.as_ref())); + + latch_failed_written_delivery( + &mut failed, + &mut latch, + &delivery_id, + &event_id, + "Relay message from Lead [evt]: parked", + ); + + assert_eq!(failed.get(&delivery_id), Some(&event_id)); + assert!( + composer_owned_by_delivery(true, true, latch.as_ref()), + "a failed body still in the composer must block the next injection" + ); + let mut pending = VecDeque::new(); + pending.push_back(test_pending_injection("evt_next")); + assert_eq!( + next_injection_index( + &pending, + false, + composer_owned_by_delivery(true, true, latch.as_ref()), + false, + 0, + &HashSet::new(), + ), + None + ); + } + + #[test] + fn only_an_admitted_nonempty_human_write_takes_ownership() { + assert!( + !human_write_takes_ownership(0, true), + "empty write_pty frame" + ); + assert!( + !human_write_takes_ownership(5, false), + "write rejected by a full queue" + ); + assert!(human_write_takes_ownership(5, true)); + } + #[test] fn requeued_targeted_exemption_returns_to_its_event_id() { // A failed write requeues the injection. A targeted exemption must go diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index 5ee6155686..2d6ab65b17 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -4483,7 +4483,10 @@ async fn terminal_failure_without_remaining_work_does_not_block_the_worker() { )) .await; - assert!(!fixture.runtime.pending_deliveries.contains_key(&delivery_id)); + assert!(!fixture + .runtime + .pending_deliveries + .contains_key(&delivery_id)); assert_eq!( fixture.runtime.workers.workers[worker_name].state, AgentWorkState::Working, @@ -4556,7 +4559,10 @@ async fn pty_delivery_verified_requires_harness_acceptance() { )) .await; assert!( - fixture.runtime.pending_deliveries.contains_key(&delivery_id), + fixture + .runtime + .pending_deliveries + .contains_key(&delivery_id), "PTY verification {legacy:?} must not clear delivery custody" ); } @@ -4577,7 +4583,10 @@ async fn pty_delivery_verified_requires_harness_acceptance() { Some("harness_acceptance"), )) .await; - assert!(!fixture.runtime.pending_deliveries.contains_key(&delivery_id)); + assert!(!fixture + .runtime + .pending_deliveries + .contains_key(&delivery_id)); cleanup_worker_registry(fixture.runtime.workers).await; } diff --git a/crates/broker/src/wrap.rs b/crates/broker/src/wrap.rs index 67a8584a3e..de6ba1a08e 100644 --- a/crates/broker/src/wrap.rs +++ b/crates/broker/src/wrap.rs @@ -2654,13 +2654,15 @@ mod tests { let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); tokio::time::sleep(Duration::from_millis(100)).await; - let (first_ack, _) = submit_injection_recovery(&pty, "codex", 1, Default::default()).unwrap(); + let (first_ack, _) = + submit_injection_recovery(&pty, "codex", 1, Default::default()).unwrap(); tokio::time::timeout(Duration::from_secs(3), first_ack) .await .expect("End+CR recovery timed out") .expect("drainer exited") .expect("End+CR recovery failed"); - let (second_ack, _) = submit_injection_recovery(&pty, "codex", 2, Default::default()).unwrap(); + let (second_ack, _) = + submit_injection_recovery(&pty, "codex", 2, Default::default()).unwrap(); tokio::time::timeout(Duration::from_secs(3), second_ack) .await .expect("LF recovery timed out") diff --git a/crates/relay-pty/src/pty.rs b/crates/relay-pty/src/pty.rs index 8a76beb0d3..32ebb34805 100644 --- a/crates/relay-pty/src/pty.rs +++ b/crates/relay-pty/src/pty.rs @@ -2086,6 +2086,7 @@ mod tests { b"same-echo\n".to_vec(), Duration::ZERO, None, + None, ) .unwrap(); submitted_tx.send((boundary, ack)).unwrap(); @@ -2846,6 +2847,7 @@ mod tests { b"queued\n".to_vec(), Duration::ZERO, None, + None, ) .expect("submit path accepts the write"); // Block until the drainer has dequeued the message and is wedged inside @@ -3198,7 +3200,7 @@ mod tests { super::drain_write_queue(writer, rx, Arc::new(AtomicU32::new(0))) }); - let cancel = Arc::new(AtomicBool::new(false)); + let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false)); let (ack_tx, ack_rx) = oneshot::channel::>(); tx.send(WriteMsg::UserInput { bytes: b"\x1b[F".to_vec(), @@ -3219,11 +3221,12 @@ mod tests { .expect("drainer resolves the cancelled write") .expect("ack sender not dropped"); assert_eq!( - ack.expect_err("cancelled follow-up must not ack success").kind(), + ack.expect_err("cancelled follow-up must not ack success") + .kind(), std::io::ErrorKind::Interrupted ); - let skipped = Arc::new(AtomicBool::new(true)); + let skipped = Arc::new(std::sync::atomic::AtomicBool::new(true)); let (skip_tx, skip_rx) = oneshot::channel::>(); tx.send(WriteMsg::UserInput { bytes: b"\r".to_vec(), @@ -3244,7 +3247,10 @@ mod tests { ack: human_tx, }) .expect("queue accepts the human write"); - assert!(human_rx.await.unwrap().is_ok(), "drainer survives cancellation"); + assert!( + human_rx.await.unwrap().is_ok(), + "drainer survives cancellation" + ); drop(tx); drainer.join().expect("drainer thread joins cleanly"); From a80f9202550b165c0c53be1a67bdc66484627029 Mon Sep 17 00:00:00 2001 From: Miya Date: Fri, 9 Oct 2026 21:47:50 +0200 Subject: [PATCH 08/24] fix(broker): keep parked drafts with prompt-like rows detectable current_composer anchored on the last prompt-looking row, so a relay body containing a line such as `> b` cut the live composer short at that row. The tail check then missed a parked draft: observe_visible_composer never saw the echo, and assess_harness_acceptance returned Inconclusive instead of Parked, so the delivery failed without submit-key recovery. composer_holds_tail keeps the latest-anchor containment check and also accepts an earlier prompt anchor, but only when its composer ends with the body's tail at the cursor. A submitted body in the transcript above a fresh prompt does not end at the cursor and is not treated as parked. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 2a0590dc-a82a-4174-9eff-c23d59571061 --- .../src/broker/delivery_verification.rs | 152 +++++++++++++----- 1 file changed, 114 insertions(+), 38 deletions(-) diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 55c8aa8553..ce908654df 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -163,11 +163,7 @@ impl PendingVerification { if self.echo_seen { return; } - let tail = expected_tail(&self.expected_echo); - if !tail.is_empty() - && current_composer(snapshot, cli) - .is_some_and(|composer| compact_render(&composer).contains(&tail)) - { + if composer_holds_tail(snapshot, cli, &expected_tail(&self.expected_echo)) { self.echo_seen = true; self.activity_buffer.clear(); } @@ -250,12 +246,18 @@ fn gemini_composer_row(line: &str) -> &str { .unwrap_or(bordered) } -fn current_composer(snapshot: &Snapshot, cli: &str) -> Option { +/// Live-composer readings for the rows through the cursor, latest prompt +/// anchor first. Earlier anchors only matter when a draft itself contains a +/// prompt-looking row (for example a quoted `> b` line); see +/// [`composer_holds_tail`]. +fn composer_candidates(snapshot: &Snapshot, cli: &str) -> Vec { let plain = snapshot.to_plain_through_cursor(); let lines: Vec<_> = plain.lines().collect(); - let end = snapshot.cursor.0.checked_sub(1)? as usize; + let Some(end) = snapshot.cursor.0.checked_sub(1).map(|row| row as usize) else { + return Vec::new(); + }; if end >= lines.len() { - return None; + return Vec::new(); } let lower = cli.to_ascii_lowercase(); if lower.contains("gemini") { @@ -263,26 +265,29 @@ fn current_composer(snapshot: &Snapshot, cli: &str) -> Option { .get(end) .is_some_and(|line| cursor_row_is_activity(cli, line)) { - return None; + return Vec::new(); } // Gemini renders its live input with a `> ` prefix (inside a `│` // border on terminals that cannot use the background-colour frame). // Anchor to that boundary instead of treating arbitrary transcript // rows near the cursor as the composer. Keeping every row through the // cursor also preserves long, wrapped drafts beyond four rows. - let start = lines.iter().take(end + 1).rposition(|line| { - let input = gemini_composer_row(line); - input == ">" || input.starts_with("> ") - })?; - return Some( - lines[start..=end] - .iter() - .map(|line| gemini_composer_row(line)) - .collect::>() - .join("\n"), - ); + return (0..=end) + .rev() + .filter(|&start| { + let input = gemini_composer_row(lines[start]); + input == ">" || input.starts_with("> ") + }) + .map(|start| { + lines[start..=end] + .iter() + .map(|line| gemini_composer_row(line)) + .collect::>() + .join("\n") + }) + .collect(); } - let is_prompt = |line: &&str| { + let is_prompt = |line: &str| { let trimmed = line.trim_start(); if lower.contains("codex") { trimmed == "›" @@ -309,20 +314,56 @@ fn current_composer(snapshot: &Snapshot, cli: &str) -> Option { || trimmed.starts_with("❯ ") } }; - let start = lines.iter().take(end + 1).rposition(is_prompt)?; - let composer_lines = &lines[start..=end]; - if composer_lines.len() > 1 - && composer_lines - .last() - .is_some_and(|line| cursor_row_is_activity(cli, line)) + let starts: Vec = (0..=end) + .rev() + .filter(|&row| is_prompt(lines[row])) + .collect(); + let Some(&latest) = starts.first() else { + return Vec::new(); + }; + if lines + .get(end) + .is_some_and(|line| cursor_row_is_activity(cli, line)) { - // The latest prompt-looking line is part of the submitted transcript; - // a distinctive harness status row owns the cursor. Do not use the - // broader ActivityDetector patterns here because ordinary multiline - // draft text can contain Tool:, Write(, or shell prompts. - return None; + if latest < end { + // The latest prompt-looking line is part of the submitted + // transcript; a distinctive harness status row owns the cursor. Do + // not use the broader ActivityDetector patterns here because + // ordinary multiline draft text can contain Tool:, Write(, or shell + // prompts. + return Vec::new(); + } + return vec![lines[latest].to_string()]; + } + starts + .into_iter() + .map(|start| lines[start..=end].join("\n")) + .collect() +} + +fn current_composer(snapshot: &Snapshot, cli: &str) -> Option { + composer_candidates(snapshot, cli).into_iter().next() +} + +/// Whether the delivery's tail is still in the live composer. +/// +/// The latest prompt anchor keeps the original containment check. A draft +/// can itself contain prompt-looking rows, which moves that anchor into the +/// body, so an earlier anchor also counts, but only when the composer it +/// yields *ends* with the tail at the cursor. A submitted body in the +/// transcript above a fresh prompt never ends at the cursor. +fn composer_holds_tail(snapshot: &Snapshot, cli: &str, tail: &str) -> bool { + if tail.is_empty() { + return false; + } + let mut candidates = composer_candidates(snapshot, cli) + .into_iter() + .map(|composer| compact_render(&composer)); + match candidates.next() { + Some(latest) if latest.contains(tail) => true, + Some(_) => candidates.any(|composer| composer.ends_with(tail)), + None => false, } - Some(composer_lines.join("\n")) } fn cursor_row_is_activity(cli: &str, line: &str) -> bool { @@ -391,11 +432,7 @@ pub(crate) fn assess_harness_acceptance( snapshot: &Snapshot, ) -> HarnessAcceptance { let tail = expected_tail(&verification.expected_echo); - let parked = !tail.is_empty() - && current_composer(snapshot, cli) - .map(|composer| compact_render(&composer).contains(&tail)) - .unwrap_or(false); - if parked { + if composer_holds_tail(snapshot, cli, &tail) { return HarnessAcceptance::Parked; } if let Some(pattern) = verification.accepted_activity() { @@ -830,6 +867,45 @@ mod tests { pty.shutdown().unwrap(); } + #[cfg(unix)] + #[tokio::test] + async fn prompt_like_body_rows_do_not_hide_a_parked_draft() { + let expected = "Relay message from Lead [evt]: quote this\n> b\nthen continue"; + let screen = "❯ Relay message from Lead [evt]: quote this\n> b\nthen continue"; + let (pty, snapshot) = codex_snapshot(screen).await; + let mut verification = codex_verification(expected); + verification.echo_seen = false; + verification.detector = ActivityDetector::for_cli("claude"); + + verification.observe_visible_composer(&snapshot, "claude"); + assert!( + verification.echo_seen, + "a `> ` row inside the body must not cut the draft off from its prompt" + ); + assert_eq!( + assess_harness_acceptance("claude", &verification, &snapshot), + HarnessAcceptance::Parked + ); + pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn submitted_prompt_like_body_above_a_fresh_prompt_is_not_parked() { + let expected = "Relay message from Lead [evt]: quote this\n> b\nthen continue"; + let screen = "❯ Relay message from Lead [evt]: quote this\n> b\nthen continue\n\nDone.\n❯ "; + let (pty, snapshot) = codex_snapshot(screen).await; + let mut verification = codex_verification(expected); + verification.detector = ActivityDetector::for_cli("claude"); + + assert_ne!( + assess_harness_acceptance("claude", &verification, &snapshot), + HarnessAcceptance::Parked, + "the transcript copy of a submitted body is not the live composer" + ); + pty.shutdown().unwrap(); + } + #[cfg(unix)] #[tokio::test] async fn claude_interrupt_status_is_activity_not_a_parked_transcript() { From 7c6bfa32bb6498263781ca3b72ed232a0810185b Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Fri, 9 Oct 2026 14:02:43 -0700 Subject: [PATCH 09/24] fix(broker): fail closed on long or off-screen failed drafts; unbounded replay fence - tail anywhere after the cursor, or text right of it, blocks generic acceptance - the failed-draft latch releases only on a provably idle composer, or an operator-wide flush_injections - failed-written delivery ids are retained for the PTY lifetime Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- .../src/broker/delivery_verification.rs | 60 +++++++++++++------ crates/broker/src/pty_worker.rs | 21 ++++++- 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 982d47073b..9680cfd696 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -251,17 +251,18 @@ fn end_of_compact_match(text: &str, compact_needle: &str) -> Option { ends.get(matched_chars - 1).copied() } -/// Whether the delivery's compact tail sits immediately at the cursor: -/// ending just before it, starting at it, or straddling it. Any of these +/// Whether the delivery's compact tail may still be in an editor around the +/// cursor: ending just before it, straddling it, or anywhere after it (the +/// cursor can sit at the start of a draft longer than the tail). Any of these /// means the editor may still hold the draft, regardless of which prompt /// glyph (or body line that looks like one) the composer parser anchored on. -fn tail_touches_cursor(snapshot: &Snapshot, tail: &str) -> bool { +fn tail_near_cursor(snapshot: &Snapshot, tail: &str) -> bool { if tail.is_empty() { return false; } let before = compact_render(&snapshot.to_plain_through_cursor()); let after = compact_render(&snapshot.to_plain_from_cursor()); - if before.ends_with(tail) || after.starts_with(tail) { + if before.ends_with(tail) || after.contains(tail) { return true; } tail.char_indices() @@ -424,20 +425,15 @@ fn composer_is_idle(snapshot: &Snapshot, cli: &str) -> bool { .unwrap_or(false) } -/// Whether a terminally failed delivery's body has left the live composer. +/// Whether a terminally failed delivery's body has provably left the composer. /// /// A failed body can remain typed in the editor. Injecting the next delivery /// on top of it would append to that draft or submit both as one turn, so the -/// worker holds injection until the composer is proven idle, or the failed -/// tail is neither in the parsed composer nor touching the cursor. +/// worker holds injection until the composer is proven idle. The tail being +/// absent from the viewport is not proof: a long draft can scroll it out of +/// view. An operator `flush_injections` is the explicit override. pub(crate) fn failed_draft_released(cli: &str, expected_echo: &str, snapshot: &Snapshot) -> bool { - let tail = expected_tail(expected_echo); - if tail.is_empty() || composer_is_idle(snapshot, cli) { - return true; - } - let in_composer = current_composer(snapshot, cli) - .is_some_and(|composer| compact_render(&composer).contains(&tail)); - !in_composer && !tail_touches_cursor(snapshot, &tail) + expected_tail(expected_echo).is_empty() || composer_is_idle(snapshot, cli) } /// Distinguish terminal echo from actual harness acceptance. @@ -473,11 +469,13 @@ pub(crate) fn assess_harness_acceptance( } // Harnesses without a turn-start marker fall back to post-echo output, but // a repaint of a still-unsent draft is output too. Only accept it once the - // body has left the cursor: a tail ending at, starting at, or straddling - // the cursor stays inconclusive. + // body has left the cursor: a tail ending at, straddling, or anywhere after + // the cursor, or any text to the cursor's right (a draft whose tail has + // scrolled out of view), stays inconclusive. if verification.echo_seen && !verification.detector.has_explicit_patterns() - && !tail_touches_cursor(snapshot, &tail) + && !tail_near_cursor(snapshot, &tail) + && !snapshot.has_visible_text_at_or_after_cursor() { if let Some(pattern) = verification .detector @@ -1056,6 +1054,34 @@ mod tests { idle_pty.shutdown().unwrap(); } + #[cfg(unix)] + #[tokio::test] + async fn generic_output_cannot_confirm_a_long_draft_with_cursor_at_its_start() { + let expected = format!("Relay message from Lead [evt]: {}", "x".repeat(170)); + let (pty, snapshot) = codex_snapshot(&format!("› {expected}\x1b[1;3H")).await; + let mut verification = codex_verification(&expected); + verification.detector = ActivityDetector::for_cli("muse"); + verification.activity_buffer.push_str("repaint"); + + assert_eq!( + assess_harness_acceptance("muse", &verification, &snapshot), + HarnessAcceptance::Inconclusive, + "a tail far to the right of the cursor is still an unsent draft" + ); + pty.shutdown().unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn a_tail_missing_from_the_viewport_does_not_release_the_latch() { + // The failed draft's head is visible but its tail has scrolled away; + // only a provably idle composer may release the latch. + let expected = "Relay message from Lead [evt]: head ... TAIL_SCROLLED_OFF"; + let (pty, snapshot) = codex_snapshot("› Relay message from Lead [evt]: head").await; + assert!(!failed_draft_released("codex", expected, &snapshot)); + pty.shutdown().unwrap(); + } + #[test] fn wrapped_echo_preserves_same_chunk_acceptance_activity() { let expected = "Relay message from Lead [evt]: Reply with exactly WRAPPED_CODEX_ACK"; diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index 40b1d4341d..be13733c10 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -116,7 +116,7 @@ struct FailedComposerLatch { /// composer: fence its id against replay for the PTY's lifetime and latch /// injection until the composer releases the draft. fn latch_failed_written_delivery( - failed_written_deliveries: &mut CompletedWorkerDeliveries, + failed_written_deliveries: &mut HashMap, failed_composer_latch: &mut Option, delivery_id: &DeliveryId, event_id: &EventId, @@ -1098,7 +1098,9 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { let mut completed_worker_deliveries = CompletedWorkerDeliveries::default(); // Deliveries written to this PTY that then failed terminally. A replay of // one of these ids is refused rather than pasted over the failed draft. - let mut failed_written_deliveries = CompletedWorkerDeliveries::default(); + // Unbounded on purpose: entries exist only for failures, and an evicted id + // would let a late retry paste the same body again. + let mut failed_written_deliveries: HashMap = HashMap::new(); let mut failed_composer_latch: Option = None; // The injection currently being written across paced stages, if any. Only // one injection is in flight at a time; the pending-injection interval arm @@ -1658,6 +1660,19 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { .and_then(Value::as_str) .filter(|event_id| !event_id.is_empty()) .map(str::to_string); + // An operator-wide flush is the explicit override + // for a failed draft the worker cannot prove has + // left the composer. + if targeted_event_id.is_none() { + if let Some(latch) = failed_composer_latch.take() { + tracing::warn!( + target: "agent_relay::worker::pty", + worker = %worker_name, + delivery_id = %latch.delivery_id, + "operator flush released the failed-draft injection latch" + ); + } + } if pty_auto.interactive_hold { if let Some(event_id) = targeted_event_id { if let Some(inj) = active_injection.as_mut() { @@ -4416,7 +4431,7 @@ mod tests { #[test] fn failed_written_delivery_latches_injection_and_fences_replay() { - let mut failed = CompletedWorkerDeliveries::default(); + let mut failed = HashMap::new(); let mut latch = None; let delivery_id = DeliveryId::from("del_parked_failure"); let event_id = EventId::from("evt_parked_failure"); From a3cbca912f0b0bdde3add188174f9bb4cd21ddbc Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Fri, 9 Oct 2026 14:33:38 -0700 Subject: [PATCH 10/24] fix(broker): satisfy clippy contains_key on the failed-delivery fence Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- crates/broker/src/pty_worker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index be13733c10..ea83a8649c 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -1411,7 +1411,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { "retryable": false, })).await; } - } else if failed_written_deliveries.get(&delivery.delivery_id).is_some() { + } else if failed_written_deliveries.contains_key(&delivery.delivery_id) { tracing::warn!( delivery_id = %delivery.delivery_id, event_id = %delivery.event_id, From 09d1f99532462e9baceb283df425039e40158fd5 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Fri, 9 Oct 2026 14:59:58 -0700 Subject: [PATCH 11/24] fix(broker): latch partial chunked drafts; publish non-stuck state; ignore cursor-row chrome - an incomplete chunked Codex initial write latches its partial draft - the last terminal delivery failure publishes working/delivery_failed - only body text right of the cursor blocks generic acceptance Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- .../src/broker/delivery_verification.rs | 43 +++++++++++- crates/broker/src/pty_worker.rs | 9 +++ crates/broker/src/runtime/worker_events.rs | 65 +++++++++++++------ 3 files changed, 96 insertions(+), 21 deletions(-) diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 9680cfd696..870aa6632d 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -270,6 +270,23 @@ fn tail_near_cursor(snapshot: &Snapshot, tail: &str) -> bool { .any(|(split, _)| before.ends_with(&tail[..split]) && after.starts_with(&tail[split..])) } +/// Whether the cells right of the cursor on its row are part of this body. +/// A cursor parked at the start of a draft whose tail has scrolled out of +/// view shows body text there; harness chrome (a placeholder or status hint) +/// does not appear in the body and is ignored. +fn body_text_right_of_cursor(snapshot: &Snapshot, expected_echo: &str) -> bool { + let from_cursor = snapshot.to_plain_from_cursor(); + let row = compact_render(from_cursor.lines().next().unwrap_or_default()); + if row.is_empty() { + return false; + } + let probe_end = row + .char_indices() + .nth(32) + .map_or(row.len(), |(index, _)| index); + compact_render(expected_echo).contains(&row[..probe_end]) +} + fn is_cat_process(cli: &str) -> bool { cli.rsplit(['/', '\\']) .next() @@ -471,11 +488,12 @@ pub(crate) fn assess_harness_acceptance( // a repaint of a still-unsent draft is output too. Only accept it once the // body has left the cursor: a tail ending at, straddling, or anywhere after // the cursor, or any text to the cursor's right (a draft whose tail has - // scrolled out of view), stays inconclusive. + // scrolled out of view), stays inconclusive. Placeholder or status text + // on the cursor row is not body text and does not block acceptance. if verification.echo_seen && !verification.detector.has_explicit_patterns() && !tail_near_cursor(snapshot, &tail) - && !snapshot.has_visible_text_at_or_after_cursor() + && !body_text_right_of_cursor(snapshot, &verification.expected_echo) { if let Some(pattern) = verification .detector @@ -1071,6 +1089,27 @@ mod tests { pty.shutdown().unwrap(); } + #[cfg(unix)] + #[tokio::test] + async fn cursor_row_placeholder_does_not_block_generic_acceptance() { + let expected = "Relay message from Lead [evt]: fix idle injection"; + let (pty, snapshot) = + codex_snapshot("Processing accepted turn\n› Type a message\x1b[2;3H").await; + assert!(snapshot.has_visible_text_at_or_after_cursor()); + let mut verification = codex_verification(expected); + verification.detector = ActivityDetector::for_cli("muse"); + verification + .activity_buffer + .push_str("Processing accepted turn"); + + assert_eq!( + assess_harness_acceptance("muse", &verification, &snapshot), + HarnessAcceptance::Accepted("activity:any_output".to_string()), + "placeholder chrome right of the cursor is not an unsent draft" + ); + pty.shutdown().unwrap(); + } + #[cfg(unix)] #[tokio::test] async fn a_tail_missing_from_the_viewport_does_not_release_the_latch() { diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index ea83a8649c..0b97ca8225 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -2447,6 +2447,15 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { "event_id": inj.pending.delivery.event_id, "reason": "initial_injection_incomplete; body will not be replayed" })).await; + // A partial chunked body may sit in the composer; the + // next delivery must not be pasted onto it. + latch_failed_written_delivery( + &mut failed_written_deliveries, + &mut failed_composer_latch, + &inj.pending.delivery.delivery_id, + &inj.pending.delivery.event_id, + inj.injection_text.as_deref().unwrap_or_default(), + ); let _ = send_frame(&out_tx, "worker_error", inj.pending.request_id, json!({ "code": "initial_injection_incomplete", "retryable": false, "message": "Initial task delivery interrupted or timed out; inspect the composer before retrying" diff --git a/crates/broker/src/runtime/worker_events.rs b/crates/broker/src/runtime/worker_events.rs index 1003793834..bbaca3aa30 100644 --- a/crates/broker/src/runtime/worker_events.rs +++ b/crates/broker/src/runtime/worker_events.rs @@ -199,6 +199,38 @@ fn accepted_delivery_verification(verification: Option<&str>, is_pty: bool) -> O } } +/// Worker state after a terminal `delivery_failed`, as `(state, published +/// state, reason)`. The failed delivery is already dead-lettered, so the +/// worker stays blocked only while another delivery for it awaits acceptance; +/// otherwise later output and idle events must be able to move it on. +fn state_after_terminal_delivery_failure( + pending_delivery_count: usize, +) -> (AgentWorkState, &'static str, &'static str) { + if pending_delivery_count > 0 { + (AgentWorkState::BlockedOnSend, "stuck", "blocked_on_send") + } else { + (AgentWorkState::Working, "working", "delivery_failed") + } +} + +#[cfg(test)] +mod terminal_delivery_failure_state_tests { + use super::*; + + #[test] + fn last_failed_delivery_publishes_a_non_stuck_state() { + assert_eq!( + state_after_terminal_delivery_failure(0), + (AgentWorkState::Working, "working", "delivery_failed"), + "a worker with no remaining deliveries must publish its way out of `stuck`" + ); + assert_eq!( + state_after_terminal_delivery_failure(2), + (AgentWorkState::BlockedOnSend, "stuck", "blocked_on_send") + ); + } +} + fn worker_event_is_current(current_generation: Option, event_generation: Uuid) -> bool { current_generation == Some(event_generation) } @@ -1096,21 +1128,13 @@ impl BrokerRuntime { .values() .filter(|candidate| candidate.worker_name == name) .count(); - // The failed delivery is already dead-lettered. - // Stay blocked only while another delivery for - // this worker still awaits acceptance; otherwise - // later output and idle events must be able to - // move the worker on. - let still_blocked = pending_delivery_count > 0; + let (state, published_state, transition_reason) = + state_after_terminal_delivery_failure(pending_delivery_count); if let Some(handle) = workers.workers.get_mut(&name) { handle.last_activity_at = Instant::now(); - handle.state = if still_blocked { - AgentWorkState::BlockedOnSend - } else { - AgentWorkState::Working - }; + handle.state = state; } - if still_blocked { + if state == AgentWorkState::BlockedOnSend { let _ = send_broker_event( sdk_out_tx, BrokerEvent::AgentBlockedOnSend { @@ -1120,14 +1144,17 @@ impl BrokerRuntime { }, ) .await; - publish_agent_state_transition( - ws_control_tx, - &name, - "stuck", - Some("blocked_on_send"), - ) - .await; } + // Always publish: an earlier delivery_unconfirmed + // may have published `stuck`, and an in-memory + // reset alone would leave that status visible. + publish_agent_state_transition( + ws_control_tx, + &name, + published_state, + Some(transition_reason), + ) + .await; let _ = emit_dropped_delivery_failures( sdk_out_tx, dead_letters, From faba2e708f2200434e2e2d83baa41f375acb7cfe Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Fri, 9 Oct 2026 15:25:51 -0700 Subject: [PATCH 12/24] test(broker): off-screen draft tail keeps generic acceptance inconclusive Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- .../src/broker/delivery_verification.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 870aa6632d..157334a7ed 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -1089,6 +1089,29 @@ mod tests { pty.shutdown().unwrap(); } + #[cfg(unix)] + #[tokio::test] + async fn generic_output_cannot_confirm_a_draft_whose_tail_scrolled_away() { + // Only the draft's head is in the viewport, right of the cursor; its + // tail has scrolled out of view, so no tail match is possible. + let expected = format!( + "Relay message from Lead [evt]: head {} TAIL_SCROLLED_OFF", + "y".repeat(160) + ); + let (pty, snapshot) = + codex_snapshot("› Relay message from Lead [evt]: head\x1b[1;3H").await; + let mut verification = codex_verification(&expected); + verification.detector = ActivityDetector::for_cli("muse"); + verification.activity_buffer.push_str("repaint"); + + assert_eq!( + assess_harness_acceptance("muse", &verification, &snapshot), + HarnessAcceptance::Inconclusive, + "body text right of the cursor is an unsent draft even without its tail" + ); + pty.shutdown().unwrap(); + } + #[cfg(unix)] #[tokio::test] async fn cursor_row_placeholder_does_not_block_generic_acceptance() { From 5447d54c53d0ba8a9cb61e5ff37439c7ee626046 Mon Sep 17 00:00:00 2001 From: Miya Date: Sat, 10 Oct 2026 02:25:12 +0200 Subject: [PATCH 13/24] fix: address #1955 review (option guard, changelog split, ack comment) - real-codex.mjs: a supplied option with no value now fails with " requires a value" instead of falling back, so `--broker --timeout-seconds 20` can no longer silently run the broker from RELAY_REAL_CODEX_BROKER_BINARY. - CHANGELOG: split the PTY delivery entry into three impact-first bullets and state the limit as three total attempts. - fleet.rs: process echo is acceptance evidence only for a plain `cat` recipient, where it proves transport receipt, not harness acceptance. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 2a0590dc-a82a-4174-9eff-c23d59571061 --- CHANGELOG.md | 4 +++- crates/broker/src/runtime/fleet.rs | 7 ++++--- .../1891-codex-parked-composer-recovery/real-codex.mjs | 6 ++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 676e57e343..4641c90cca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Broker-managed PTY messages now stay pending until the harness accepts them instead of being acknowledged while still parked in the composer. A delivery that is never accepted blocks further sends to that agent, gets up to two submit-only retries without resending the body, and is then reported as failed; typing into the agent's terminal takes over and cancels the retries. +- Broker-managed PTY messages now stay pending until the harness accepts them, instead of being acknowledged while still parked in the composer. +- A PTY delivery the harness never accepts blocks further sends to that agent and is retried by pressing submit only, never by resending the body, for up to three total attempts before it is reported as failed. +- Typing into an agent's terminal while a delivery is being retried takes over the terminal and cancels the retries. ## [13.2.0] - 2026-10-08 diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index 418390dabc..faaec81682 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -1940,9 +1940,10 @@ pub(super) async fn enqueue_delivery_ack( /// confirmation of a specific PTY injection (relay#1310: the ack must not /// fire before the worker confirms the write landed). Called with the /// `delivery_id`/`event_id` from the worker's own internal `delivery_ack` -/// event (`worker_events.rs`), which pty_worker.rs sends only after activity, -/// a cleared composer, or process echo proves harness acceptance — never at -/// write-enqueue time. +/// event (`worker_events.rs`), which pty_worker.rs sends only after activity +/// or a cleared composer proves harness acceptance (or, for a plain `cat` +/// recipient with no harness, after its PTY echo proves transport receipt) — +/// never at write-enqueue time. /// /// Returns the `(agent, up_to_seq)` to send to the engine once resolved, or /// `None` when there is nothing withheld for `delivery_id` (already diff --git a/tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs b/tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs index dcac3761fc..a8c47e72bb 100644 --- a/tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs +++ b/tests/relayflows/cases/1891-codex-parked-composer-recovery/real-codex.mjs @@ -415,8 +415,10 @@ function sendFrame(child, frame) { function option(name, fallback) { const index = process.argv.indexOf(name); - const value = index >= 0 ? process.argv[index + 1] : undefined; - return value === undefined || value.startsWith('--') ? fallback : value; + if (index < 0) return fallback; + const value = process.argv[index + 1]; + if (value === undefined || value.startsWith('--')) throw new Error(`${name} requires a value.`); + return value; } async function assertExecutable(candidate, name) { From c749f74ed97d427017620b29ce90eeedf12796a4 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Fri, 9 Oct 2026 17:45:43 -0700 Subject: [PATCH 14/24] fix(broker): ignore prompt and border chrome when probing body text at the cursor Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- .../src/broker/delivery_verification.rs | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 157334a7ed..ddd538c9b4 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -273,10 +273,15 @@ fn tail_near_cursor(snapshot: &Snapshot, tail: &str) -> bool { /// Whether the cells right of the cursor on its row are part of this body. /// A cursor parked at the start of a draft whose tail has scrolled out of /// view shows body text there; harness chrome (a placeholder or status hint) -/// does not appear in the body and is ignored. +/// does not appear in the body and is ignored. A placeholder whose exact text +/// also appears in the body still reads as a draft: that errs inconclusive, +/// never toward confirming an unsent message. fn body_text_right_of_cursor(snapshot: &Snapshot, expected_echo: &str) -> bool { let from_cursor = snapshot.to_plain_from_cursor(); - let row = compact_render(from_cursor.lines().next().unwrap_or_default()); + let compact_row = compact_render(from_cursor.lines().next().unwrap_or_default()); + // Prompt glyphs and box borders (`›`, `┃`, `│`) can bracket the text + // under the cursor; they are never part of the body, so ignore them. + let row = compact_row.trim_matches(|character: char| !character.is_alphanumeric()); if row.is_empty() { return false; } @@ -1112,6 +1117,29 @@ mod tests { pty.shutdown().unwrap(); } + #[cfg(unix)] + #[tokio::test] + async fn bordered_draft_head_right_of_cursor_stays_inconclusive() { + // The cursor sits on a box border; the draft's head follows it and its + // tail has scrolled out of view. + let expected = format!( + "Relay message from Lead [evt]: head {} TAIL_SCROLLED_OFF", + "y".repeat(160) + ); + let (pty, snapshot) = + codex_snapshot("┃ Relay message from Lead [evt]: head ┃\x1b[1;1H").await; + let mut verification = codex_verification(&expected); + verification.detector = ActivityDetector::for_cli("muse"); + verification.activity_buffer.push_str("repaint"); + + assert_eq!( + assess_harness_acceptance("muse", &verification, &snapshot), + HarnessAcceptance::Inconclusive, + "border chrome around the cursor must not hide an unsent draft" + ); + pty.shutdown().unwrap(); + } + #[cfg(unix)] #[tokio::test] async fn cursor_row_placeholder_does_not_block_generic_acceptance() { From 0202e9f81b81bea4d5b9f1396724f1d9306d162d Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Fri, 9 Oct 2026 18:17:23 -0700 Subject: [PATCH 15/24] fix(broker): strip only prompt and border glyphs from the cursor-row body probe Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- .../src/broker/delivery_verification.rs | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index ddd538c9b4..97ab93d426 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -270,6 +270,11 @@ fn tail_near_cursor(snapshot: &Snapshot, tail: &str) -> bool { .any(|(split, _)| before.ends_with(&tail[..split]) && after.starts_with(&tail[split..])) } +/// Prompt glyphs and box-drawing borders that frame a composer row. +const CURSOR_ROW_CHROME: &[char] = &[ + '›', '❯', '❭', '>', '$', '│', '┃', '║', '▌', '▎', '▏', '╎', '╏', '┆', '┇', +]; + /// Whether the cells right of the cursor on its row are part of this body. /// A cursor parked at the start of a draft whose tail has scrolled out of /// view shows body text there; harness chrome (a placeholder or status hint) @@ -279,9 +284,9 @@ fn tail_near_cursor(snapshot: &Snapshot, tail: &str) -> bool { fn body_text_right_of_cursor(snapshot: &Snapshot, expected_echo: &str) -> bool { let from_cursor = snapshot.to_plain_from_cursor(); let compact_row = compact_render(from_cursor.lines().next().unwrap_or_default()); - // Prompt glyphs and box borders (`›`, `┃`, `│`) can bracket the text - // under the cursor; they are never part of the body, so ignore them. - let row = compact_row.trim_matches(|character: char| !character.is_alphanumeric()); + // Prompt glyphs and box borders can bracket the text under the cursor. + // Strip only those: a draft can itself start with punctuation or symbols. + let row = compact_row.trim_matches(CURSOR_ROW_CHROME); if row.is_empty() { return false; } @@ -1140,6 +1145,24 @@ mod tests { pty.shutdown().unwrap(); } + #[cfg(unix)] + #[tokio::test] + async fn symbolic_draft_head_right_of_cursor_stays_inconclusive() { + // The visible head is punctuation only; the tail has scrolled away. + let expected = format!("~~~ *** ### {} TAIL_SCROLLED_OFF", "y".repeat(160)); + let (pty, snapshot) = codex_snapshot("┃ ~~~ *** ### ┃\x1b[1;1H").await; + let mut verification = codex_verification(&expected); + verification.detector = ActivityDetector::for_cli("muse"); + verification.activity_buffer.push_str("repaint"); + + assert_eq!( + assess_harness_acceptance("muse", &verification, &snapshot), + HarnessAcceptance::Inconclusive, + "a draft head made of symbols is still body text" + ); + pty.shutdown().unwrap(); + } + #[cfg(unix)] #[tokio::test] async fn cursor_row_placeholder_does_not_block_generic_acceptance() { From 2354c373096c9f6e768cc18dba9f5da57ffce8e3 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Fri, 9 Oct 2026 22:43:48 -0700 Subject: [PATCH 16/24] fix(broker): failed draft ending in a bare prompt row stays latched MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit composer_is_idle alone releases the failed-draft latch when the body's last line is a bare prompt glyph (`›`, `>`): the parser anchors on that row and sees an idle prompt while the whole draft is still in the editor. Release now also requires composer_holds_tail to no longer find the failed tail. Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- .../src/broker/delivery_verification.rs | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 650b2158d8..7e8394cabb 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -500,8 +500,15 @@ fn composer_is_idle(snapshot: &Snapshot, cli: &str) -> bool { /// worker holds injection until the composer is proven idle. The tail being /// absent from the viewport is not proof: a long draft can scroll it out of /// view. An operator `flush_injections` is the explicit override. +/// +/// An idle-looking composer is not enough on its own: a body whose last line +/// is a bare prompt glyph parses as an idle prompt while the whole draft is +/// still in the editor, so the tail must also be absent from every composer +/// candidate that ends at the cursor. pub(crate) fn failed_draft_released(cli: &str, expected_echo: &str, snapshot: &Snapshot) -> bool { - expected_tail(expected_echo).is_empty() || composer_is_idle(snapshot, cli) + let tail = expected_tail(expected_echo); + tail.is_empty() + || (composer_is_idle(snapshot, cli) && !composer_holds_tail(snapshot, cli, &tail)) } /// Distinguish terminal echo from actual harness acceptance. @@ -1145,6 +1152,26 @@ mod tests { pty.shutdown().unwrap(); } + #[cfg(unix)] + #[tokio::test] + async fn failed_draft_ending_in_a_bare_prompt_line_stays_latched() { + // The failed body's last line is a bare prompt glyph. The composer + // parser anchors on that line and sees an idle prompt, but the whole + // draft is still in the editor and ends at the cursor. + for (cli, glyph) in [("codex", "›"), ("claude", ">")] { + let expected = format!("Relay message from Lead [evt]: quote this\n{glyph}"); + let (pty, snapshot) = codex_snapshot(&format!( + "{glyph} Relay message from Lead [evt]: quote this\n{glyph}" + )) + .await; + assert!( + !failed_draft_released(cli, &expected, &snapshot), + "{cli}: a draft ending in a bare `{glyph}` row must not release the latch" + ); + pty.shutdown().unwrap(); + } + } + #[cfg(unix)] #[tokio::test] async fn failed_draft_latch_holds_until_the_composer_releases_it() { From 39a80d003448c1bb7ffca0e24faaa0311d9f7590 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Fri, 9 Oct 2026 22:43:49 -0700 Subject: [PATCH 17/24] fix(broker): partial chunked initial write latches until operator flush (optional) An incomplete chunked Codex initial write latches the full body while only a prefix reached the composer, so composer observation cannot prove release. Mark that latch operator-release-only (released by an operator-wide flush_injections) and send agent_blocked_on_send so the broker publishes `stuck` instead of `working`. Optional pending sign-off; revertable on its own. Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- crates/broker/src/pty_worker.rs | 53 ++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index 0b97ca8225..0f79f1fc04 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -110,6 +110,15 @@ struct FailedComposerLatch { delivery_id: DeliveryId, event_id: EventId, expected_echo: String, + /// Only a written prefix of `expected_echo` reached the composer (an + /// incomplete chunked initial write), so its tail cannot prove the draft + /// is gone. Such a latch releases only on an operator `flush_injections`. + operator_release_only: bool, +} + +/// Whether observing the composer may release this latch automatically. +fn failed_latch_auto_releasable(latch: &FailedComposerLatch) -> bool { + !latch.operator_release_only } /// Record a terminal failure for a delivery whose body may remain in the @@ -127,6 +136,7 @@ fn latch_failed_written_delivery( delivery_id: delivery_id.clone(), event_id: event_id.clone(), expected_echo: expected_echo.to_string(), + operator_release_only: false, }); } @@ -2456,10 +2466,28 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { &inj.pending.delivery.event_id, inj.injection_text.as_deref().unwrap_or_default(), ); + // Bytes were written (an oversize body rejected before + // any write has no injection text): only an unknown + // prefix is in the composer, so only an operator may + // release the latch. + let partial_body_written = inj.injection_text.is_some(); + if let Some(latch) = failed_composer_latch.as_mut() { + latch.operator_release_only = partial_body_written; + } let _ = send_frame(&out_tx, "worker_error", inj.pending.request_id, json!({ "code": "initial_injection_incomplete", "retryable": false, "message": "Initial task delivery interrupted or timed out; inspect the composer before retrying" })).await; + if partial_body_written { + // Surface the hold: without this the broker shows + // the worker as working while every later delivery + // waits for the operator's flush. + let _ = send_frame(&out_tx, "agent_blocked_on_send", None, json!({ + "reason": "failed_draft_requires_flush", + "blocked_secs": 0, + "pending_delivery_count": pending_worker_injections.len(), + })).await; + } continue; } if inj.human_input_generation != human_input_generation { @@ -2972,7 +3000,8 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { // holds that body (the harness took it late, or a human // submitted or cleared it). Until then queued deliveries wait. if let Some(latch) = failed_composer_latch.as_ref() { - if pending_verifications.is_empty() + if failed_latch_auto_releasable(latch) + && pending_verifications.is_empty() && pending_recovery_writes.is_empty() && active_injection.is_none() && failed_draft_released( @@ -4438,6 +4467,28 @@ mod tests { ); } + #[test] + fn partially_written_initial_task_latch_waits_for_operator_flush() { + let mut failed = HashMap::new(); + let mut latch = None; + latch_failed_written_delivery( + &mut failed, + &mut latch, + &DeliveryId::from("del_partial_initial"), + &EventId::from("evt_partial_initial"), + "Relay message from Lead [evt]: long initial task", + ); + assert!( + failed_latch_auto_releasable(latch.as_ref().unwrap()), + "a fully written failed draft may be released by an idle composer" + ); + latch.as_mut().unwrap().operator_release_only = true; + assert!( + !failed_latch_auto_releasable(latch.as_ref().unwrap()), + "a partially written draft's tail proves nothing; only a flush releases it" + ); + } + #[test] fn failed_written_delivery_latches_injection_and_fences_replay() { let mut failed = HashMap::new(); From 07893566693137ec13a76676315c8b8badc7fed4 Mon Sep 17 00:00:00 2001 From: Miya Date: Sat, 10 Oct 2026 11:18:19 +0200 Subject: [PATCH 18/24] docs(changelog): list delivery_unconfirmed / delivery_resubmitted broker events under Added Addresses #1959 review (CodeRabbit): the minor bump is driven by additive protocol surface that the Fixed bullets did not name. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 2a0590dc-a82a-4174-9eff-c23d59571061 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cac790b9ed..809e636110 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The `agent-relay mcp` server adds `upload_file` (local path or base64) and `download_file`, and `send_group_dm` accepts `attachments`. - Agents spawned by the broker receive message attachments (up to 25 MiB each) as local files: the injected message ends with an `Attachments:` list giving each file's path under `.agent-relay/attachments/` in the agent's working directory, or an `agent-relay message file download ` command when a file could not be fetched. Messages that carry only attachments are now delivered too. - `@agent-relay/sdk` adds `files.upload` / `files.get` / `files.download` on `AgentRelay` and `RelaycastMessagingClient`, plus `uploadRelayFile` / `downloadRelayFile` helpers. +- Broker events report PTY submit recovery: `delivery_unconfirmed` (the harness has not accepted a write yet, with `attempts` / `max_attempts`) and `delivery_resubmitted` (a submit-only retry, with `attempt` and `strategy`); `delivery_verified` adds `evidence` and `attempts`. `@agent-relay/harness-driver` and the Python and Swift SDK protocol types include both new events. ### Fixed From 2c187fd127915687b842a5e7975a112dd6a297bb Mon Sep 17 00:00:00 2001 From: Khaliq Date: Sat, 10 Oct 2026 05:17:01 -0700 Subject: [PATCH 19/24] fix(broker): wrap failed-draft latch, Devin parked deferral, report latched queueing - wrap mode latches a delivery whose body was written but never accepted and holds injection until the composer is provably idle or the human submits it - a parked wrap delivery defers while a Devin prompt is briefly unready instead of failing, without spending retry budget - queueing a delivery behind the failed-draft latch alone sends agent_blocked_on_send so the broker does not report the worker as working Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- crates/broker/src/pty_worker.rs | 65 +++++++++++++++++++++++++++++++++ crates/broker/src/wrap.rs | 62 ++++++++++++++++++++++++++----- 2 files changed, 117 insertions(+), 10 deletions(-) diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index 0f79f1fc04..6760f4def9 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -153,6 +153,21 @@ fn composer_owned_by_delivery( || failed_composer_latch.is_some() } +/// Whether the failed-draft latch alone holds a newly queued delivery. Another +/// blocker (a verification, a recovery write, an active injection) already +/// explains the wait; reporting the latch then would be redundant. +fn failed_draft_latch_is_sole_blocker( + failed_composer_latch: Option<&FailedComposerLatch>, + pending_verifications_empty: bool, + pending_recovery_writes_empty: bool, + active_injection_present: bool, +) -> bool { + failed_composer_latch.is_some() + && pending_verifications_empty + && pending_recovery_writes_empty + && !active_injection_present +} + /// Only a non-empty `write_pty` that the drainer admitted can change the /// composer, so only that write takes ownership away from relay deliveries. fn human_write_takes_ownership(byte_len: usize, admitted: bool) -> bool { @@ -1451,6 +1466,21 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { request_id: frame.request_id, queued_at: Instant::now(), }); + if failed_draft_latch_is_sole_blocker( + failed_composer_latch.as_ref(), + pending_verifications.is_empty(), + pending_recovery_writes.is_empty(), + active_injection.is_some(), + ) { + // The broker marks a worker Working on + // delivery_queued, but this delivery + // will wait behind the failed draft. + let _ = send_frame(&out_tx, "agent_blocked_on_send", None, json!({ + "reason": "failed_draft_latched", + "blocked_secs": 0, + "pending_delivery_count": pending_worker_injections.len(), + })).await; + } } else { tracing::debug!( delivery_id = %delivery.delivery_id, @@ -4525,6 +4555,41 @@ mod tests { ); } + #[test] + fn latched_worker_reports_blocked_only_when_the_latch_is_the_sole_blocker() { + let latch = FailedComposerLatch { + delivery_id: DeliveryId::from("del_failed"), + event_id: EventId::from("evt_failed"), + expected_echo: "Relay message from Lead [evt]: failed".to_string(), + operator_release_only: false, + }; + assert!(failed_draft_latch_is_sole_blocker( + Some(&latch), + true, + true, + false + )); + assert!(!failed_draft_latch_is_sole_blocker(None, true, true, false)); + assert!(!failed_draft_latch_is_sole_blocker( + Some(&latch), + false, + true, + false + )); + assert!(!failed_draft_latch_is_sole_blocker( + Some(&latch), + true, + false, + false + )); + assert!(!failed_draft_latch_is_sole_blocker( + Some(&latch), + true, + true, + true + )); + } + #[test] fn only_an_admitted_nonempty_human_write_takes_ownership() { assert!( diff --git a/crates/broker/src/wrap.rs b/crates/broker/src/wrap.rs index c23cceca00..1478b52aee 100644 --- a/crates/broker/src/wrap.rs +++ b/crates/broker/src/wrap.rs @@ -27,9 +27,10 @@ use tokio::{sync::mpsc, time::MissedTickBehavior}; use crate::broker::{ delivery_verification::{ - assess_harness_acceptance, DeliveryOutcome, HarnessAcceptance, PendingActivity, - PendingVerification, ThrottleState, VerificationOutput, ACTIVITY_BUFFER_KEEP_BYTES, - ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW, MAX_VERIFICATION_ATTEMPTS, VERIFICATION_WINDOW, + assess_harness_acceptance, failed_draft_released, DeliveryOutcome, HarnessAcceptance, + PendingActivity, PendingVerification, ThrottleState, VerificationOutput, + ACTIVITY_BUFFER_KEEP_BYTES, ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW, + MAX_VERIFICATION_ATTEMPTS, VERIFICATION_WINDOW, }, injection_format::{format_injection_for_worker_with_workspace, McpReminderThrottle}, }; @@ -301,6 +302,12 @@ fn wrap_injection_timer_allowed(has_pending_write_ack: bool, has_pending_accepta !has_pending_write_ack && !has_pending_acceptance } +/// Human stdin that submits the composer (Enter) takes whatever it holds, +/// including a failed relay draft, so it releases the wrap failed-draft latch. +fn human_input_submits_composer(data: &[u8]) -> bool { + data.iter().any(|byte| matches!(byte, b'\r' | b'\n')) +} + // Readiness deferrals must not spend the bounded delivery retry budget. fn prepare_wrap_retry( verification: &mut PendingVerification, @@ -1589,6 +1596,11 @@ pub(crate) async fn run_wrap( let mut stdin_pending: VecDeque> = VecDeque::new(); let mut human_input_generation = 0u64; let mut recovery_cancels: Vec> = Vec::new(); + // Expected echo of a delivery whose body was written but never proven + // accepted. It may still be typed in the composer, so no later relay + // message is injected on top of it until the composer is provably idle or + // the human submits it. + let mut wrap_failed_draft: Option = None; let mut stdin_retry_deadline: Option = None; const STDIN_RETRY_INTERVAL: Duration = Duration::from_millis(4); @@ -1606,6 +1618,13 @@ pub(crate) async fn run_wrap( Some(data) = stdin_rx.recv() => { human_input_generation = human_input_generation.saturating_add(1); cancel_recovery_writes(&mut recovery_cancels); + if wrap_failed_draft.is_some() && human_input_submits_composer(&data) { + tracing::info!( + target = "agent_relay::worker::wrap", + "wrap: human submitted the composer; releasing the failed-draft latch" + ); + wrap_failed_draft = None; + } if !pending_verifications.is_empty() { for verification in pending_verifications.drain(..) { tracing::warn!( @@ -2165,7 +2184,7 @@ pub(crate) async fn run_wrap( _ = pending_injection_interval.tick(), if wrap_injection_timer_allowed( !pending_wrap_writes.is_empty(), - !pending_verifications.is_empty(), + !pending_verifications.is_empty() || wrap_failed_draft.is_some(), ) => { // Give backlogged human keystrokes priority onto the PTY FIFO // over a new automated injection — see the auto-responder gate @@ -2278,6 +2297,7 @@ pub(crate) async fn run_wrap( "wrap: human input followed the injection write; automatic verification cancelled" ); throttle.record(DeliveryOutcome::Failed); + wrap_failed_draft = Some(injection.clone()); continue; } @@ -2342,6 +2362,7 @@ pub(crate) async fn run_wrap( "wrap: human input followed submit-key recovery; further automatic recovery cancelled" ); throttle.record(DeliveryOutcome::Failed); + wrap_failed_draft = Some(verification.expected_echo.clone()); continue; } @@ -2352,6 +2373,7 @@ pub(crate) async fn run_wrap( "wrap: submit-key recovery was not confirmed; body left untouched" ); throttle.record(DeliveryOutcome::Failed); + wrap_failed_draft = Some(verification.expected_echo.clone()); continue; } @@ -2448,19 +2470,21 @@ pub(crate) async fn run_wrap( "wrap: failed to queue submit-key recovery; body left untouched" ); throttle.record(DeliveryOutcome::Failed); + wrap_failed_draft = Some(pv.expected_echo.clone()); } } } HarnessAcceptance::Parked if !pv.acceptance_expired() && (!stdin_pending.is_empty() - || !pending_wrap_writes.is_empty()) => + || !pending_wrap_writes.is_empty() + || !crate::devin::can_inject(&resolved_cli, &pty)) => { - // An in-flight writer takes priority. Human - // input already drains these verifications in - // the stdin arm, so this deferral never resets - // the total acceptance lifetime. - pv.injected_at = Instant::now(); + // An in-flight writer or a briefly unready + // Devin prompt takes priority. Deferring spends + // no retry budget and never resets the total + // acceptance lifetime. + prepare_wrap_retry(&mut pv, false, Instant::now()); pending_verifications.push_back(pv); } HarnessAcceptance::Parked => { @@ -2470,6 +2494,7 @@ pub(crate) async fn run_wrap( "wrap: body remained parked after bounded submit-key recovery" ); throttle.record(DeliveryOutcome::Failed); + wrap_failed_draft = Some(pv.expected_echo.clone()); } HarnessAcceptance::Inconclusive => { tracing::error!( @@ -2478,6 +2503,7 @@ pub(crate) async fn run_wrap( "wrap: harness acceptance could not be proven; body left untouched" ); throttle.record(DeliveryOutcome::Failed); + wrap_failed_draft = Some(pv.expected_echo.clone()); } } } else { @@ -2485,6 +2511,19 @@ pub(crate) async fn run_wrap( } } + if let Some(expected_echo) = wrap_failed_draft.as_deref() { + if pending_verifications.is_empty() + && pending_wrap_writes.is_empty() + && failed_draft_released(&resolved_cli, expected_echo, &Snapshot::capture(&pty)) + { + tracing::info!( + target = "agent_relay::worker::wrap", + "wrap: failed draft left the composer; resuming injection" + ); + wrap_failed_draft = None; + } + } + if activity_detector.is_some() { let mut i = 0; while i < pending_activities.len() { @@ -2821,6 +2860,9 @@ sys.stdout.flush()"#; #[test] fn wrap_injection_timer_waits_for_write_ack_and_harness_acceptance() { + assert!(super::human_input_submits_composer(b"\r")); + assert!(super::human_input_submits_composer(b"ok\n")); + assert!(!super::human_input_submits_composer(b"typing")); assert!(wrap_injection_timer_allowed(false, false)); assert!(!wrap_injection_timer_allowed(true, false)); assert!(!wrap_injection_timer_allowed(false, true)); From ce7dca57ef4d845bca357820a37cd5e2f2c631dd Mon Sep 17 00:00:00 2001 From: Khaliq Date: Sat, 10 Oct 2026 05:17:02 -0700 Subject: [PATCH 20/24] test(broker): read-only attachments root test checks the mode is enforced The StarSling Linux runner lets the test write into a 0o555 directory, so prepare_root correctly accepts it. Assert refusal of a read-only root only where the mode is enforced, and otherwise refuse a root that cannot be a directory. Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- crates/broker/src/attachments.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/broker/src/attachments.rs b/crates/broker/src/attachments.rs index 473e2c2f76..9d2260ae2d 100644 --- a/crates/broker/src/attachments.rs +++ b/crates/broker/src/attachments.rs @@ -1347,9 +1347,23 @@ mod tests { std::fs::create_dir_all(&primary).unwrap(); std::fs::set_permissions(&primary, std::fs::Permissions::from_mode(0o555)).unwrap(); let fallback = temp.path().join("fallback"); + // Some CI runners bypass directory permission bits (root, a granted + // CAP_DAC_OVERRIDE, or a filesystem that ignores modes). There the + // 0o555 directory really is writable, so accepting it is correct and + // refusal must be exercised with a root that cannot be written at all. + let mode_enforced = std::fs::write(primary.join(".mode-probe"), b"").is_err(); let refused = prepare_root(&primary).await.is_err(); std::fs::set_permissions(&primary, std::fs::Permissions::from_mode(0o755)).unwrap(); - assert!(refused, "a read-only attachments root must be refused"); + if mode_enforced { + assert!(refused, "a read-only attachments root must be refused"); + } else { + let not_a_directory = temp.path().join("not-a-directory"); + std::fs::write(¬_a_directory, b"").unwrap(); + assert!( + prepare_root(¬_a_directory).await.is_err(), + "an unwritable attachments root must be refused" + ); + } assert!(prepare_root(&fallback).await.is_ok()); } From 1055cb49d1725bda5ae97375c5ec84591263643b Mon Sep 17 00:00:00 2001 From: Khaliq Date: Sat, 10 Oct 2026 15:01:01 -0700 Subject: [PATCH 21/24] fix(broker): three-outcome PTY acceptance per the #1959 policy (P2-P5) Accepted / Unconfirmed / StuckInComposer, as approved by Khaliq on 2026-10-10. - P2: Inconclusive (written, not visibly parked, no harness evidence) is now Unconfirmed: the worker releases custody with delivery_ack and reports a terminal delivery_unconfirmed {outcome, terminal, reason}, never delivery_failed. The broker never sets BlockedOnSend or publishes `stuck` for it. Wrap mode treats it the same way. - P3: the failed-draft latch (pty and wrap) arms only on positive evidence that the body is still in the composer (draft_visible_at_cursor: tail in the composer or around the cursor, or body text either side of the cursor). The partial chunked initial write uses the same rule; the operator-only latch from 39a80d003 is removed. - P4: a visibly stuck draft holds injection (never typed over); after FAILED_DRAFT_ESCALATION_TIMEOUT (60s) the worker sends one terminal agent_blocked_on_send {failed_draft_latched}. Human input still cancels a pending recovery. - P5: a verified spawn whose initial task is Unconfirmed now succeeds with warning.code = "spawn_task_unconfirmed"; only a real delivery failure (spawn_task_failed) fails it. Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- .../src/broker/delivery_verification.rs | 60 ++++++ crates/broker/src/pty_worker.rs | 188 ++++++++++++------ crates/broker/src/runtime/fleet.rs | 46 +++-- crates/broker/src/runtime/tests.rs | 146 +++++++++----- crates/broker/src/runtime/worker_events.rs | 67 +++++++ crates/broker/src/wrap.rs | 28 ++- 6 files changed, 401 insertions(+), 134 deletions(-) diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index bdb985569e..59a43ee7d1 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -494,6 +494,40 @@ fn composer_is_idle(snapshot: &Snapshot, cli: &str) -> bool { .unwrap_or(false) } +/// Whether the cursor row's text left of the cursor is the end of this body. +/// A partially written or unsubmitted draft leaves its latest characters just +/// before the cursor; prompt glyphs and borders are ignored. Short rows are not +/// evidence: a handful of characters can match any body by coincidence. +fn body_text_left_of_cursor(snapshot: &Snapshot, expected_echo: &str) -> bool { + let through_cursor = snapshot.to_plain_through_cursor(); + let compact_row = compact_render(through_cursor.lines().last().unwrap_or_default()); + let row = compact_row.trim_matches(CURSOR_ROW_CHROME); + if row.chars().count() < 8 { + return false; + } + let probe_start = row + .char_indices() + .rev() + .nth(31) + .map_or(0, |(index, _)| index); + compact_render(expected_echo).contains(&row[probe_start..]) +} + +/// Positive evidence that a delivery's body is still typed in the composer: +/// its tail is in the parsed composer or around the cursor, or body text sits +/// directly left or right of the cursor. Only this arms the failed-draft +/// latch; a delivery the broker merely cannot verify never does. +pub(crate) fn draft_visible_at_cursor(snapshot: &Snapshot, cli: &str, expected_echo: &str) -> bool { + let tail = expected_tail(expected_echo); + if tail.is_empty() { + return false; + } + composer_holds_tail(snapshot, cli, &tail) + || tail_near_cursor(snapshot, &tail) + || body_text_right_of_cursor(snapshot, expected_echo) + || body_text_left_of_cursor(snapshot, expected_echo) +} + /// Whether a terminally failed delivery's body has provably left the composer. /// /// A failed body can remain typed in the editor. Injecting the next delivery @@ -1323,6 +1357,32 @@ mod tests { pty.shutdown().unwrap(); } + #[cfg(unix)] + #[tokio::test] + async fn draft_visibility_is_positive_evidence_only() { + let expected = "Relay message from Lead [evt]: please finish the parked draft"; + let (typed_pty, typed) = codex_snapshot(&format!("› {expected}")).await; + assert!( + draft_visible_at_cursor(&typed, "codex", expected), + "draft left of the cursor" + ); + typed_pty.shutdown().unwrap(); + + let (cleared_pty, cleared) = codex_snapshot("› Ask Codex to do anything\x1b[1;3H").await; + assert!( + !draft_visible_at_cursor(&cleared, "codex", expected), + "an idle composer" + ); + cleared_pty.shutdown().unwrap(); + + let (other_pty, other) = codex_snapshot("› an unrelated human draft here").await; + assert!( + !draft_visible_at_cursor(&other, "codex", expected), + "someone else's text" + ); + other_pty.shutdown().unwrap(); + } + #[test] fn wrapped_echo_preserves_same_chunk_acceptance_activity() { let expected = "Relay message from Lead [evt]: Reply with exactly WRAPPED_CODEX_ACK"; diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index b83bad347f..21f2f9ce7c 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -29,9 +29,9 @@ use crate::broker::{ continuity::parse_continuity_command, delivery_verification::{ assess_harness_acceptance, current_timestamp_ms, delivery_injected_event_payload, - delivery_queued_event_payload, failed_draft_released, DeliveryOutcome, HarnessAcceptance, - PendingActivity, PendingVerification, ThrottleState, VerificationOutput, - ACTIVITY_BUFFER_KEEP_BYTES, ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW, + delivery_queued_event_payload, draft_visible_at_cursor, failed_draft_released, + DeliveryOutcome, HarnessAcceptance, PendingActivity, PendingVerification, ThrottleState, + VerificationOutput, ACTIVITY_BUFFER_KEEP_BYTES, ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW, MAX_VERIFICATION_ATTEMPTS, VERIFICATION_WINDOW, }, injection_format::{format_injection_for_worker_with_workspace, McpReminderThrottle}, @@ -112,33 +112,55 @@ struct FailedComposerLatch { delivery_id: DeliveryId, event_id: EventId, expected_echo: String, - /// Only a written prefix of `expected_echo` reached the composer (an - /// incomplete chunked initial write), so its tail cannot prove the draft - /// is gone. Such a latch releases only on an operator `flush_injections`. - operator_release_only: bool, + latched_at: Instant, + /// The terminal stuck signal has been sent for this latch. + escalated: bool, } -/// Whether observing the composer may release this latch automatically. -fn failed_latch_auto_releasable(latch: &FailedComposerLatch) -> bool { - !latch.operator_release_only +/// How long a visibly stuck draft may hold injection before the worker sends +/// the terminal `failed_draft_latched` signal to the sender and operator. The +/// draft is still never typed over; queued deliveries keep waiting. +const FAILED_DRAFT_ESCALATION_TIMEOUT: Duration = Duration::from_secs(60); + +/// [`FAILED_DRAFT_ESCALATION_TIMEOUT`], overridable in milliseconds through +/// `RELAY_FAILED_DRAFT_ESCALATION_MS` so integration tests need not wait. +fn failed_draft_escalation_timeout() -> Duration { + std::env::var("RELAY_FAILED_DRAFT_ESCALATION_MS") + .ok() + .and_then(|value| value.trim().parse::().ok()) + .map(Duration::from_millis) + .unwrap_or(FAILED_DRAFT_ESCALATION_TIMEOUT) } -/// Record a terminal failure for a delivery whose body may remain in the -/// composer: fence its id against replay for the PTY's lifetime and latch -/// injection until the composer releases the draft. +/// Whether a still-latched draft has outlived the escalation timeout and has +/// not been reported yet. +fn failed_draft_escalation_due(latch: &FailedComposerLatch, timeout: Duration) -> bool { + !latch.escalated && latch.latched_at.elapsed() >= timeout +} + +/// Record a terminal failure for a delivery whose body was written: fence its +/// id against replay for the PTY's lifetime and, only on positive evidence +/// that the body is still in the composer (`draft_visible`), latch injection +/// until the composer releases the draft. A failure without that evidence +/// never blocks later deliveries. fn latch_failed_written_delivery( failed_written_deliveries: &mut HashMap, failed_composer_latch: &mut Option, delivery_id: &DeliveryId, event_id: &EventId, expected_echo: &str, + draft_visible: bool, ) { failed_written_deliveries.insert(delivery_id.clone(), event_id.clone()); + if !draft_visible { + return; + } *failed_composer_latch = Some(FailedComposerLatch { delivery_id: delivery_id.clone(), event_id: event_id.clone(), expected_echo: expected_echo.to_string(), - operator_release_only: false, + latched_at: Instant::now(), + escalated: false, }); } @@ -1226,6 +1248,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { // Cancellation flags for queued submit-key recoveries. Human input sets // them so a delayed recovery key cannot submit the human's draft. let mut recovery_cancels: Vec> = Vec::new(); + let failed_draft_escalation_timeout = failed_draft_escalation_timeout(); let mut startup_output = String::new(); let mut startup_total_bytes = 0usize; let mut init_request_id: Option = None; @@ -1620,6 +1643,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { &delivery_id, &pv.event_id, &pv.expected_echo, + draft_visible_at_cursor(&Snapshot::capture(&pty), &resolved_cli, &pv.expected_echo), ); } if cancelled_verification { @@ -2546,29 +2570,12 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { &inj.pending.delivery.delivery_id, &inj.pending.delivery.event_id, inj.injection_text.as_deref().unwrap_or_default(), + draft_visible_at_cursor(&Snapshot::capture(&pty), &resolved_cli, inj.injection_text.as_deref().unwrap_or_default()), ); - // Bytes were written (an oversize body rejected before - // any write has no injection text): only an unknown - // prefix is in the composer, so only an operator may - // release the latch. - let partial_body_written = inj.injection_text.is_some(); - if let Some(latch) = failed_composer_latch.as_mut() { - latch.operator_release_only = partial_body_written; - } let _ = send_frame(&out_tx, "worker_error", inj.pending.request_id, json!({ "code": "initial_injection_incomplete", "retryable": false, "message": "Initial task delivery interrupted or timed out; inspect the composer before retrying" })).await; - if partial_body_written { - // Surface the hold: without this the broker shows - // the worker as working while every later delivery - // waits for the operator's flush. - let _ = send_frame(&out_tx, "agent_blocked_on_send", None, json!({ - "reason": "failed_draft_requires_flush", - "blocked_secs": 0, - "pending_delivery_count": pending_worker_injections.len(), - })).await; - } continue; } if inj.human_input_generation != human_input_generation { @@ -2585,6 +2592,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { &delivery_id, &inj.pending.delivery.event_id, inj.injection_text.as_deref().unwrap_or_default(), + draft_visible_at_cursor(&Snapshot::capture(&pty), &resolved_cli, inj.injection_text.as_deref().unwrap_or_default()), ); throttle.record(DeliveryOutcome::Failed); continue; @@ -2826,6 +2834,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { &delivery_id, &event_id, &pv.expected_echo, + draft_visible_at_cursor(&Snapshot::capture(&pty), &resolved_cli, &pv.expected_echo), ); } } @@ -2852,6 +2861,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { &delivery_id, &event_id, &pv.expected_echo, + draft_visible_at_cursor(&Snapshot::capture(&pty), &resolved_cli, &pv.expected_echo), ); } Err(_) => { @@ -2870,6 +2880,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { &delivery_id, &event_id, &pv.expected_echo, + draft_visible_at_cursor(&Snapshot::capture(&pty), &resolved_cli, &pv.expected_echo), ); } } @@ -3017,6 +3028,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { &delivery_id, &event_id, &pv.expected_echo, + draft_visible_at_cursor(&Snapshot::capture(&pty), &resolved_cli, &pv.expected_echo), ); } } @@ -3040,34 +3052,52 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { pv.injected_at = Instant::now(); pending_verifications.push_back(pv); } - state @ (HarnessAcceptance::Parked - | HarnessAcceptance::Inconclusive) => { - let reason = match state { - HarnessAcceptance::Parked => { - "body remained parked after bounded submit-key recovery" - } - HarnessAcceptance::Inconclusive => { - crate::broker::delivery_verification::HARNESS_ACCEPTANCE_UNPROVEN - } - HarnessAcceptance::Accepted(_) => unreachable!(), - }; + HarnessAcceptance::Inconclusive => { + // Unconfirmed: the body was written and is not + // visibly parked, but nothing proved the + // harness took the turn. That is a delivery, + // not a failure: release custody, report it + // unconfirmed, and keep delivering. Never latch. + let _ = send_frame( + &out_tx, + "delivery_ack", + pv.request_id.clone(), + json!({ "delivery_id": delivery_id, "event_id": event_id }), + ) + .await; + let _ = send_frame(&out_tx, "delivery_unconfirmed", None, json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "outcome": "unconfirmed", + "terminal": true, + "reason": crate::broker::delivery_verification::HARNESS_ACCEPTANCE_UNPROVEN, + "attempts": pv.attempts, + })).await; + throttle.record(DeliveryOutcome::Unverified); + pending_worker_delivery_ids.remove(&delivery_id); + completed_worker_deliveries.insert(delivery_id, event_id); + } + HarnessAcceptance::Parked => { + // StuckInComposer: the body is still visibly + // parked after bounded submit-key recovery. let _ = send_frame(&out_tx, "delivery_failed", None, json!({ "delivery_id": delivery_id, "event_id": event_id, - "reason": reason, + "reason": "body remained parked after bounded submit-key recovery", "attempts": pv.attempts, })).await; throttle.record(DeliveryOutcome::Failed); // Latch before removing the pending id: the body - // may still be in the composer, so neither a - // replay nor the next queued delivery may be - // typed on top of it. + // is still in the composer, so neither a replay + // nor the next queued delivery may be typed on + // top of it. Parked is the positive evidence. latch_failed_written_delivery( &mut failed_written_deliveries, &mut failed_composer_latch, &delivery_id, &event_id, &pv.expected_echo, + true, ); pending_worker_delivery_ids.remove(&delivery_id); } @@ -3081,8 +3111,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { // holds that body (the harness took it late, or a human // submitted or cleared it). Until then queued deliveries wait. if let Some(latch) = failed_composer_latch.as_ref() { - if failed_latch_auto_releasable(latch) - && pending_verifications.is_empty() + if pending_verifications.is_empty() && pending_recovery_writes.is_empty() && active_injection.is_none() && failed_draft_released( @@ -3099,6 +3128,27 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { failed_composer_latch = None; } } + // A draft still stuck after the bounded timeout gets one + // terminal signal to the sender and operator. It is still + // never typed over; queued deliveries keep waiting. + if let Some(latch) = failed_composer_latch.as_mut() { + if failed_draft_escalation_due(latch, failed_draft_escalation_timeout) { + latch.escalated = true; + tracing::warn!( + delivery_id = %latch.delivery_id, + event_id = %latch.event_id, + "failed draft still stuck in the composer; escalating" + ); + let _ = send_frame(&out_tx, "agent_blocked_on_send", None, json!({ + "reason": "failed_draft_latched", + "terminal": true, + "delivery_id": latch.delivery_id, + "event_id": latch.event_id, + "blocked_secs": latch.latched_at.elapsed().as_secs(), + "pending_delivery_count": pending_worker_injections.len(), + })).await; + } + } if activity_detector.is_some() { let mut i = 0; @@ -4755,24 +4805,40 @@ mod tests { } #[test] - fn partially_written_initial_task_latch_waits_for_operator_flush() { + fn latch_arms_only_on_visible_draft_and_escalates_once_after_timeout() { let mut failed = HashMap::new(); let mut latch = None; + let delivery_id = DeliveryId::from("del_unproven"); + let event_id = EventId::from("evt_unproven"); latch_failed_written_delivery( &mut failed, &mut latch, - &DeliveryId::from("del_partial_initial"), - &EventId::from("evt_partial_initial"), - "Relay message from Lead [evt]: long initial task", + &delivery_id, + &event_id, + "Relay message from Lead [evt]: unproven", + false, ); - assert!( - failed_latch_auto_releasable(latch.as_ref().unwrap()), - "a fully written failed draft may be released by an idle composer" + assert!(failed.contains_key(&delivery_id), "replay stays fenced"); + assert!(latch.is_none(), "no positive evidence, no latch"); + + latch_failed_written_delivery( + &mut failed, + &mut latch, + &delivery_id, + &event_id, + "Relay message from Lead [evt]: stuck", + true, ); - latch.as_mut().unwrap().operator_release_only = true; + let latch = latch.as_mut().expect("visible draft latches"); + assert!(!failed_draft_escalation_due( + latch, + Duration::from_secs(3600) + )); + assert!(failed_draft_escalation_due(latch, Duration::ZERO)); + latch.escalated = true; assert!( - !failed_latch_auto_releasable(latch.as_ref().unwrap()), - "a partially written draft's tail proves nothing; only a flush releases it" + !failed_draft_escalation_due(latch, Duration::ZERO), + "escalates once" ); } @@ -4790,6 +4856,7 @@ mod tests { &delivery_id, &event_id, "Relay message from Lead [evt]: parked", + true, ); assert_eq!(failed.get(&delivery_id), Some(&event_id)); @@ -4818,7 +4885,8 @@ mod tests { delivery_id: DeliveryId::from("del_failed"), event_id: EventId::from("evt_failed"), expected_echo: "Relay message from Lead [evt]: failed".to_string(), - operator_release_only: false, + latched_at: Instant::now(), + escalated: false, }; assert!(failed_draft_latch_is_sole_blocker( Some(&latch), diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index d9057e94a4..1286e0de36 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -249,31 +249,41 @@ pub(super) fn verified_spawn_ready_result( } } -/// A live, ready agent whose initial task was never confirmed as fully -/// received. +/// A live, ready agent whose initial task was delivered but not confirmed. /// -/// This is deliberately neither a success nor an ordinary failure. The spawn -/// did happen — the agent is registered, ready and holding the node's only -/// slot for that name — so the caller must not retry, or it duplicates the -/// agent. But the task it was spawned to run cannot be shown to have arrived -/// intact, and `spawned:true, ready:true` is exactly the proof `fleet spawn` -/// uses to report success, so the only honest answer is an error that names -/// the live agent and says what to do instead (relay#1893 review, P1). +/// Per the #1959 acceptance policy, only a real delivery failure fails a +/// verified spawn. An unconfirmed task (written, not visibly parked, but no +/// harness evidence either way, or bound to no verdict at all) is a success: +/// the agent is registered, ready and holding the name, so a retry would +/// duplicate it. The result carries `warning.code = "spawn_task_unconfirmed"` +/// so callers can tell it apart from a task the harness was seen to accept. pub(super) fn verified_spawn_task_unconfirmed_result( invocation_id: String, name: &WorkerName, verification: Option<&str>, ) -> ActionResult { - verified_spawn_failed_result( + ActionResult { + task: None, + v: FLEET_WIRE_VERSION, + id: None, invocation_id, - &format!( - "spawn_task_unconfirmed: agent '{name}' is live and ready, but full receipt of its \ - initial task was not confirmed (verification: {}). Do not retry this spawn — it \ - would duplicate the agent. Resend the task, or write the brief to a file and send a \ - short pointer, then check the agent's transcript.", - verification.unwrap_or("none") - ), - ) + result: ActionResultPayload::Output(ActionResultOutput { + output: json!({ + "spawned": true, + "ready": true, + "name": name.as_str(), + "warning": { + "code": "spawn_task_unconfirmed", + "verification": verification.unwrap_or("none"), + "message": format!( + "agent '{name}' is live and ready and its initial task was delivered, but \ + the harness never confirmed accepting it. Do not retry this spawn; check \ + the agent's transcript." + ), + }, + }), + }), + } } /// Result for a verified spawn whose worker had already reported diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index e6a9ae35bf..8f80d356e3 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -884,9 +884,39 @@ fn delivery_verified_worker_event( /// looking at *what* confirmed the delivery. A worker that acked through the /// timeout fallback — i.e. an agent that may have swallowed the whole task — /// produced a result identical to one that echoed every byte. +/// #1959 acceptance policy: an unconfirmed initial task is a delivered task, so +/// the verified spawn succeeds and names the live agent with a warning. +fn assert_unconfirmed_spawn_success( + result: &crate::fleet_wire::ActionResultPayload, + worker_name: &str, + verification: &str, +) { + let crate::fleet_wire::ActionResultPayload::Output(output) = result else { + panic!("an unconfirmed task must resolve the spawn as success: {result:?}"); + }; + assert_eq!(output.output["spawned"], true, "{:?}", output.output); + assert_eq!(output.output["ready"], true, "{:?}", output.output); + assert_eq!(output.output["name"], worker_name, "{:?}", output.output); + assert_eq!( + output.output["warning"]["code"], "spawn_task_unconfirmed", + "{:?}", + output.output + ); + assert_eq!( + output.output["warning"]["verification"], verification, + "{:?}", + output.output + ); + let message = output.output["warning"]["message"] + .as_str() + .unwrap_or_default(); + assert!(message.contains(worker_name), "{message}"); + assert!(message.contains("Do not retry"), "{message}"); +} + #[tokio::test] -async fn a_spawn_task_acked_without_proof_of_receipt_never_reports_success() { - use crate::fleet_wire::{ActionResultPayload, BrokerToRelaycast}; +async fn a_spawn_task_acked_without_proof_of_receipt_succeeds_with_an_unconfirmed_warning() { + use crate::fleet_wire::BrokerToRelaycast; // Every label a worker can ack with that is not harness acceptance, // including the echo labels of workers that predate it: a body echoed in @@ -936,22 +966,10 @@ async fn a_spawn_task_acked_without_proof_of_receipt_never_reports_success() { panic!("spawn completion must send an action result"); }; assert_eq!(result.invocation_id, "inv-receipt"); - let ActionResultPayload::Error(error) = result.result else { - panic!("{verification:?} is not proof of receipt and must not succeed"); - }; - assert!( - error.error.starts_with("spawn_task_unconfirmed: "), - "{}", - error.error - ); - // The agent is live: name it so the caller can reach it, and say not to - // retry, because a retry duplicates it. - assert!(error.error.contains(worker_name), "{}", error.error); - assert!(error.error.contains("Do not retry"), "{}", error.error); - assert!( - error.error.contains(verification.unwrap_or("none")), - "{}", - error.error + assert_unconfirmed_spawn_success( + &result.result, + worker_name, + verification.unwrap_or("none"), ); // Resolved, so maintenance's readiness deadline can no longer release // the live worker this spawn produced. @@ -1100,14 +1118,7 @@ async fn a_task_verdict_before_proven_readiness_resolves_on_the_proven_ready() { output.output, json!({"spawned": true, "ready": true, "name": worker_name}) ), - (ActionResultPayload::Error(error), false) => { - assert!( - error.error.starts_with("spawn_task_unconfirmed: "), - "{}", - error.error - ); - assert!(error.error.contains(verification), "{}", error.error); - } + (result, false) => assert_unconfirmed_spawn_success(result, worker_name, verification), (other, _) => panic!("{verification}: unexpected result {other:?}"), } assert!(!fixture.runtime.pending_verified_spawns.contains_key(&name)); @@ -1176,8 +1187,7 @@ async fn a_failed_initial_task_releases_the_worker_before_failing_the_spawn() { /// must resolve as unconfirmed and keep the live worker, never release it as a /// failed task (which could kill the run and invite a duplicating retry). #[tokio::test] -async fn an_unproven_acceptance_reports_the_spawn_unconfirmed_and_keeps_the_worker() { - use crate::fleet_wire::ActionResultPayload; +async fn an_unproven_acceptance_succeeds_with_an_unconfirmed_warning_and_keeps_the_worker() { let worker_name = "spawn-task-unproven"; let registry = make_worker_registry_with_worker(worker_name).await; let generation = registry.workers[worker_name].generation; @@ -1200,15 +1210,11 @@ async fn an_unproven_acceptance_reports_the_spawn_unconfirmed_and_keeps_the_work .await; let results = drain_action_results(&mut fixture.fleet_control_rx); assert_eq!(results.len(), 1, "resolved exactly once"); - let ActionResultPayload::Error(error) = &results[0].result else { - panic!("an unproven task must not succeed: {:?}", results[0]); - }; - assert!( - error.error.starts_with("spawn_task_unconfirmed: "), - "{}", - error.error + assert_unconfirmed_spawn_success( + &results[0].result, + name.as_str(), + "harness_acceptance_unproven", ); - assert!(error.error.contains("Do not retry"), "{}", error.error); assert!(!fixture.runtime.pending_verified_spawns.contains_key(&name)); fixture.runtime.handle_maintenance_tick().await; assert!(fixture.runtime.workers.has_worker(&name), "worker kept"); @@ -1218,18 +1224,13 @@ async fn an_unproven_acceptance_reports_the_spawn_unconfirmed_and_keeps_the_work /// returns has its PTY task queued without a binding to this action, so the /// action cannot learn the task's verdict and must not report plain success. #[test] -fn an_already_ready_worker_with_an_unbound_pty_task_is_not_reported_ready() { +fn an_already_ready_worker_with_an_unbound_pty_task_succeeds_with_an_unconfirmed_warning() { use crate::fleet_wire::ActionResultPayload; let name = WorkerName::from("already-ready"); - let ActionResultPayload::Error(error) = - super::fleet::already_ready_spawn_result("inv".into(), &name, true).result - else { - panic!("an unbound PTY task must not report success"); - }; - assert!( - error.error.starts_with("spawn_task_unconfirmed: "), - "{}", - error.error + assert_unconfirmed_spawn_success( + &super::fleet::already_ready_spawn_result("inv".into(), &name, true).result, + name.as_str(), + "unbound", ); assert!(matches!( super::fleet::already_ready_spawn_result("inv".into(), &name, false).result, @@ -5474,6 +5475,59 @@ async fn pty_delivery_verified_requires_harness_acceptance() { cleanup_worker_registry(fixture.runtime.workers).await; } +#[tokio::test] +async fn terminal_unconfirmed_delivery_never_blocks_the_worker() { + let worker_name = "worker-a"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let delivery_id = DeliveryId::new("del_unconfirmed_terminal"); + let pending = make_pending_delivery(delivery_id.as_str(), worker_name); + let event_id = pending.delivery.event_id.clone(); + let mut fixture = + worker_event_runtime_fixture(registry, HashMap::from([(delivery_id.clone(), pending)])); + + fixture + .runtime + .handle_worker_event(WorkerEvent::Message { + name: WorkerName::from(worker_name), + generation, + value: json!({"type": "delivery_unconfirmed", "payload": { + "delivery_id": delivery_id.as_str(), + "event_id": event_id.as_str(), + "outcome": "unconfirmed", + "terminal": true, + "reason": crate::broker::delivery_verification::HARNESS_ACCEPTANCE_UNPROVEN, + "attempts": 1, + }}), + }) + .await; + + assert_ne!( + fixture.runtime.workers.workers[worker_name].state, + AgentWorkState::BlockedOnSend, + "Unconfirmed is a delivery, not a blocked send" + ); + let mut kinds = Vec::new(); + while let Ok(frame) = fixture._sdk_out_rx.try_recv() { + if let Some(kind) = frame.payload.get("kind").and_then(Value::as_str) { + kinds.push(kind.to_string()); + } + } + assert!( + kinds.iter().any(|kind| kind == "delivery_unconfirmed"), + "{kinds:?}" + ); + assert!( + !kinds.iter().any(|kind| kind == "agent_blocked_on_send"), + "{kinds:?}" + ); + assert!( + !kinds.iter().any(|kind| kind == "delivery_failed"), + "{kinds:?}" + ); + cleanup_worker_registry(fixture.runtime.workers).await; +} + #[tokio::test] async fn submit_recovery_progress_defers_broker_delivery_retry() { let worker_name = "worker-a"; diff --git a/crates/broker/src/runtime/worker_events.rs b/crates/broker/src/runtime/worker_events.rs index 8374fce138..7ccd520003 100644 --- a/crates/broker/src/runtime/worker_events.rs +++ b/crates/broker/src/runtime/worker_events.rs @@ -231,6 +231,13 @@ mod terminal_delivery_failure_state_tests { } } +/// Whether a worker's `delivery_unconfirmed` frame is the terminal Unconfirmed +/// outcome (delivered without harness evidence) rather than a mid-recovery +/// progress report for a body still awaiting acceptance. +fn is_terminal_unconfirmed(payload: &Value) -> bool { + payload.get("terminal").and_then(Value::as_bool) == Some(true) +} + fn worker_event_is_current(current_generation: Option, event_generation: Uuid) -> bool { current_generation == Some(event_generation) } @@ -919,6 +926,66 @@ impl BrokerRuntime { .get("event_id") .and_then(Value::as_str) .unwrap_or(""); + // A terminal `delivery_unconfirmed` is the + // Unconfirmed outcome: the body was delivered + // (custody already released by the preceding + // delivery_ack) without harness evidence. It is + // never a blocked worker and never `stuck`. + let terminal_unconfirmed = msg_type == "delivery_unconfirmed" + && is_terminal_unconfirmed(payload); + if terminal_unconfirmed { + if let Some(pending) = pending_verified_spawns + .get_mut(&name) + .filter(|pending| pending.matches_task(generation, event_id)) + { + pending.record_task_verification( + false, + Some("harness_acceptance_unproven"), + ); + } + let resolved = pending_verified_spawns + .get(&name) + .is_some_and(|pending| pending.can_report_ready(generation)) + .then(|| pending_verified_spawns.remove(&name)) + .flatten(); + if let Some(pending) = resolved { + let _ = fleet_control_tx + .send(FleetControlCommand::Send( + crate::fleet_wire::BrokerToRelaycast::ActionResult( + pending.completion(&name), + ), + )) + .await; + } + if let Some(handle) = workers.workers.get_mut(&name) { + handle.last_activity_at = Instant::now(); + if handle.state == AgentWorkState::BlockedOnSend { + handle.state = AgentWorkState::Working; + } + } + tracing::info!( + target = "agent_relay::broker", + worker = %name, + delivery_id = %delivery_id, + event_id = %event_id, + "delivery unconfirmed: delivered without harness evidence" + ); + let _ = send_event( + sdk_out_tx, + json!({ + "kind": "delivery_unconfirmed", + "name": name, + "delivery_id": payload.get("delivery_id"), + "event_id": payload.get("event_id"), + "outcome": "unconfirmed", + "terminal": true, + "reason": payload.get("reason"), + "attempts": payload.get("attempts"), + }), + ) + .await; + return; + } let matching_pending = pending_deliveries .get_mut(delivery_id) .filter(|pending| pending.delivery.event_id.as_str() == event_id) diff --git a/crates/broker/src/wrap.rs b/crates/broker/src/wrap.rs index 2e789bdf1e..f7cca2f042 100644 --- a/crates/broker/src/wrap.rs +++ b/crates/broker/src/wrap.rs @@ -27,8 +27,8 @@ use tokio::{sync::mpsc, time::MissedTickBehavior}; use crate::broker::{ delivery_verification::{ - assess_harness_acceptance, failed_draft_released, DeliveryOutcome, HarnessAcceptance, - PendingActivity, PendingVerification, ThrottleState, VerificationOutput, + assess_harness_acceptance, draft_visible_at_cursor, failed_draft_released, DeliveryOutcome, + HarnessAcceptance, PendingActivity, PendingVerification, ThrottleState, VerificationOutput, ACTIVITY_BUFFER_KEEP_BYTES, ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW, MAX_VERIFICATION_ATTEMPTS, VERIFICATION_WINDOW, }, @@ -2380,7 +2380,9 @@ pub(crate) async fn run_wrap( "wrap: human input followed the injection write; automatic verification cancelled" ); throttle.record(DeliveryOutcome::Failed); - wrap_failed_draft = Some(injection.clone()); + if draft_visible_at_cursor(&Snapshot::capture(&pty), &resolved_cli, &injection) { + wrap_failed_draft = Some(injection.clone()); + } continue; } @@ -2445,7 +2447,9 @@ pub(crate) async fn run_wrap( "wrap: human input followed submit-key recovery; further automatic recovery cancelled" ); throttle.record(DeliveryOutcome::Failed); - wrap_failed_draft = Some(verification.expected_echo.clone()); + if draft_visible_at_cursor(&Snapshot::capture(&pty), &resolved_cli, &verification.expected_echo) { + wrap_failed_draft = Some(verification.expected_echo.clone()); + } continue; } @@ -2456,7 +2460,9 @@ pub(crate) async fn run_wrap( "wrap: submit-key recovery was not confirmed; body left untouched" ); throttle.record(DeliveryOutcome::Failed); - wrap_failed_draft = Some(verification.expected_echo.clone()); + if draft_visible_at_cursor(&Snapshot::capture(&pty), &resolved_cli, &verification.expected_echo) { + wrap_failed_draft = Some(verification.expected_echo.clone()); + } continue; } @@ -2553,7 +2559,9 @@ pub(crate) async fn run_wrap( "wrap: failed to queue submit-key recovery; body left untouched" ); throttle.record(DeliveryOutcome::Failed); - wrap_failed_draft = Some(pv.expected_echo.clone()); + if draft_visible_at_cursor(&Snapshot::capture(&pty), &resolved_cli, &pv.expected_echo) { + wrap_failed_draft = Some(pv.expected_echo.clone()); + } } } } @@ -2577,16 +2585,16 @@ pub(crate) async fn run_wrap( "wrap: body remained parked after bounded submit-key recovery" ); throttle.record(DeliveryOutcome::Failed); + // Parked is the positive evidence. wrap_failed_draft = Some(pv.expected_echo.clone()); } HarnessAcceptance::Inconclusive => { - tracing::error!( + tracing::info!( event_id = %pv.event_id, attempts = pv.attempts, - "wrap: harness acceptance could not be proven; body left untouched" + "wrap: delivery unconfirmed; delivered without harness evidence" ); - throttle.record(DeliveryOutcome::Failed); - wrap_failed_draft = Some(pv.expected_echo.clone()); + throttle.record(DeliveryOutcome::Unverified); } } } else { From 54fbd3afc71f06bee165f710503f4081d56b2997 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Sat, 10 Oct 2026 15:01:02 -0700 Subject: [PATCH 22/24] test(broker): opaque and stuck harness integration tests; fleet stub repaints its prompt (P6) - pty_injection_integrity: an opaque harness (consumes input, shows nothing) keeps receiving three consecutive deliveries, each a terminal delivery_unconfirmed; a visibly stuck draft fails as parked, latches, never has the next delivery typed over it, and escalates with a terminal failed_draft_latched signal (RELAY_FAILED_DRAFT_ESCALATION_MS=2000). - fleet stub-agent: Enter repaints the prompt, like a real harness. Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- .../broker/tests/pty_injection_integrity.rs | 101 ++++++++++++++++++ tests/e2e/fleet/nodes/stub-agent.cjs | 3 + 2 files changed, 104 insertions(+) diff --git a/crates/broker/tests/pty_injection_integrity.rs b/crates/broker/tests/pty_injection_integrity.rs index 57d71c3803..a8caa171b7 100644 --- a/crates/broker/tests/pty_injection_integrity.rs +++ b/crates/broker/tests/pty_injection_integrity.rs @@ -37,6 +37,14 @@ data=b'' while True: chunk=os.read(0,65536) with open(os.environ['TRANSCRIPT'],'ab') as f: f.write(chunk) + if mode == 'opaque': + # Consumes input but never shows it, never repaints: no evidence either way. + continue + if mode == 'stuck': + # Shows typed text in the composer but swallows every submit key. + shown=chunk.replace(b'\x1b[200~',b'').replace(b'\x1b[201~',b'').replace(b'\r',b'').replace(b'\n',b'') + if shown: os.write(1,shown) + continue data+=chunk if mode == 'hostile' and b'\x1b\x1b' in data: os.write(1,b'\x1b[2J\x1b[HTrust prompt, no composer') @@ -53,6 +61,7 @@ while True: let mut child = Command::new(env!("CARGO_BIN_EXE_agent-relay-broker")) .args(["pty", "--agent-name", "integrity", script.to_str().unwrap()]) .env("FIXTURE_MODE", mode) + .env("RELAY_FAILED_DRAFT_ESCALATION_MS", "2000") .env("TRANSCRIPT", dir.path().join("transcript")) .env("AGENT_RELAY_LOCAL_ONLY", "1") .env( @@ -101,6 +110,21 @@ while True: } } } + /// The next frame whose type is one of `kinds`, skipping everything else. + fn wait_any(&self, kinds: &[&str], secs: u64) -> Value { + let deadline = Instant::now() + Duration::from_secs(secs); + loop { + let line = self + .rx + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .unwrap_or_else(|_| panic!("timed out waiting for {kinds:?}")); + if let Ok(frame) = serde_json::from_str::(&line) { + if kinds.iter().any(|kind| frame["type"] == *kind) { + return frame; + } + } + } + } fn deliver(&mut self, id: &str, body: &str) { self.send("deliver_relay",json!({"delivery_id":format!("del_{id}"),"event_id":id,"from":"broker","target":"integrity","body":body,"priority":2,"injection_mode":"wait"})); } @@ -187,3 +211,80 @@ fn disappearing_composer_fails_before_body_injection() { ); assert!(!String::from_utf8_lossy(&f.transcript()).contains("must not type")); } + +/// Khaliq's #1959 policy: a harness that gives no evidence either way still +/// receives every delivery, each reported unconfirmed, never failed, and an +/// unproven delivery never latches the next one. +#[test] +fn opaque_harness_keeps_receiving_consecutive_deliveries_as_unconfirmed() { + let mut f = Fixture::new("opaque"); + for id in ["opaque_one", "opaque_two", "opaque_three"] { + let body = format!("OPAQUE-BODY-{id}"); + f.deliver(id, &body); + let frame = f.wait_any( + &[ + "delivery_unconfirmed", + "delivery_failed", + "delivery_verified", + ], + 30, + ); + assert_eq!(frame["type"], "delivery_unconfirmed", "{frame}"); + assert_eq!(frame["payload"]["terminal"], true, "{frame}"); + assert_eq!(frame["payload"]["outcome"], "unconfirmed", "{frame}"); + assert!( + String::from_utf8_lossy(&f.transcript()).contains(&body), + "every delivery is written to an opaque harness" + ); + } +} + +/// A body still visibly parked after bounded recovery latches the composer: +/// the next delivery queues and is never typed over the draft, and after the +/// bounded timeout the worker sends one terminal stuck signal. +#[test] +fn visibly_stuck_draft_latches_queue_and_escalates_without_typing_over_it() { + let mut f = Fixture::new("stuck"); + f.deliver("stuck_first", "STUCK-DRAFT-first-delivery"); + let failed = f.wait_any( + &[ + "delivery_failed", + "delivery_verified", + "delivery_unconfirmed", + ], + 60, + ); + // Mid-recovery progress frames are non-terminal delivery_unconfirmed. + let failed = + if failed["type"] == "delivery_unconfirmed" && failed["payload"]["terminal"] != true { + f.wait_any(&["delivery_failed", "delivery_verified"], 60) + } else { + failed + }; + assert_eq!(failed["type"], "delivery_failed", "{failed}"); + assert!( + failed["payload"]["reason"] + .as_str() + .unwrap_or_default() + .contains("parked"), + "{failed}" + ); + f.deliver("stuck_second", "QUEUED-BEHIND-the-stuck-draft"); + let blocked = f.wait_any(&["agent_blocked_on_send"], 30); + let blocked = if blocked["payload"]["reason"] == "failed_draft_latched" + && blocked["payload"]["terminal"] != true + { + f.wait_any(&["agent_blocked_on_send"], 30) + } else { + blocked + }; + assert_eq!( + blocked["payload"]["reason"], "failed_draft_latched", + "{blocked}" + ); + assert_eq!(blocked["payload"]["terminal"], true, "{blocked}"); + assert!( + !String::from_utf8_lossy(&f.transcript()).contains("QUEUED-BEHIND"), + "the queued delivery must never be typed over the stuck draft" + ); +} diff --git a/tests/e2e/fleet/nodes/stub-agent.cjs b/tests/e2e/fleet/nodes/stub-agent.cjs index f8631dd37f..1f17624e2e 100644 --- a/tests/e2e/fleet/nodes/stub-agent.cjs +++ b/tests/e2e/fleet/nodes/stub-agent.cjs @@ -59,6 +59,9 @@ try { process.stdin.on('data', (chunk) => { if (!ready) return; if (codexComposer) renderCodexComposer(chunk.toString()); + // Like a real harness, Enter submits the turn and repaints the prompt, so + // the broker can see the composer clear and confirm acceptance. + else if (/[\r\n]/.test(chunk.toString())) process.stdout.write('\r\n❯ '); input += chunk.toString(); // Require a delimiter after the nonce. PTY chunks can split anywhere, so // treating the current buffer end as a complete token could record a From ebe5e4d4784567745877033477e5d1ff52f9d944 Mon Sep 17 00:00:00 2001 From: Miya Date: Sun, 11 Oct 2026 00:24:10 +0200 Subject: [PATCH 23/24] docs(changelog): describe the approved #1959 acceptance policy against 13.3.0 Unconfirmed PTY deliveries and unconfirmed spawn tasks now succeed (Changed); the stuck-draft bullet names the 60 s escalation; the delivery_unconfirmed event bullet mentions its terminal form. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 2a0590dc-a82a-4174-9eff-c23d59571061 --- CHANGELOG.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abf4225d43..9d80a118b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,14 +10,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `agent-relay integration subscribe --to self` routes provider events to the broker worker's registered identity, allowing fleet workers to subscribe without desktop session detection. -- Broker events report PTY submit recovery: `delivery_unconfirmed` (the harness has not accepted a write yet, with `attempts` / `max_attempts`) and `delivery_resubmitted` (a submit-only retry, with `attempt` and `strategy`); `delivery_verified` adds `evidence` and `attempts`. `@agent-relay/harness-driver` and the Python and Swift SDK protocol types include both new events. +- Broker events report PTY submit recovery: `delivery_unconfirmed` (no acceptance evidence for a write, with `attempts` / `max_attempts`; `terminal: true` when the delivery ends unconfirmed) and `delivery_resubmitted` (a submit-only retry, with `attempt` and `strategy`); `delivery_verified` adds `evidence` and `attempts`. `@agent-relay/harness-driver` and the Python and Swift SDK protocol types include both new events. + +### Changed + +- A PTY delivery the broker can neither confirm nor see still parked in the agent's input box is now delivered as unconfirmed (a terminal `delivery_unconfirmed` with `outcome: "unconfirmed"`) instead of failing, and later messages to that agent are no longer held behind it. Agents whose terminal shows no acceptance signal keep receiving messages. +- A confirmed `fleet spawn --task` whose task was written but never visibly accepted now succeeds (`spawned: true`) with `warning.code: "spawn_task_unconfirmed"`, instead of failing with `spawn_task_unconfirmed` and exit 8. Only a task that could not be delivered fails the spawn, with `spawn_task_failed`. ### Fixed - `agent-relay fleet spawn` explains how to reuse a retained agent name with `fleet release --delete-agent --wait` when registration fails with `agent_already_exists`. - The installer's Quick Start now prints `agent-relay node up --background` and `agent-relay node down`, which exist, instead of the removed `agent-relay up` form that failed with "unknown command 'up'". An install pinned with `AGENT_RELAY_VERSION` to a release older than 9.2.2 still prints the top-level `up`/`down` forms that release has. -- Broker-managed PTY messages now stay pending until the harness accepts them, instead of being acknowledged while still parked in the composer. -- A PTY delivery the harness never accepts blocks further sends to that agent and is retried by pressing submit only, never by resending the body, for up to three total attempts before it is reported as failed. +- A PTY message still visible in the agent's input box after submit is retried by pressing submit only, never by resending the body, for up to three total attempts. Later messages queue behind it instead of being typed on top of it, and if it is still stuck after 60 seconds the broker reports a terminal `agent_blocked_on_send` with reason `failed_draft_latched`. - Typing into an agent's terminal while a delivery is being retried takes over the terminal and cancels the retries. ## [13.3.0] - 2026-10-10 From 4cb0e54715321afae92453ebecdb6508e465903c Mon Sep 17 00:00:00 2001 From: Khaliq Date: Sat, 10 Oct 2026 17:12:56 -0700 Subject: [PATCH 24/24] fix(broker): visible drafts are stuck, not unconfirmed; only a lone Enter releases the wrap latch - assess_harness_acceptance: a draft still visible at the cursor (draft_visible_at_cursor) is Parked / StuckInComposer even when the through-cursor composer parse missed it (cursor at the start or middle of the draft). It is never Unconfirmed, so it latches and the next delivery is never typed onto it. Tests that asserted Inconclusive for a visible draft now assert Parked. - wrap: only a lone Enter keystroke (CR/LF bytes alone) releases the failed-draft latch; a typed line or bracketed multiline paste carrying newlines adds text, not a submit. Co-Authored-By: Claude Opus 5.5 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 Session-Id: a4d98f2e-cd4b-47a9-87f8-1deb0f44eb41 --- .../src/broker/delivery_verification.rs | 51 +++++++++++-------- crates/broker/src/wrap.rs | 17 +++++-- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 59a43ee7d1..0ceb1dd080 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -593,6 +593,13 @@ pub(crate) fn assess_harness_acceptance( return HarnessAcceptance::Accepted(format!("activity:{pattern}")); } } + // A draft still visible at the cursor is StuckInComposer even when the + // through-cursor composer parse missed it (the cursor sits at the start or + // middle of the draft). Treating it as Unconfirmed would leave it unlatched + // and let the next delivery be typed onto it. + if draft_visible_at_cursor(snapshot, cli, &verification.expected_echo) { + return HarnessAcceptance::Parked; + } HarnessAcceptance::Inconclusive } @@ -909,7 +916,7 @@ mod tests { verification.echo_seen = false; assert_eq!( assess_harness_acceptance("cat", &verification, &snapshot), - HarnessAcceptance::Inconclusive, + HarnessAcceptance::Parked, ); let mut output = VerificationOutput::default(); output.push_str("raw PTY echo: LOCAL_WORK_PROOF\r\n"); @@ -1139,18 +1146,20 @@ mod tests { #[cfg(unix)] #[tokio::test] - async fn text_right_of_cursor_cannot_look_like_a_cleared_composer() { - let expected = "Relay message from Lead [evt]: hidden draft"; - let screen = format!("› {expected}\x1b[1;3H"); - let (pty, snapshot) = codex_snapshot(&screen).await; - let verification = codex_verification(expected); - - assert_eq!( - assess_harness_acceptance("codex", &verification, &snapshot), - HarnessAcceptance::Inconclusive, - "right-side draft text prevents composer-cleared acceptance while recovery stays fail-closed" - ); - pty.shutdown().unwrap(); + async fn a_visible_draft_with_the_cursor_at_its_start_or_middle_is_stuck() { + let expected = "Relay message from Lead [evt]: hidden draft that is still parked here"; + for cursor in ["\x1b[1;3H", "\x1b[1;25H"] { + let screen = format!("› {expected}{cursor}"); + let (pty, snapshot) = codex_snapshot(&screen).await; + let verification = codex_verification(expected); + + assert_eq!( + assess_harness_acceptance("codex", &verification, &snapshot), + HarnessAcceptance::Parked, + "a draft still in the composer is StuckInComposer, never Unconfirmed ({cursor:?})" + ); + pty.shutdown().unwrap(); + } } #[cfg(unix)] @@ -1186,7 +1195,7 @@ mod tests { assert_eq!( assess_harness_acceptance("muse", &verification, &snapshot), - HarnessAcceptance::Inconclusive, + HarnessAcceptance::Parked, "generic output must not confirm a body that still sits at the cursor" ); pty.shutdown().unwrap(); @@ -1205,7 +1214,7 @@ mod tests { assert_eq!( assess_harness_acceptance("opencode", &verification, &snapshot), - HarnessAcceptance::Inconclusive, + HarnessAcceptance::Parked, "OpenCode transcript words must not confirm a typed-but-unsent body" ); pty.shutdown().unwrap(); @@ -1255,7 +1264,7 @@ mod tests { assert_eq!( assess_harness_acceptance("muse", &verification, &snapshot), - HarnessAcceptance::Inconclusive, + HarnessAcceptance::Parked, "a tail far to the right of the cursor is still an unsent draft" ); pty.shutdown().unwrap(); @@ -1278,7 +1287,7 @@ mod tests { assert_eq!( assess_harness_acceptance("muse", &verification, &snapshot), - HarnessAcceptance::Inconclusive, + HarnessAcceptance::Parked, "body text right of the cursor is an unsent draft even without its tail" ); pty.shutdown().unwrap(); @@ -1286,7 +1295,7 @@ mod tests { #[cfg(unix)] #[tokio::test] - async fn bordered_draft_head_right_of_cursor_stays_inconclusive() { + async fn bordered_draft_head_right_of_cursor_is_stuck_in_the_composer() { // The cursor sits on a box border; the draft's head follows it and its // tail has scrolled out of view. let expected = format!( @@ -1301,7 +1310,7 @@ mod tests { assert_eq!( assess_harness_acceptance("muse", &verification, &snapshot), - HarnessAcceptance::Inconclusive, + HarnessAcceptance::Parked, "border chrome around the cursor must not hide an unsent draft" ); pty.shutdown().unwrap(); @@ -1309,7 +1318,7 @@ mod tests { #[cfg(unix)] #[tokio::test] - async fn symbolic_draft_head_right_of_cursor_stays_inconclusive() { + async fn symbolic_draft_head_right_of_cursor_is_stuck_in_the_composer() { // The visible head is punctuation only; the tail has scrolled away. let expected = format!("~~~ *** ### {} TAIL_SCROLLED_OFF", "y".repeat(160)); let (pty, snapshot) = codex_snapshot("┃ ~~~ *** ### ┃\x1b[1;1H").await; @@ -1319,7 +1328,7 @@ mod tests { assert_eq!( assess_harness_acceptance("muse", &verification, &snapshot), - HarnessAcceptance::Inconclusive, + HarnessAcceptance::Parked, "a draft head made of symbols is still body text" ); pty.shutdown().unwrap(); diff --git a/crates/broker/src/wrap.rs b/crates/broker/src/wrap.rs index f7cca2f042..d23ff3e783 100644 --- a/crates/broker/src/wrap.rs +++ b/crates/broker/src/wrap.rs @@ -315,10 +315,12 @@ fn wrap_injection_timer_allowed(has_pending_write_ack: bool, has_pending_accepta !has_pending_write_ack && !has_pending_acceptance } -/// Human stdin that submits the composer (Enter) takes whatever it holds, +/// A lone Enter keystroke submits the composer and takes whatever it holds, /// including a failed relay draft, so it releases the wrap failed-draft latch. +/// A multiline or bracketed paste also carries CR/LF bytes but adds text to +/// the composer rather than submitting it, so it never releases the latch. fn human_input_submits_composer(data: &[u8]) -> bool { - data.iter().any(|byte| matches!(byte, b'\r' | b'\n')) + !data.is_empty() && data.iter().all(|byte| matches!(byte, b'\r' | b'\n')) } // Readiness deferrals must not spend the bounded delivery retry budget. @@ -2952,7 +2954,16 @@ sys.stdout.flush()"#; #[test] fn wrap_injection_timer_waits_for_write_ack_and_harness_acceptance() { assert!(super::human_input_submits_composer(b"\r")); - assert!(super::human_input_submits_composer(b"ok\n")); + assert!(super::human_input_submits_composer(b"\r\n")); + assert!( + !super::human_input_submits_composer(b"ok\n"), + "typed line plus newline is text" + ); + assert!( + !super::human_input_submits_composer(b"\x1b[200~line one\rline two\x1b[201~"), + "a bracketed multiline paste is not a submit" + ); + assert!(!super::human_input_submits_composer(b"")); assert!(!super::human_input_submits_composer(b"typing")); assert!(wrap_injection_timer_allowed(false, false)); assert!(!wrap_injection_timer_allowed(true, false));