From 3521cb41de3408c6e109c0f65aaae162c30dd62b Mon Sep 17 00:00:00 2001 From: Relayflow Date: Sun, 4 Oct 2026 07:25:10 +0000 Subject: [PATCH 01/42] fix(broker): atomic bracketed-paste injection for paste-capable harnesses --- crates/broker/src/devin.rs | 16 +- crates/broker/src/injection_wire.rs | 82 +++++++++ crates/broker/src/lib.rs | 1 + crates/broker/src/pty_worker.rs | 28 ++- crates/broker/src/wrap.rs | 29 ++- .../broker/tests/pty_injection_integrity.rs | 174 ++++++++++++++++++ crates/relay-pty/src/pty.rs | 66 ++++++- 7 files changed, 379 insertions(+), 17 deletions(-) create mode 100644 crates/broker/src/injection_wire.rs create mode 100644 crates/broker/tests/pty_injection_integrity.rs diff --git a/crates/broker/src/devin.rs b/crates/broker/src/devin.rs index 7ee027891b..1d78af38ae 100644 --- a/crates/broker/src/devin.rs +++ b/crates/broker/src/devin.rs @@ -9,12 +9,16 @@ use anyhow::{Context, Result}; use serde_json::Value; use std::{fs, path::Path}; -pub(crate) fn injection_bytes(cli: &str, text: &str) -> Vec { - if is_devin_cli(cli) { - format!("\x1b[200~{}\x1b[201~", text.replace('\x1b', "")).into_bytes() - } else { - text.as_bytes().to_vec() - } +#[cfg(test)] +fn injection_bytes(cli: &str, text: &str) -> Vec { + crate::injection_wire::injection_bytes( + if is_devin_cli(cli) { + crate::injection_wire::InjectionWire::Paste + } else { + crate::injection_wire::InjectionWire::Typed + }, + text, + ) } pub(crate) fn can_inject(cli: &str, pty: &PtySession) -> bool { diff --git a/crates/broker/src/injection_wire.rs b/crates/broker/src/injection_wire.rs new file mode 100644 index 0000000000..d213527ec2 --- /dev/null +++ b/crates/broker/src/injection_wire.rs @@ -0,0 +1,82 @@ +//! Shared, bounded PTY injection wire format for fleet and wrap. +use crate::pty::PtySession; +use std::time::Duration; + +pub(crate) const MAX_INJECTION_BODY_BYTES: usize = 16 * 1024; +// At the default 5 ms/byte, leaves room for readiness, prompt recheck and verification. +pub(crate) const MAX_TYPED_WRITE_TIME: Duration = Duration::from_millis(7680); +pub(crate) const MAX_TYPED_BYTES: usize = MAX_TYPED_WRITE_TIME.as_millis() as usize / 5; +pub(crate) const INJECTION_PROMPT_WAIT: Duration = Duration::from_secs(15); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum InjectionWire { + Paste, + Typed, +} + +pub(crate) fn injection_wire(cli: &str, pty: &PtySession) -> InjectionWire { + select_wire( + cli, + pty.bracketed_paste_enabled(), + std::env::var("RELAY_INJECT_PASTE").as_deref() == Ok("0"), + ) +} +fn select_wire(cli: &str, capable: bool, disabled: bool) -> InjectionWire { + if !disabled && (capable || crate::readiness::is_devin_cli(cli)) { + InjectionWire::Paste + } else { + InjectionWire::Typed + } +} +pub(crate) fn injection_bytes(wire: InjectionWire, text: &str) -> Vec { + let text = text.replace("\r\n", "\n").replace(['\r', '\x1b'], ""); + match wire { + InjectionWire::Paste => format!("\x1b[200~{text}\x1b[201~").into_bytes(), + InjectionWire::Typed => text.into_bytes(), + } +} +pub(crate) fn effective_limit(wire: InjectionWire, pace: Duration) -> usize { + if wire == InjectionWire::Paste || pace.is_zero() { + MAX_INJECTION_BODY_BYTES + } else { + MAX_TYPED_BYTES.min((MAX_TYPED_WRITE_TIME.as_millis() / pace.as_millis().max(1)) as usize) + } +} +pub(crate) fn can_inject(cli: &str, pty: &PtySession) -> bool { + let lower = cli.to_lowercase(); + let known = ["claude", "codex", "gemini"] + .iter() + .any(|name| lower.contains(name)) + || crate::readiness::is_devin_cli(cli); + !known + || crate::readiness::cli_prompt_ready( + cli, + crate::readiness::GridReadinessSnapshot { + screen: &pty.screen_text(), + cursor: Some(pty.cursor_position()), + }, + ) +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn body_cannot_submit_or_close_paste() { + for ch in (0..=0x10ffff).filter_map(char::from_u32) { + let body = format!("head{ch}\x1b[201~\r\nend\r"); + for wire in [InjectionWire::Paste, InjectionWire::Typed] { + let bytes = injection_bytes(wire, &body); + assert!(!bytes.contains(&b'\r')); + let expected_escapes = if wire == InjectionWire::Paste { 2 } else { 0 }; + assert_eq!(bytes.iter().filter(|&&b| b == 27).count(), expected_escapes); + } + } + } + #[test] + fn capability_and_override() { + assert_eq!(select_wire("claude", true, false), InjectionWire::Paste); + assert_eq!(select_wire("claude", true, true), InjectionWire::Typed); + assert_eq!(select_wire("other", false, false), InjectionWire::Typed); + assert_eq!(injection_bytes(InjectionWire::Typed, "a\r\nb\rc"), b"a\nbc"); + } +} diff --git a/crates/broker/src/lib.rs b/crates/broker/src/lib.rs index ca30af129b..ce71df3b04 100644 --- a/crates/broker/src/lib.rs +++ b/crates/broker/src/lib.rs @@ -9,6 +9,7 @@ mod devin; pub mod fleet_wire; pub mod ids; +mod injection_wire; pub mod protocol; pub mod snippets; diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index 2a007e9123..a11afb774f 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -164,6 +164,7 @@ enum InjectionStage { struct ActiveInjection { pending: PendingWorkerInjection, stage: InjectionStage, + prompt_wait_started: Instant, next_at: tokio::time::Instant, injection_text: Option, /// Receive-time PTY-output sequence captured atomically with submitting @@ -1976,6 +1977,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { active_injection = Some(ActiveInjection { pending, stage: InjectionStage::Escape, + prompt_wait_started: Instant::now(), next_at: tokio::time::Instant::now() + throttle.delay(), injection_text: None, output_boundary: None, @@ -2045,7 +2047,14 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { InjectionStage::Body => { // Recheck after throttling/steer delay: never paste // into a dialog that replaced the previously idle UI. - if !crate::devin::can_inject(&resolved_cli, &pty) { + if !crate::injection_wire::can_inject(&resolved_cli, &pty) { + if inj.prompt_wait_started.elapsed() >= crate::injection_wire::INJECTION_PROMPT_WAIT { + let delivery = &inj.pending.delivery; + let _ = send_frame(&out_tx, "delivery_failed", None, json!({"delivery_id": delivery.delivery_id, "event_id": delivery.event_id, "reason": "prompt_unproven"})).await; + let _ = send_frame(&out_tx, "worker_error", inj.pending.request_id, json!({"code": "prompt_unproven", "retryable": false, "message": "Composer readiness could not be proven; body was not written"})).await; + // Retain the pending id: a retry must never turn failure into an ack. + continue; + } inj.next_at = tokio::time::Instant::now() + Duration::from_millis(50); active_injection = Some(inj); continue; @@ -2064,9 +2073,22 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { inj.pending.delivery.workspace_id.as_deref(), inj.pending.delivery.workspace_alias.as_deref(), ); + let injection = String::from_utf8(crate::injection_wire::injection_bytes(crate::injection_wire::InjectionWire::Typed, &injection))?; if include_mcp_reminder { mcp_reminder_throttle.note_sent(Instant::now()); } + let wire = crate::injection_wire::injection_wire(&resolved_cli, &pty); + let limit = if initial_codex_delivery(&resolved_cli, &inj.pending.delivery) { + initial_codex_max_body_bytes().min(crate::injection_wire::MAX_INJECTION_BODY_BYTES) + } else { crate::injection_wire::effective_limit(wire, inject_rate) }; + if injection.len() > limit { + let delivery = &inj.pending.delivery; + let reason = format!("injection_too_large: {resolved_cli} effective limit is {limit} bytes including envelope; use a brief file pointer"); + let _ = send_frame(&out_tx, "delivery_failed", None, json!({"delivery_id": delivery.delivery_id, "event_id": delivery.event_id, "reason": reason})).await; + let _ = send_frame(&out_tx, "worker_error", inj.pending.request_id, json!({"code": "injection_too_large", "retryable": false, "message": reason})).await; + // Retain the pending id; terminal failure is never a successful replay. + continue; + } if initial_codex_delivery(&resolved_cli, &inj.pending.delivery) { // Reject before any byte is written rather than after // guaranteeing a partial-write timeout: a body needing @@ -2125,7 +2147,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { // bracketed-paste body shape; the shared helper then // selects the harness-specific delayed Enter. Finalization // still waits for this ack in the injection-ack arm. - let bytes = crate::devin::injection_bytes(&resolved_cli, &injection); + let bytes = crate::injection_wire::injection_bytes(wire, &injection); let write = submit_injection_body(&pty, &resolved_cli, bytes, inject_rate); match write { @@ -3463,6 +3485,7 @@ mod tests { let targeted = ActiveInjection { pending: test_pending_injection("init_task"), stage: InjectionStage::Escape, + prompt_wait_started: Instant::now(), next_at: tokio::time::Instant::now(), injection_text: None, output_boundary: None, @@ -3502,6 +3525,7 @@ mod tests { let mut injection = ActiveInjection { pending: test_pending_injection("init_task"), stage: InjectionStage::Escape, + prompt_wait_started: Instant::now(), next_at: tokio::time::Instant::now(), injection_text: None, output_boundary: None, diff --git a/crates/broker/src/wrap.rs b/crates/broker/src/wrap.rs index c38b63e2cb..3bb97c3fa8 100644 --- a/crates/broker/src/wrap.rs +++ b/crates/broker/src/wrap.rs @@ -103,7 +103,20 @@ pub(crate) fn submit_injection_body( bytes: Vec, pace: Duration, ) -> anyhow::Result<(tokio::sync::oneshot::Receiver>, u64)> { - if let Some(delay) = injection_submit_followup_delay(resolved_cli) { + // Follow the already-selected wire. Capability can latch between body + // construction and queue admission; re-probing here could bulk-type raw bytes. + let wire = if bytes.starts_with(b"\x1b[200~") && bytes.ends_with(b"\x1b[201~") { + crate::injection_wire::InjectionWire::Paste + } else { + crate::injection_wire::InjectionWire::Typed + }; + let limit = crate::injection_wire::effective_limit(wire, pace); + anyhow::ensure!(bytes.len().saturating_sub(if wire == crate::injection_wire::InjectionWire::Paste { 12 } else { 0 }) <= limit, "injection_too_large: {resolved_cli} effective limit is {limit} bytes; use a brief file pointer"); + let paste = wire == crate::injection_wire::InjectionWire::Paste; + let pace = if paste { Duration::ZERO } else { pace }; + if let Some(delay) = injection_submit_followup_delay(resolved_cli) + .or_else(|| paste.then_some(PASTE_INJECTION_SUBMIT_DELAY)) + { pty.submit_write_paced_with_followup_and_output_boundary(bytes, pace, delay, b"\r".to_vec()) } else { let mut burst = bytes; @@ -2144,7 +2157,12 @@ pub(crate) async fn run_wrap( pending.workspace_id.as_deref(), pending.workspace_alias.as_deref(), ); - let bytes = crate::devin::injection_bytes(&resolved_cli, &injection); + if injection.len() > crate::injection_wire::MAX_INJECTION_BODY_BYTES { + tracing::warn!(limit = crate::injection_wire::MAX_INJECTION_BODY_BYTES, "wrap: injection_too_large; body was not written and will not be replayed"); + throttle.record(DeliveryOutcome::Failed); + continue; + } + let bytes = crate::injection_wire::injection_bytes(crate::injection_wire::injection_wire(&resolved_cli, &pty), &injection); let write = submit_injection_body(&pty, &resolved_cli, bytes, Duration::ZERO); match write { Ok((ack_rx, output_boundary)) => { @@ -2353,7 +2371,12 @@ pub(crate) async fn run_wrap( pv.workspace_id.as_deref(), pv.workspace_alias.as_deref(), ); - let bytes = crate::devin::injection_bytes(&resolved_cli, &injection); + if injection.len() > crate::injection_wire::MAX_INJECTION_BODY_BYTES { + tracing::warn!(limit = crate::injection_wire::MAX_INJECTION_BODY_BYTES, "wrap: injection_too_large; body was not written and will not be replayed"); + throttle.record(DeliveryOutcome::Failed); + continue; + } + let bytes = crate::injection_wire::injection_bytes(crate::injection_wire::injection_wire(&resolved_cli, &pty), &injection); let write = submit_injection_body(&pty, &resolved_cli, bytes, Duration::ZERO); match write { Ok((ack_rx, output_boundary)) => { diff --git a/crates/broker/tests/pty_injection_integrity.rs b/crates/broker/tests/pty_injection_integrity.rs new file mode 100644 index 0000000000..ee8423a91a --- /dev/null +++ b/crates/broker/tests/pty_injection_integrity.rs @@ -0,0 +1,174 @@ +//! Real broker, raw-mode fake Claude TUI, byte-level transcript assertions. +#![cfg(unix)] +use serde_json::{json, Value}; +use std::{ + io::{BufRead, BufReader, Write}, + os::unix::fs::PermissionsExt, + process::{Child, Command, Stdio}, + sync::mpsc, + time::{Duration, Instant}, +}; +struct Fixture { + child: Child, + rx: mpsc::Receiver, + dir: tempfile::TempDir, +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} +impl Fixture { + fn new(mode: &str) -> Self { + let dir = tempfile::tempdir().unwrap(); + let script = dir.path().join("claude-fixture"); + std::fs::write( + &script, + r#"#!/usr/bin/env python3 +import os, tty, time, select +# Canonical mode silently discards input after MAX_CANON. Real TUIs use raw mode. +tty.setraw(0) +mode=os.environ['FIXTURE_MODE'] +time.sleep(0.2) +os.write(1,b'\x1b[?2004h\x1b[?2004l' if mode != 'typed' else b'') +os.write(1,'Claude Code v2.1.261\r\n❯ '.encode()) +data=b'' +while True: + chunk=os.read(0,65536) + with open(os.environ['TRANSCRIPT'],'ab') as f: f.write(chunk) + data+=chunk + if mode == 'hostile' and b'\x1b\x1b' in data: + os.write(1,b'\x1b[2J\x1b[HTrust prompt, no composer') + data=b'' + if data.endswith(b'\r'): + body=data.replace(b'\x1b[200~',b'').replace(b'\x1b[201~',b'').rstrip(b'\r') + if mode == 'tail': body=body[-200:] + if mode != 'silent': os.write(1,b'\r\n'+body+b'\r\n') + os.write(1,'\r\n❯ '.encode()) + data=b'' +"#, + ) + .unwrap(); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_agent-relay-broker")) + .args(["pty", "--agent-name", "integrity", script.to_str().unwrap()]) + .env("FIXTURE_MODE", mode) + .env("TRANSCRIPT", dir.path().join("transcript")) + .env("AGENT_RELAY_LOCAL_ONLY", "1") + .env( + "RELAY_INJECT_PASTE", + if mode == "typed" { "0" } else { "1" }, + ) + .current_dir(dir.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + let stdout = child.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + if tx.send(line.unwrap()).is_err() { + break; + } + } + }); + let mut f = Self { child, rx, dir }; + f.send("init_worker", json!({"agent":{"name":"integrity"}})); + assert_eq!(f.wait("worker_ready")["payload"]["readiness_proven"], true); + f + } + fn send(&mut self, kind: &str, payload: Value) { + writeln!( + self.child.stdin.as_mut().unwrap(), + "{}", + json!({"v":2,"type":kind,"payload":payload}) + ) + .unwrap(); + } + fn wait(&self, kind: &str) -> Value { + let deadline = Instant::now() + Duration::from_secs(25); + loop { + let line = self + .rx + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .expect(kind); + if let Ok(frame) = serde_json::from_str::(&line) { + if frame["type"] == kind { + return frame; + } + } + } + } + fn deliver(&mut self, id: &str, body: &str) { + self.send("deliver_relay",json!({"delivery_id":format!("del_{id}"),"event_id":id,"from":"broker","target":"integrity","body":body,"priority":2,"injection_mode":"wait"})); + } + fn transcript(&self) -> Vec { + std::fs::read(self.dir.path().join("transcript")).unwrap_or_default() + } +} +fn long_pair() { + let mut f = Fixture::new("echo"); + for id in ["init_task", "relay_message"] { + let body = format!("HEAD-{id}\n{}\nTAIL-{id}", "éabc xyz\n".repeat(1100)); + assert!(body.len() > 10_000); + let before = f.transcript().len(); + f.deliver(id, &body); + let verified = f.wait("delivery_verified"); + assert_ne!(verified["payload"]["verification"], "timeout_fallback"); + let transcript = f.transcript(); + let bytes = &transcript[before..]; + assert!(bytes.windows(body.len()).any(|w| w == body.as_bytes())); + assert_eq!(bytes.windows(6).filter(|w| *w == b"\x1b[200~").count(), 1); + assert_eq!(bytes.windows(6).filter(|w| *w == b"\x1b[201~").count(), 1); + assert_eq!(bytes.iter().filter(|&&b| b == b'\r').count(), 1); + } +} +#[test] +fn ten_kib_task_and_relay_message_are_intact() { + long_pair(); +} +#[test] +fn over_limit_and_typed_fallback_reject_before_writing() { + for mode in ["echo", "typed"] { + let mut f = Fixture::new(mode); + f.deliver( + "init_limit", + &"x".repeat(if mode == "typed" { 10_000 } else { 17_000 }), + ); + assert!(f.wait("delivery_failed")["payload"]["reason"] + .as_str() + .unwrap() + .contains("injection_too_large")); + assert!(f.transcript().is_empty()); + } +} +#[test] +fn absent_echo_preserves_timeout_fallback() { + let mut f = Fixture::new("silent"); + f.deliver("init_silent", "hello"); + assert_eq!( + f.wait("delivery_verified")["payload"]["verification"], + "timeout_fallback" + ); +} +#[test] +#[ignore = "20 cold starts; run separately"] +fn twenty_cold_starts() { + for _ in 0..20 { + long_pair(); + } +} + +#[test] +fn disappearing_composer_fails_before_body_injection() { + let mut f = Fixture::new("hostile"); + f.send("deliver_relay", json!({"delivery_id":"del_hostile", "event_id":"init_hostile", "from":"broker", "target":"integrity", "body":"must not type this into a dialog", "priority":2,"injection_mode":"steer"})); + assert_eq!( + f.wait("delivery_failed")["payload"]["reason"], + "prompt_unproven" + ); + assert!(!String::from_utf8_lossy(&f.transcript()).contains("must not type")); +} diff --git a/crates/relay-pty/src/pty.rs b/crates/relay-pty/src/pty.rs index 36e521604d..916da21f83 100644 --- a/crates/relay-pty/src/pty.rs +++ b/crates/relay-pty/src/pty.rs @@ -431,6 +431,7 @@ pub struct PtySession { /// buffered stream chunks the snapshot already reflects and apply only /// the ones that came after. consumed_offset: Arc, + paste_capable: Arc, /// Sequence assigned as soon as the reader obtains a PTY output chunk, /// before grid parsing or async queue admission. Consumers sample this /// producer watermark when submitting input so output that was already @@ -1057,6 +1058,8 @@ impl PtySession { let (tx, rx) = mpsc::channel(256); let term_clone = term.clone(); let processor_clone = processor.clone(); + let paste_capable = Arc::new(AtomicBool::new(false)); + let paste_capable_reader = paste_capable.clone(); let consumed_offset = Arc::new(AtomicU64::new(0)); let consumed_offset_reader = consumed_offset.clone(); let output_sequence = Arc::new(AtomicU64::new(0)); @@ -1130,7 +1133,20 @@ impl PtySession { // writer lock taken by the drainer thread. let mut processor_guard = processor_clone.lock(); let mut term_guard = term_clone.lock(); - processor_guard.advance(&mut *term_guard, bytes); + // Observe every mode transition, including enable/disable in one read. + if paste_capable_reader.load(Ordering::Acquire) { + processor_guard.advance(&mut *term_guard, bytes); + } else { + for byte in bytes { + processor_guard.advance(&mut *term_guard, std::slice::from_ref(byte)); + if term_guard + .mode() + .contains(alacritty_terminal::term::TermMode::BRACKETED_PASTE) + { + paste_capable_reader.store(true, Ordering::Release); + } + } + } // Publish the grid's consumed byte offset while // still holding the term lock, so a concurrent // snapshot reader (which also locks `term`) reads @@ -1171,6 +1187,7 @@ impl PtySession { term, processor, consumed_offset, + paste_capable, output_sequence, output_order, #[cfg(unix)] @@ -1451,6 +1468,11 @@ impl PtySession { /// with a grid render. /// /// [`with_term_and_offset`]: PtySession::with_term_and_offset + /// Sticky capability, independent of whether the composer is currently ready. + pub fn bracketed_paste_enabled(&self) -> bool { + self.paste_capable.load(Ordering::Acquire) + } + pub fn consumed_offset(&self) -> u64 { self.consumed_offset.load(Ordering::Acquire) } @@ -1807,6 +1829,37 @@ mod tests { let _ = pty.shutdown(); } + #[cfg(unix)] + #[tokio::test] + async fn paste_capability_latches_across_disable_in_same_read() { + let (pty, mut rx) = PtySession::spawn( + "sh", + &[ + "-c".into(), + "printf '\\033[?2004h\\033[?2004lREADY'; sleep 2".into(), + ], + 24, + 80, + ) + .unwrap(); + let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); + tokio::time::timeout(Duration::from_secs(3), async { + while !pty.screen_text().contains("READY") { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!(pty.bracketed_paste_enabled()); + assert!(!pty + .term + .lock() + .mode() + .contains(alacritty_terminal::term::TermMode::BRACKETED_PASTE)); + pty.shutdown().unwrap(); + drain.abort(); + } + #[tokio::test] async fn consumed_offset_tracks_total_bytes_parsed_into_grid() { let (pty, mut rx) = PtySession::spawn("echo", &["offset-line".into()], 24, 80).unwrap(); @@ -2964,20 +3017,21 @@ mod tests { ack: ack_tx, }) .expect("queue accepts paced user input"); + tx.send(WriteMsg::Reply(b"terminal-reply".to_vec())) + .unwrap(); let ack = tokio::time::timeout(Duration::from_secs(2), ack_rx) .await .expect("drainer acks paced write") .expect("ack sender not dropped"); assert!(ack.is_ok(), "paced write must succeed"); + drop(tx); + drainer.join().expect("drainer thread joins cleanly"); assert_eq!( out.lock().unwrap().as_slice(), - b"go\x1b[A\r", - "paced write must deliver every byte in order" + b"go\x1b[A\rterminal-reply", + "terminal replies queue behind the entire typed body" ); - - drop(tx); - drainer.join().expect("drainer thread joins cleanly"); } /// A delayed follow-up must be a distinct PTY write while remaining inside From a973af981c7c8160c3d56ad3e9c66600bf55b17a Mon Sep 17 00:00:00 2001 From: Relayflow Date: Sun, 4 Oct 2026 07:25:10 +0000 Subject: [PATCH 02/42] fix(broker): detect truncated injection echoes without replaying tasks --- .../src/broker/delivery_verification.rs | 109 +++++++++++++++++- crates/broker/src/pty_worker.rs | 16 ++- crates/broker/src/runtime/delivery.rs | 24 +++- crates/broker/src/wrap.rs | 31 ++++- .../broker/tests/pty_injection_integrity.rs | 25 ++++ docs/harnesses/injection.md | 48 ++++++++ 6 files changed, 244 insertions(+), 9 deletions(-) create mode 100644 docs/harnesses/injection.md diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 93da568903..363b61e286 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -15,8 +15,8 @@ use crate::{ pub(crate) const ACTIVITY_WINDOW: Duration = Duration::from_secs(5); pub(crate) const ACTIVITY_BUFFER_MAX_BYTES: usize = 16_000; pub(crate) const ACTIVITY_BUFFER_KEEP_BYTES: usize = 12_000; -const VERIFICATION_OUTPUT_MAX_BYTES: usize = 16_000; -const VERIFICATION_OUTPUT_KEEP_BYTES: usize = 12_000; +const VERIFICATION_OUTPUT_MAX_BYTES: usize = 64 * 1024; +const VERIFICATION_OUTPUT_KEEP_BYTES: usize = 48 * 1024; #[derive(Debug, Clone, Copy)] pub(crate) enum DeliveryOutcome { @@ -190,6 +190,14 @@ impl VerificationOutput { } } + /// A missing head is evidence only if we retained the entire observation window. + pub(crate) fn retains_boundary(&self, boundary: u64) -> bool { + self.segments.front().is_none_or(|segment| { + segment.sequence <= boundary.saturating_add(1) + && segment.start_offset >= self.base_offset + }) + } + /// Retained output read after the supplied producer sequence. pub(crate) fn since(&self, boundary: u64) -> Cow<'_, str> { let Some(segment) = self @@ -280,10 +288,69 @@ pub(crate) fn queue_or_take_detected_activity( } } -/// Check if the expected echo string appears in PTY output (after stripping ANSI). -pub(crate) fn check_echo_in_output(output: &str, expected: &str) -> bool { +/// One evidence ladder shared by fleet and wrap; timeout policy stays at the caller. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum EchoVerdict { + Exact, + Anchors, + PasteSummary, + HeadMissing, + Absent, +} +impl EchoVerdict { + pub(crate) fn confirmed(&self) -> bool { + matches!(self, Self::Exact | Self::Anchors | Self::PasteSummary) + } + pub(crate) fn label(&self) -> &'static str { + match self { + Self::Exact => "echo", + Self::Anchors => "echo_anchors", + Self::PasteSummary => "paste_summary", + Self::HeadMissing => "echo_head_missing", + Self::Absent => "timeout_fallback", + } + } +} +pub(crate) fn echo_verdict(output: &str, expected: &str) -> EchoVerdict { let clean = strip_ansi(output); - clean.contains(expected) + if !expected.is_empty() && clean.contains(expected) { + return EchoVerdict::Exact; + } + let compact = |s: &str| s.chars().filter(|c| !c.is_whitespace()).collect::(); + let expected = expected + .find("Relay message from ") + .map_or(expected, |i| &expected[i..]); + let expected = compact(expected); + let observed = compact(&clean); + // Short echoes need an exact match; overlapping anchors cannot prove loss. + if expected.chars().count() >= 240 { + let head: String = expected.chars().take(120).collect(); + let tail: String = expected + .chars() + .rev() + .take(120) + .collect::>() + .into_iter() + .rev() + .collect(); + if observed.contains(&tail) { + return if observed.contains(&head) { + EchoVerdict::Anchors + } else { + EchoVerdict::HeadMissing + }; + } + } + if clean.split("[Pasted text #").skip(1).any(|s| { + s.split_once(']') + .is_some_and(|(marker, _)| marker.starts_with(|c: char| c.is_ascii_digit())) + }) { + return EchoVerdict::PasteSummary; + } + EchoVerdict::Absent +} +pub(crate) fn check_echo_in_output(output: &str, expected: &str) -> bool { + echo_verdict(output, expected).confirmed() } pub(crate) fn current_timestamp_ms() -> u64 { @@ -325,6 +392,38 @@ pub(crate) fn delivery_injected_event_payload( mod tests { use super::*; + #[test] + fn trimmed_observation_cannot_prove_head_loss() { + let mut output = VerificationOutput::default(); + output.push_str(&"x".repeat(VERIFICATION_OUTPUT_MAX_BYTES + 1)); + assert!(!output.retains_boundary(0)); + let boundary = output.boundary(); + output.push_str("next delivery"); + // Conservative: a partially retained first segment never proves loss. + assert!(!output.retains_boundary(boundary)); + } + + #[test] + fn evidence_ladder_preserves_absent_fallback_and_detects_tail_loss() { + let expected = format!( + "Reminder changes each time\nRelay message from broker [init_1]: HEAD{}TAIL", + "abc xyz ".repeat(1280) + ); + assert_eq!(echo_verdict(&expected, &expected), EchoVerdict::Exact); + let wrapped = expected.replace(' ', "\r\n "); + assert_eq!(echo_verdict(&wrapped, &expected), EchoVerdict::Anchors); + assert_eq!( + echo_verdict(&expected[expected.len() - 200..], &expected), + EchoVerdict::HeadMissing + ); + assert_eq!( + echo_verdict("[Pasted text #1 +120 lines]", &expected), + EchoVerdict::PasteSummary + ); + assert_eq!(echo_verdict("Working…", &expected), EchoVerdict::Absent); + assert_eq!(echo_verdict("", &expected).label(), "timeout_fallback"); + } + #[test] fn check_echo_clean_text() { let output = "some preamble\nRelay message from Alice [evt_1]: hello world\nmore output"; diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index a11afb774f..3dfa4dc5ab 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -1799,7 +1799,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { json!({ "delivery_id": delivery_id, "event_id": event_id, - "verification": "echo" + "verification": crate::broker::delivery_verification::echo_verdict(&echo_buffer.since(pv.output_boundary), &pv.expected_echo).label() }), ) .await; @@ -2294,7 +2294,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { json!({ "delivery_id": delivery_id, "event_id": event_id, - "verification": "echo" + "verification": crate::broker::delivery_verification::echo_verdict(&echo_buffer.since(pv.output_boundary), &pv.expected_echo).label() }), ) .await; @@ -2392,6 +2392,18 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { let pv = pending_verifications.remove(i).unwrap(); let delivery_id = pv.delivery_id.clone(); let event_id = pv.event_id.clone(); + let verdict = crate::broker::delivery_verification::echo_verdict( + &echo_buffer.since(pv.output_boundary), &pv.expected_echo); + if verdict == crate::broker::delivery_verification::EchoVerdict::HeadMissing && echo_buffer.retains_boundary(pv.output_boundary) { + let _ = send_frame(&out_tx, "delivery_failed", None, json!({"delivery_id": delivery_id, "event_id": event_id, "reason": "echo_head_missing"})).await; + let _ = send_frame(&out_tx, "worker_error", pv.request_id, json!({"code": "echo_head_missing", "retryable": false, "message": "Task echo contains its tail without its head; inspect the agent before retrying"})).await; + throttle.record(DeliveryOutcome::Failed); + // Retain the pending id, matching initial_injection_incomplete. + continue; + } + static TIMEOUT_FALLBACKS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let fallback_count = TIMEOUT_FALLBACKS.fetch_add(1, Ordering::Relaxed) + 1; + tracing::warn!(fallback_count, %delivery_id, "delivery echo absent; preserving unverified timeout fallback"); // Do not re-inject on verification timeout. Re-injection can duplicate // already-delivered messages when terminal echo parsing is noisy. tracing::info!( diff --git a/crates/broker/src/runtime/delivery.rs b/crates/broker/src/runtime/delivery.rs index 9f4630c339..e597e40280 100644 --- a/crates/broker/src/runtime/delivery.rs +++ b/crates/broker/src/runtime/delivery.rs @@ -861,6 +861,11 @@ pub(crate) async fn queue_and_try_delivery_raw( withheld_fleet_ack: Option, withheld_fleet_ack_floor: Option, ) -> Result { + anyhow::ensure!( + body.len() <= crate::injection_wire::MAX_INJECTION_BODY_BYTES, + "injection_too_large: global body limit is {} bytes; use a brief file pointer", + crate::injection_wire::MAX_INJECTION_BODY_BYTES + ); // Fleet delivery IDs are stable across Relaycast retries. Preserve that // identity all the way into the worker so its completed-delivery cache can // re-ACK a replay without pasting the instruction a second time. Local @@ -1060,7 +1065,12 @@ pub(crate) fn delivery_ack_timeout( ) -> Duration { let minimum = match injection_mode { MessageInjectionMode::Wait => WAIT_DELIVERY_ACK_TIMEOUT, - MessageInjectionMode::Steer => crate::broker::delivery_verification::VERIFICATION_WINDOW, + MessageInjectionMode::Steer => { + crate::injection_wire::INJECTION_PROMPT_WAIT + + crate::injection_wire::MAX_TYPED_WRITE_TIME + + Duration::from_millis(250) + + crate::broker::delivery_verification::VERIFICATION_WINDOW + } }; std::cmp::max(retry_interval, minimum) } @@ -1300,3 +1310,15 @@ mod reply_target_tests { } } } + +#[cfg(test)] +mod injection_budget_tests { + use super::*; + #[test] + fn steer_ack_covers_prompt_typing_submit_and_verification() { + let required = crate::injection_wire::INJECTION_PROMPT_WAIT + + Duration::from_millis(crate::injection_wire::MAX_TYPED_BYTES as u64 * 5 + 250) + + crate::broker::delivery_verification::VERIFICATION_WINDOW; + assert!(delivery_ack_timeout(&MessageInjectionMode::Steer, Duration::ZERO) >= required); + } +} diff --git a/crates/broker/src/wrap.rs b/crates/broker/src/wrap.rs index 3bb97c3fa8..56d4cadd58 100644 --- a/crates/broker/src/wrap.rs +++ b/crates/broker/src/wrap.rs @@ -275,6 +275,18 @@ fn prepare_wrap_retry( true } +// Wrap has no delivery protocol. Preserve its existing failed-throttle timeout +// policy, including absent echoes; fleet alone uses the compatibility fallback. +fn wrap_timeout_outcome( + verdict: crate::broker::delivery_verification::EchoVerdict, +) -> DeliveryOutcome { + if verdict.confirmed() { + DeliveryOutcome::Success + } else { + DeliveryOutcome::Failed + } +} + /// Start echo verification after a PTY write ack without missing output that /// raced ahead of the ack select arm. Returns `true` when the echo was already /// present and the delivery was confirmed immediately. @@ -2324,7 +2336,8 @@ pub(crate) async fn run_wrap( attempts = pv.attempts, "wrap: delivery verification failed after max retries" ); - throttle.record(DeliveryOutcome::Failed); + throttle.record(wrap_timeout_outcome(crate::broker::delivery_verification::echo_verdict( + &echo_buffer.since(pv.output_boundary), &pv.expected_echo))); } } else { i += 1; @@ -2731,6 +2744,22 @@ sys.stdout.flush()"#; assert_eq!(verification.attempts, 2); } + #[test] + fn wrap_timeout_stays_failed_without_confirming_evidence() { + use crate::broker::delivery_verification::{DeliveryOutcome, EchoVerdict}; + for verdict in [EchoVerdict::Absent, EchoVerdict::HeadMissing] { + assert!(matches!( + super::wrap_timeout_outcome(verdict), + DeliveryOutcome::Failed + )); + } + // Wrap always uses bulk writes, including its Typed fallback. Its + // write acknowledgement budget must cover the delayed submit only. + assert!( + Duration::ZERO + super::PASTE_INJECTION_SUBMIT_DELAY < super::WRAP_WRITE_ACK_TIMEOUT + ); + } + #[test] fn echo_arriving_before_write_ack_is_confirmed_immediately() { let injection = "multiline task\nwith a delayed submit"; diff --git a/crates/broker/tests/pty_injection_integrity.rs b/crates/broker/tests/pty_injection_integrity.rs index ee8423a91a..69cfa03d93 100644 --- a/crates/broker/tests/pty_injection_integrity.rs +++ b/crates/broker/tests/pty_injection_integrity.rs @@ -131,6 +131,31 @@ fn ten_kib_task_and_relay_message_are_intact() { long_pair(); } #[test] +fn tail_only_echo_fails_without_replay() { + let mut f = Fixture::new("tail"); + f.deliver("init_tail", &format!("HEAD{}TAIL", "abc xyz".repeat(1500))); + assert_eq!( + f.wait("delivery_failed")["payload"]["reason"], + "echo_head_missing" + ); + assert_eq!(f.wait("worker_error")["payload"]["retryable"], false); + let before = f.transcript(); + f.deliver("init_tail", &format!("HEAD{}TAIL", "abc xyz".repeat(1500))); + let deadline = Instant::now() + Duration::from_millis(300); + while let Ok(line) = + f.rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) + { + if let Ok(frame) = serde_json::from_str::(&line) { + assert_ne!( + frame["type"], "delivery_verified", + "failed replay must never become success" + ); + assert_ne!(frame["type"], "delivery_ack"); + } + } + assert_eq!(f.transcript(), before, "failed body must never be replayed"); +} +#[test] fn over_limit_and_typed_fallback_reject_before_writing() { for mode in ["echo", "typed"] { let mut f = Fixture::new(mode); diff --git a/docs/harnesses/injection.md b/docs/harnesses/injection.md new file mode 100644 index 0000000000..9a10066e4c --- /dev/null +++ b/docs/harnesses/injection.md @@ -0,0 +1,48 @@ +# PTY task and relay-message delivery + +Fleet workers wait for a recognized composer before injecting. If the composer +cannot be proven within 15 seconds of starting an injection, delivery fails with +`prompt_unproven` without writing the body. Unknown harnesses retain their existing +input behavior. + +A harness that has enabled terminal bracketed-paste mode (DECSET 2004) receives one +bulk bracketed paste followed by a separate, delayed Enter. Capability stays latched +for the session, independently of composer readiness. Devin also uses this wire. +`RELAY_INJECT_PASTE=0` forces the typed fallback. Wrap sessions retain bulk writes. +Codex initial tasks retain their existing chunked delivery path. + +The broker-wide ceiling is **16,384 UTF-8 bytes**. The formatted envelope must fit +this ceiling as well, so leave room for attribution and broker reminders. Paced +fallback input has an effective limit of **1,536 bytes including the envelope** at +the default 5 ms pace. A slower `RELAY_INJECT_RATE_MS` lowers that limit to keep +writing within 7.68 seconds. Unpaced fallback uses the global ceiling. Codex uses +the smaller of the global ceiling and its deadline-derived limit. Rejections name +the harness and effective limit; oversized bodies are never partially written. +For longer briefs, create a file on the node and send a short instruction to read it. + +The input wire normalizes CRLF to LF and strips bare carriage returns and ESC +characters. These bytes cannot submit the composer early or close a paste from +inside its body. Other UTF-8 content is preserved. + +```sh +agent-relay fleet spawn claude --node my-node --name reviewer --task-file ./brief.md +agent-relay node agent spawn claude --name reviewer --task-file ./brief.md +``` + +`--task-file` reads a UTF-8 file on the **requester's machine**, then sends the +contents through the usual task transport. It does not create a file on the node +or bypass size limits. Fleet requires exactly one of `--task` and `--task-file`; +local `agent spawn` and `node agent spawn` allow neither. The integration command's +existing task option remains unchanged. + +Verification recognizes exact echoes, whitespace-normalized head/tail anchors, +and paste summaries. A tail without its head fails with `echo_head_missing`; the +body is never automatically replayed. If the observation buffer has discarded the +head, that is not treated as proof of truncation. No echo retains the existing +`timeout_fallback` behavior, with a warning and process-local fallback counter. +This compatibility verdict **does not prove byte-for-byte receipt**. + +Verified fleet PTY spawns with a task wait for its delivery verdict after proven +startup readiness. A failed task fails the spawn action; timeout fallback still +counts as success for compatibility. Wrap uses the same echo evidence but keeps +its existing failed-throttle policy on absent echoes. From 924171f5a067f9f350b98c81cc6d333ab2ade00b Mon Sep 17 00:00:00 2001 From: Relayflow Date: Sun, 4 Oct 2026 07:27:21 +0000 Subject: [PATCH 03/42] feat(cli): add task files and confirm fleet spawn task delivery --- .../compact_fe43i9idd2e5_2026-10-04.json | 51 ++++++++++++ .../compact_fe43i9idd2e5_2026-10-04.md | 18 +++++ CHANGELOG.md | 3 + crates/broker/src/runtime/fleet.rs | 46 ++++++++++- crates/broker/src/runtime/worker_events.rs | 72 +++++++++++++++-- packages/cli/src/cli/commands/fleet.test.ts | 34 ++++++++ packages/cli/src/cli/commands/fleet.ts | 6 +- packages/cli/src/cli/commands/local-agent.ts | 7 +- packages/cli/src/cli/commands/message.ts | 4 + packages/cli/src/cli/lib/task-input.test.ts | 33 ++++++++ packages/cli/src/cli/lib/task-input.ts | 28 +++++++ summary.md | 79 +++++++++++++++++++ 12 files changed, 371 insertions(+), 10 deletions(-) create mode 100644 .agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.json create mode 100644 .agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.md create mode 100644 packages/cli/src/cli/lib/task-input.test.ts create mode 100644 packages/cli/src/cli/lib/task-input.ts create mode 100644 summary.md diff --git a/.agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.json b/.agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.json new file mode 100644 index 0000000000..ac1bfab2cf --- /dev/null +++ b/.agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.json @@ -0,0 +1,51 @@ +{ + "id": "compact_fe43i9idd2e5", + "version": 1, + "type": "compacted", + "compactedAt": "2026-10-04T07:26:54.566Z", + "sourceTrajectories": [ + "traj_mrt7zp5f4emg" + ], + "dateRange": { + "start": "2026-10-04T07:16:26.257Z", + "end": "2026-10-04T07:26:54.151Z" + }, + "summary": { + "totalDecisions": 1, + "totalEvents": 1, + "uniqueAgents": [ + "default" + ] + }, + "decisionGroups": [ + { + "category": "other", + "decisions": [ + { + "question": "Use a latched paste capability, a 16 KiB ceiling and a 1536-byte default typed cap; preserve absent-echo fallback, fail only positive tail-loss evidence", + "chosen": "Use a latched paste capability, a 16 KiB ceiling and a 1536-byte default typed cap; preserve absent-echo fallback, fail only positive tail-loss evidence", + "reasoning": "Reviewed plan requires a bounded 90-second spawn budget and compatibility for TUIs that do not echo. Wrap already bulk-writes. Live Claude enabled DECSET 2004 during startup.", + "fromTrajectory": "traj_mrt7zp5f4emg" + } + ] + } + ], + "keyLearnings": [], + "keyFindings": [], + "filesAffected": [ + "crates/broker/src/broker/delivery_verification.rs", + "crates/broker/src/devin.rs", + "crates/broker/src/injection_wire.rs", + "crates/broker/src/lib.rs", + "crates/broker/src/pty_worker.rs", + "crates/broker/src/runtime/delivery.rs", + "crates/broker/src/wrap.rs", + "crates/broker/tests/pty_injection_integrity.rs", + "crates/relay-pty/src/pty.rs", + "docs/harnesses/injection.md" + ], + "commits": [ + "a973af9", + "3521cb4" + ] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.md b/.agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.md new file mode 100644 index 0000000000..338921b06d --- /dev/null +++ b/.agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.md @@ -0,0 +1,18 @@ +# Trajectory Compaction: Oct 4, 2026 - Oct 4, 2026 + +## Summary +- Sessions: 1 +- Decisions: 1 +- Events: 1 +- Agents: default +- Files: 10 +- Commits: 2 + +## Other +- Use a latched paste capability, a 16 KiB ceiling and a 1536-byte default typed cap; preserve absent-echo fallback, fail only positive tail-loss evidence -> Use a latched paste capability, a 16 KiB ceiling and a 1536-byte default typed cap; preserve absent-echo fallback, fail only positive tail-loss evidence (traj_mrt7zp5f4emg) + +## Key Learnings +- None + +## Key Findings +- None \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 285057e7cd..2430231b69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `fleet spawn`, `agent spawn`, and `node agent spawn` accept `--task-file` to read an initial brief from a local UTF-8 file. Fleet requires exactly one of `--task` and `--task-file`. - `node agent list|spawn|new|release|set-model` and `node tail` accept `--state-dir`, `--broker-url`, and `--api-key`, so brokers started with `node up --state-dir` — such as fleet nodes — can be managed from any directory. Without these flags, `RELAY_BROKER_URL` / `RELAY_BROKER_API_KEY` or the enclosing project's broker is used. ### Fixed +- PTY tasks and relay messages use atomic bracketed paste when supported, wait for the composer, and prevent embedded carriage returns from submitting partial input. Oversized bodies and tail-only echoes fail explicitly without replay. +- Verified fleet PTY spawns wait for the initial task delivery verdict before reporting readiness; unverified timeout fallback remains compatible. - Broker fleet control keeps a healthy node WebSocket open when Relaycast returns retryable `d1_pressure` for `node.register` or `inventory.sync`, retrying the same frame with bounded exponential backoff while the existing registration and application-liveness deadlines remain fail-closed. - `--state-dir` on `node agent` commands, `node status`, and `node down` also accepts a fleet node directory whose broker state lives in `state/`. - An explicit `--state-dir` on `node agent` commands is no longer overridden by `RELAY_BROKER_URL` / `RELAY_BROKER_API_KEY`. diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index 42ff384bef..03bbb18a1e 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -153,6 +153,17 @@ pub(super) struct PendingVerifiedSpawn { pub(super) deadline: Instant, pub(super) started: Instant, pub(super) generation: Uuid, + pub(super) readiness_proven: bool, + pub(super) task_event_id: Option, +} + +impl PendingVerifiedSpawn { + pub(super) fn matches_task(&self, generation: Uuid, event_id: &str) -> bool { + self.generation == generation && self.task_event_id.as_deref() == Some(event_id) + } + pub(super) fn can_report_ready(&self, generation: Uuid) -> bool { + self.generation == generation && self.readiness_proven && self.task_event_id.is_none() + } } pub(super) fn verified_spawn_ready_result( @@ -1661,6 +1672,8 @@ impl BrokerRuntime { deadline: Instant::now() + VERIFIED_SPAWN_READY_TIMEOUT, started, generation, + readiness_proven: false, + task_event_id: None, }, ); } @@ -3268,10 +3281,41 @@ mod tests { ); } // CLI/SDK default confirmation budget is 120s (fleet.ts / relaycast.ts). - assert!(crate::pty_worker::STARTUP_READY_TIMEOUT < VERIFIED_SPAWN_READY_TIMEOUT); + assert!( + crate::pty_worker::STARTUP_READY_TIMEOUT + + crate::injection_wire::INJECTION_PROMPT_WAIT + + Duration::from_millis(crate::injection_wire::MAX_TYPED_BYTES as u64 * 5 + 250) + + crate::broker::delivery_verification::VERIFICATION_WINDOW + < VERIFIED_SPAWN_READY_TIMEOUT + ); assert!(VERIFIED_SPAWN_READY_TIMEOUT < Duration::from_secs(120)); } + #[test] + fn verified_task_spawn_waits_for_matching_delivery_and_proven_prompt() { + let generation = Uuid::new_v4(); + let mut pending = PendingVerifiedSpawn { + invocation_id: "test".into(), + deadline: Instant::now(), + started: Instant::now(), + generation, + readiness_proven: false, + task_event_id: Some("init_test".into()), + }; + assert!(!pending.can_report_ready(generation)); + pending.readiness_proven = true; + assert!(!pending.can_report_ready(generation)); + assert!(!pending.matches_task(Uuid::new_v4(), "init_test")); + assert!(!pending.matches_task(generation, "another_message")); + assert!(pending.matches_task(generation, "init_test")); + // Exact echo, anchors, paste summary and compatibility fallback all retire + // the matching task; failed deliveries resolve the action as failure. + pending.task_event_id = None; + assert!(pending.can_report_ready(generation)); + pending.readiness_proven = false; + assert!(!pending.can_report_ready(generation)); + } + fn test_agent_spec(session_id: Option<&str>, harness_session_id: Option<&str>) -> AgentSpec { AgentSpec { name: WorkerName::from("agent-a"), diff --git a/crates/broker/src/runtime/worker_events.rs b/crates/broker/src/runtime/worker_events.rs index 1ea5d1ff36..5d8191f89c 100644 --- a/crates/broker/src/runtime/worker_events.rs +++ b/crates/broker/src/runtime/worker_events.rs @@ -869,6 +869,23 @@ impl BrokerRuntime { .get("event_id") .and_then(Value::as_str) .unwrap_or(""); + if pending_verified_spawns.get(&name).is_some_and(|pending| { + pending.readiness_proven + && pending.matches_task(generation, event_id) + }) { + if let Some(pending) = pending_verified_spawns.remove(&name) { + let result = + verified_spawn_ready_result(pending.invocation_id, &name); + let _ = fleet_control_tx + .send(FleetControlCommand::Send( + crate::fleet_wire::BrokerToRelaycast::ActionResult( + result, + ), + )) + .await; + } + } + // "echo" when the injection was confirmed in PTY // output; "timeout_fallback" when the worker acked // without ever seeing the echo. @@ -961,6 +978,28 @@ impl BrokerRuntime { .get("event_id") .and_then(Value::as_str) .unwrap_or(""); + if pending_verified_spawns + .get(&name) + .is_some_and(|pending| pending.matches_task(generation, event_id)) + { + if let Some(pending) = pending_verified_spawns.remove(&name) { + let result = super::fleet::verified_spawn_failed_result( + pending.invocation_id, + payload + .get("reason") + .and_then(Value::as_str) + .unwrap_or("initial_task_failed"), + ); + let _ = fleet_control_tx + .send(FleetControlCommand::Send( + crate::fleet_wire::BrokerToRelaycast::ActionResult( + result, + ), + )) + .await; + } + } + let reason = payload .get("reason") .and_then(Value::as_str) @@ -1414,15 +1453,25 @@ impl BrokerRuntime { .get(&name) .map(|handle| handle.spec.runtime == AgentRuntime::Pty) .unwrap_or(false); + let initial_task = workers.take_initial_task_for_injection(&name); + let initial_event_id = initial_task + .as_ref() + .map(|_| format!("init_{}", Uuid::new_v4().simple())); + if let Some(pending) = pending_verified_spawns.get_mut(&name) { + if pending.generation == generation { + pending.readiness_proven |= readiness_proven; + if is_pty_worker && initial_event_id.is_some() { + pending.task_event_id = initial_event_id.clone(); + } + } + } // Resolve the verified Fleet action before optional SDK // notifications and initial-task work. A congested SDK // output queue must not turn a ready worker into an // action timeout. let pending = pending_verified_spawns .get(&name) - .is_some_and(|pending| { - readiness_proven && pending.generation == generation - }) + .is_some_and(|pending| pending.can_report_ready(generation)) .then(|| pending_verified_spawns.remove(&name)) .flatten(); if let Some(pending) = pending { @@ -1459,8 +1508,8 @@ impl BrokerRuntime { ); } } - if let Some(task_text) = workers.take_initial_task_for_injection(&name) { - let event_id = format!("init_{}", Uuid::new_v4().simple()); + if let (Some(task_text), Some(event_id)) = (initial_task, initial_event_id) + { if let Err(e) = queue_and_try_delivery_raw( workers, pending_deliveries, @@ -1481,6 +1530,19 @@ impl BrokerRuntime { .await { tracing::warn!(worker = %name, error = %e, "failed to deliver initial_task"); + if let Some(pending) = pending_verified_spawns.remove(&name) { + let result = super::fleet::verified_spawn_failed_result( + pending.invocation_id, + &e.to_string(), + ); + let _ = fleet_control_tx + .send(FleetControlCommand::Send( + crate::fleet_wire::BrokerToRelaycast::ActionResult( + result, + ), + )) + .await; + } } // The initial task bypasses the delivery-mode queue, but the // hold replayed above freezes injection pops inside the PTY diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 7236424cb8..e5eab7ab77 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -3601,6 +3601,7 @@ describe('fleet command support', () => { { from: 'user' } ); return { + program, cwd, projectRoot, local, @@ -3615,6 +3616,39 @@ describe('fleet command support', () => { }; } + it('reads --task-file before spawning through the existing task transport', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-fleet-brief-')); + try { + const brief = path.join(dir, 'brief.md'); + fs.writeFileSync(brief, ' Full brief\r\nsecond line\n'); + const fixture = setup(); + await fixture.program.parseAsync( + ['fleet', 'spawn', 'claude', '--name', 'file-worker', '--task-file', brief], + { from: 'user' } + ); + expect(fixture.errors).toEqual([]); + expect(fixture.local.spawnPty).toHaveBeenCalledWith( + expect.objectContaining({ task: ' Full brief\r\nsecond line\n' }) + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it.each([[], ['--task', 'brief', '--task-file', 'brief.md']])( + 'rejects missing or conflicting task sources: %j', + async (...args) => { + const fixture = setup(); + await expect( + fixture.program.parseAsync(['fleet', 'spawn', 'claude', '--name', 'file-worker', ...args], { + from: 'user', + }) + ).rejects.toThrow('__exit__'); + expect(fixture.errors.join('\n')).toContain('exactly one'); + expect(fixture.local.spawnPty).not.toHaveBeenCalled(); + } + ); + it('uses the caller nested directory and local broker despite ambient hosted credentials', async () => { vi.stubEnv('RELAY_WORKSPACE_KEY', 'rk_live_ambient_test'); vi.stubEnv('RELAY_AGENT_TOKEN', 'at_live_ambient_test'); diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index ba8a884e1d..425854029d 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -1,3 +1,4 @@ +import { readTaskInput } from '../lib/task-input.js'; import { randomUUID } from 'node:crypto'; import path from 'node:path'; @@ -424,7 +425,8 @@ export function registerFleetCommands( .description('Spawn locally by default, or select a fleet node or Cloud sandbox explicitly') .argument('', 'AI CLI to launch', parseFleetCli) .requiredOption('--name ', 'Worker agent name') - .requiredOption('--task ', 'Initial task instructions') + .option('--task ', 'Initial task instructions') + .option('--task-file ', 'Read the initial task from a local UTF-8 file') .option('--auto-place', 'Request automatic eligible-node placement in the Relay workspace') .option('--node ', 'Target a specific fleet node') .option('--target-node ', 'Alias for --node') @@ -474,7 +476,7 @@ export function registerFleetCommands( await runSdk(deps.sdk, async () => { const clientOptions = sdkOptionsFromOpts(options); const name = requiredText(options.name, 'Worker name'); - const task = requiredText(options.task, 'Task'); + const task = (await readTaskInput(options.task, options.taskFile, true))!; let targetNode = optionalText(options.targetNode, 'Target node') ?? optionalText(options.node, 'Node'); const useSandbox = options.sandbox === true; // Explicit hosted credentials/transport and personas retain their legacy diff --git a/packages/cli/src/cli/commands/local-agent.ts b/packages/cli/src/cli/commands/local-agent.ts index eeb23cbd79..48fa1a62ce 100644 --- a/packages/cli/src/cli/commands/local-agent.ts +++ b/packages/cli/src/cli/commands/local-agent.ts @@ -1,3 +1,4 @@ +import { readTaskInput } from '../lib/task-input.js'; import type { Command } from 'commander'; import { AGENT37_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; @@ -849,6 +850,7 @@ export function registerLocalAgentCommands( .option('--name ', 'Agent name (defaults to the provider)') .option('--channels ', 'Channels to join', ['general']) .option('--task ', 'Initial task prompt') + .option('--task-file ', 'Read the initial task from a local UTF-8 file') .option('--model ', 'Model override') .option('--runtime ', 'Harness runtime: auto | native | pty', 'auto') .option('--cwd ', 'Working directory for the spawned agent') @@ -873,7 +875,7 @@ export function registerLocalAgentCommands( const resolved = resolveAutoSpawn( provider, baseName, - opts.task as string | undefined, + await readTaskInput(opts.task, opts.taskFile), opts.model as string | undefined ); await spawnAgentWithClient(client, { @@ -904,6 +906,7 @@ export function registerLocalAgentCommands( .option('--mode ', 'Attach mode: drive | view | passthrough', 'drive') .option('--channels ', 'Channels to join', ['general']) .option('--task ', 'Initial task prompt') + .option('--task-file ', 'Read the initial task from a local UTF-8 file') .option('--model ', 'Model override') .option('--runtime ', 'Harness runtime: auto | native | pty', 'auto') .option('--cwd ', 'Working directory for the spawned agent') @@ -933,7 +936,7 @@ export function registerLocalAgentCommands( const resolved = resolveAutoSpawn( provider, baseName, - options.task as string | undefined, + await readTaskInput(options.task, options.taskFile), options.model as string | undefined ); await run( diff --git a/packages/cli/src/cli/commands/message.ts b/packages/cli/src/cli/commands/message.ts index 7fc9d9fa45..5a923890b4 100644 --- a/packages/cli/src/cli/commands/message.ts +++ b/packages/cli/src/cli/commands/message.ts @@ -1,3 +1,4 @@ +import { validateInjectionSize } from '../lib/task-input.js'; import { InvalidArgumentError, type Command } from 'commander'; import { @@ -53,6 +54,7 @@ export function registerMessageCommands( .argument('', 'Message text') ).action(async (channel: string, text: string, o: Record) => { await runSdk(deps, async () => { + validateInjectionSize(text); printJson(deps, await deps.createAgentRelay(opts(o)).messages.send({ channel, text })); }); }); @@ -80,6 +82,7 @@ export function registerMessageCommands( .argument('', 'Reply text') ).action(async (messageId: string, text: string, o: Record) => { await runSdk(deps, async () => { + validateInjectionSize(text); printJson(deps, await deps.createAgentRelay(opts(o)).messages.reply({ messageId, text })); }); }); @@ -132,6 +135,7 @@ export function registerMessageCommands( ) ).action(async (agent: string, text: string, o: Record) => { await runSdk(deps, async () => { + validateInjectionSize(text); const mode = o.mode as 'wait' | 'steer' | undefined; const options = opts(o); const relay = deps.createAgentRelay(options); diff --git a/packages/cli/src/cli/lib/task-input.test.ts b/packages/cli/src/cli/lib/task-input.test.ts new file mode 100644 index 0000000000..149f05b12e --- /dev/null +++ b/packages/cli/src/cli/lib/task-input.test.ts @@ -0,0 +1,33 @@ +import { mkdtemp, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it, expect } from 'vitest'; +import { readTaskInput, MAX_INJECTION_BODY_BYTES } from './task-input.js'; +describe('task input', () => { + it('requires exactly one source for fleet, allows neither locally', async () => { + await expect(readTaskInput(undefined, undefined, true)).rejects.toThrow('exactly one'); + await expect(readTaskInput('hi', 'brief.md')).rejects.toThrow('exactly one'); + expect(await readTaskInput(undefined, undefined)).toBeUndefined(); + }); + it('reads requester files without stripping whitespace or normalizing CRLF', async () => { + const dir = await mkdtemp(join(tmpdir(), 'relay-task-')); + try { + const file = join(dir, 'brief.md'); + await writeFile(file, ' héllo\r\nworld\n'); + expect(await readTaskInput(undefined, file, true)).toBe(' héllo\r\nworld\n'); + await writeFile(file, ' '); + await expect(readTaskInput(undefined, file)).rejects.toThrow('empty'); + await expect(readTaskInput(undefined, join(dir, 'missing'))).rejects.toThrow(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + it('counts UTF-8 bytes', async () => { + expect(await readTaskInput('a'.repeat(MAX_INJECTION_BODY_BYTES), undefined)).toHaveLength( + MAX_INJECTION_BODY_BYTES + ); + await expect(readTaskInput('é'.repeat(MAX_INJECTION_BODY_BYTES), undefined)).rejects.toThrow( + 'UTF-8 bytes' + ); + }); +}); diff --git a/packages/cli/src/cli/lib/task-input.ts b/packages/cli/src/cli/lib/task-input.ts new file mode 100644 index 0000000000..e1c759650f --- /dev/null +++ b/packages/cli/src/cli/lib/task-input.ts @@ -0,0 +1,28 @@ +import { readFile } from 'node:fs/promises'; + +/** Broker-wide ceiling; a harness may reject a lower effective wire limit. */ +export const MAX_INJECTION_BODY_BYTES = 16 * 1024; +export async function readTaskInput( + task: unknown, + taskFile: unknown, + required = false +): Promise { + if (task !== undefined && taskFile !== undefined) + throw new Error('Specify exactly one of --task or --task-file'); + if (task === undefined && taskFile === undefined) { + if (required) throw new Error('Specify exactly one of --task or --task-file'); + return undefined; + } + const text = taskFile === undefined ? task : await readFile(String(taskFile), 'utf8'); + if (typeof text !== 'string' || !text.trim()) throw new Error('Task must not be empty'); + validateInjectionSize(text); + return text; +} + +export function validateInjectionSize(text: string): void { + if (Buffer.byteLength(text, 'utf8') > MAX_INJECTION_BODY_BYTES) { + throw new Error( + `Injection exceeds ${MAX_INJECTION_BODY_BYTES} UTF-8 bytes; use a brief file on the node and send a short pointer` + ); + } +} diff --git a/summary.md b/summary.md new file mode 100644 index 0000000000..21d0e86cb0 --- /dev/null +++ b/summary.md @@ -0,0 +1,79 @@ +# PTY injection integrity and fleet task confirmation + +Long fleet tasks and relay messages were typed into Claude one character every +5 ms. A 6.7 KB brief occupied the PTY write drainer for roughly 33 seconds, delaying +terminal-query replies and startup auto-responses while the composer could change. +The existing verification timeout acknowledged even a tail-only echo. + +This change sends one bracketed paste to harnesses that have advertised DECSET +2004, followed by the existing delayed Enter. The capability is latched across +mode resets; composer readiness is checked independently before writing. A missing +composer, oversized body, or tail-only echo produces an explicit non-retryable +failure. Failed deliveries retain their identity and are never replayed or +subsequently acknowledged as successful duplicates. CRLF becomes LF; bare CR and +ESC cannot submit or terminate the body early. + +The verification ladder is shared with wrap: exact echo, normalized head/tail +anchors, then paste summary. Missing heads only count as loss when the observation +window is retained. Absent echoes preserve `timeout_fallback`, now with a warning +and process-local counter. Wrap retains its existing failed-throttle timeout policy. + +Verified fleet PTY spawns with a task remain pending after readiness until the +matching task verdict arrives; explicit task failures fail the spawn action. +Native and argv-task startup paths retain their existing readiness contract. +`fleet spawn`, `agent spawn`, and `node agent spawn` accept `--task-file`, read on +the requester and forwarded through existing task transport. Fleet requires +exactly one task source. Message commands and task input validate UTF-8 byte size. + +Limits are 16 KiB globally, including the formatted envelope at the PTY boundary, +and 1,536 bytes for the default paced fallback. A slower explicit pace lowers that +cap. Codex retains its chunked initial-task path and uses the smaller derived cap. +The ordinary typed-path budget closes inside 90 seconds: +60 s startup + 15 s prompt recheck + 7.68 s typing + 0.25 s submit + 5 s verification += 87.93 s. Steer acknowledgements include the injection budget. + +## Validation + +- Full broker and relay-pty run: **1,627 passed, 0 failed, 7 ignored**. +- Integrity integration suite, including the ignored stress test: **6 passed**. + Twenty cold starts each receive a >10 KB task and a >10 KB follow-up message, + verified byte-for-byte in the fake harness's input transcript with one paste + pair and one submit. Other arms cover tail loss/no replay, explicit size + rejection, disappearing composer, and absent-echo compatibility. +- CLI regression suites: **188 passed**; CLI TypeScript checking passed. +- `cargo fmt --all --check` and `cargo clippy --all-targets` passed. Clippy reports + five pre-existing sliced-string warnings in Codex tests. +- Live Claude startup probe observed DECSET 2004; no disable appeared during the + six-second observation. Parser regression covers enable and disable in one read. + +Commands (debug symbols disabled to fit the workspace disk): + +```sh +env -u GIT_CONFIG_COUNT CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 CARGO_INCREMENTAL=0 cargo test -p agent-relay-broker -p relay-pty +CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 CARGO_INCREMENTAL=0 cargo test -p agent-relay-broker --test pty_injection_integrity -- --include-ignored +CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 CARGO_INCREMENTAL=0 cargo clippy --all-targets +cargo fmt --all --check +npx vitest run packages/cli/src/cli/lib/task-input.test.ts packages/cli/src/cli/commands/fleet.test.ts packages/cli/src/cli/commands/local-agent.test.ts packages/cli/src/cli/commands/relaycast-groups.test.ts +npx tsc --noEmit -p packages/cli/tsconfig.json +``` + +This workspace injects `core.hooksPath=/dev/null` through `GIT_CONFIG_COUNT`. +Removing that setting only for the Rust test process lets the existing hook-chain +regressions exercise their own temporary repositories; all four then pass. + +## Compatibility and remaining work + +Per `reviewed-plan.md`, absent-echo fallback still counts as spawn success. It is +**not cryptographic or byte-for-byte receipt confirmation**; the stronger ticket +acceptance needs a harness receipt protocol. The 20-run proof uses a raw-mode fake +Claude TUI through the real broker, not 20 authenticated Claude transcripts. + +Wrap already bulk-wrote Claude input before this change. Fleet gains that behavior +plus explicit paste framing and readiness checks, rather than simply reverting +the 10.4.0 pacing fix. Large Claude inputs can now appear as pasted-content blocks; +the attribution envelope is unchanged. + +Duplicate desktop/broker registrations remain a separate investigation requiring +the reporter's ambient MCP configuration. A node-side brief-file writer and +integration-command `--task-file` support are follow-ups. No workflow files were +changed, and no release or merge was performed. From d735b8da34a60675ce5b8bb882aa892f7c8b5798 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Sun, 4 Oct 2026 08:18:23 +0000 Subject: [PATCH 04/42] fix(broker): confirm task receipt from whole-payload echo only Address the three P1 findings on PR #1893. Endpoint anchors and a collapsed-paste marker were treated as positive delivery confirmation, and the verified fleet spawn branch reported spawned:true, ready:true without inspecting the verification label at all. An agent that swallowed its whole task, received only its head and tail, or echoed nothing produced the same successful spawn result as one that received every byte. Only whole-payload evidence confirms a delivery now: `echo` (verbatim) or `echo_normalized` (verbatim once the TUI's wrapping whitespace is removed from both sides, which replaces the head/tail `echo_anchors` rung). Matching endpoints around an unobserved middle become `echo_incomplete` and a collapsed paste stays `paste_summary`; both ack the delivery without claiming receipt. A tail without its head still fails with `echo_head_missing`, and an absent echo still reports `timeout_fallback`. `verification_label_confirms_receipt` is the single source of truth for which wire labels count as receipt, kept in lockstep with `EchoVerdict::confirmed` by test. A verified spawn whose initial task is acked without it resolves as `spawn_task_unconfirmed`, which names the live agent and warns against retrying (a retry duplicates the agent) rather than reporting success or a retryable failure; the pending entry is still retired so the readiness deadline cannot release a live worker. Wrap keeps its failed-throttle policy on absent echoes and on evidence of loss, but records a collapsed paste as unverified: that verdict indicates nothing about loss, so it must not back the injection delay off on every delivery to a paste-collapsing TUI. Regressions: missing, changed and reversed anchors; a partial paste and a redrawn earlier paste marker; label/verdict agreement; timeout dispositions; runtime arms feeding timeout_fallback, paste_summary, echo_incomplete and a missing label through the spawn-completion handler. The PTY integration suite gains the reviewer's `middle_lost` and `paste_tail` fixture modes as negative-confirmation assertions. Co-Authored-By: Claude Opus 5 --- .../active/traj_ynyux9pee7x7/trajectory.json | 102 +++++++ CHANGELOG.md | 6 +- .../src/broker/delivery_verification.rs | 266 ++++++++++++++++-- crates/broker/src/protocol.rs | 15 +- crates/broker/src/pty_worker.rs | 87 ++++-- crates/broker/src/runtime/fleet.rs | 33 ++- crates/broker/src/runtime/tests.rs | 153 ++++++++++ crates/broker/src/runtime/worker_events.rs | 52 +++- crates/broker/src/wrap.rs | 30 +- .../broker/tests/pty_injection_integrity.rs | 42 ++- crates/relay-pty/src/pty.rs | 77 ++++- docs/harnesses/injection.md | 28 +- packages/harness-driver/src/protocol.ts | 18 +- summary.md | 79 ------ 14 files changed, 819 insertions(+), 169 deletions(-) delete mode 100644 summary.md diff --git a/.agentworkforce/trajectories/active/traj_ynyux9pee7x7/trajectory.json b/.agentworkforce/trajectories/active/traj_ynyux9pee7x7/trajectory.json index ae39159f30..3fc9615228 100644 --- a/.agentworkforce/trajectories/active/traj_ynyux9pee7x7/trajectory.json +++ b/.agentworkforce/trajectories/active/traj_ynyux9pee7x7/trajectory.json @@ -31,6 +31,108 @@ "reasoning": "npm ci pinned SDK 8.0.7 without metadata while Publish Package freshly resolves 8.14.0 with relay.action-caller=v1. Align CI with release resolution without weakening assertions or changing production code." }, "significance": "high" + }, + { + "ts": 1791096850177, + "type": "decision", + "content": "Root-caused tail truncation to paced per-codepoint typing (5ms/atom) with no bracketed-paste framing for Claude; plan fixes it with capability-gated atomic paste + head/tail-anchored confirmation: Root-caused tail truncation to paced per-codepoint typing (5ms/atom) with no bracketed-paste framing for Claude; plan fixes it with capability-gated atomic paste + head/tail-anchored confirmation", + "raw": { + "question": "Root-caused tail truncation to paced per-codepoint typing (5ms/atom) with no bracketed-paste framing for Claude; plan fixes it with capability-gated atomic paste + head/tail-anchored confirmation", + "chosen": "Root-caused tail truncation to paced per-codepoint typing (5ms/atom) with no bracketed-paste framing for Claude; plan fixes it with capability-gated atomic paste + head/tail-anchored confirmation", + "alternatives": [], + "reasoning": "write_paced emits one VT atom per write_all, so a 6.7KB task spends ~33s only in the TUI composer; any composer clear in that window loses the head. devin.rs:12 wraps paste only for Devin. check_echo_in_output cannot detect the loss, so pty_worker.rs:2368 acks it as timeout_fallback success." + }, + "significance": "high" + }, + { + "ts": 1791099654001, + "type": "decision", + "content": "Reserve per-byte VTE parsing for chunks carrying the DECSET 2004 parameter: Reserve per-byte VTE parsing for chunks carrying the DECSET 2004 parameter", + "raw": { + "question": "Reserve per-byte VTE parsing for chunks carrying the DECSET 2004 parameter", + "chosen": "Reserve per-byte VTE parsing for chunks carrying the DECSET 2004 parameter", + "alternatives": [], + "reasoning": "The paste-capability latch parsed every PTY output byte with its own processor.advance() call until the mode was seen, which is permanent for any harness that never enables 2004. Scanning the chunk for the literal 2004 and otherwise bulk-parsing with one post-chunk mode check keeps enable+disable-in-one-read detection (both complete sequences must contain 2004) and still latches a sequence split across reads." + }, + "significance": "high" + }, + { + "ts": 1791099654447, + "type": "decision", + "content": "Treat the two failing CLI suites and four Rust hook-chain tests as sandbox environment, not branch regressions: Treat the two failing CLI suites and four Rust hook-chain tests as sandbox environment, not branch regressions", + "raw": { + "question": "Treat the two failing CLI suites and four Rust hook-chain tests as sandbox environment, not branch regressions", + "chosen": "Treat the two failing CLI suites and four Rust hook-chain tests as sandbox environment, not branch regressions", + "alternatives": [], + "reasoning": "sdk-client.test.ts and fleet-lifecycle-integration.test.ts assert no ambient gateway is inherited and read the real process env; this sandbox exports RELAY_BASE_URL, which no CI workflow sets. The spawner hook-chain tests are defeated by the sandbox's GIT_CONFIG_COUNT core.hooksPath=/dev/null override. Both fixed in .relayflow/check.sh (uncommitted) rather than by changing tests." + }, + "significance": "high" + }, + { + "ts": 1791100397805, + "type": "decision", + "content": "Review current PR and staged repair together; withhold clean signoff for unverified task success: Review current PR and staged repair together; withhold clean signoff for unverified task success", + "raw": { + "question": "Review current PR and staged repair together; withhold clean signoff for unverified task success", + "chosen": "Review current PR and staged repair together; withhold clean signoff for unverified task success", + "alternatives": [], + "reasoning": "The user requires full receipt confirmation; timeout_fallback still completes a verified spawn successfully." + }, + "significance": "high" + }, + { + "ts": 1791100495394, + "type": "decision", + "content": "Review PR 1893 requests changes: timeout success and incomplete-echo confirmations remain: Review PR 1893 requests changes: timeout success and incomplete-echo confirmations remain", + "raw": { + "question": "Review PR 1893 requests changes: timeout success and incomplete-echo confirmations remain", + "chosen": "Review PR 1893 requests changes: timeout success and incomplete-echo confirmations remain", + "alternatives": [], + "reasoning": "188 CLI tests and six PTY tests pass; temporary adversarial PTY integration reproduces echo_anchors for a missing middle and paste_summary for a marker plus 40-byte tail. Findings recorded in review.md; no review.clean." + }, + "significance": "high" + }, + { + "ts": 1791100867791, + "type": "decision", + "content": "Positive delivery confirmation requires whole-payload echo evidence; endpoint anchors and collapsed-paste markers are inconclusive, and an unconfirmed initial task resolves a verified fleet spawn as an explicit non-retryable unconfirmed outcome rather than success: Positive delivery confirmation requires whole-payload echo evidence; endpoint anchors and collapsed-paste markers are inconclusive, and an unconfirmed initial task resolves a verified fleet spawn as an explicit non-retryable unconfirmed outcome rather than success", + "raw": { + "question": "Positive delivery confirmation requires whole-payload echo evidence; endpoint anchors and collapsed-paste markers are inconclusive, and an unconfirmed initial task resolves a verified fleet spawn as an explicit non-retryable unconfirmed outcome rather than success", + "chosen": "Positive delivery confirmation requires whole-payload echo evidence; endpoint anchors and collapsed-paste markers are inconclusive, and an unconfirmed initial task resolves a verified fleet spawn as an explicit non-retryable unconfirmed outcome rather than success", + "alternatives": [], + "reasoning": "review.md P1 x3: timeout_fallback/anchors/paste_summary all reached the spawn success branch, so a swallowed or middle-truncated task was indistinguishable from a received one" + }, + "significance": "high" + }, + { + "ts": 1791101890744, + "type": "decision", + "content": "Report what the terminal actually showed (echo_incomplete / paste_summary) instead of labelling every unconfirmed ack timeout_fallback: Report what the terminal actually showed (echo_incomplete / paste_summary) instead of labelling every unconfirmed ack timeout_fallback", + "raw": { + "question": "Report what the terminal actually showed (echo_incomplete / paste_summary) instead of labelling every unconfirmed ack timeout_fallback", + "chosen": "Report what the terminal actually showed (echo_incomplete / paste_summary) instead of labelling every unconfirmed ack timeout_fallback", + "alternatives": [], + "reasoning": "Operators need to tell a collapsed paste apart from a silent agent; the runtime gates on verification_label_confirms_receipt, so richer labels cost nothing in correctness" + }, + "significance": "high" + }, + { + "ts": 1791101891202, + "type": "reflection", + "content": "Three P1s shared one root cause: confirmed() conflated 'some evidence' with 'full receipt'. Narrowing confirmation to whole-payload echo and gating the spawn result on the label fixed all three; 20 cold starts and the two adversarial fixture modes pass.", + "raw": { + "focalPoints": [ + "verification-policy", + "spawn-confirmation" + ], + "confidence": 0.85 + }, + "significance": "high", + "tags": [ + "focal:verification-policy", + "focal:spawn-confirmation", + "confidence:0.85" + ] } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 2430231b69..b2c7b4c5e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `fleet spawn`, `agent spawn`, and `node agent spawn` accept `--task-file` to read an initial brief from a local UTF-8 file. Fleet requires exactly one of `--task` and `--task-file`. - `node agent list|spawn|new|release|set-model` and `node tail` accept `--state-dir`, `--broker-url`, and `--api-key`, so brokers started with `node up --state-dir` — such as fleet nodes — can be managed from any directory. Without these flags, `RELAY_BROKER_URL` / `RELAY_BROKER_API_KEY` or the enclosing project's broker is used. +### Changed + +- Task and relay-message bodies are capped at 16 KiB. `message send|reply|dm`, `fleet spawn`, and the broker reject larger input naming the limit, and the paced fallback wire rejects above 1,536 bytes; write the brief to a file and send a pointer instead. + ### Fixed - PTY tasks and relay messages use atomic bracketed paste when supported, wait for the composer, and prevent embedded carriage returns from submitting partial input. Oversized bodies and tail-only echoes fail explicitly without replay. -- Verified fleet PTY spawns wait for the initial task delivery verdict before reporting readiness; unverified timeout fallback remains compatible. +- `fleet spawn --task` reports success only when the agent's terminal echoed the whole task. A task acked without that proof fails with `spawn_task_unconfirmed`, which names the live agent and says not to retry, and `delivery_verified` now reports what was observed (`echo_incomplete`, `paste_summary`, `timeout_fallback`) instead of treating head/tail anchors or a collapsed-paste marker as receipt. - Broker fleet control keeps a healthy node WebSocket open when Relaycast returns retryable `d1_pressure` for `node.register` or `inventory.sync`, retrying the same frame with bounded exponential backoff while the existing registration and application-liveness deadlines remain fail-closed. - `--state-dir` on `node agent` commands, `node status`, and `node down` also accepts a fleet node directory whose broker state lives in `state/`. - An explicit `--state-dir` on `node agent` commands is no longer overridden by `RELAY_BROKER_URL` / `RELAY_BROKER_API_KEY`. diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 363b61e286..33b5269bd7 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -289,28 +289,123 @@ pub(crate) fn queue_or_take_detected_activity( } /// One evidence ladder shared by fleet and wrap; timeout policy stays at the caller. -#[derive(Debug, PartialEq, Eq)] +/// +/// Only whole-payload evidence confirms a delivery. Endpoint anchors and a +/// collapsed-paste marker are reported, because they tell an operator what the +/// terminal actually showed, but neither establishes that the harness received +/// every byte: anchors say nothing about the bytes between them, and a paste +/// marker is emitted for any paste, including a truncated one (relay#1893 +/// review, P1 x2). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum EchoVerdict { + /// The whole expected envelope appears verbatim in post-boundary output. Exact, - Anchors, - PasteSummary, + /// The whole expected envelope appears once the TUI's wrapping whitespace + /// is removed from both sides. Equivalent to `Exact` as integrity evidence. + Normalized, + /// The tail anchor is present without the head anchor: the injection lost + /// its head, which is the truncation this verification exists to catch. HeadMissing, + /// Both anchors are present but the payload between them is not, or the + /// echo is otherwise a strict subset. Evidence of a delivery, never + /// evidence of a complete one. + Incomplete, + /// The harness collapsed the paste into a `[Pasted text #N …]` marker, so + /// no content was echoed at all and nothing about it can be verified. + PasteSummary, + /// No echo evidence whatsoever. Absent, } impl EchoVerdict { + /// Whether this verdict proves the harness received the whole payload. pub(crate) fn confirmed(&self) -> bool { - matches!(self, Self::Exact | Self::Anchors | Self::PasteSummary) + matches!(self, Self::Exact | Self::Normalized) } pub(crate) fn label(&self) -> &'static str { match self { Self::Exact => "echo", - Self::Anchors => "echo_anchors", - Self::PasteSummary => "paste_summary", + Self::Normalized => "echo_normalized", Self::HeadMissing => "echo_head_missing", + Self::Incomplete => "echo_incomplete", + Self::PasteSummary => "paste_summary", Self::Absent => "timeout_fallback", } } } + +/// Whether a `delivery_verified` label proves full receipt. +/// +/// The broker runtime sees deliveries only as the worker's wire label, so this +/// is the one place that decides which labels a caller — notably the verified +/// fleet spawn completion in `runtime::worker_events` — may treat as receipt. +/// Kept in lockstep with [`EchoVerdict::confirmed`] by +/// `verification_labels_agree_with_the_verdict_ladder`. +pub(crate) fn verification_label_confirms_receipt(label: &str) -> bool { + label == EchoVerdict::Exact.label() || label == EchoVerdict::Normalized.label() +} + +/// What a verification timeout should report for a final verdict. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum VerificationTimeout { + /// Head loss proven against a fully retained observation window. Terminal. + HeadLoss, + /// Whole-payload evidence that the output arm did not get to act on. + /// Practically unreachable — that arm checks every pending verification on + /// each read — but a late confirmation is still a confirmation, so it must + /// not be downgraded to an unverified ack. + Confirmed { label: &'static str }, + /// Acked without proof of full receipt. `label` is the wire `verification` + /// value; `reason` states what the observed output actually showed. The + /// label is never one `verification_label_confirms_receipt` accepts. + Unconfirmed { label: &'static str, reason: String }, +} + +/// Classify a timed-out verification. +/// +/// `retains_boundary` must come from [`VerificationOutput::retains_boundary`]: +/// a trimmed window cannot prove head loss, so a head-missing verdict without +/// it degrades to an unconfirmed ack rather than failing a delivery that may +/// well have landed. +pub(crate) fn verification_timeout( + verdict: EchoVerdict, + retains_boundary: bool, + window: Duration, +) -> VerificationTimeout { + match verdict { + EchoVerdict::HeadMissing if retains_boundary => VerificationTimeout::HeadLoss, + EchoVerdict::HeadMissing => VerificationTimeout::Unconfirmed { + label: EchoVerdict::Incomplete.label(), + reason: "echo tail observed without its head, and the observation window was trimmed" + .to_string(), + }, + EchoVerdict::Exact | EchoVerdict::Normalized => VerificationTimeout::Confirmed { + label: verdict.label(), + }, + EchoVerdict::Incomplete => VerificationTimeout::Unconfirmed { + label: verdict.label(), + reason: + "echo matched only its head and tail; the payload between them was not observed" + .to_string(), + }, + EchoVerdict::PasteSummary => VerificationTimeout::Unconfirmed { + label: verdict.label(), + reason: "harness collapsed the paste, so no content was echoed to verify".to_string(), + }, + EchoVerdict::Absent => VerificationTimeout::Unconfirmed { + label: verdict.label(), + reason: format!("echo not detected within {}s window", window.as_secs()), + }, + } +} + +/// Whether `output` carries a `[Pasted text #N …]` collapsed-paste marker. +fn has_paste_marker(output: &str) -> bool { + output.split("[Pasted text #").skip(1).any(|rest| { + rest.split_once(']') + .is_some_and(|(marker, _)| marker.starts_with(|c: char| c.is_ascii_digit())) + }) +} + pub(crate) fn echo_verdict(output: &str, expected: &str) -> EchoVerdict { let clean = strip_ansi(output); if !expected.is_empty() && clean.contains(expected) { @@ -322,7 +417,16 @@ pub(crate) fn echo_verdict(output: &str, expected: &str) -> EchoVerdict { .map_or(expected, |i| &expected[i..]); let expected = compact(expected); let observed = compact(&clean); - // Short echoes need an exact match; overlapping anchors cannot prove loss. + // A TUI wraps and re-indents what it echoes, so whitespace cannot be + // compared — but every other byte can, and only the whole payload confirms + // receipt. Anchors are then diagnosis, not confirmation: a present tail + // without its head is the reported truncation, and a present head and tail + // around an unobserved middle is simply incomplete. + if !expected.is_empty() && observed.contains(&expected) { + return EchoVerdict::Normalized; + } + // Short echoes need a whole-payload match; overlapping anchors cannot + // distinguish loss from a near-complete echo. if expected.chars().count() >= 240 { let head: String = expected.chars().take(120).collect(); let tail: String = expected @@ -335,16 +439,13 @@ pub(crate) fn echo_verdict(output: &str, expected: &str) -> EchoVerdict { .collect(); if observed.contains(&tail) { return if observed.contains(&head) { - EchoVerdict::Anchors + EchoVerdict::Incomplete } else { EchoVerdict::HeadMissing }; } } - if clean.split("[Pasted text #").skip(1).any(|s| { - s.split_once(']') - .is_some_and(|(marker, _)| marker.starts_with(|c: char| c.is_ascii_digit())) - }) { + if has_paste_marker(&clean) { return EchoVerdict::PasteSummary; } EchoVerdict::Absent @@ -403,27 +504,150 @@ mod tests { assert!(!output.retains_boundary(boundary)); } - #[test] - fn evidence_ladder_preserves_absent_fallback_and_detects_tail_loss() { - let expected = format!( + /// A long envelope and the pieces of it an adversarial or lossy TUI can + /// echo. `HEAD`/`TAIL` sentinels sit inside the anchors so a verdict can be + /// built from endpoints alone. + fn long_envelope() -> String { + format!( "Reminder changes each time\nRelay message from broker [init_1]: HEAD{}TAIL", "abc xyz ".repeat(1280) - ); + ) + } + + #[test] + fn whole_payload_echoes_confirm_and_a_lost_head_fails() { + let expected = long_envelope(); assert_eq!(echo_verdict(&expected, &expected), EchoVerdict::Exact); + // Wrapping and re-indenting is all a TUI may do to a confirmed echo. let wrapped = expected.replace(' ', "\r\n "); - assert_eq!(echo_verdict(&wrapped, &expected), EchoVerdict::Anchors); + assert_eq!(echo_verdict(&wrapped, &expected), EchoVerdict::Normalized); + assert!(echo_verdict(&wrapped, &expected).confirmed()); assert_eq!( echo_verdict(&expected[expected.len() - 200..], &expected), EchoVerdict::HeadMissing ); - assert_eq!( - echo_verdict("[Pasted text #1 +120 lines]", &expected), - EchoVerdict::PasteSummary - ); assert_eq!(echo_verdict("Working…", &expected), EchoVerdict::Absent); assert_eq!(echo_verdict("", &expected).label(), "timeout_fallback"); } + /// relay#1893 review, P1: matching endpoints certified a payload whose + /// middle was missing, and that verdict completed a verified fleet spawn. + #[test] + fn endpoint_anchors_never_confirm_a_payload_whose_middle_is_unobserved() { + let expected = long_envelope(); + let anchor_head = &expected[..expected.find("HEAD").unwrap() + 300]; + let anchor_tail = &expected[expected.len() - 300..]; + + // Missing middle: exactly the reviewer's `middle_lost` reproduction. + let missing_middle = format!("{anchor_head}{anchor_tail}"); + assert_eq!( + echo_verdict(&missing_middle, &expected), + EchoVerdict::Incomplete + ); + // Changed middle: every byte between the anchors is a substitution. + let changed_middle = format!("{anchor_head}{}{anchor_tail}", "q".repeat(9_000)); + assert_eq!( + echo_verdict(&changed_middle, &expected), + EchoVerdict::Incomplete + ); + // Reversed anchors: the endpoints are both present in the wrong order, + // so nothing about the payload's shape was actually observed. + let reversed = format!("{anchor_tail}{anchor_head}"); + assert_eq!(echo_verdict(&reversed, &expected), EchoVerdict::Incomplete); + for observed in [missing_middle, changed_middle, reversed] { + assert!( + !echo_verdict(&observed, &expected).confirmed(), + "endpoint evidence must stay unconfirmed" + ); + assert!(!check_echo_in_output(&observed, &expected)); + } + } + + /// relay#1893 review, P1: any pasted-text marker verified any delivery. + #[test] + fn a_collapsed_paste_marker_never_confirms_a_delivery() { + let expected = long_envelope(); + for observed in [ + // A full collapse: the harness echoed no content at all. + "[Pasted text #1 +120 lines]".to_string(), + // The reviewer's `paste_tail` reproduction: a partial paste's + // marker plus the payload's last bytes. + format!( + "[Pasted text #1 +3 lines]{}", + &expected[expected.len() - 40..] + ), + // An earlier, smaller paste's marker redrawn during this delivery. + "❯ [Pasted text #1 +2 lines] previous turn\n❯ ".to_string(), + ] { + let verdict = echo_verdict(&observed, &expected); + assert_eq!(verdict, EchoVerdict::PasteSummary); + assert!( + !verdict.confirmed(), + "a paste marker is not a receipt for {observed:?}" + ); + assert!(!check_echo_in_output(&observed, &expected)); + } + } + + /// The runtime sees labels, not verdicts; the two must not drift apart. + #[test] + fn verification_labels_agree_with_the_verdict_ladder() { + for verdict in [ + EchoVerdict::Exact, + EchoVerdict::Normalized, + EchoVerdict::HeadMissing, + EchoVerdict::Incomplete, + EchoVerdict::PasteSummary, + EchoVerdict::Absent, + ] { + assert_eq!( + verification_label_confirms_receipt(verdict.label()), + verdict.confirmed(), + "{:?} label and confirmation disagree", + verdict + ); + } + // An absent or unknown label is never receipt. + assert!(!verification_label_confirms_receipt("")); + assert!(!verification_label_confirms_receipt("echo_anchors")); + } + + #[test] + fn timeout_disposition_fails_only_on_proven_head_loss() { + let window = Duration::from_secs(5); + assert_eq!( + verification_timeout(EchoVerdict::HeadMissing, true, window), + VerificationTimeout::HeadLoss + ); + // A trimmed window cannot prove head loss, so it cannot fail a delivery. + for (verdict, label) in [ + (EchoVerdict::HeadMissing, "echo_incomplete"), + (EchoVerdict::Incomplete, "echo_incomplete"), + (EchoVerdict::PasteSummary, "paste_summary"), + (EchoVerdict::Absent, "timeout_fallback"), + ] { + let retains = verdict != EchoVerdict::HeadMissing; + match verification_timeout(verdict, retains, window) { + VerificationTimeout::Unconfirmed { label: got, reason } => { + assert_eq!(got, label, "{verdict:?}"); + assert!(!reason.is_empty()); + assert!(!verification_label_confirms_receipt(got), "{verdict:?}"); + } + other => panic!("{verdict:?} must not fail a delivery: {other:?}"), + } + } + // A late whole-payload echo keeps its confirmation rather than being + // downgraded to an unverified ack. + for verdict in [EchoVerdict::Exact, EchoVerdict::Normalized] { + assert_eq!( + verification_timeout(verdict, true, window), + VerificationTimeout::Confirmed { + label: verdict.label() + } + ); + } + } + #[test] fn check_echo_clean_text() { let output = "some preamble\nRelay message from Alice [evt_1]: hello world\nmore output"; diff --git a/crates/broker/src/protocol.rs b/crates/broker/src/protocol.rs index 1f95ac4cba..67e88dca50 100644 --- a/crates/broker/src/protocol.rs +++ b/crates/broker/src/protocol.rs @@ -415,8 +415,15 @@ pub enum BrokerEvent { name: WorkerName, delivery_id: DeliveryId, event_id: EventId, - /// "echo" when confirmed in PTY output, "timeout_fallback" when the - /// delivery was acked without echo verification. + /// What the PTY output proved about this delivery. Only "echo" (the + /// whole envelope verbatim) and "echo_normalized" (the whole envelope + /// once the TUI's wrapping whitespace is removed) confirm full + /// receipt. "echo_incomplete" (head and tail without the payload + /// between them), "paste_summary" (the harness collapsed the paste, so + /// no content was echoed) and "timeout_fallback" (no echo at all) are + /// acks without proof of receipt. A tail without its head is a + /// `DeliveryFailed` with reason "echo_head_missing", never a + /// verification label. #[serde(default, skip_serializing_if = "Option::is_none")] verification: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -637,6 +644,10 @@ pub enum WorkerToBroker { DeliveryFailed { delivery_id: DeliveryId, event_id: EventId, + /// Terminal and never replayed. PTY injection adds + /// "prompt_unproven" (no composer was recognized in time), + /// "echo_head_missing" (the echo lost its head) and + /// "injection_too_large: …" (rejected before any byte was written). reason: String, }, WorkerStream { diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index 3dfa4dc5ab..cd4fcae7de 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -714,6 +714,28 @@ fn restore_hold_exemption( } } +/// Wire `verification` label for a delivery the output arm just confirmed. +/// +/// Both output arms confirm through `pending_verification_echo_seen`, which +/// accepts only whole-payload evidence, so re-deriving the label here cannot +/// report anything weaker. The assertion keeps that true if the ladder gains a +/// rung: a label the runtime does not treat as receipt must never be emitted +/// from a path that already acted on confirmation (relay#1893 review). +fn confirmed_verification_label( + output: &crate::broker::delivery_verification::VerificationOutput, + pv: &PendingVerification, +) -> &'static str { + let verdict = crate::broker::delivery_verification::echo_verdict( + &output.since(pv.output_boundary), + &pv.expected_echo, + ); + debug_assert!( + verdict.confirmed(), + "output-arm confirmation emitted a non-confirming verdict: {verdict:?}" + ); + verdict.label() +} + /// Relay command detection must stay disabled for the entire injection lifecycle. /// The paced writer can echo command-like text before its post-write verification /// is queued, so checking only `pending_verifications` leaves a false-positive gap. @@ -1799,7 +1821,10 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { json!({ "delivery_id": delivery_id, "event_id": event_id, - "verification": crate::broker::delivery_verification::echo_verdict(&echo_buffer.since(pv.output_boundary), &pv.expected_echo).label() + // Output-arm confirmation is gated on + // `confirmed()`, so this label can only be + // whole-payload evidence. + "verification": confirmed_verification_label(&echo_buffer, &pv) }), ) .await; @@ -2294,7 +2319,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { json!({ "delivery_id": delivery_id, "event_id": event_id, - "verification": crate::broker::delivery_verification::echo_verdict(&echo_buffer.since(pv.output_boundary), &pv.expected_echo).label() + "verification": confirmed_verification_label(&echo_buffer, &pv) }), ) .await; @@ -2394,22 +2419,41 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { let event_id = pv.event_id.clone(); let verdict = crate::broker::delivery_verification::echo_verdict( &echo_buffer.since(pv.output_boundary), &pv.expected_echo); - if verdict == crate::broker::delivery_verification::EchoVerdict::HeadMissing && echo_buffer.retains_boundary(pv.output_boundary) { - let _ = send_frame(&out_tx, "delivery_failed", None, json!({"delivery_id": delivery_id, "event_id": event_id, "reason": "echo_head_missing"})).await; - let _ = send_frame(&out_tx, "worker_error", pv.request_id, json!({"code": "echo_head_missing", "retryable": false, "message": "Task echo contains its tail without its head; inspect the agent before retrying"})).await; - throttle.record(DeliveryOutcome::Failed); - // Retain the pending id, matching initial_injection_incomplete. - continue; + let disposition = crate::broker::delivery_verification::verification_timeout( + verdict, + echo_buffer.retains_boundary(pv.output_boundary), + verification_window, + ); + let (verification, reason) = match disposition { + crate::broker::delivery_verification::VerificationTimeout::HeadLoss => { + let _ = send_frame(&out_tx, "delivery_failed", None, json!({"delivery_id": delivery_id, "event_id": event_id, "reason": "echo_head_missing"})).await; + let _ = send_frame(&out_tx, "worker_error", pv.request_id, json!({"code": "echo_head_missing", "retryable": false, "message": "Task echo contains its tail without its head; inspect the agent before retrying"})).await; + throttle.record(DeliveryOutcome::Failed); + // Retain the pending id, matching initial_injection_incomplete. + continue; + } + crate::broker::delivery_verification::VerificationTimeout::Confirmed { label } => (label, None), + crate::broker::delivery_verification::VerificationTimeout::Unconfirmed { label, reason } => (label, Some(reason)), + }; + let outcome = match reason { + None => DeliveryOutcome::Success, + // An unconfirmed ack is not a verified delivery: + // keep it out of the throttle's success signal. + Some(_) => DeliveryOutcome::Unverified, + }; + if let Some(reason) = reason.as_deref() { + static TIMEOUT_FALLBACKS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let fallback_count = TIMEOUT_FALLBACKS.fetch_add(1, Ordering::Relaxed) + 1; + tracing::warn!(fallback_count, %delivery_id, %verification, %reason, "delivery receipt unconfirmed; acking without claiming verification"); } - static TIMEOUT_FALLBACKS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - let fallback_count = TIMEOUT_FALLBACKS.fetch_add(1, Ordering::Relaxed) + 1; - tracing::warn!(fallback_count, %delivery_id, "delivery echo absent; preserving unverified timeout fallback"); // Do not re-inject on verification timeout. Re-injection can duplicate // already-delivered messages when terminal echo parsing is noisy. tracing::info!( delivery_id = %delivery_id, attempts = pv.attempts, - "delivery echo not detected within verification window; acknowledging via timeout fallback (unverified)" + %verification, + confirmed = reason.is_none(), + "verification window closed; acknowledging the delivery" ); let _ = send_frame( &out_tx, @@ -2421,21 +2465,22 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { }), ) .await; + let mut verified = json!({ + "delivery_id": delivery_id, + "event_id": event_id, + "verification": verification, + }); + if let Some(reason) = reason { + verified["reason"] = json!(reason); + } let _ = send_frame( &out_tx, "delivery_verified", pv.request_id.clone(), - json!({ - "delivery_id": delivery_id, - "event_id": event_id, - "verification": "timeout_fallback", - "reason": format!("echo not detected within {}s window", verification_window.as_secs()) - }), + verified, ) .await; - // Timeout-fallback acks are not verified deliveries: - // keep them out of the throttle's success signal. - throttle.record(DeliveryOutcome::Unverified); + throttle.record(outcome); pending_worker_delivery_ids.remove(&delivery_id); completed_worker_deliveries.insert(delivery_id, event_id); } else { diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index 03bbb18a1e..7526b45e03 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -181,6 +181,33 @@ pub(super) fn verified_spawn_ready_result( } } +/// A live, ready agent whose initial task was never confirmed as fully +/// received. +/// +/// This is deliberately neither a success nor an ordinary failure. The spawn +/// did happen — the agent is registered, ready and holding the node's only +/// slot for that name — so the caller must not retry, or it duplicates the +/// agent. But the task it was spawned to run cannot be shown to have arrived +/// intact, and `spawned:true, ready:true` is exactly the proof `fleet spawn` +/// uses to report success, so the only honest answer is an error that names +/// the live agent and says what to do instead (relay#1893 review, P1). +pub(super) fn verified_spawn_task_unconfirmed_result( + invocation_id: String, + name: &WorkerName, + verification: Option<&str>, +) -> ActionResult { + verified_spawn_failed_result( + invocation_id, + &format!( + "spawn_task_unconfirmed: agent '{name}' is live and ready, but full receipt of its \ + initial task was not confirmed (verification: {}). Do not retry this spawn — it \ + would duplicate the agent. Resend the task, or write the brief to a file and send a \ + short pointer, then check the agent's transcript.", + verification.unwrap_or("none") + ), + ) +} + pub(super) fn verified_spawn_failed_result(invocation_id: String, error: &str) -> ActionResult { ActionResult { task: None, @@ -3308,8 +3335,10 @@ mod tests { assert!(!pending.matches_task(Uuid::new_v4(), "init_test")); assert!(!pending.matches_task(generation, "another_message")); assert!(pending.matches_task(generation, "init_test")); - // Exact echo, anchors, paste summary and compatibility fallback all retire - // the matching task; failed deliveries resolve the action as failure. + // The matching task retires this entry whatever its verdict; whether + // that resolves the action as success or as `spawn_task_unconfirmed` + // is `worker_events`' receipt gate, covered by + // `a_spawn_task_acked_without_proof_of_receipt_never_reports_success`. pending.task_event_id = None; assert!(pending.can_report_ready(generation)); pending.readiness_proven = false; diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index 6d5e66376a..a24a368b0e 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -847,6 +847,159 @@ fn delivery_lifecycle_worker_event( } } +fn delivery_verified_worker_event( + name: &str, + generation: Uuid, + event_id: &str, + verification: Option<&str>, +) -> WorkerEvent { + let mut payload = json!({ + "delivery_id": format!("del_{event_id}"), + "event_id": event_id, + }); + if let Some(verification) = verification { + payload["verification"] = json!(verification); + } + WorkerEvent::Message { + name: WorkerName::from(name), + generation, + value: json!({ "type": "delivery_verified", "payload": payload }), + } +} + +/// relay#1893 review, P1: the spawn-completion branch checked readiness and the +/// task's event id, then reported `spawned:true, ready:true` without ever +/// looking at *what* confirmed the delivery. A worker that acked through the +/// timeout fallback — i.e. an agent that may have swallowed the whole task — +/// produced a result identical to one that echoed every byte. +#[tokio::test] +async fn a_spawn_task_acked_without_proof_of_receipt_never_reports_success() { + use crate::fleet_wire::{ActionResultPayload, BrokerToRelaycast}; + + // Every label a worker can ack with that is not whole-payload evidence. + for verification in [ + Some("timeout_fallback"), + Some("paste_summary"), + Some("echo_incomplete"), + // A worker frame that omits the field entirely claims nothing. + None, + ] { + let worker_name = "spawn-receipt"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let mut fixture = worker_event_runtime_fixture(registry, HashMap::new()); + let event_id = "init_receipt"; + fixture.runtime.pending_verified_spawns.insert( + WorkerName::from(worker_name), + super::fleet::PendingVerifiedSpawn { + invocation_id: "inv-receipt".to_string(), + deadline: Instant::now() + Duration::from_secs(90), + started: Instant::now(), + generation, + readiness_proven: true, + task_event_id: Some(event_id.to_string()), + }, + ); + + fixture + .runtime + .handle_worker_event(delivery_verified_worker_event( + worker_name, + generation, + event_id, + verification, + )) + .await; + + let frame = fixture + .fleet_control_rx + .try_recv() + .expect("an unconfirmed task must still resolve the open spawn action"); + let FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) = frame else { + panic!("spawn completion must send an action result"); + }; + assert_eq!(result.invocation_id, "inv-receipt"); + let ActionResultPayload::Error(error) = result.result else { + panic!("{verification:?} is not proof of receipt and must not succeed"); + }; + assert!( + error.error.starts_with("spawn_task_unconfirmed: "), + "{}", + error.error + ); + // The agent is live: name it so the caller can reach it, and say not to + // retry, because a retry duplicates it. + assert!(error.error.contains(worker_name), "{}", error.error); + assert!(error.error.contains("Do not retry"), "{}", error.error); + assert!( + error.error.contains(verification.unwrap_or("none")), + "{}", + error.error + ); + // Resolved, so maintenance's readiness deadline can no longer release + // the live worker this spawn produced. + assert!(!fixture + .runtime + .pending_verified_spawns + .contains_key(&WorkerName::from(worker_name))); + } +} + +/// The other half of the gate: whole-payload evidence still succeeds, and the +/// labels that do so are exactly the ones the verdict ladder confirms. +#[tokio::test] +async fn a_spawn_task_proven_received_reports_spawned_and_ready() { + use crate::fleet_wire::{ActionResultPayload, BrokerToRelaycast}; + + for verification in ["echo", "echo_normalized"] { + assert!( + crate::broker::delivery_verification::verification_label_confirms_receipt(verification), + "{verification} must be a confirming label" + ); + let worker_name = "spawn-receipt-ok"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let mut fixture = worker_event_runtime_fixture(registry, HashMap::new()); + let event_id = "init_receipt_ok"; + fixture.runtime.pending_verified_spawns.insert( + WorkerName::from(worker_name), + super::fleet::PendingVerifiedSpawn { + invocation_id: "inv-receipt-ok".to_string(), + deadline: Instant::now() + Duration::from_secs(90), + started: Instant::now(), + generation, + readiness_proven: true, + task_event_id: Some(event_id.to_string()), + }, + ); + + fixture + .runtime + .handle_worker_event(delivery_verified_worker_event( + worker_name, + generation, + event_id, + Some(verification), + )) + .await; + + let frame = fixture + .fleet_control_rx + .try_recv() + .expect("a confirmed task must resolve the open spawn action"); + let FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) = frame else { + panic!("spawn completion must send an action result"); + }; + let ActionResultPayload::Output(output) = result.result else { + panic!("{verification} proves receipt and must succeed"); + }; + assert_eq!( + output.output, + json!({"spawned": true, "ready": true, "name": worker_name}) + ); + } +} + fn inbound_ctx<'a>(event_id: &'a str) -> InboundContext<'a> { InboundContext { from: "Alice", diff --git a/crates/broker/src/runtime/worker_events.rs b/crates/broker/src/runtime/worker_events.rs index 5d8191f89c..01ff3fc775 100644 --- a/crates/broker/src/runtime/worker_events.rs +++ b/crates/broker/src/runtime/worker_events.rs @@ -869,13 +869,46 @@ impl BrokerRuntime { .get("event_id") .and_then(Value::as_str) .unwrap_or(""); + // "echo" / "echo_normalized" when the whole payload + // was observed in PTY output. Every other label — + // "echo_incomplete", "paste_summary", + // "timeout_fallback" — is an ack without proof of + // receipt. `verification_label_confirms_receipt` is + // the single source of truth for which is which. + let verification = payload.get("verification").and_then(Value::as_str); + let reason = payload.get("reason").and_then(Value::as_str); + let receipt_confirmed = verification.is_some_and( + crate::broker::delivery_verification::verification_label_confirms_receipt, + ); if pending_verified_spawns.get(&name).is_some_and(|pending| { pending.readiness_proven && pending.matches_task(generation, event_id) }) { if let Some(pending) = pending_verified_spawns.remove(&name) { - let result = - verified_spawn_ready_result(pending.invocation_id, &name); + // A spawn carrying a task succeeds only when + // that task is proven received. An ack the + // worker could not verify leaves the agent + // live, so the action resolves as explicitly + // unconfirmed rather than as either a success + // or a retryable failure. + let result = if receipt_confirmed { + verified_spawn_ready_result(pending.invocation_id, &name) + } else { + tracing::warn!( + target = "agent_relay::broker", + worker = %name, + delivery_id = %delivery_id, + event_id = %event_id, + verification = verification.unwrap_or("none"), + "verified spawn's initial task was acked without \ + proof of full receipt; reporting it unconfirmed" + ); + super::fleet::verified_spawn_task_unconfirmed_result( + pending.invocation_id, + &name, + verification, + ) + }; let _ = fleet_control_tx .send(FleetControlCommand::Send( crate::fleet_wire::BrokerToRelaycast::ActionResult( @@ -886,22 +919,15 @@ impl BrokerRuntime { } } - // "echo" when the injection was confirmed in PTY - // output; "timeout_fallback" when the worker acked - // without ever seeing the echo. - let verification = payload - .get("verification") - .and_then(Value::as_str) - .unwrap_or("echo"); - let reason = payload.get("reason").and_then(Value::as_str); - if verification == "timeout_fallback" { + if !receipt_confirmed { tracing::info!( target = "agent_relay::broker", worker = %name, delivery_id = %delivery_id, event_id = %event_id, + verification = verification.unwrap_or("none"), reason = reason.unwrap_or(""), - "delivery acked via timeout fallback — echo never verified" + "delivery acked without confirmed receipt" ); } else { tracing::debug!( @@ -924,7 +950,7 @@ impl BrokerRuntime { "name": name, "delivery_id": delivery_id, "event_id": event_id, - "verification": verification, + "verification": verification.unwrap_or("echo"), }); if let (Some(reason), Some(map)) = (reason, verified_event.as_object_mut()) diff --git a/crates/broker/src/wrap.rs b/crates/broker/src/wrap.rs index 56d4cadd58..c71441f71e 100644 --- a/crates/broker/src/wrap.rs +++ b/crates/broker/src/wrap.rs @@ -276,14 +276,18 @@ fn prepare_wrap_retry( } // Wrap has no delivery protocol. Preserve its existing failed-throttle timeout -// policy, including absent echoes; fleet alone uses the compatibility fallback. +// policy, including absent echoes and evidence of loss. A collapsed paste is +// the one unconfirmed verdict that is not evidence of anything going wrong — +// the harness simply never echoes content — so it must not drive the injection +// delay up on every delivery to a paste-collapsing TUI. fn wrap_timeout_outcome( verdict: crate::broker::delivery_verification::EchoVerdict, ) -> DeliveryOutcome { - if verdict.confirmed() { - DeliveryOutcome::Success - } else { - DeliveryOutcome::Failed + use crate::broker::delivery_verification::EchoVerdict; + match verdict { + _ if verdict.confirmed() => DeliveryOutcome::Success, + EchoVerdict::PasteSummary => DeliveryOutcome::Unverified, + _ => DeliveryOutcome::Failed, } } @@ -2747,12 +2751,26 @@ sys.stdout.flush()"#; #[test] fn wrap_timeout_stays_failed_without_confirming_evidence() { use crate::broker::delivery_verification::{DeliveryOutcome, EchoVerdict}; - for verdict in [EchoVerdict::Absent, EchoVerdict::HeadMissing] { + for verdict in [ + EchoVerdict::Absent, + EchoVerdict::HeadMissing, + EchoVerdict::Incomplete, + ] { assert!(matches!( super::wrap_timeout_outcome(verdict), DeliveryOutcome::Failed )); } + // Neither a success (nothing was verified) nor a failure (nothing + // indicates loss) — exactly what `Unverified` exists for. + assert!(matches!( + super::wrap_timeout_outcome(EchoVerdict::PasteSummary), + DeliveryOutcome::Unverified + )); + assert!(matches!( + super::wrap_timeout_outcome(EchoVerdict::Normalized), + DeliveryOutcome::Success + )); // Wrap always uses bulk writes, including its Typed fallback. Its // write acknowledgement budget must cover the delayed submit only. assert!( diff --git a/crates/broker/tests/pty_injection_integrity.rs b/crates/broker/tests/pty_injection_integrity.rs index 69cfa03d93..a046e43168 100644 --- a/crates/broker/tests/pty_injection_integrity.rs +++ b/crates/broker/tests/pty_injection_integrity.rs @@ -44,6 +44,8 @@ while True: if data.endswith(b'\r'): body=data.replace(b'\x1b[200~',b'').replace(b'\x1b[201~',b'').rstrip(b'\r') if mode == 'tail': body=body[-200:] + if mode == 'middle_lost': body=body[:body.index(b'Relay message from ')+300]+body[-300:] + if mode == 'paste_tail': body=b'[Pasted text #1 +3 lines]'+body[-40:] if mode != 'silent': os.write(1,b'\r\n'+body+b'\r\n') os.write(1,'\r\n❯ '.encode()) data=b'' @@ -117,7 +119,13 @@ fn long_pair() { let before = f.transcript().len(); f.deliver(id, &body); let verified = f.wait("delivery_verified"); - assert_ne!(verified["payload"]["verification"], "timeout_fallback"); + // Whole-payload evidence, which is the only verdict a verified fleet + // spawn accepts as receipt of its task. + assert!( + ["echo", "echo_normalized"] + .contains(&verified["payload"]["verification"].as_str().unwrap()), + "{verified}" + ); let transcript = f.transcript(); let bytes = &transcript[before..]; assert!(bytes.windows(body.len()).any(|w| w == body.as_bytes())); @@ -179,6 +187,38 @@ fn absent_echo_preserves_timeout_fallback() { "timeout_fallback" ); } +/// relay#1893 review, P1 x2: a tail-preserving echo was certified by matching +/// endpoints, and a collapsed-paste marker was certified by its mere presence. +/// Neither observes the payload, so neither may report a confirmed delivery. +#[test] +fn partial_echoes_are_acked_without_claiming_verification() { + for (mode, expected) in [ + // Head and tail echoed, ~10 KB of the middle gone. + ("middle_lost", "echo_incomplete"), + // A partial paste's marker plus the body's last 40 bytes. + ("paste_tail", "paste_summary"), + ] { + let mut f = Fixture::new(mode); + f.deliver( + "init_partial", + &format!("HEAD{}TAIL", "important task content ".repeat(450)), + ); + let verified = f.wait("delivery_verified"); + assert_eq!( + verified["payload"]["verification"], expected, + "{mode} must report what was actually observed" + ); + // The labels a verified fleet spawn accepts as receipt. + for confirming in ["echo", "echo_normalized"] { + assert_ne!( + verified["payload"]["verification"], confirming, + "{mode} must never confirm full receipt" + ); + } + assert!(verified["payload"]["reason"].is_string(), "{mode}"); + } +} + #[test] #[ignore = "20 cold starts; run separately"] fn twenty_cold_starts() { diff --git a/crates/relay-pty/src/pty.rs b/crates/relay-pty/src/pty.rs index 916da21f83..620dbe76e7 100644 --- a/crates/relay-pty/src/pty.rs +++ b/crates/relay-pty/src/pty.rs @@ -928,6 +928,12 @@ fn read_was_cancelled(error: &io::Error) -> bool { error.raw_os_error() == Some(ERROR_OPERATION_ABORTED as i32) } +/// Whether the grid currently has bracketed-paste mode (DECSET 2004) set. +fn paste_mode_enabled(term: &Term) -> bool { + term.mode() + .contains(alacritty_terminal::term::TermMode::BRACKETED_PASTE) +} + impl PtySession { fn enqueue_write_with_output_boundary( &self, @@ -1133,18 +1139,30 @@ impl PtySession { // writer lock taken by the drainer thread. let mut processor_guard = processor_clone.lock(); let mut term_guard = term_clone.lock(); - // Observe every mode transition, including enable/disable in one read. + // Observe every mode transition, including an enable and a + // disable inside one read. Stepping the parser one byte at a + // time is the only way to see a transition that does not + // survive to the end of the chunk, but it costs the reader + // thread a parser call per byte, so it is reserved for chunks + // that can actually carry the DECSET 2004 parameter. Every + // other chunk is parsed in bulk and then checked once, which + // still latches a `2004h` split across reads. if paste_capable_reader.load(Ordering::Acquire) { processor_guard.advance(&mut *term_guard, bytes); } else { - for byte in bytes { - processor_guard.advance(&mut *term_guard, std::slice::from_ref(byte)); - if term_guard - .mode() - .contains(alacritty_terminal::term::TermMode::BRACKETED_PASTE) - { - paste_capable_reader.store(true, Ordering::Release); + if bytes.windows(4).any(|window| window == b"2004") { + for byte in bytes { + processor_guard + .advance(&mut *term_guard, std::slice::from_ref(byte)); + if paste_mode_enabled(&term_guard) { + paste_capable_reader.store(true, Ordering::Release); + } } + } else { + processor_guard.advance(&mut *term_guard, bytes); + } + if paste_mode_enabled(&term_guard) { + paste_capable_reader.store(true, Ordering::Release); } } // Publish the grid's consumed byte offset while @@ -1468,15 +1486,19 @@ impl PtySession { /// with a grid render. /// /// [`with_term_and_offset`]: PtySession::with_term_and_offset - /// Sticky capability, independent of whether the composer is currently ready. - pub fn bracketed_paste_enabled(&self) -> bool { - self.paste_capable.load(Ordering::Acquire) - } - pub fn consumed_offset(&self) -> u64 { self.consumed_offset.load(Ordering::Acquire) } + /// Whether the child has ever enabled bracketed-paste mode (DECSET 2004). + /// + /// Sticky: a TUI that turns the mode off while a dialog is open still + /// understands `ESC[200~` framing, so the capability is independent of + /// whether the composer is currently ready. + pub fn bracketed_paste_enabled(&self) -> bool { + self.paste_capable.load(Ordering::Acquire) + } + /// Check if the child process has exited without blocking. /// Returns true if the child has exited (or was already reaped). /// @@ -1860,6 +1882,35 @@ mod tests { drain.abort(); } + /// Only a chunk carrying the `2004` parameter is parsed byte-by-byte, so a + /// DECSET split across reads must still latch from the bulk-parse path. + #[cfg(unix)] + #[tokio::test] + async fn paste_capability_latches_when_decset_spans_two_reads() { + let (pty, mut rx) = PtySession::spawn( + "sh", + &[ + "-c".into(), + "printf '\\033[?20'; sleep 0.3; printf '04hREADY'; sleep 2".into(), + ], + 24, + 80, + ) + .unwrap(); + let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); + // `READY` trails the DECSET, so the mode was parsed by the time it renders. + tokio::time::timeout(Duration::from_secs(3), async { + while !pty.screen_text().contains("READY") { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!(pty.bracketed_paste_enabled()); + pty.shutdown().unwrap(); + drain.abort(); + } + #[tokio::test] async fn consumed_offset_tracks_total_bytes_parsed_into_grid() { let (pty, mut rx) = PtySession::spawn("echo", &["offset-line".into()], 24, 80).unwrap(); diff --git a/docs/harnesses/injection.md b/docs/harnesses/injection.md index 9a10066e4c..bb7a6fa108 100644 --- a/docs/harnesses/injection.md +++ b/docs/harnesses/injection.md @@ -35,14 +35,24 @@ or bypass size limits. Fleet requires exactly one of `--task` and `--task-file`; local `agent spawn` and `node agent spawn` allow neither. The integration command's existing task option remains unchanged. -Verification recognizes exact echoes, whitespace-normalized head/tail anchors, -and paste summaries. A tail without its head fails with `echo_head_missing`; the -body is never automatically replayed. If the observation buffer has discarded the -head, that is not treated as proof of truncation. No echo retains the existing -`timeout_fallback` behavior, with a warning and process-local fallback counter. -This compatibility verdict **does not prove byte-for-byte receipt**. +Only a whole-payload echo confirms receipt: `echo` (verbatim) or +`echo_normalized` (verbatim once the TUI's wrapping whitespace is removed from +both sides). A tail without its head fails with `echo_head_missing`; the body is +never automatically replayed. If the observation buffer has discarded the head, +that is not treated as proof of truncation. + +Every other verdict is an ack that **does not prove byte-for-byte receipt**, and +each says what was actually observed: `echo_incomplete` (head and tail without +the payload between them — matching endpoints say nothing about the bytes +between them), `paste_summary` (the harness collapsed the paste, so no content +was echoed at all) and `timeout_fallback` (no echo), the last with a warning and +a process-local fallback counter. Verified fleet PTY spawns with a task wait for its delivery verdict after proven -startup readiness. A failed task fails the spawn action; timeout fallback still -counts as success for compatibility. Wrap uses the same echo evidence but keeps -its existing failed-throttle policy on absent echoes. +startup readiness. A failed task fails the spawn action. A task acked without +confirmed receipt resolves the action as `spawn_task_unconfirmed`, naming the +live agent: the spawn must not be retried (that duplicates the agent), so resend +the task or point the agent at a brief file. Wrap uses the same echo evidence and +keeps its failed-throttle policy on absent echoes and on evidence of loss; a +collapsed paste is recorded as unverified instead, since it indicates nothing +about loss. diff --git a/packages/harness-driver/src/protocol.ts b/packages/harness-driver/src/protocol.ts index 1e524c9c75..c27d02c359 100644 --- a/packages/harness-driver/src/protocol.ts +++ b/packages/harness-driver/src/protocol.ts @@ -613,7 +613,16 @@ export type BrokerEvent = name: string; delivery_id: string; event_id: string; - /** 'echo' when confirmed in PTY output, 'timeout_fallback' when acked unverified. */ + /** + * What the PTY output proved about this delivery. Only 'echo' (the whole + * envelope verbatim) and 'echo_normalized' (the whole envelope once the + * TUI's wrapping whitespace is removed) confirm full receipt. + * 'echo_incomplete' (head and tail without the payload between them), + * 'paste_summary' (the harness collapsed the paste, so no content was + * echoed) and 'timeout_fallback' (no echo at all) are acks without proof + * of receipt. A tail without its head is a 'delivery_failed' with reason + * 'echo_head_missing'. + */ verification?: string; reason?: string; } @@ -838,10 +847,17 @@ export type WorkerToBroker = } | { type: 'delivery_verified'; + /** See the `delivery_verified` broker event for `verification` values. */ payload: { delivery_id: string; event_id: string; verification?: string; reason?: string }; } | { type: 'delivery_failed'; + /** + * Terminal and never replayed. PTY injection adds 'prompt_unproven' (no + * composer was recognized in time), 'echo_head_missing' (the echo lost + * its head) and 'injection_too_large: …' (rejected before any byte was + * written). + */ payload: { delivery_id: string; event_id: string; reason: string }; } | { diff --git a/summary.md b/summary.md deleted file mode 100644 index 21d0e86cb0..0000000000 --- a/summary.md +++ /dev/null @@ -1,79 +0,0 @@ -# PTY injection integrity and fleet task confirmation - -Long fleet tasks and relay messages were typed into Claude one character every -5 ms. A 6.7 KB brief occupied the PTY write drainer for roughly 33 seconds, delaying -terminal-query replies and startup auto-responses while the composer could change. -The existing verification timeout acknowledged even a tail-only echo. - -This change sends one bracketed paste to harnesses that have advertised DECSET -2004, followed by the existing delayed Enter. The capability is latched across -mode resets; composer readiness is checked independently before writing. A missing -composer, oversized body, or tail-only echo produces an explicit non-retryable -failure. Failed deliveries retain their identity and are never replayed or -subsequently acknowledged as successful duplicates. CRLF becomes LF; bare CR and -ESC cannot submit or terminate the body early. - -The verification ladder is shared with wrap: exact echo, normalized head/tail -anchors, then paste summary. Missing heads only count as loss when the observation -window is retained. Absent echoes preserve `timeout_fallback`, now with a warning -and process-local counter. Wrap retains its existing failed-throttle timeout policy. - -Verified fleet PTY spawns with a task remain pending after readiness until the -matching task verdict arrives; explicit task failures fail the spawn action. -Native and argv-task startup paths retain their existing readiness contract. -`fleet spawn`, `agent spawn`, and `node agent spawn` accept `--task-file`, read on -the requester and forwarded through existing task transport. Fleet requires -exactly one task source. Message commands and task input validate UTF-8 byte size. - -Limits are 16 KiB globally, including the formatted envelope at the PTY boundary, -and 1,536 bytes for the default paced fallback. A slower explicit pace lowers that -cap. Codex retains its chunked initial-task path and uses the smaller derived cap. -The ordinary typed-path budget closes inside 90 seconds: -60 s startup + 15 s prompt recheck + 7.68 s typing + 0.25 s submit + 5 s verification -= 87.93 s. Steer acknowledgements include the injection budget. - -## Validation - -- Full broker and relay-pty run: **1,627 passed, 0 failed, 7 ignored**. -- Integrity integration suite, including the ignored stress test: **6 passed**. - Twenty cold starts each receive a >10 KB task and a >10 KB follow-up message, - verified byte-for-byte in the fake harness's input transcript with one paste - pair and one submit. Other arms cover tail loss/no replay, explicit size - rejection, disappearing composer, and absent-echo compatibility. -- CLI regression suites: **188 passed**; CLI TypeScript checking passed. -- `cargo fmt --all --check` and `cargo clippy --all-targets` passed. Clippy reports - five pre-existing sliced-string warnings in Codex tests. -- Live Claude startup probe observed DECSET 2004; no disable appeared during the - six-second observation. Parser regression covers enable and disable in one read. - -Commands (debug symbols disabled to fit the workspace disk): - -```sh -env -u GIT_CONFIG_COUNT CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 CARGO_INCREMENTAL=0 cargo test -p agent-relay-broker -p relay-pty -CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 CARGO_INCREMENTAL=0 cargo test -p agent-relay-broker --test pty_injection_integrity -- --include-ignored -CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 CARGO_INCREMENTAL=0 cargo clippy --all-targets -cargo fmt --all --check -npx vitest run packages/cli/src/cli/lib/task-input.test.ts packages/cli/src/cli/commands/fleet.test.ts packages/cli/src/cli/commands/local-agent.test.ts packages/cli/src/cli/commands/relaycast-groups.test.ts -npx tsc --noEmit -p packages/cli/tsconfig.json -``` - -This workspace injects `core.hooksPath=/dev/null` through `GIT_CONFIG_COUNT`. -Removing that setting only for the Rust test process lets the existing hook-chain -regressions exercise their own temporary repositories; all four then pass. - -## Compatibility and remaining work - -Per `reviewed-plan.md`, absent-echo fallback still counts as spawn success. It is -**not cryptographic or byte-for-byte receipt confirmation**; the stronger ticket -acceptance needs a harness receipt protocol. The 20-run proof uses a raw-mode fake -Claude TUI through the real broker, not 20 authenticated Claude transcripts. - -Wrap already bulk-wrote Claude input before this change. Fleet gains that behavior -plus explicit paste framing and readiness checks, rather than simply reverting -the 10.4.0 pacing fix. Large Claude inputs can now appear as pasted-content blocks; -the attribution envelope is unchanged. - -Duplicate desktop/broker registrations remain a separate investigation requiring -the reporter's ambient MCP configuration. A node-side brief-file writer and -integration-command `--task-file` support are follow-ups. No workflow files were -changed, and no release or merge was performed. From 5c8b79826b0ddacc28d61704c1f6f9c3d793339c Mon Sep 17 00:00:00 2001 From: Relayflow Date: Sun, 4 Oct 2026 14:46:41 +0000 Subject: [PATCH 05/42] fix(fleet): report pending spawns and wait for name clearance --- .../active/traj_ro3v7jmydpo5/trajectory.json | 82 +++++++ CHANGELOG.md | 17 +- crates/broker/src/runtime/relaycast_events.rs | 17 +- packages/cli/README.md | 17 ++ packages/cli/src/cli/commands/agent.test.ts | 34 +++ packages/cli/src/cli/commands/agent.ts | 40 ++-- packages/cli/src/cli/commands/fleet.test.ts | 146 +++++++++++- packages/cli/src/cli/commands/fleet.ts | 160 ++++++++++--- .../cli/src/cli/lib/agent-registration.ts | 4 +- .../cli/src/cli/lib/agent-removal.test.ts | 65 ++++++ packages/cli/src/cli/lib/agent-removal.ts | 81 +++++++ packages/cli/src/cli/lib/fleet-live-agents.ts | 7 + packages/cli/src/cli/lib/sdk-command.test.ts | 19 ++ packages/cli/src/cli/lib/sdk-command.ts | 10 +- .../cli/src/cli/lib/spawn-liveness.test.ts | 133 +++++++++++ packages/cli/src/cli/lib/spawn-liveness.ts | 211 ++++++++++++++++++ 16 files changed, 980 insertions(+), 63 deletions(-) create mode 100644 .agentworkforce/trajectories/fleet-spawn-fix/active/traj_ro3v7jmydpo5/trajectory.json create mode 100644 packages/cli/src/cli/lib/agent-removal.test.ts create mode 100644 packages/cli/src/cli/lib/agent-removal.ts create mode 100644 packages/cli/src/cli/lib/spawn-liveness.test.ts create mode 100644 packages/cli/src/cli/lib/spawn-liveness.ts diff --git a/.agentworkforce/trajectories/fleet-spawn-fix/active/traj_ro3v7jmydpo5/trajectory.json b/.agentworkforce/trajectories/fleet-spawn-fix/active/traj_ro3v7jmydpo5/trajectory.json new file mode 100644 index 0000000000..bb498656c8 --- /dev/null +++ b/.agentworkforce/trajectories/fleet-spawn-fix/active/traj_ro3v7jmydpo5/trajectory.json @@ -0,0 +1,82 @@ +{ + "id": "traj_ro3v7jmydpo5", + "version": 1, + "task": { + "title": "Fix fleet spawn pending outcomes and removal waits" + }, + "status": "active", + "startedAt": "2026-10-04T14:42:43.542Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-10-04T14:42:55.330Z" + } + ], + "chapters": [ + { + "id": "chap_349pedkimqde", + "title": "Work", + "agentName": "default", + "startedAt": "2026-10-04T14:42:55.330Z", + "events": [ + { + "ts": 1791124975331, + "type": "decision", + "content": "Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes: Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes", + "raw": { + "question": "Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes", + "chosen": "Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes", + "alternatives": [], + "reasoning": "Preserve SDK behavior and sandbox safety; uncertain accepted spawns exit 8 with diagnostics." + }, + "significance": "high" + }, + { + "ts": 1791124975634, + "type": "decision", + "content": "Ship removal waits opt-in; use get not-found and deadline list absence, never released status: Ship removal waits opt-in; use get not-found and deadline list absence, never released status", + "raw": { + "question": "Ship removal waits opt-in; use get not-found and deadline list absence, never released status", + "chosen": "Ship removal waits opt-in; use get not-found and deadline list absence, never released status", + "alternatives": [], + "reasoning": "No workspace credential environment variables or machine-global agent-relay configuration are available for live deletion verification. Reviewed-plan section 3.7 requires opt-in in this case. Exit 8 requires observed presence; all-read failures keep the asynchronous acknowledgement." + }, + "significance": "high" + }, + { + "ts": 1791125201843, + "type": "decision", + "content": "Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance: Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance", + "raw": { + "question": "Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance", + "chosen": "Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance", + "alternatives": [], + "reasoning": "Cleanup is already scheduled for owned failed spawns. A collision never grants ownership of the incumbent identity. Fleet release adds removal evidence to its existing JSON only with --wait." + }, + "significance": "high" + }, + { + "ts": 1791125587977, + "type": "decision", + "content": "Final invocation reads preserve the actual requested readiness contract: Final invocation reads preserve the actual requested readiness contract", + "raw": { + "question": "Final invocation reads preserve the actual requested readiness contract", + "chosen": "Final invocation reads preserve the actual requested readiness contract", + "alternatives": [], + "reasoning": "SDK confirm:true also requests verify_ready by default. A late terminal result must have ready:true; live heartbeat recovery remains explicitly confirmed:false. Late persona child acknowledgements stay pending and retain their child invocation/node." + }, + "significance": "high" + } + ] + } + ], + "commits": [], + "filesChanged": [], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "3cfc0fd6cbc68c45a8dd455d7917a3c223ad74fd", + "endRef": "3cfc0fd6cbc68c45a8dd455d7917a3c223ad74fd" + } +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f29516327..4403e3eabd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,22 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Minor] + +### Added + +- `agent remove --wait` and `fleet release --delete-agent --wait` verify registration clearance before name reuse, with `--wait-timeout` and `--no-wait` controls. Fleet release adds `removal` evidence when waiting. +- `fleet agent list` JSON includes node heartbeat timestamps and ages, with warnings for stale snapshots. + +### Changed + +- Unconfirmed `fleet spawn` results report `spawn_pending` with invocation diagnostics and exit 8. A fresh target-node heartbeat can return exit 0 with `confirmed:false`, extending the existing accepted-placement shape without claiming harness readiness. +- Removal waits exit 8 if the registration remains present; scripts must handle nonzero exits beyond `$? -eq 1`. Waits remain opt-in, and unavailable verification reads preserve the asynchronous acknowledgement. + +### Fixed + +- Fleet registration collisions use readable `spawn_name_taken` guidance covering asynchronous removal and failed-spawn cleanup, instead of Rust `Fatal(AlreadyExists {...})` text. +- Pending fleet spawns preserve provisioned sandboxes and distinguish live, stale, elsewhere, registered, absent, and unknown evidence before advising a retry. ## [13.1.1] - 2026-10-04 diff --git a/crates/broker/src/runtime/relaycast_events.rs b/crates/broker/src/runtime/relaycast_events.rs index 64c0050971..9cf1e2c776 100644 --- a/crates/broker/src/runtime/relaycast_events.rs +++ b/crates/broker/src/runtime/relaycast_events.rs @@ -802,7 +802,9 @@ pub(super) async fn spawn_worker_from_request( } Some(token) } - Err(error) => anyhow::bail!("WS spawn registration failed: {error:?}"), + Err(RegRetryOutcome::RetryableExhausted(error) | RegRetryOutcome::Fatal(error)) => { + anyhow::bail!("{}", format_worker_preregistration_error(&name, &error)); + } } } } @@ -984,6 +986,19 @@ pub(super) async fn spawn_worker_from_request( #[cfg(test)] mod tests { use super::*; + + #[test] + fn duplicate_registration_keeps_already_exists_for_debug_log_classification() { + let error = ::relaycast::AgentRegistrationError::AlreadyExists { + agent_name: "worker-a".to_string(), + }; + let message = format_worker_preregistration_error("worker-a", &error); + assert!(message.contains("already exists"), "{message}"); + assert!(message.contains("worker-a")); + assert!(!message.contains("Fatal(")); + assert!(!message.contains("AlreadyExists")); + } + use crate::terminal_control::TerminalToCloud; use ::relaycast::WsEvent; diff --git a/packages/cli/README.md b/packages/cli/README.md index afa86e8e9c..29c8fb8b73 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -573,3 +573,20 @@ Hosted equivalents live under `agent-relay cloud …`. - `@agent-relay/sdk`: messaging, delivery contracts, and actions. - `@agent-relay/harness-driver`: optional managed harness runtime. - `agent-relay`: CLI and MCP entry point. + +### Pending fleet spawns and name reuse + +An accepted spawn whose result is still unknown returns `spawn_pending` (exit 8), +with its invocation ID, dispatch state, and available liveness evidence. Check +`agent-relay fleet agent list --node ` before retrying. A fresh heartbeat +showing the worker on the requested node can return `confirmed:false` with exit 0; +this proves observed worker presence, not harness readiness. Heartbeat snapshots +can change between listings; JSON output includes their timestamps and ages. + +If no worker is running and you intend to reclaim the name, run +`agent-relay agent remove --wait` before respawning. Waiting is opt-in; +`--wait-timeout ` defaults to 30000, followed by one bounded membership read. +`fleet release --delete-agent --wait` offers the same check and adds +`removal` evidence to its JSON. A remaining registration exits 8; unavailable +verification reads retain the asynchronous acknowledgement. A name collision +can also mean cleanup from a previous unsuccessful spawn is still in flight. diff --git a/packages/cli/src/cli/commands/agent.test.ts b/packages/cli/src/cli/commands/agent.test.ts index 775d576da8..0d67f55e97 100644 --- a/packages/cli/src/cli/commands/agent.test.ts +++ b/packages/cli/src/cli/commands/agent.test.ts @@ -287,6 +287,40 @@ describe('agent identity lifecycle commands', () => { expect(rendered).not.toContain('Removed agent'); }); + it('waits for not-found even when the release acknowledgement says completed', async () => { + const { program, workspaceRelay, log } = createHarness(); + workspaceRelay.agents.get.mockRejectedValueOnce({ status: 404 }); + await program.parseAsync(['agent', 'remove', 'worker', '--wait'], { from: 'user' }); + expect(workspaceRelay.agents.get).toHaveBeenCalledWith('worker'); + expect(log).toHaveBeenCalledWith('Removed agent worker.'); + }); + + it('exits 8 while a released registration still owns the name', async () => { + const { program, workspaceRelay, log } = createHarness(); + workspaceRelay.agents.get.mockResolvedValue({ id: 'old', name: 'worker', status: 'released' }); + await expect( + program.parseAsync(['agent', 'remove', 'worker', '--wait', '--wait-timeout', '1'], { from: 'user' }) + ).rejects.toThrow('exit:8'); + expect(log.mock.calls.flat().join('')).not.toContain('Removed agent'); + }); + + it('keeps the asynchronous acknowledgement when every verification read is unavailable', async () => { + const { program, workspaceRelay, log, error } = createHarness(); + workspaceRelay.agents.get.mockRejectedValue(new Error('403 forbidden')); + await program.parseAsync(['agent', 'remove', 'worker', '--wait', '--wait-timeout', '1'], { + from: 'user', + }); + expect(log.mock.calls.flat().join('')).toContain('initiated'); + expect(error.mock.calls.flat().join('')).toContain('Could not verify'); + }); + + it('does not claim clearance or poll with --no-wait', async () => { + const { program, workspaceRelay, log } = createHarness(); + await program.parseAsync(['agent', 'remove', 'worker', '--no-wait'], { from: 'user' }); + expect(workspaceRelay.agents.get).not.toHaveBeenCalled(); + expect(log.mock.calls.flat().join('')).toContain('initiated'); + }); + it('redacts SQL and bound parameters when removal fails', async () => { const { program, workspaceRelay, error } = createHarness(); workspaceRelay.workspace.release.mockRejectedValueOnce( diff --git a/packages/cli/src/cli/commands/agent.ts b/packages/cli/src/cli/commands/agent.ts index bd7aff33fb..d9958ec170 100644 --- a/packages/cli/src/cli/commands/agent.ts +++ b/packages/cli/src/cli/commands/agent.ts @@ -8,20 +8,11 @@ import { withSdkDefaults, type SdkCommandDeps, } from '../lib/sdk-command.js'; -import { RelayError } from '@agent-relay/sdk'; +import { AgentRemovalPendingError, isNotFoundError, waitForAgentRemoval } from '../lib/agent-removal.js'; import { withAgentRegistrationDeadline, withDeadline } from '../lib/agent-registration.js'; import { attributableReleaseReason } from '../lib/release-reason.js'; -function isNotFoundError(error: unknown): boolean { - if (error instanceof RelayError) return error.code === 'not_found' || error.statusCode === 404; - const statusCode = - error && typeof error === 'object' - ? ((error as { statusCode?: unknown }).statusCode ?? (error as { status?: unknown }).status) - : undefined; - return Number(statusCode) === 404; -} - export type AgentCommandDependencies = SdkCommandDeps; function withAgentDefaults(overrides: Partial = {}): AgentCommandDependencies { @@ -152,6 +143,9 @@ export function registerAgentCommands( .description('Remove an agent while preserving attributed message history') .argument('', 'Agent name') .option('--reason ', 'Removal reason') + .option('--wait', 'Wait until the registration is cleared', false) + .option('--no-wait', 'Return after the removal is accepted') + .option('--wait-timeout ', 'Registration clearance timeout', '30000') ).action(async (name: string, opts: Record) => { await runSdk(deps, async () => { const relay = deps.createWorkspaceRelay(sdkOptionsFromOpts(opts)); @@ -161,17 +155,23 @@ export function registerAgentCommands( 'agent removed' ); const result = await relay.workspace.release({ name, reason, deleteAgent: true }); - // The release endpoint acknowledges an async action invocation — a - // resolved promise means the request was accepted, not that the - // deletion has finished. Only claim "Removed" once the invocation - // itself reports completion; otherwise say what actually happened. - if (result.status === 'completed') { - deps.log(`Removed agent ${name}.`); - } else { - deps.log( - `Removal of agent ${name} was initiated (status: ${result.status ?? 'pending'}) and is processed asynchronously.` - ); + if (opts.wait === true) { + const removal = await waitForAgentRemoval({ + name, + getAgent: (name) => relay.agents.get(name), + listAgents: () => relay.agents.list(), + timeoutMs: Number(opts.waitTimeout), + }); + if (removal.cleared) { + deps.log(`Removed agent ${name}.`); + return; + } + if (removal.observedPresent) throw new AgentRemovalPendingError(name); + deps.error(`Could not verify removal: ${removal.readError ?? 'registration unavailable'}`); } + deps.log( + `Removal of agent ${name} was initiated (status: ${result.status ?? 'pending'}) and is processed asynchronously. Use --wait to verify the name is reusable.` + ); }); }); } diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 7236424cb8..47ceb5899d 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1085,12 +1085,104 @@ describe('fleet command support', () => { const rendered = errors.join('\n'); expect(rendered).toContain('do not retry blindly'); - expect(rendered).toContain('"code":"spawn_unconfirmed"'); - expect(rendered).toContain('"state":"unconfirmed_may_be_running"'); + expect(rendered).toContain('"code":"spawn_pending"'); + expect(rendered).toContain('"state":"pending"'); expect(rendered).toContain('"dispatchState":"dispatched"'); expect(rendered).toContain(`"invocationId":"${invocationId}"`); }); + it.each(['live', 'terminal', 'collision', 'missing_ready'] as const)( + 'resolves timed-out targeted spawn using %s evidence', + async (evidence) => { + const invocationId = 'inv_late'; + const pending = new RelayPlacementError('spawn_unconfirmed', 'timeout', { + capability: 'spawn:codex', + attempts: 1, + node: 'target', + invocationId, + state: 'unconfirmed_may_be_running', + dispatchState: 'dispatched', + }); + const getInvocation = vi.fn(async () => + evidence === 'live' + ? { status: 'running' } + : evidence === 'terminal' || evidence === 'missing_ready' + ? { status: 'completed', output: { spawned: true, ready: evidence === 'terminal' } } + : { + status: 'failed', + error: "failed to pre-register worker 'worker': agent 'worker' already exists and registration is create-only; use a unique name", + } + ); + const nodes = { + get: vi.fn(async () => ({ + name: 'target', + status: 'online', + lastHeartbeatAt: new Date().toISOString(), + capabilities: [{ name: 'relay:live-agents:v1', metadata: { names: ['worker'] } }], + })), + list: vi.fn(), + }; + const logs: string[] = []; + const errors: string[] = []; + const exit = vi.fn(); + const warn = vi.fn(); + const program = new Command(); + registerFleetCommands(program, { + resolveSandboxRepository: () => undefined, + sdk: { + createAgentRelay: vi.fn(() => ({ + messaging: { + placement: { + spawn: vi.fn(async () => { + throw pending; + }), + }, + commands: { getInvocation }, + }, + })) as never, + createWorkspaceRelay: vi.fn(() => ({ nodes, agents: {} })) as never, + createWorkspace: vi.fn() as never, + log: (message) => logs.push(String(message)), + error: (message) => errors.push(String(message)), + exit: exit as never, + }, + warn, + log: vi.fn(), + error: vi.fn(), + }); + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--name', + 'worker', + '--task', + 'work', + '--node', + 'target', + '--token', + 'at_test', + ], + { from: 'user' } + ); + expect(getInvocation).toHaveBeenCalledWith('spawn', invocationId); + if (evidence === 'collision' || evidence === 'missing_ready') { + expect(exit).toHaveBeenCalledWith(1); + expect(errors.join('')).toContain(evidence === 'collision' ? '"code":"spawn_name_taken"' : '"code":"spawn_failed"'); + } else { + expect(exit).not.toHaveBeenCalled(); + expect(JSON.parse(logs[0]!).invocation.placement).toMatchObject({ + state: evidence === 'terminal' ? 'ready' : 'accepted', + confirmed: evidence === 'terminal', + dispatchState: 'dispatched', + }); + if (evidence === 'live') + expect(warn).toHaveBeenCalledWith(expect.stringContaining('harness readiness was not confirmed')); + } + } + ); + it('fleet spawn --sandbox-provider agent37 provisions the isolated canary and uses a temporary launcher', async () => { vi.stubEnv('RELAY_AGENT_TOKEN', undefined); const placement = { @@ -2458,6 +2550,7 @@ describe('fleet command support', () => { ).rejects.toThrow('__exit__'); expect(errors.join('\n')).toContain('dispatch may still be running'); + expect(errors.join('\n')).toContain('"code":"spawn_pending"'); expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( expect.objectContaining({ workloadProfile: 'long-running-agent', @@ -4172,6 +4265,55 @@ describe('fleet command support', () => { ); }); + it('fleet release --delete-agent --wait adds clearance evidence on the selected transport', async () => { + const createWorkspaceRelay = vi.fn(() => ({ + agents: { + get: vi.fn(async () => { + throw { status: 404 }; + }), + list: vi.fn(), + }, + })); + const log = vi.fn(); + const program = new Command(); + registerFleetCommands(program, { + sdk: { + createWorkspaceRelay: createWorkspaceRelay as never, + createAgentRelay: vi.fn() as never, + createWorkspace: vi.fn() as never, + log, + error: vi.fn(), + exit: vi.fn() as never, + }, + createFleetWorkspaceClient: vi.fn(() => ({ + agents: { release: vi.fn(async () => ({ status: 'dispatched' })) }, + })) as never, + retireOwnedBindings: vi.fn(async () => undefined), + warn: vi.fn(), + }); + await program.parseAsync( + [ + 'fleet', + 'release', + 'worker', + '--delete-agent', + '--wait', + '--workspace-key', + 'rk_test', + '--base-url', + 'https://isolated.example', + ], + { from: 'user' } + ); + expect(createWorkspaceRelay).toHaveBeenCalledWith( + expect.objectContaining({ workspaceKey: 'rk_test', baseUrl: 'https://isolated.example' }) + ); + expect(JSON.parse(log.mock.calls[0]![0])).toMatchObject({ + status: 'dispatched', + removal: { cleared: true }, + }); + }); + it('fleet release --delete-agent aborts without deleting when binding retirement fails', async () => { const release = vi.fn(async () => ({ name: 'api-worker', diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index ba8a884e1d..f0d066ed5e 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -35,7 +35,16 @@ import { readBrokerConnection } from '../lib/broker-lifecycle.js'; import { spawnAgentWithClient } from '../lib/client-factory.js'; import { formatRelativeTime, sanitizeForTerminalLine } from '../lib/formatting.js'; import { connectProjectBrokerClient } from '../lib/project-broker-client.js'; -import { isAvailableFleetNode } from '../lib/fleet-live-agents.js'; +import { AgentRemovalPendingError, waitForAgentRemoval } from '../lib/agent-removal.js'; +import { + classifySpawnFailure, + mayStillBeRunning, + nameTakenSpawnError, + pendingSpawnError, + probeSpawnLiveness, + terminalSpawnOutcome, +} from '../lib/spawn-liveness.js'; +import { heartbeatAgeMs, MAX_LIVE_HEARTBEAT_AGE_MS, isAvailableFleetNode } from '../lib/fleet-live-agents.js'; import { declaredWorkforceMetadata } from '../lib/registration-metadata.js'; import { redactSecrets } from '../lib/redact.js'; import { attributableReleaseReason } from '../lib/release-reason.js'; @@ -193,13 +202,13 @@ function spawnInvocationWithMergedPlacement(invocation: Record) ...invocation, placement: { ...(sdkPlacement ?? {}), - dispatchState: receipt.dispatchState, + dispatchState: sdkPlacement?.dispatchState ?? receipt.dispatchState, ...(invocationId ? { invocationId } : {}), }, }; } -function throwForTerminalSpawnFailure(invocation: Record): void { +function throwForTerminalSpawnFailure(invocation: Record, name: string): void { const placement = spawnPlacementReceipt(invocation); if (placement.state !== 'failed') return; const invocationId = typeof placement.invocationId === 'string' ? placement.invocationId : undefined; @@ -207,7 +216,7 @@ function throwForTerminalSpawnFailure(invocation: Record): void typeof invocation.error === 'string' && invocation.error.trim() ? invocation.error : 'Fleet spawn invocation reported a terminal failure.'; - throw new RelayPlacementError('spawn_failed', message, { + const error = new RelayPlacementError('spawn_failed', message, { capability: typeof invocation.capability === 'string' ? invocation.capability : 'spawn', attempts: 1, ...(invocationId ? { invocationId } : {}), @@ -215,6 +224,7 @@ function throwForTerminalSpawnFailure(invocation: Record): void dispatchState: placement.dispatchState as 'dispatched' | 'not_dispatched' | 'unknown', receipt: invocation, }); + throw classifySpawnFailure(message) ? nameTakenSpawnError(name, error) : error; } export interface FleetCommandDependencies { @@ -990,32 +1000,82 @@ export function registerFleetCommands( '/.skills' )}. The Relayfile daemon synchronizes this tree; it intentionally has no .git directory.` : undefined; - const invocation = await relay.messaging.placement.spawn({ - capability: `spawn:${cli}`, - node: targetNode, - failFast: true, - confirm, - ...(confirm ? { confirmTimeoutMs: confirmTimeoutMs } : {}), - input: { + let invocation; + try { + invocation = await relay.messaging.placement.spawn({ + capability: `spawn:${cli}`, + node: targetNode, + failFast: true, + confirm, + ...(confirm ? { confirmTimeoutMs: confirmTimeoutMs } : {}), + input: { + name, + cli, + task: + sandbox && + checkoutRepository && + sandboxRepository && + mountSandboxRelayfile && + sandbox.outcome === 'provisioned' + ? `${task}\n\nAgent Relay sandbox context: Relayfile records are available at ${sandbox.relayfileMountPath ?? '/workspace'}. The source checkout is separate; use ${workerCwd ?? 'the worker checkout'} for repository files and the mount for Relayfile records.` + : liveSandboxContext + ? `${task}\n\n${liveSandboxContext}` + : task, + ...(channel ? { channels: [channel] } : {}), + ...(model ? { model } : {}), + ...(workerCwd ? { worker_cwd: workerCwd } : {}), + ...registrationMetadata, + ...(sessionRef ? { session_ref: sessionRef } : {}), + }, + }); + } catch (error) { + if (!(error instanceof RelayPlacementError)) throw error; + if (error.code === 'spawn_failed' && classifySpawnFailure(error.message)) + throw nameTakenSpawnError(name, error); + if (error.code !== 'spawn_unconfirmed') throw error; + const liveness = await probeSpawnLiveness({ name, - cli, - task: - sandbox && - checkoutRepository && - sandboxRepository && - mountSandboxRelayfile && - sandbox.outcome === 'provisioned' - ? `${task}\n\nAgent Relay sandbox context: Relayfile records are available at ${sandbox.relayfileMountPath ?? '/workspace'}. The source checkout is separate; use ${workerCwd ?? 'the worker checkout'} for repository files and the mount for Relayfile records.` - : liveSandboxContext - ? `${task}\n\n${liveSandboxContext}` - : task, - ...(channel ? { channels: [channel] } : {}), - ...(model ? { model } : {}), - ...(workerCwd ? { worker_cwd: workerCwd } : {}), - ...registrationMetadata, - ...(sessionRef ? { session_ref: sessionRef } : {}), - }, - }); + targetNode, + ...(error.invocationId + ? { + getInvocation: () => relay.messaging.commands.getInvocation('spawn', error.invocationId!), + } + : {}), + createClient: () => { + const workspace = deps.sdk.createWorkspaceRelay(relaycastClientOptions); + return { nodes: workspace.nodes, agents: workspace.agents }; + }, + }); + if (liveness.evidence === 'invocation_terminal') { + try { + invocation = terminalSpawnOutcome(liveness.invocation!, error, confirm); + } catch (terminalError) { + if ( + terminalError instanceof RelayPlacementError && + classifySpawnFailure(terminalError.message) + ) + throw nameTakenSpawnError(name, terminalError); + throw terminalError; + } + } else if (liveness.evidence === 'live') { + deps.warn( + `A worker named ${JSON.stringify(name)} is running on ${JSON.stringify(liveness.node)} according to its fresh heartbeat; harness readiness was not confirmed within the budget.` + ); + invocation = { + status: 'accepted', + invocationId: error.invocationId, + placement: { + state: 'accepted', + confirmed: false, + invocationId: error.invocationId, + dispatchState: error.dispatchState, + liveness, + }, + }; + } else { + throw pendingSpawnError(name, error, liveness, error.message); + } + } const printableSandbox = (sandbox?.outcome === 'provisioned' || sandbox?.outcome === 'reused') && sandbox.relaycastTarget ? { @@ -1037,11 +1097,7 @@ export function registerFleetCommands( invocation: spawnInvocationWithMergedPlacement(invocation as unknown as Record), }); } catch (error) { - if ( - shouldCleanupSandbox && - sandbox?.outcome === 'provisioned' && - !(error instanceof RelayPlacementError && error.state === 'unconfirmed_may_be_running') - ) { + if (shouldCleanupSandbox && sandbox?.outcome === 'provisioned' && !mayStillBeRunning(error)) { await deps .deleteCloudFleetSandbox({ cloudWorkspaceId: sandbox.cloudWorkspaceId, @@ -1122,7 +1178,7 @@ export function registerFleetCommands( } : {}), }); - throwForTerminalSpawnFailure(invocation); + throwForTerminalSpawnFailure(invocation, name); printJson(deps.sdk, { invocation: spawnInvocationWithPlacement(invocation) }); }); }); @@ -1134,6 +1190,9 @@ export function registerFleetCommands( .argument('', 'Worker agent name') .option('--reason ', 'Release reason') .option('--delete-agent', 'Permanently delete the agent after release') + .option('--wait', 'With --delete-agent, wait for registration clearance', false) + .option('--no-wait', 'Return after release is accepted') + .option('--wait-timeout ', 'Registration clearance timeout', '30000') .option( '--unsubscribe-bindings', 'Retire provider bindings owned by this identity (implied by --delete-agent)' @@ -1191,6 +1250,22 @@ export function registerFleetCommands( reason, deleteAgent: true, }); + if (options.wait === true) { + // Construct/read on the same selected transport; unavailable credentials are uncertainty. + const removal = await waitForAgentRemoval({ + name: workerName, + getAgent: (name) => deps.sdk.createWorkspaceRelay(transport).agents.get(name), + listAgents: () => deps.sdk.createWorkspaceRelay(transport).agents.list(), + timeoutMs: Number(options.waitTimeout), + }); + printJson(deps.sdk, { ...deleted, removal }); + if (!removal.cleared && removal.observedPresent) throw new AgentRemovalPendingError(workerName); + if (!removal.cleared) + deps.warn( + `Removal was initiated asynchronously; could not verify clearance: ${removal.readError}` + ); + return; + } printJson(deps.sdk, deleted); return; } @@ -1523,12 +1598,27 @@ async function runFleetAgentList( const now = new Date(); const output = buildRows({ contributions, roster }, now); + const heartbeatNodes = contributions.map(({ node }) => ({ + name: node.name, + lastHeartbeatAt: node.lastHeartbeatAt ?? null, + heartbeatAgeMs: heartbeatAgeMs(node, now.getTime()), + })); + if ( + heartbeatNodes.some( + (node) => node.heartbeatAgeMs !== null && node.heartbeatAgeMs > MAX_LIVE_HEARTBEAT_AGE_MS + ) + ) { + deps.warn( + 'Some node heartbeats are older than 36 seconds. Listings are heartbeat snapshots and can differ within one heartbeat window.' + ); + } if (options.pretty === true) { deps.log(formatPretty(output)); return; } printJson(deps.sdk, { generatedAt: now.toISOString(), + nodes: heartbeatNodes, localNode: localNodeName ?? null, perNode: output.perNode, unplacedRoster: output.unplacedRoster, diff --git a/packages/cli/src/cli/lib/agent-registration.ts b/packages/cli/src/cli/lib/agent-registration.ts index 132aa68a98..f8883b9d5c 100644 --- a/packages/cli/src/cli/lib/agent-registration.ts +++ b/packages/cli/src/cli/lib/agent-registration.ts @@ -6,7 +6,7 @@ export const DEFAULT_AGENT_REGISTRATION_TIMEOUT_MS = 15_000; // "bounded registration" into "registration always times out immediately". const MAX_SETTIMEOUT_DELAY_MS = 2_147_483_647; -function normalizeTimeoutMs(timeoutMs: number): number { +export function normalizeTimeoutMs(timeoutMs: number): number { if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { return DEFAULT_AGENT_REGISTRATION_TIMEOUT_MS; } @@ -17,7 +17,7 @@ function normalizeTimeoutMs(timeoutMs: number): number { // close the quote, emit an escaped literal quote, reopen. Without this, a // name containing `$(...)` or backticks would execute command substitution // if a user copies the recovery command as-is into a shell. -function shellQuote(value: string): string { +export function shellQuote(value: string): string { return `'${value.replace(/'/g, `'\\''`)}'`; } diff --git a/packages/cli/src/cli/lib/agent-removal.test.ts b/packages/cli/src/cli/lib/agent-removal.test.ts new file mode 100644 index 0000000000..30312984b6 --- /dev/null +++ b/packages/cli/src/cli/lib/agent-removal.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it, vi } from 'vitest'; +import { waitForAgentRemoval } from './agent-removal.js'; + +function harness() { + let time = 0; + return { + name: 'worker', + timeoutMs: 1000, + pollIntervalMs: 500, + now: () => time, + sleep: vi.fn(async (ms: number) => { + time += ms; + }), + getAgent: vi.fn(async (): Promise => ({ name: 'worker', status: 'released' })), + listAgents: vi.fn(async () => [{ name: 'worker' }]), + }; +} +describe('registration removal wait', () => { + it('polls released tombstones until confirmed not-found', async () => { + const h = harness(); + h.getAgent.mockResolvedValueOnce({ status: 'released' }).mockRejectedValueOnce({ statusCode: 404 }); + expect(await waitForAgentRemoval(h)).toMatchObject({ cleared: true, waitedMs: 500 }); + expect(h.listAgents).not.toHaveBeenCalled(); + }); + it('uses one deadline membership check to clear persistent tombstones', async () => { + const h = harness(); + h.listAgents.mockResolvedValueOnce([]); + expect(await waitForAgentRemoval(h)).toMatchObject({ cleared: true, waitedMs: 1000 }); + expect(h.listAgents).toHaveBeenCalledTimes(1); + }); + it('never clears on released status alone', async () => { + expect(await waitForAgentRemoval(harness())).toMatchObject({ cleared: false, observedPresent: true }); + }); + it('does not confuse transport/auth errors with presence or deletion', async () => { + const h = harness(); + h.getAgent.mockRejectedValue(new Error('403')); + h.listAgents.mockRejectedValue(new Error('offline')); + expect(await waitForAgentRemoval(h)).toMatchObject({ + cleared: false, + observedPresent: false, + readError: 'offline', + }); + }); + it('does not sleep past the deadline', async () => { + const h = harness(); + h.timeoutMs = 750; + await waitForAgentRemoval(h); + expect(h.sleep.mock.calls.map(([ms]) => ms)).toEqual([500, 250]); + }); + it('bounds never-settling reads including the final list read', async () => { + vi.useFakeTimers(); + try { + const result = waitForAgentRemoval({ + name: 'worker', + timeoutMs: 1000, + getAgent: () => new Promise(() => {}), + listAgents: () => new Promise(() => {}), + }); + await vi.advanceTimersByTimeAsync(2000); + expect(await result).toMatchObject({ cleared: false, observedPresent: false, waitedMs: 2000 }); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/packages/cli/src/cli/lib/agent-removal.ts b/packages/cli/src/cli/lib/agent-removal.ts new file mode 100644 index 0000000000..5b367dce68 --- /dev/null +++ b/packages/cli/src/cli/lib/agent-removal.ts @@ -0,0 +1,81 @@ +import { RelayError, safeRelayErrorMessage } from '@agent-relay/sdk'; +import { normalizeTimeoutMs, shellQuote, withDeadline } from './agent-registration.js'; + +export const DEFAULT_REMOVAL_WAIT_TIMEOUT_MS = 30_000; +export const REMOVAL_POLL_INTERVAL_MS = 500; + +export function isNotFoundError(error: unknown): boolean { + if (error instanceof RelayError) return error.code === 'not_found' || error.statusCode === 404; + const status = + error && typeof error === 'object' + ? ((error as { statusCode?: unknown }).statusCode ?? (error as { status?: unknown }).status) + : undefined; + return Number(status) === 404; +} + +export interface RemovalResult { + cleared: boolean; + waitedMs: number; + observedPresent: boolean; + readError?: string; +} + +export class AgentRemovalPendingError extends Error { + readonly exitCode = 8; + constructor(name: string) { + super( + `Removal of agent ${JSON.stringify(name)} was accepted, but its registration is still present. Re-check \`agent-relay agent list\` or run \`agent-relay agent remove ${shellQuote(name)} --wait\` before respawning.` + ); + } +} + +export async function waitForAgentRemoval(options: { + name: string; + getAgent: (name: string) => Promise; + listAgents: () => Promise>; + timeoutMs?: number; + pollIntervalMs?: number; + now?: () => number; + sleep?: (ms: number) => Promise; +}): Promise { + const now = options.now ?? Date.now; + const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); + const started = now(); + const budget = normalizeTimeoutMs(options.timeoutMs ?? DEFAULT_REMOVAL_WAIT_TIMEOUT_MS); + const deadline = started + budget; + let observedPresent = false; + let readError: string | undefined; + const result = (cleared: boolean): RemovalResult => ({ + cleared, + waitedMs: now() - started, + observedPresent, + ...(readError ? { readError } : {}), + }); + // Reserve one bounded read for membership corroboration at the deadline. + // A released/offline status still owns the name and is never clearance. + const read = (fn: () => Promise, ms: number) => + withDeadline(fn, () => new Error('Removal verification read timed out.'), ms); + while (now() < deadline) { + try { + await read(() => options.getAgent(options.name), Math.max(1, deadline - now())); + observedPresent = true; + readError = undefined; + } catch (error) { + if (isNotFoundError(error)) return result(true); + readError = safeRelayErrorMessage(error); + } + const remaining = deadline - now(); + if (remaining > 0) + await sleep( + Math.min(normalizeTimeoutMs(options.pollIntervalMs ?? REMOVAL_POLL_INTERVAL_MS), remaining) + ); + } + try { + const agents = await read(options.listAgents, Math.min(1_000, budget)); + if (!agents.some((agent) => agent.name === options.name)) return result(true); + observedPresent = true; + } catch (error) { + readError = safeRelayErrorMessage(error); + } + return result(false); +} diff --git a/packages/cli/src/cli/lib/fleet-live-agents.ts b/packages/cli/src/cli/lib/fleet-live-agents.ts index 41e7056b06..94d58c2bfe 100644 --- a/packages/cli/src/cli/lib/fleet-live-agents.ts +++ b/packages/cli/src/cli/lib/fleet-live-agents.ts @@ -79,3 +79,10 @@ export function isAvailableFleetNode(node: { const isLive = node.live === undefined ? node.status === 'online' : node.live === true; return isLive && node.handlersLive !== false && !isDirectPseudoNode; } + +export const MAX_LIVE_HEARTBEAT_AGE_MS = 36_000; + +export function heartbeatAgeMs(node: { lastHeartbeatAt?: string }, now = Date.now()): number | null { + const timestamp = Date.parse(node.lastHeartbeatAt ?? ''); + return Number.isFinite(timestamp) ? Math.max(0, now - timestamp) : null; +} diff --git a/packages/cli/src/cli/lib/sdk-command.test.ts b/packages/cli/src/cli/lib/sdk-command.test.ts index 01c4ab88bd..b56b906482 100644 --- a/packages/cli/src/cli/lib/sdk-command.test.ts +++ b/packages/cli/src/cli/lib/sdk-command.test.ts @@ -1,3 +1,4 @@ +import { pendingSpawnError } from './spawn-liveness.js'; import { describe, expect, it, vi } from 'vitest'; import { RelayPlacementError } from '@agent-relay/sdk'; @@ -87,6 +88,24 @@ describe('runSdk structured error output', () => { }); }); + it('renders pending evidence and honors only known error exit codes', async () => { + const { deps: sdkDeps, errors } = deps(); + await runSdk(sdkDeps, async () => { + throw pendingSpawnError( + 'worker', + { invocationId: 'inv', dispatchState: 'dispatched' }, + { evidence: 'unknown' } + ); + }); + expect(sdkDeps.exit).toHaveBeenCalledWith(8); + expect(errors.join('')).toContain('"code":"spawn_pending"'); + expect(errors.join('')).toContain('"liveness":{"evidence":"unknown"}'); + await runSdk(sdkDeps, async () => { + throw Object.assign(new Error('plain'), { exitCode: 0 }); + }); + expect(sdkDeps.exit).toHaveBeenLastCalledWith(1); + }); + it('does not attach a structured receipt for a plain Error', async () => { const { deps: sdkDeps, errors } = deps(); diff --git a/packages/cli/src/cli/lib/sdk-command.ts b/packages/cli/src/cli/lib/sdk-command.ts index cdf84799dd..74b2a345d6 100644 --- a/packages/cli/src/cli/lib/sdk-command.ts +++ b/packages/cli/src/cli/lib/sdk-command.ts @@ -7,6 +7,8 @@ import { type AgentRelayAgent, } from '@agent-relay/sdk'; +import { AgentRemovalPendingError } from './agent-removal.js'; +import { FleetSpawnError } from './spawn-liveness.js'; import { defaultExit } from './exit.js'; import { createAgentRelay, createWorkspaceRelay, type SdkClientOptions } from './sdk-client.js'; import { sanitizedSpawnReceipt } from './spawn-lifecycle.js'; @@ -77,7 +79,7 @@ export async function runSdk(deps: SdkCommandDeps, fn: () => Promise): Pro await fn(); } catch (err) { const message = safeRelayErrorMessage(err); - if (err instanceof RelayPlacementError) { + if (err instanceof RelayPlacementError || err instanceof FleetSpawnError) { const structured = { error: { code: err.code, @@ -86,6 +88,10 @@ export async function runSdk(deps: SdkCommandDeps, fn: () => Promise): Pro ...(err.node ? { node: err.node } : {}), ...(err.dispatchState ? { dispatchState: err.dispatchState } : {}), ...(err.receipt ? { receipt: sanitizedSpawnReceipt(err.receipt) } : {}), + ...(err instanceof FleetSpawnError && err.liveness ? { liveness: err.liveness } : {}), + ...(err instanceof FleetSpawnError && err.diagnostic + ? { diagnostic: safeRelayErrorMessage(new Error(err.diagnostic)) } + : {}), message, }, }; @@ -93,6 +99,6 @@ export async function runSdk(deps: SdkCommandDeps, fn: () => Promise): Pro } else { deps.error(message); } - deps.exit(1); + deps.exit(err instanceof FleetSpawnError || err instanceof AgentRemovalPendingError ? err.exitCode : 1); } } diff --git a/packages/cli/src/cli/lib/spawn-liveness.test.ts b/packages/cli/src/cli/lib/spawn-liveness.test.ts new file mode 100644 index 0000000000..9ff75a6bb9 --- /dev/null +++ b/packages/cli/src/cli/lib/spawn-liveness.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it, vi } from 'vitest'; +import { RelayPlacementError, type RelayNode } from '@agent-relay/sdk'; +import { + classifySpawnFailure, + mayStillBeRunning, + pendingSpawnError, + probeSpawnLiveness, + terminalSpawnOutcome, +} from './spawn-liveness.js'; + +const now = Date.parse('2026-10-04T12:00:00Z'); +const node = (name = 'target', age = 0): RelayNode => ({ + name, + status: 'online', + lastHeartbeatAt: new Date(now - age).toISOString(), + capabilities: [{ name: 'relay:live-agents:v1', metadata: { names: ['worker'] } }], +}); +function client(nodes: RelayNode[] = []) { + return { + nodes: { + get: vi.fn(async (name: string) => nodes.find((n) => n.name === name) ?? null), + list: vi.fn(async () => nodes), + }, + agents: { + get: vi.fn(async (): Promise => { + throw { status: 404 }; + }), + }, + }; +} + +describe('spawn liveness resolution', () => { + it.each([ + [node(), 'live'], + [node('other'), 'live_elsewhere'], + [node('target', 36_001), 'stale'], + [{ ...node(), lastHeartbeatAt: undefined }, 'stale'], + [{ ...node(), status: 'offline' }, 'absent'], + ] as const)('requires fresh target-node evidence (%j)', async (record, evidence) => { + const workspace = client([record]); + expect( + await probeSpawnLiveness({ + name: 'worker', + targetNode: 'target', + createClient: () => workspace, + now: () => now, + }) + ).toMatchObject({ evidence }); + if (evidence === 'live') expect(workspace.nodes.list).not.toHaveBeenCalled(); + }); + it('folds construction errors into unknown without replacing accepted dispatch evidence', async () => { + const result = await probeSpawnLiveness({ + name: 'worker', + createClient: () => { + throw new Error('no key'); + }, + }); + expect(result).toEqual({ evidence: 'unknown', readError: 'no key' }); + const pending = pendingSpawnError('worker', { invocationId: 'inv', dispatchState: 'dispatched' }, result); + expect(pending.exitCode).toBe(8); + expect(pending.message).toContain('--wait'); + expect(pending.message).not.toContain('failed'); + expect(mayStillBeRunning(pending)).toBe(true); + }); + it('preserves SDK unconfirmed errors as potentially running', () => { + expect( + mayStillBeRunning( + new RelayPlacementError('spawn_unconfirmed', 'pending', { + capability: 'spawn', + attempts: 1, + state: 'unconfirmed_may_be_running', + }) + ) + ).toBe(true); + expect(mayStillBeRunning(new Error('failure'))).toBe(false); + }); + it('reads the invocation before constructing a workspace client', async () => { + const createClient = vi.fn(() => { + throw new Error('no key'); + }); + const invocation = { status: 'completed', output: { spawned: true, ready: false } }; + const result = await probeSpawnLiveness({ + name: 'worker', + getInvocation: async () => invocation, + createClient, + }); + expect(result).toEqual({ evidence: 'invocation_terminal', invocation }); + expect(createClient).not.toHaveBeenCalled(); + expect(terminalSpawnOutcome(invocation, {}, false)).toMatchObject({ placement: { confirmed: true } }); + expect(() => terminalSpawnOutcome(invocation, {}, true)).toThrow('proof'); + expect(() => terminalSpawnOutcome({ status: 'failed', error: 'launch error' }, {}, false)).toThrow( + 'launch error' + ); + }); + it('distinguishes registered, absent and unavailable reads', async () => { + const workspace = client(); + workspace.agents.get.mockResolvedValueOnce({ status: 'released' }); + expect(await probeSpawnLiveness({ name: 'worker', createClient: () => workspace })).toMatchObject({ + evidence: 'registered', + }); + expect(await probeSpawnLiveness({ name: 'worker', createClient: () => workspace })).toMatchObject({ + evidence: 'absent', + }); + workspace.agents.get.mockRejectedValueOnce(new Error('403 forbidden')); + expect(await probeSpawnLiveness({ name: 'worker', createClient: () => workspace })).toMatchObject({ + evidence: 'unknown', + }); + }); + it('bounds a hung invocation read and still probes worker evidence', async () => { + vi.useFakeTimers(); + try { + const result = probeSpawnLiveness({ + name: 'worker', + getInvocation: () => new Promise(() => {}), + createClient: () => client([node()]), + now: () => now, + }); + await vi.advanceTimersByTimeAsync(2_000); + expect(await result).toMatchObject({ evidence: 'live' }); + } finally { + vi.useRealTimers(); + } + }); + it('classifies only broker pre-registration collisions', () => { + expect( + classifySpawnFailure( + "failed to pre-register worker 'worker': agent 'worker' already exists and registration is create-only; use a unique agent name" + ) + ).toBe('spawn_name_taken'); + expect(classifySpawnFailure('failed to create sandbox: directory already exists')).toBeUndefined(); + expect(classifySpawnFailure("failed to pre-register worker 'worker': cache entry already exists")).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/cli/lib/spawn-liveness.ts b/packages/cli/src/cli/lib/spawn-liveness.ts new file mode 100644 index 0000000000..74f1a7a428 --- /dev/null +++ b/packages/cli/src/cli/lib/spawn-liveness.ts @@ -0,0 +1,211 @@ +import { RelayPlacementError, safeRelayErrorMessage, type RelayNode } from '@agent-relay/sdk'; +import { shellQuote, withDeadline } from './agent-registration.js'; +import { isNotFoundError } from './agent-removal.js'; +import { + heartbeatAgeMs, + isAvailableFleetNode, + MAX_LIVE_HEARTBEAT_AGE_MS, + readRemoteLiveAgents, +} from './fleet-live-agents.js'; + +export interface SpawnLiveness { + evidence: 'invocation_terminal' | 'live' | 'live_elsewhere' | 'stale' | 'registered' | 'absent' | 'unknown'; + node?: string; + heartbeatAgeMs?: number | null; + readError?: string; + invocation?: Record; +} + +interface SpawnContext { + invocationId?: string; + node?: string; + dispatchState?: string; + receipt?: Record; +} + +export class FleetSpawnError extends Error { + readonly state: 'pending' | 'failed'; + readonly exitCode: number; + readonly invocationId?: string; + readonly node?: string; + readonly dispatchState?: string; + readonly receipt?: Record; + constructor( + readonly code: 'spawn_pending' | 'spawn_name_taken', + message: string, + context: SpawnContext, + readonly liveness?: SpawnLiveness, + readonly diagnostic?: string + ) { + super(message); + this.state = code === 'spawn_pending' ? 'pending' : 'failed'; + this.exitCode = code === 'spawn_pending' ? 8 : 1; + this.invocationId = context.invocationId; + this.node = context.node; + this.dispatchState = context.dispatchState; + this.receipt = context.receipt; + } +} + +export function mayStillBeRunning(error: unknown): boolean { + return ( + (error instanceof RelayPlacementError && error.state === 'unconfirmed_may_be_running') || + (error instanceof FleetSpawnError && error.code === 'spawn_pending') + ); +} + +export function pendingSpawnError( + name: string, + context: SpawnContext, + liveness: SpawnLiveness, + diagnostic?: string +): FleetSpawnError { + const nodeOption = context.node ? ` --node ${shellQuote(context.node)}` : ''; + return new FleetSpawnError( + 'spawn_pending', + `Spawn of ${JSON.stringify(name)} was accepted; its outcome is pending. ` + + `Evidence: ${liveness.evidence}${liveness.node ? ` on ${JSON.stringify(liveness.node)}` : ''}. ` + + `Invocation: ${context.invocationId ?? 'unavailable'}; dispatch: ${context.dispatchState ?? 'unknown'}. ` + + `Check \`agent-relay fleet agent list${nodeOption}\` before retrying. ` + + `Only if no worker is running and you intend to reclaim the name, run \`agent-relay agent remove ${shellQuote(name)} --wait\` and wait for clearance before respawning.`, + context, + liveness, + diagnostic + ); +} + +export function classifySpawnFailure(message: string): 'spawn_name_taken' | undefined { + // Anchored to the broker formatter; unrelated "already exists" failures are not registration collisions. + return /(?:^|:\s)failed to pre-register worker '[^\n]+': agent '[^\n]+' already exists and registration is create-only(?:;|$)/i.test( + message + ) + ? 'spawn_name_taken' + : undefined; +} + +export function nameTakenSpawnError(name: string, error: Error & SpawnContext): FleetSpawnError { + return new FleetSpawnError( + 'spawn_name_taken', + `${safeRelayErrorMessage(error)} A removal or cleanup from a previous unsuccessful spawn may still be in flight. Re-check \`agent-relay agent list\`; if the name is present with no worker running, run \`agent-relay agent remove ${shellQuote(name)} --wait\`, wait for clearance, then retry.`, + error + ); +} + +/** Reads are bounded and client construction is fallible (agent-token-only callers are supported). */ +export async function probeSpawnLiveness(options: { + name: string; + targetNode?: string; + getInvocation?: () => Promise; + createClient: () => { + nodes: { get: (name: string) => Promise; list: () => Promise }; + agents: { get: (name: string) => Promise }; + }; + now?: () => number; +}): Promise { + let readError: string | undefined; + const read = (fn: () => Promise) => + withDeadline(fn, () => new Error('Spawn evidence read timed out.'), 2_000); + if (options.getInvocation) { + try { + const value = await read(options.getInvocation); + if (value && typeof value === 'object') { + const invocation = value as Record; + if ( + /^(completed|succeeded|success|failed|error|cancelled|canceled|denied)$/i.test( + String(invocation.status) + ) + ) { + return { evidence: 'invocation_terminal', invocation }; + } + } + } catch (error) { + readError = safeRelayErrorMessage(error); + } + } + try { + const client = options.createClient(); + try { + const target = options.targetNode ? await read(() => client.nodes.get(options.targetNode!)) : undefined; + const nodes = options.targetNode && target ? [target] : await read(() => client.nodes.list()); + // Prefer the targeted read; consult the fleet only if it doesn't claim this name. + if ( + options.targetNode && + target && + !readRemoteLiveAgents(target).agents.some((a) => a.name === options.name) + ) { + nodes.push(...(await read(() => client.nodes.list()))); + } + let weaker: SpawnLiveness | undefined; + for (const node of nodes) { + if ( + !isAvailableFleetNode(node) || + !readRemoteLiveAgents(node).agents.some((agent) => agent.name === options.name) + ) + continue; + const age = heartbeatAgeMs(node, options.now?.() ?? Date.now()); + const evidence = + age === null || age > MAX_LIVE_HEARTBEAT_AGE_MS + ? 'stale' + : options.targetNode && node.name !== options.targetNode + ? 'live_elsewhere' + : 'live'; + const result: SpawnLiveness = { evidence, node: node.name, heartbeatAgeMs: age }; + if (evidence === 'live') return result; + weaker ??= result; + } + if (weaker) return weaker; + } catch (error) { + readError = safeRelayErrorMessage(error); + } + try { + await read(() => client.agents.get(options.name)); + return { evidence: 'registered', ...(readError ? { readError } : {}) }; + } catch (error) { + if (isNotFoundError(error)) return { evidence: 'absent', ...(readError ? { readError } : {}) }; + readError = safeRelayErrorMessage(error); + } + } catch (error) { + readError = safeRelayErrorMessage(error); + } + return { evidence: 'unknown', readError }; +} + +/** Preserve the original request's launch/readiness contract on the final invocation read. */ +export function terminalSpawnOutcome( + invocation: Record, + context: SpawnContext, + requireReady: boolean +): Record { + const output = invocation.output as Record | undefined; + if ( + !/^(completed|succeeded|success)$/i.test(String(invocation.status)) || + output?.spawned !== true || + typeof output.ready !== 'boolean' || + (requireReady && output.ready !== true) + ) { + throw new RelayPlacementError( + 'spawn_failed', + typeof invocation.error === 'string' + ? invocation.error + : 'Spawn completed without the requested launch/readiness proof.', + { + capability: 'spawn', + attempts: 1, + state: 'failed', + invocationId: context.invocationId, + node: context.node, + dispatchState: context.dispatchState as 'dispatched' | 'not_dispatched' | 'unknown', + receipt: invocation, + } + ); + } + return { + ...invocation, + placement: { + state: output.ready ? 'ready' : 'accepted', + confirmed: true, + invocationId: context.invocationId, + dispatchState: context.dispatchState, + }, + }; +} From 68ea64b76833f3f58cd3ee4bdbf572bd7a82a81c Mon Sep 17 00:00:00 2001 From: Relayflow Date: Sun, 4 Oct 2026 14:55:01 +0000 Subject: [PATCH 06/42] fix(mcp): mirror pending spawn evidence and removal waits --- .../active/traj_ro3v7jmydpo5/trajectory.json | 82 --------- .../compact_o598x260wugy_2026-10-04.json | 89 ++++++++++ .../compact_o598x260wugy_2026-10-04.md | 33 ++++ CHANGELOG.md | 1 + .../src/cli/agent-relay-mcp.startup.test.ts | 155 ++++++++++++++++-- packages/cli/src/cli/agent-relay-mcp.ts | 144 +++++++++++++++- 6 files changed, 400 insertions(+), 104 deletions(-) delete mode 100644 .agentworkforce/trajectories/fleet-spawn-fix/active/traj_ro3v7jmydpo5/trajectory.json create mode 100644 .agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.json create mode 100644 .agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.md diff --git a/.agentworkforce/trajectories/fleet-spawn-fix/active/traj_ro3v7jmydpo5/trajectory.json b/.agentworkforce/trajectories/fleet-spawn-fix/active/traj_ro3v7jmydpo5/trajectory.json deleted file mode 100644 index bb498656c8..0000000000 --- a/.agentworkforce/trajectories/fleet-spawn-fix/active/traj_ro3v7jmydpo5/trajectory.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "id": "traj_ro3v7jmydpo5", - "version": 1, - "task": { - "title": "Fix fleet spawn pending outcomes and removal waits" - }, - "status": "active", - "startedAt": "2026-10-04T14:42:43.542Z", - "agents": [ - { - "name": "default", - "role": "lead", - "joinedAt": "2026-10-04T14:42:55.330Z" - } - ], - "chapters": [ - { - "id": "chap_349pedkimqde", - "title": "Work", - "agentName": "default", - "startedAt": "2026-10-04T14:42:55.330Z", - "events": [ - { - "ts": 1791124975331, - "type": "decision", - "content": "Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes: Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes", - "raw": { - "question": "Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes", - "chosen": "Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes", - "alternatives": [], - "reasoning": "Preserve SDK behavior and sandbox safety; uncertain accepted spawns exit 8 with diagnostics." - }, - "significance": "high" - }, - { - "ts": 1791124975634, - "type": "decision", - "content": "Ship removal waits opt-in; use get not-found and deadline list absence, never released status: Ship removal waits opt-in; use get not-found and deadline list absence, never released status", - "raw": { - "question": "Ship removal waits opt-in; use get not-found and deadline list absence, never released status", - "chosen": "Ship removal waits opt-in; use get not-found and deadline list absence, never released status", - "alternatives": [], - "reasoning": "No workspace credential environment variables or machine-global agent-relay configuration are available for live deletion verification. Reviewed-plan section 3.7 requires opt-in in this case. Exit 8 requires observed presence; all-read failures keep the asynchronous acknowledgement." - }, - "significance": "high" - }, - { - "ts": 1791125201843, - "type": "decision", - "content": "Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance: Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance", - "raw": { - "question": "Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance", - "chosen": "Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance", - "alternatives": [], - "reasoning": "Cleanup is already scheduled for owned failed spawns. A collision never grants ownership of the incumbent identity. Fleet release adds removal evidence to its existing JSON only with --wait." - }, - "significance": "high" - }, - { - "ts": 1791125587977, - "type": "decision", - "content": "Final invocation reads preserve the actual requested readiness contract: Final invocation reads preserve the actual requested readiness contract", - "raw": { - "question": "Final invocation reads preserve the actual requested readiness contract", - "chosen": "Final invocation reads preserve the actual requested readiness contract", - "alternatives": [], - "reasoning": "SDK confirm:true also requests verify_ready by default. A late terminal result must have ready:true; live heartbeat recovery remains explicitly confirmed:false. Late persona child acknowledgements stay pending and retain their child invocation/node." - }, - "significance": "high" - } - ] - } - ], - "commits": [], - "filesChanged": [], - "projectId": "AgentWorkforce/relay", - "tags": [], - "_trace": { - "startRef": "3cfc0fd6cbc68c45a8dd455d7917a3c223ad74fd", - "endRef": "3cfc0fd6cbc68c45a8dd455d7917a3c223ad74fd" - } -} \ No newline at end of file diff --git a/.agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.json b/.agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.json new file mode 100644 index 0000000000..ac1cfda54a --- /dev/null +++ b/.agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.json @@ -0,0 +1,89 @@ +{ + "id": "compact_o598x260wugy", + "version": 1, + "type": "compacted", + "compactedAt": "2026-10-04T14:54:40.616Z", + "sourceTrajectories": [ + "traj_ro3v7jmydpo5" + ], + "dateRange": { + "start": "2026-10-04T14:42:43.542Z", + "end": "2026-10-04T14:54:28.331Z" + }, + "summary": { + "totalDecisions": 4, + "totalEvents": 4, + "uniqueAgents": [ + "default" + ] + }, + "decisionGroups": [ + { + "category": "other", + "decisions": [ + { + "question": "Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes", + "chosen": "Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes", + "reasoning": "Preserve SDK behavior and sandbox safety; uncertain accepted spawns exit 8 with diagnostics.", + "fromTrajectory": "traj_ro3v7jmydpo5" + } + ] + }, + { + "category": "security", + "decisions": [ + { + "question": "Ship removal waits opt-in; use get not-found and deadline list absence, never released status", + "chosen": "Ship removal waits opt-in; use get not-found and deadline list absence, never released status", + "reasoning": "No workspace credential environment variables or machine-global agent-relay configuration are available for live deletion verification. Reviewed-plan section 3.7 requires opt-in in this case. Exit 8 requires observed presence; all-read failures keep the asynchronous acknowledgement.", + "fromTrajectory": "traj_ro3v7jmydpo5" + } + ] + }, + { + "category": "naming", + "decisions": [ + { + "question": "Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance", + "chosen": "Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance", + "reasoning": "Cleanup is already scheduled for owned failed spawns. A collision never grants ownership of the incumbent identity. Fleet release adds removal evidence to its existing JSON only with --wait.", + "fromTrajectory": "traj_ro3v7jmydpo5" + } + ] + }, + { + "category": "api", + "decisions": [ + { + "question": "Final invocation reads preserve the actual requested readiness contract", + "chosen": "Final invocation reads preserve the actual requested readiness contract", + "reasoning": "SDK confirm:true also requests verify_ready by default. A late terminal result must have ready:true; live heartbeat recovery remains explicitly confirmed:false. Late persona child acknowledgements stay pending and retain their child invocation/node.", + "fromTrajectory": "traj_ro3v7jmydpo5" + } + ] + } + ], + "keyLearnings": [], + "keyFindings": [], + "filesAffected": [ + ".agentworkforce/trajectories/fleet-spawn-fix/active/traj_ro3v7jmydpo5/trajectory.json", + "CHANGELOG.md", + "crates/broker/src/runtime/relaycast_events.rs", + "packages/cli/README.md", + "packages/cli/src/cli/commands/agent.test.ts", + "packages/cli/src/cli/commands/agent.ts", + "packages/cli/src/cli/commands/fleet.test.ts", + "packages/cli/src/cli/commands/fleet.ts", + "packages/cli/src/cli/lib/agent-registration.ts", + "packages/cli/src/cli/lib/agent-removal.test.ts", + "packages/cli/src/cli/lib/agent-removal.ts", + "packages/cli/src/cli/lib/fleet-live-agents.ts", + "packages/cli/src/cli/lib/sdk-command.test.ts", + "packages/cli/src/cli/lib/sdk-command.ts", + "packages/cli/src/cli/lib/spawn-liveness.test.ts", + "packages/cli/src/cli/lib/spawn-liveness.ts" + ], + "commits": [ + "5c8b798" + ] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.md b/.agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.md new file mode 100644 index 0000000000..de15c369d4 --- /dev/null +++ b/.agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.md @@ -0,0 +1,33 @@ +# Trajectory Compaction: Oct 4, 2026 - Oct 4, 2026 + +## Summary +- Sessions: 1 +- Decisions: 4 +- Events: 4 +- Agents: default +- Files: 16 +- Commits: 1 + +## Other +- Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes -> Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes (traj_ro3v7jmydpo5) + +## Security +- Ship removal waits opt-in; use get not-found and deadline list absence, never released status -> Ship removal waits opt-in; use get not-found and deadline list absence, never released status (traj_ro3v7jmydpo5) + +## Naming +- Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance -> Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance (traj_ro3v7jmydpo5) + +## Api +- Final invocation reads preserve the actual requested readiness contract -> Final invocation reads preserve the actual requested readiness contract (traj_ro3v7jmydpo5) + +## Key Learnings +- None + +## Key Findings +- None + +## Completion evidence + +CLI and MCP pending spawn evidence, fresh target-node heartbeat checks, sandbox preservation, opt-in name-clearance waits, readable registration collisions, and roster heartbeat ages are implemented. SDK behavior and workflow files are unchanged. + +Full CLI suite: 2,021 passed, 17 skipped with inherited RELAY_BASE_URL unset. Typechecking and the latest 72 MCP regression tests passed. Cargo is unavailable, so the Rust formatter regression was not run. No live workspace credentials were available; removal waits remain opt-in under reviewed-plan section 3.7. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4403e3eabd..b1be9dc08f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- MCP `spawn` reports pending/liveness evidence, and `remove_agent` accepts optional registration-clearance waits. - `agent remove --wait` and `fleet release --delete-agent --wait` verify registration clearance before name reuse, with `--wait-timeout` and `--no-wait` controls. Fleet release adds `removal` evidence when waiting. - `fleet agent list` JSON includes node heartbeat timestamps and ages, with warnings for stale snapshots. diff --git a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts index 96d615bc59..7cc39900d9 100644 --- a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts +++ b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts @@ -287,9 +287,17 @@ async function loadAgentRelayMcpModule(options: LoadOptions = {}) { output: { spawned: true, ready: true }, })), }; + const agentRelayNodesGet = vi.fn( + async (name: string) => (await agentRelayNodesList()).find((node) => node.name === name) ?? null + ); + const agentRelayAgentsGet = vi.fn(async (): Promise => { + throw { status: 404 }; + }); + const agentRelayAgentsList = vi.fn(async (): Promise> => []); const AgentRelayMock = vi.fn(function (this: unknown) { return { - nodes: { list: agentRelayNodesList }, + nodes: { list: agentRelayNodesList, get: agentRelayNodesGet }, + agents: { get: agentRelayAgentsGet, list: agentRelayAgentsList }, messaging: { commands: agentRelayMessagingCommands, }, @@ -333,7 +341,8 @@ async function loadAgentRelayMcpModule(options: LoadOptions = {}) { }, }; }); - vi.doMock('@relaycast/sdk', () => ({ + vi.doMock('@relaycast/sdk', async () => ({ + ...(await vi.importActual>('@relaycast/sdk')), RelayCast, SDK_VERSION: 'test-sdk-version', })); @@ -376,6 +385,10 @@ async function loadAgentRelayMcpModule(options: LoadOptions = {}) { RelayCast, FakeTransport, agentRelayMessagingCommands, + agentRelayNodesList, + agentRelayNodesGet, + agentRelayAgentsGet, + agentRelayAgentsList, agentClients, }, }; @@ -1149,17 +1162,17 @@ describe('createAgentRelayMcpServer', () => { (error: unknown) => error ), new Promise((resolve) => - setTimeout(() => resolve('nested timeout test did not finish'), 130_001) + setTimeout(() => resolve('nested timeout test did not finish'), 132_001) ), ]); - await vi.advanceTimersByTimeAsync(130_001); + await vi.advanceTimersByTimeAsync(132_001); await expect(outcome).resolves.toMatchObject({ isError: true, structuredContent: { error: { - code: 'spawn_unconfirmed', + code: 'spawn_pending', invocationId: 'inv_nested', dispatchState: 'dispatched', node: 'nested-node', @@ -1194,20 +1207,20 @@ describe('createAgentRelayMcpServer', () => { (error: unknown) => error ), new Promise((resolve) => - setTimeout(() => resolve('still pending after the persona spawn deadline'), 130_001) + setTimeout(() => resolve('still pending after the persona spawn deadline'), 132_001) ), ]); - await vi.advanceTimersByTimeAsync(130_001); + await vi.advanceTimersByTimeAsync(132_001); await expect(outcome).resolves.toMatchObject({ isError: true, structuredContent: { error: { - code: 'spawn_unconfirmed', - state: 'unconfirmed_may_be_running', + code: 'spawn_pending', + state: 'pending', dispatchState: 'unknown', - message: expect.stringContaining('may still be running'), + message: expect.stringContaining('outcome is pending'), }, }, }); @@ -1218,7 +1231,7 @@ describe('createAgentRelayMcpServer', () => { it('preserves an unconfirmed receipt when authorization is revoked after acceptance', async () => { const { mod, mocks } = await loadAgentRelayMcpModule(); - mocks.agentRelayMessagingCommands.getInvocation.mockRejectedValueOnce(new Error('401 unauthorized')); + mocks.agentRelayMessagingCommands.getInvocation.mockRejectedValue(new Error('401 unauthorized')); mod.createAgentRelayMcpServer({ workspaceKey: 'rk_live_existing', agentToken: 'at_live_fleet', @@ -1233,16 +1246,128 @@ describe('createAgentRelayMcpServer', () => { isError: true, structuredContent: { error: { - code: 'spawn_unconfirmed', - state: 'unconfirmed_may_be_running', + code: 'spawn_pending', + state: 'pending', dispatchState: 'unknown', invocationId: 'inv_1', - message: expect.stringContaining('may still be running'), + message: expect.stringContaining('outcome is pending'), }, }, }); }); + it('recovers a terminal spawn result on the final invocation read', async () => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mocks.agentRelayMessagingCommands.getInvocation.mockRejectedValueOnce(new Error('401 unauthorized')); + mod.createAgentRelayMcpServer({ agentToken: 'at_live_fleet', agentName: 'orchestrator' }); + const result = await mocks.serverInstances[0].tools + .get('spawn')! + .handler({ name: 'worker', cli: 'codex' }); + expect(result.isError).not.toBe(true); + expect(result.structuredContent.placement).toMatchObject({ state: 'ready', confirmed: true }); + expect(mocks.agentRelayMessagingCommands.getInvocation).toHaveBeenCalledTimes(2); + }); + + it('keeps a late persona child acknowledgement pending with its own correlation', async () => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mocks.agentRelayMessagingCommands.getInvocation + .mockRejectedValueOnce(new Error('401 unauthorized')) + .mockResolvedValueOnce({ + status: 'completed', + output: { + invocationId: 'inv_child', + actionName: 'spawn', + status: 'dispatched', + handlerNodeId: 'child-node', + }, + } as never); + mod.createAgentRelayMcpServer({ agentToken: 'at_live_fleet', agentName: 'orchestrator' }); + const result = await mocks.serverInstances[0].tools + .get('spawn')! + .handler({ name: 'worker', persona: 'reviewer' }); + expect(result).toMatchObject({ + isError: true, + structuredContent: { error: { code: 'spawn_pending', invocationId: 'inv_child', node: 'child-node' } }, + }); + }); + + it.each(['live', 'live_elsewhere', 'stale', 'unknown'] as const)( + 'reports %s evidence without manufacturing readiness', + async (evidence) => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mocks.agentRelayMessagingCommands.invoke.mockResolvedValueOnce({} as never); + const record = { + name: evidence === 'live_elsewhere' ? 'other' : 'node-a', + status: 'online', + lastHeartbeatAt: new Date(Date.now() - (evidence === 'stale' ? 40_000 : 0)).toISOString(), + capabilities: [{ name: 'relay:live-agents:v1', metadata: { names: ['worker'] } }], + }; + mocks.agentRelayNodesList.mockResolvedValue([record]); + mod.createAgentRelayMcpServer({ + agentToken: 'at_live_fleet', + agentName: 'orchestrator', + ...(evidence !== 'unknown' ? { workspaceKey: 'rk_test' } : {}), + }); + const result = await mocks.serverInstances[0].tools + .get('spawn')! + .handler({ name: 'worker', cli: 'codex', target_node: 'node-a' }); + if (evidence === 'live') { + expect(result.isError).not.toBe(true); + expect(result.structuredContent.placement).toMatchObject({ + state: 'accepted', + confirmed: false, + liveness: { evidence }, + }); + } else { + expect(result).toMatchObject({ + isError: true, + structuredContent: { error: { code: 'spawn_pending', state: 'pending', liveness: { evidence } } }, + }); + } + } + ); + + it('classifies MCP registration collisions without exposing Rust debug text', async () => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mocks.agentRelayMessagingCommands.getInvocation.mockResolvedValueOnce({ + status: 'failed', + error: + "failed to pre-register worker 'worker': agent 'worker' already exists and registration is create-only; use a unique name", + } as never); + mod.createAgentRelayMcpServer({ agentToken: 'at_live_fleet', agentName: 'orchestrator' }); + const result = await mocks.serverInstances[0].tools + .get('spawn')! + .handler({ name: 'worker', cli: 'codex' }); + expect(result).toMatchObject({ + isError: true, + structuredContent: { error: { code: 'spawn_name_taken', message: expect.stringContaining('--wait') } }, + }); + }); + + it.each(['cleared', 'present', 'unavailable'] as const)( + 'remove_agent wait reports registration %s after acknowledgement', + async (evidence) => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + if (evidence === 'present') { + mocks.agentRelayAgentsGet.mockResolvedValue({ name: 'worker', status: 'released' }); + mocks.agentRelayAgentsList.mockResolvedValue([{ name: 'worker' }]); + } else if (evidence === 'unavailable') { + mocks.agentRelayAgentsGet.mockRejectedValue(new Error('403 forbidden')); + mocks.agentRelayAgentsList.mockRejectedValue(new Error('403 forbidden')); + } + mod.createAgentRelayMcpServer({ + workspaceKey: 'rk_test', + agentToken: 'at_live_fleet', + agentName: 'orchestrator', + }); + const result = await mocks.serverInstances[0].tools + .get('remove_agent')! + .handler({ name: 'worker', delete_agent: true, wait: true, wait_timeout_ms: 1 }); + expect(result.structuredContent.removal).toMatchObject({ cleared: evidence === 'cleared' }); + expect(result.isError === true).toBe(evidence === 'present'); + } + ); + it('correlates nested authorization failures to the nested route', async () => { const { mod, mocks } = await loadAgentRelayMcpModule(); mocks.agentRelayMessagingCommands.invoke.mockResolvedValueOnce({ @@ -1284,7 +1409,7 @@ describe('createAgentRelayMcpServer', () => { isError: true, structuredContent: { error: { - code: 'spawn_unconfirmed', + code: 'spawn_pending', invocationId: 'inv_nested', dispatchState: 'dispatched', node: 'nested-node', diff --git a/packages/cli/src/cli/agent-relay-mcp.ts b/packages/cli/src/cli/agent-relay-mcp.ts index ecd4ac37be..cb18f30c7d 100644 --- a/packages/cli/src/cli/agent-relay-mcp.ts +++ b/packages/cli/src/cli/agent-relay-mcp.ts @@ -1,4 +1,13 @@ #!/usr/bin/env node +import { waitForAgentRemoval } from './lib/agent-removal.js'; +import { + FleetSpawnError, + classifySpawnFailure, + nameTakenSpawnError, + pendingSpawnError, + probeSpawnLiveness, + terminalSpawnOutcome, +} from './lib/spawn-liveness.js'; import fs from 'node:fs'; import path from 'node:path'; @@ -97,6 +106,7 @@ const VERIFIED_SPAWN_FAILURE_STATUSES = new Set(['failed', 'error', 'cancelled', class VerifiedSpawnError extends Error { readonly code: 'spawn_unconfirmed' | 'spawn_failed'; + readonly actionName?: string; readonly invocationId?: string; readonly state: SpawnLifecycleState; readonly dispatchState: SpawnDispatchState; @@ -107,6 +117,7 @@ class VerifiedSpawnError extends Error { message: string, context: { code?: 'spawn_unconfirmed' | 'spawn_failed'; + actionName?: string; state: SpawnLifecycleState; dispatchState?: SpawnDispatchState; invocationId?: string; @@ -116,6 +127,7 @@ class VerifiedSpawnError extends Error { ) { super(message); this.name = 'VerifiedSpawnError'; + this.actionName = context.actionName; this.code = context.code ?? (context.state === 'failed' ? 'spawn_failed' : 'spawn_unconfirmed'); this.state = context.state; this.dispatchState = context.dispatchState ?? 'unknown'; @@ -270,6 +282,7 @@ async function pollInvocation( code: 'spawn_unconfirmed', state: 'unconfirmed_may_be_running', invocationId: current.invocationId, + actionName: current.actionName, dispatchState: current.dispatchState, ...(current.node ? { node: current.node } : {}), } @@ -281,6 +294,7 @@ async function pollInvocation( { state: 'unconfirmed_may_be_running', invocationId: current.invocationId, + actionName: current.actionName, dispatchState: current.dispatchState, ...(current.node ? { node: current.node } : {}), receipt: { status: 'unknown' }, @@ -335,6 +349,7 @@ async function waitForVerifiedSpawn( state: 'failed', dispatchState: dispatchStateForRecord(record, current.dispatchState), invocationId: current.invocationId, + actionName: current.actionName, receipt: record, }); } @@ -349,6 +364,7 @@ async function waitForVerifiedSpawn( state: 'failed', dispatchState: dispatchStateForRecord(record, current.dispatchState), invocationId: current.invocationId, + actionName: current.actionName, node: invocationText(record, 'handlerNodeId', 'handler_node_id') ?? invocationText(record, 'dispatchedNodeId', 'dispatched_node_id') ?? @@ -364,6 +380,7 @@ async function waitForVerifiedSpawn( state: 'failed', dispatchState: dispatchStateForRecord(record, current.dispatchState), invocationId: current.invocationId, + actionName: current.actionName, node: invocationText(record, 'handlerNodeId', 'handler_node_id') ?? invocationText(record, 'dispatchedNodeId', 'dispatched_node_id') ?? @@ -379,6 +396,7 @@ async function waitForVerifiedSpawn( state: 'unconfirmed_may_be_running', dispatchState: dispatchStateForRecord(record, current.dispatchState), invocationId: current.invocationId, + actionName: current.actionName, node: invocationText(record, 'handlerNodeId', 'handler_node_id') ?? invocationText(record, 'dispatchedNodeId', 'dispatched_node_id') ?? @@ -876,15 +894,92 @@ async function invokeVerifiedSpawn( } const commands = new AgentRelay({ agentToken, baseUrl }).messaging.commands; const invocation = await commands.invoke('spawn', actionInput); - return waitForVerifiedSpawn(commands, invocation); + try { + return await waitForVerifiedSpawn(commands, invocation); + } catch (error) { + if (!(error instanceof VerifiedSpawnError)) throw error; + const name = String(actionInput.name); + if (error.code === 'spawn_failed' && classifySpawnFailure(error.message)) + throw nameTakenSpawnError(name, error); + if (error.code !== 'spawn_unconfirmed') throw error; + const requestedNode = typeof actionInput.target_node === 'string' ? actionInput.target_node : undefined; + const targetNode = actionInput.persona ? (error.node ?? requestedNode) : (requestedNode ?? error.node); + const context = { ...error, node: targetNode }; + const liveness = await probeSpawnLiveness({ + name, + targetNode, + ...(error.invocationId + ? { getInvocation: () => commands.getInvocation(error.actionName ?? 'spawn', error.invocationId!) } + : {}), + createClient: () => { + requireWorkspaceKey(session); + const workspace = new AgentRelay({ workspaceKey: session.workspaceKey!, baseUrl }); + return { nodes: workspace.nodes, agents: workspace.agents }; + }, + }); + if (liveness.evidence === 'invocation_terminal') { + const nested = VERIFIED_SPAWN_SUCCESS_STATUSES.has(String(liveness.invocation!.status).toLowerCase()) + ? nestedPersonaSpawnRef(liveness.invocation!) : undefined; + if (nested) { + // A late persona result can be only an acknowledgement of its child. + // It is neither missing readiness proof nor a terminal worker failure. + throw pendingSpawnError( + name, + { + ...context, + invocationId: nested.invocationId, + node: nested.node ?? targetNode, + dispatchState: nested.dispatchState, + }, + { evidence: 'unknown' }, + 'The persona handler returned a nested spawn invocation after the confirmation budget.' + ); + } + try { + return terminalSpawnOutcome(liveness.invocation!, context, true); + } catch (terminalError) { + const message = terminalError instanceof Error ? terminalError.message : String(terminalError); + if (classifySpawnFailure(message)) + throw nameTakenSpawnError(name, Object.assign(new Error(message), context)); + throw new VerifiedSpawnError(message, { + ...error, + code: 'spawn_failed', + state: 'failed', + receipt: liveness.invocation, + }); + } + } + if (liveness.evidence === 'live') { + return { + invocationId: error.invocationId, + status: 'accepted', + placement: { + state: 'accepted', + confirmed: false, + dispatchState: error.dispatchState, + invocationId: error.invocationId, + liveness, + warning: `A worker named ${JSON.stringify(name)} is running on ${JSON.stringify(liveness.node)} according to its fresh heartbeat; harness readiness was not confirmed within the budget.`, + }, + }; + } + throw pendingSpawnError(name, context, liveness, error.message); + } } -function verifiedSpawnErrorResult(error: VerifiedSpawnError) { +function verifiedSpawnErrorResult(error: VerifiedSpawnError | FleetSpawnError) { return { ...jsonContent({ ok: false, error: { code: error.code, + ...(error instanceof FleetSpawnError + ? { + exitCode: error.exitCode, + liveness: error.liveness, + diagnostic: error.diagnostic ? safeRelayErrorMessage(new Error(error.diagnostic)) : undefined, + } + : {}), state: error.state, dispatchState: error.dispatchState, ...(error.invocationId ? { invocationId: error.invocationId } : {}), @@ -1372,7 +1467,7 @@ function registerAgentRelayTools( title: 'Spawn Agent', description: 'Invoke the fleet spawn action with either a raw `cli` or an AgentWorkforce `persona` name/path. ' + - 'Persona requests route to a node exposing `spawn:persona` (for example, `defineWorkforcePersonaSpawnNode` from `@agentworkforce/local-surface`). Both forms return only after broker registration and harness readiness are verified.', + 'Persona requests route to a node exposing `spawn:persona` (for example, `defineWorkforcePersonaSpawnNode` from `@agentworkforce/local-surface`). Both forms wait for broker registration and harness readiness. Unconfirmed outcomes report pending evidence; fresh worker presence may return confirmed:false without readiness proof.', inputSchema: { name: z.string().describe('Agent name'), cli: z @@ -1473,10 +1568,14 @@ function registerAgentRelayTools( const invocation = await invokeVerifiedSpawn(getSession(), as, baseUrl, actionInput); return jsonContent({ invocation, - placement: { state: 'ready', ...spawnReceipt(recordValue(invocation)) }, + placement: recordValue(invocation).placement ?? { + state: 'ready', + ...spawnReceipt(recordValue(invocation)), + }, }); } catch (error) { - if (error instanceof VerifiedSpawnError) return verifiedSpawnErrorResult(error); + if (error instanceof VerifiedSpawnError || error instanceof FleetSpawnError) + return verifiedSpawnErrorResult(error); throw error; } }) @@ -1488,7 +1587,7 @@ function registerAgentRelayTools( title: 'Remove Agent', description: 'Release a worker agent from active duty, optionally deleting it outright. ' + - 'Returns an `invocation` record acknowledging the request, which is processed asynchronously. Releasing keeps the agent registered and re-spawnable; passing `delete_agent` removes the identity permanently.', + 'Returns an `invocation` record acknowledging the request, which is processed asynchronously. Releasing keeps the agent registered; passing `delete_agent` requests permanent deletion. Use `wait` with `delete_agent` to verify the name is reusable.', inputSchema: { name: z.string().describe('Agent name'), reason: z @@ -1496,12 +1595,18 @@ function registerAgentRelayTools( .optional() .describe('Removal reason; defaults to an attributable Agent Relay MCP reason'), delete_agent: z.boolean().optional().describe('Permanently delete the agent'), + wait: z.boolean().optional().describe('With delete_agent, wait for registration clearance'), + wait_timeout_ms: z + .number() + .positive() + .optional() + .describe('Clearance timeout in milliseconds (default 30000)'), ...identityOverrideInputShape, }, outputSchema: jsonResult, annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, }, - async ({ name, reason, delete_agent, as }) => { + async ({ name, reason, delete_agent, wait, wait_timeout_ms, as }) => { const session = getSession(); const selected = as ? session.agents.get(as) : undefined; if (as && !selected) { @@ -1531,6 +1636,31 @@ function registerAgentRelayTools( throw error; } } + if (delete_agent && wait) { + const workspace = () => { + requireWorkspaceKey(session); + return new AgentRelay({ workspaceKey: session.workspaceKey!, baseUrl }); + }; + const removal = await waitForAgentRemoval({ + name, + getAgent: (name) => workspace().agents.get(name), + listAgents: () => workspace().agents.list(), + timeoutMs: wait_timeout_ms, + }); + return { + ...jsonContent({ + invocation, + removal, + ...(!removal.cleared + ? { + message: + 'Removal was initiated asynchronously; verify registration clearance before respawning.', + } + : {}), + }), + ...(!removal.cleared && removal.observedPresent ? { isError: true as const } : {}), + }; + } return jsonContent({ invocation }); } ); From bc6cb608f85a8178ea7cda4b584b044d04d87a59 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 4 Oct 2026 14:58:35 +0000 Subject: [PATCH 07/42] style: auto-format Rust code with cargo fmt --- crates/broker/src/runtime/relaycast_events.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/broker/src/runtime/relaycast_events.rs b/crates/broker/src/runtime/relaycast_events.rs index 9cf1e2c776..5d1fbb1bdc 100644 --- a/crates/broker/src/runtime/relaycast_events.rs +++ b/crates/broker/src/runtime/relaycast_events.rs @@ -802,7 +802,10 @@ pub(super) async fn spawn_worker_from_request( } Some(token) } - Err(RegRetryOutcome::RetryableExhausted(error) | RegRetryOutcome::Fatal(error)) => { + Err( + RegRetryOutcome::RetryableExhausted(error) + | RegRetryOutcome::Fatal(error), + ) => { anyhow::bail!("{}", format_worker_preregistration_error(&name, &error)); } } From 1665d249dc0dd58a656c18ab952d2c4021ab305d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 4 Oct 2026 15:00:55 +0000 Subject: [PATCH 08/42] style: auto-format with Prettier --- packages/cli/src/cli/agent-relay-mcp.ts | 3 ++- packages/cli/src/cli/commands/fleet.test.ts | 7 +++++-- packages/cli/src/cli/lib/spawn-liveness.test.ts | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/cli/agent-relay-mcp.ts b/packages/cli/src/cli/agent-relay-mcp.ts index cb18f30c7d..876ab067cd 100644 --- a/packages/cli/src/cli/agent-relay-mcp.ts +++ b/packages/cli/src/cli/agent-relay-mcp.ts @@ -919,7 +919,8 @@ async function invokeVerifiedSpawn( }); if (liveness.evidence === 'invocation_terminal') { const nested = VERIFIED_SPAWN_SUCCESS_STATUSES.has(String(liveness.invocation!.status).toLowerCase()) - ? nestedPersonaSpawnRef(liveness.invocation!) : undefined; + ? nestedPersonaSpawnRef(liveness.invocation!) + : undefined; if (nested) { // A late persona result can be only an acknowledgement of its child. // It is neither missing readiness proof nor a terminal worker failure. diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 47ceb5899d..4ce2bbdf63 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1110,7 +1110,8 @@ describe('fleet command support', () => { ? { status: 'completed', output: { spawned: true, ready: evidence === 'terminal' } } : { status: 'failed', - error: "failed to pre-register worker 'worker': agent 'worker' already exists and registration is create-only; use a unique name", + error: + "failed to pre-register worker 'worker': agent 'worker' already exists and registration is create-only; use a unique name", } ); const nodes = { @@ -1169,7 +1170,9 @@ describe('fleet command support', () => { expect(getInvocation).toHaveBeenCalledWith('spawn', invocationId); if (evidence === 'collision' || evidence === 'missing_ready') { expect(exit).toHaveBeenCalledWith(1); - expect(errors.join('')).toContain(evidence === 'collision' ? '"code":"spawn_name_taken"' : '"code":"spawn_failed"'); + expect(errors.join('')).toContain( + evidence === 'collision' ? '"code":"spawn_name_taken"' : '"code":"spawn_failed"' + ); } else { expect(exit).not.toHaveBeenCalled(); expect(JSON.parse(logs[0]!).invocation.placement).toMatchObject({ diff --git a/packages/cli/src/cli/lib/spawn-liveness.test.ts b/packages/cli/src/cli/lib/spawn-liveness.test.ts index 9ff75a6bb9..73ac8a214a 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.test.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.test.ts @@ -128,6 +128,8 @@ describe('spawn liveness resolution', () => { ) ).toBe('spawn_name_taken'); expect(classifySpawnFailure('failed to create sandbox: directory already exists')).toBeUndefined(); - expect(classifySpawnFailure("failed to pre-register worker 'worker': cache entry already exists")).toBeUndefined(); + expect( + classifySpawnFailure("failed to pre-register worker 'worker': cache entry already exists") + ).toBeUndefined(); }); }); From d84339893dfa57ed6dd2efbda67826568d444003 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Mon, 5 Oct 2026 10:19:56 +0000 Subject: [PATCH 09/42] fix: coalesce pending replies and verify provider subscriptions --- .../compact_tggynuofmzom_2026-10-05.json | 89 +++++++++ .../compact_tggynuofmzom_2026-10-05.md | 14 ++ CHANGELOG.md | 3 + .../evidence/provider-subscription-replies.md | 68 +++++++ .../commands/integration-subscribe.test.ts | 179 +++++++++++++++++- packages/cli/src/cli/commands/integration.ts | 69 ++++++- .../src/cli/commands/relaycast-groups.test.ts | 9 +- .../cli/mcp/messaging-tools.protocol.test.ts | 38 ++++ packages/cli/src/cli/mcp/messaging-tools.ts | 37 +++- .../cli/src/cli/mcp/request-replay.test.ts | 87 +++++++++ packages/cli/src/cli/mcp/request-replay.ts | 29 ++- packages/cli/src/cli/telemetry/events.ts | 2 + .../src/ai-sdk/native-relay-tools.test.ts | 42 ++++ .../src/ai-sdk/native-relay-tools.ts | 23 ++- packages/integration-prompts/src/builders.ts | 3 + .../sdk/src/__tests__/integrations.test.ts | 22 +++ .../src/__tests__/relaycast-translate.test.ts | 21 +- packages/sdk/src/messaging/index.ts | 2 + .../sdk/src/messaging/relaycast-translate.ts | 9 +- packages/sdk/src/messaging/relaycast.ts | 17 +- packages/sdk/src/messaging/types.ts | 1 + summary.md | 36 ++++ 22 files changed, 772 insertions(+), 28 deletions(-) create mode 100644 .agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.json create mode 100644 .agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.md create mode 100644 docs/evidence/provider-subscription-replies.md create mode 100644 packages/cli/src/cli/mcp/request-replay.test.ts create mode 100644 summary.md diff --git a/.agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.json b/.agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.json new file mode 100644 index 0000000000..1788af304f --- /dev/null +++ b/.agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.json @@ -0,0 +1,89 @@ +{ + "id": "compact_tggynuofmzom", + "version": 1, + "type": "compacted", + "compactedAt": "2026-10-05T10:18:47.735Z", + "sourceTrajectories": [ + "traj_kvnuivdfl4un" + ], + "dateRange": { + "start": "2026-10-05T10:17:44.252Z", + "end": "2026-10-05T10:18:47.370Z" + }, + "summary": { + "totalDecisions": 2, + "totalEvents": 2, + "uniqueAgents": [ + "default" + ] + }, + "decisionGroups": [ + { + "category": "security", + "decisions": [ + { + "question": "Implement pending-only local coalescing and verified provisioning; leave provider scope unchanged", + "chosen": "Implement pending-only local coalescing and verified provisioning; leave provider scope unchanged", + "reasoning": "Reviewed plan confirms MCP replay asymmetry. Incident credentials and control-plane daemon are unavailable; do not widen PR globs without E2 evidence.", + "fromTrajectory": "traj_kvnuivdfl4un" + } + ] + }, + { + "category": "tooling", + "decisions": [ + { + "question": "Demote plural-events hypothesis and defer explicit message idempotency keys", + "chosen": "Demote plural-events hypothesis and defer explicit message idempotency keys", + "reasoning": "Installed Relaycast types require plural events and the SDK already sends generated Idempotency-Key headers. Engine enforcement cannot be verified here. Installed Relayfile client has no writeback ledger status or retry methods.", + "fromTrajectory": "traj_kvnuivdfl4un" + } + ] + } + ], + "keyLearnings": [], + "keyFindings": [], + "filesAffected": [ + "CHANGELOG.md", + "packages/cli/src/cli/commands/integration-subscribe.test.ts", + "packages/cli/src/cli/commands/integration.ts", + "packages/cli/src/cli/commands/relaycast-groups.test.ts", + "packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts", + "packages/cli/src/cli/mcp/messaging-tools.ts", + "packages/cli/src/cli/mcp/request-replay.ts", + "packages/cli/src/cli/telemetry/events.ts", + "packages/harnesses/src/ai-sdk/native-relay-tools.test.ts", + "packages/harnesses/src/ai-sdk/native-relay-tools.ts", + "packages/integration-prompts/src/builders.ts", + "packages/sdk/src/__tests__/integrations.test.ts", + "packages/sdk/src/__tests__/relaycast-translate.test.ts", + "packages/sdk/src/messaging/index.ts", + "packages/sdk/src/messaging/relaycast-translate.ts", + "packages/sdk/src/messaging/relaycast.ts", + "packages/sdk/src/messaging/types.ts", + "packages/cli/src/cli/mcp/request-replay.test.ts", + "summary.md", + "docs/evidence/provider-subscription-replies.md" + ], + "commits": [], + "narrative": "Implemented pending-only MCP/native reply coalescing, subscription event read-back with journaled rollback, authorization-aware output, outbound configuration diagnostics, and regression tests on the current branch. Local tests and typechecks passed. Incident workspace access and the Relayfile daemon are unavailable; actual GitHub writeback and durable retry remain unverified. See summary.md and docs/evidence/provider-subscription-replies.md.", + "decisions": [ + { + "question": "Implement pending-only local coalescing and verified provisioning; leave provider scope unchanged", + "chosen": "Implement pending-only local coalescing and verified provisioning; leave provider scope unchanged", + "reasoning": "Reviewed plan confirms MCP replay asymmetry. Incident credentials and control-plane daemon are unavailable; do not widen PR globs without E2 evidence." + }, + { + "question": "Demote plural-events hypothesis and defer explicit message idempotency keys", + "chosen": "Demote plural-events hypothesis and defer explicit message idempotency keys", + "reasoning": "Installed Relaycast types require plural events and the SDK already sends generated Idempotency-Key headers. Engine enforcement cannot be verified here. Installed Relayfile client has no writeback ledger status or retry methods." + } + ], + "conventions": [], + "lessons": [], + "openQuestions": [ + "Does subscription writeback ingress use the existing Relayfile ops ledger, keyed by Relay message ID?", + "What are the incident binding paths and persisted subscription events?", + "Does the engine enforce message/reply idempotency keys across processes?" + ] +} diff --git a/.agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.md b/.agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.md new file mode 100644 index 0000000000..7ae252bd06 --- /dev/null +++ b/.agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.md @@ -0,0 +1,14 @@ +# Provider subscription reply trajectory + +Implemented pending-only MCP/native reply coalescing, subscription event read-back with journaled rollback, authorization-aware output, outbound configuration diagnostics, and regression tests on the current branch. Local tests and typechecks passed. Incident workspace access and the Relayfile daemon are unavailable; actual GitHub writeback and durable retry remain unverified. See summary.md and docs/evidence/provider-subscription-replies.md. + +## Decisions + +- Implement pending-only local coalescing and verified provisioning; leave provider scope unchanged. Reviewed plan confirms MCP replay asymmetry. Incident credentials and control-plane daemon are unavailable; do not widen PR globs without E2 evidence. +- Demote plural-events hypothesis and defer explicit message idempotency keys. Installed Relaycast types require plural events and the SDK already sends generated Idempotency-Key headers. Engine enforcement cannot be verified here. Installed Relayfile client has no writeback ledger status or retry methods. + +## Open questions + +- Does subscription writeback ingress use the existing Relayfile ops ledger, keyed by Relay message ID? +- What are the incident binding paths and persisted subscription events? +- Does the engine enforce message/reply idempotency keys across processes? diff --git a/CHANGELOG.md b/CHANGELOG.md index 855ba982b9..176900c04a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Relay MCP and native host tools coalesce identical pending channel posts and thread replies while preserving sequential repeats. +- `integration subscribe` verifies persisted writeback events, rolls back incomplete subscriptions, and reports GitHub authorization uncertainty; `--list` distinguishes subscription configuration from delivery confirmation. + - Broker recovery now stops with a clear re-enrollment error when Relaycast refuses node-token rotation proof, while retaining the last in-memory and durable credential for operator recovery. Accepted rotations use an atomically persisted idempotency key so a lost response or restart can recover the committed replacement. - `agent-relay fleet spawn --sandbox` accepts the canonical DEV Relaycast target only when authenticated against the exact DEV Cloud API URL (`https://dev.agentrelay.com/cloud`), while production trust remains unchanged. - `agent-relay fleet spawn --sandbox` makes bounded visibility checks for a node-created identity on both the worker-token and workspace-key read paths during channel reconciliation, absorbing Relaycast read-after-write lag without rotating the new worker credential. diff --git a/docs/evidence/provider-subscription-replies.md b/docs/evidence/provider-subscription-replies.md new file mode 100644 index 0000000000..35be3a0f13 --- /dev/null +++ b/docs/evidence/provider-subscription-replies.md @@ -0,0 +1,68 @@ +# Provider subscription reply investigation (2026-10-05) + +This change implements the locally verifiable portions of `reviewed-plan.md`. +It does **not** establish that the reported GitHub writeback failure is fixed. + +## Available evidence + +- The incident's Relay workspace is inaccessible in this run: `agent-relay + message inbox check` reports no workspace key. No authenticated message, + transcript, subscription, or binding read could be made (E1–E3). +- A read-only `RelayfileControlPlaneClient({ autoStart: false })` binding read + reports `DAEMON_UNAVAILABLE`. Stored PR globs therefore remain unverified; + this change does not add an issue-comment glob or widen provider scope. +- Installed `@relaycast/types/dist/subscription.d.ts` declares plural `events`, + `is_active`, and subscription configuration. It declares no delivery-attempt + status or retry operation. Event-array support is corroborated, but the + incident subscription's stored coverage remains unknown. +- Installed `@relayfile/client/dist/client.d.ts` exposes binding, webhook, and + writeback-secret operations, but no ops-ledger read or retry API. The local + client does not establish whether cloud subscription ingress uses the + file-write ops ledger (E4 remains open). +- Installed `@relaycast/sdk/dist/agent.js` supplies `Idempotency-Key` headers for + channel posts and thread replies, including generated keys. That proves + transport behavior, not engine enforcement. No explicit `idempotency_key` + tool argument is added without E5 confirmation. +- The existing changelog also records a standalone MCP double-dispatch fix in + 13.1.0. The incident was on 12.4.0; without its transcript, parallel aliases + cannot be asserted as the incident's root cause. + +## Changes and boundaries + +MCP posts/replies use request replay and pending-only argument coalescing. +Native tools use a per-session pending map. Concurrent identical writes join; +settled writes, different identities, targets, options, and sessions remain +separate. MCP joins emit `agent_relay_write_coalesced` with only the tool name. +This cannot deduplicate distinct processes or sequential unkeyed retries after a +lost response. + +Subscription setup reads persisted event coverage before binding, rejects +unsupported events and inactive/mismatched subscriptions, and uses the existing +journaled rollback. Missing GitHub identity authorization produces an explicit +unverified message. `--list` exposes outbound configuration with +`deliveryStatus: null`; this is not a successful delivery receipt. Its +`lastDeliverySource` distinguishes inbound control-plane reports from channel +activity inferred locally. + +The deterministic tests exercise real CLI provisioning/rollback and MCP +transport against boundary fakes, plus native-tool and SDK normalization +regressions. They do not substitute a fake provider ledger for a real +GitHub inbound → fleet wake → reply → writeback integration test. + +## Remaining companion verification + +Read the incident binding and subscription, compare issue-comment and PR paths, +and inspect the cloud subscription writeback ingress. If ingress bypasses the +existing ops ledger, route it through that ledger using the originating Relay +message ID as the stable operation key. Expose durable failed/dead-lettered +status and retry through the control-plane client. Verify retry after both a +provider failure and a lost successful response, with one resulting GitHub +comment. Only then build the full seam test against those verified contracts. + +No broker reconnect behavior changed. Existing delivery-book tests cover +cumulative ACK dedupe, seeded resume cursors, and replay of unacknowledged +manual sequences after restart in `crates/broker/src/node_control.rs`. +`duplicate_and_stale_plans_still_ack_to_stop_redelivery` in +`crates/broker/src/runtime/fleet.rs` covers ACK behavior. These are existing +coverage, not a new exact-PR end-to-end proof; Rust tests were not run here +because Cargo is unavailable. diff --git a/packages/cli/src/cli/commands/integration-subscribe.test.ts b/packages/cli/src/cli/commands/integration-subscribe.test.ts index b28d090a53..d669acd9b9 100644 --- a/packages/cli/src/cli/commands/integration-subscribe.test.ts +++ b/packages/cli/src/cli/commands/integration-subscribe.test.ts @@ -29,6 +29,7 @@ interface InboundWebhook { function createRelayMock(opts: { inboundWebhooks?: InboundWebhook[] } = {}) { let counter = 0; + const subscriptions = new Map>(); return { agents: { register: vi.fn(async (i: { name: string }) => ({ id: 'a1', token: 't1', name: i.name })), @@ -36,7 +37,12 @@ function createRelayMock(opts: { inboundWebhooks?: InboundWebhook[] } = {}) { }, integrations: { subscriptions: { - create: vi.fn(async (i: unknown) => ({ id: `sub_${++counter}`, ...(i as object) })), + create: vi.fn(async (i: unknown) => { + const row = { id: `sub_${++counter}`, ...(i as object) }; + subscriptions.set(row.id, row); + return row; + }), + get: vi.fn(async (id: string) => subscriptions.get(id) ?? { id }), delete: vi.fn(async () => undefined), }, }, @@ -2064,3 +2070,174 @@ describe('confirmed agent subscription setup', () => { expect(h.relay.webhooks.createInbound).not.toHaveBeenCalled(); }); }); + +describe('subscription writeback verification', () => { + it.each([{ event: 'message.created' }, { events: ['message.created'] }, {}])( + 'rolls back incomplete persisted event coverage: %j', + async (echo) => { + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockImplementation(async (id) => ({ id, ...echo })); + const { program, relayfile, error, journal } = harness({ relay }); + await program.parseAsync(ARGS(), { from: 'user' }); + expect(error.mock.calls.flat().join(' ')).toContain('did not confirm requested events'); + expect(relayfile.bind).not.toHaveBeenCalled(); + expect(relay.integrations.subscriptions.delete).toHaveBeenCalled(); + expect(relay.webhooks.delete).toHaveBeenCalled(); + expect(relayfile.deleteWebhookSubscription).toHaveBeenCalled(); + expect(await journal.list()).toEqual([]); + } + ); + + it('journals a failed rollback delete after read-back fails', async () => { + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockRejectedValue(new Error('read unavailable')); + relay.integrations.subscriptions.delete.mockRejectedValue(new Error('delete unavailable')); + const { program, journal, relayfile } = harness({ relay }); + await program.parseAsync(ARGS(), { from: 'user' }); + expect(relayfile.bind).not.toHaveBeenCalled(); + expect(await journal.list()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'relay-subscription', id: expect.stringMatching(/^sub_/) }), + ]) + ); + }); + + it.each(['events', 'event_types', 'eventTypes'])( + 'accepts %s coverage and de-duplicates requested events', + async (field) => { + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockImplementation(async (id) => ({ + id, + [field]: ['thread.reply', 'message.created'], + })); + const { program, relayfile } = harness({ relay }); + await program.parseAsync(ARGS(['--events', 'thread.reply,message.created,thread.reply']), { + from: 'user', + }); + expect(relayfile.bind).toHaveBeenCalledOnce(); + expect(relay.integrations.subscriptions.create).toHaveBeenCalledWith( + expect.objectContaining({ + event: 'thread.reply', + events: ['thread.reply', 'message.created'], + }) + ); + } + ); + + it('rejects misspelled events before provisioning', async () => { + const { program, relay, relayfile, error } = harness(); + await program.parseAsync(ARGS(['--events', 'thread.replies']), { from: 'user' }); + expect(error.mock.calls.flat().join(' ')).toContain('Unsupported subscription events'); + expect(relay.webhooks.createInbound).not.toHaveBeenCalled(); + expect(relayfile.createWebhookSubscription).not.toHaveBeenCalled(); + }); + + it('accepts singular thread.reply coverage', async () => { + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockImplementation(async (id) => ({ id, event: 'thread.reply' })); + const { program, relayfile } = harness({ relay }); + await program.parseAsync(ARGS(['--events', 'thread.reply']), { from: 'user' }); + expect(relayfile.bind).toHaveBeenCalledOnce(); + }); + + it.each([true, false, undefined])( + 'reports GitHub identity authorization %s honestly', + async (authorized) => { + const relayfile = createRelayfileMock([], { + listWebhookSubscriptions: vi.fn(async () => ({ + workspaceId: 'rw_test', + subscriptions: [ + { + subscriptionId: 'whsub_1', + url: 'https://cast.test/inbound', + pathGlobs: ['/github/repos/AgentWorkforce/relaycast/pulls/448/**'], + githubPrIdentityAuthorized: authorized, + }, + ], + })), + }); + const { program, log } = harness({ relayfile }); + await program.parseAsync( + [ + 'integration', + 'subscribe', + 'github', + '--resource', + '/github/repos/AgentWorkforce/relaycast/pulls/448/**', + '--to', + '#general', + ], + { from: 'user' } + ); + const output = log.mock.calls.flat().join(' '); + expect(output).not.toContain('Replies will post back'); + expect(output).toContain(authorized === true ? 'writeback is configured' : 'writeback is unverified'); + expect(relayfile.bind).toHaveBeenCalledWith( + expect.objectContaining({ resource: '/github/repos/AgentWorkforce/relaycast/pulls/448/**' }) + ); + } + ); +}); + +it('reports outbound configuration separately from inferred inbound activity', async () => { + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockResolvedValue({ + id: 'sub_existing', + events: ['thread.reply'], + isActive: false, + }); + relay.messages.list.mockResolvedValue([{ createdAt: '2026-10-05T00:00:00Z' }] as never); + const relayfile = createRelayfileMock([ + { + provider: 'github', + resource: '/github/repos/AgentWorkforce/relaycast/pulls/448/**', + channel: 'general', + subscriptionId: 'sub_existing', + webhookId: 'in_existing', + }, + ]); + const { program, log } = harness({ relay, relayfile }); + await program.parseAsync(['integration', 'subscribe', '--list'], { from: 'user' }); + expect(JSON.parse(log.mock.calls[0][0]).bindings[0]).toMatchObject({ + lastDeliverySource: 'channel-inferred', + writebackSubscription: { + id: 'sub_existing', + events: ['thread.reply'], + active: false, + deliveryStatus: null, + }, + }); + relay.integrations.subscriptions.get.mockRejectedValue(new Error('unavailable')); + log.mockClear(); + await program.parseAsync(['integration', 'subscribe', '--list'], { from: 'user' }); + expect(JSON.parse(log.mock.calls[0][0]).bindings[0].writebackSubscription).toBeNull(); +}); + +it.each([ + { id: 'wrong', events: ['message.created', 'thread.reply'] }, + { isActive: false, events: ['message.created', 'thread.reply'] }, + { is_active: false, events: ['message.created', 'thread.reply'] }, +])('preserves the prior binding when read-back is invalid: %j', async (echo) => { + const prior = { + provider: 'slack', + resource: RESOURCE, + channel: 'general', + webhookId: 'in_old', + subscriptionId: 'sub_old', + }; + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockImplementation(async (id) => ({ id, ...echo })); + const relayfile = createRelayfileMock([prior]); + const { program, error } = harness({ relay, relayfile }); + await program.parseAsync(ARGS(), { from: 'user' }); + expect(error.mock.calls.flat().join(' ')).toContain('different or inactive subscription'); + expect(await relayfile.listBindings()).toEqual([prior]); + expect(relay.integrations.subscriptions.delete).not.toHaveBeenCalledWith('sub_old'); + expect(relay.webhooks.delete).not.toHaveBeenCalledWith('in_old'); +}); + +it('does not advertise thread writeback when only message.created is requested', async () => { + const { program, log } = harness(); + await program.parseAsync(ARGS(['--events', 'message.created']), { from: 'user' }); + expect(log.mock.calls.flat().join(' ')).toContain('thread.reply was not requested'); +}); diff --git a/packages/cli/src/cli/commands/integration.ts b/packages/cli/src/cli/commands/integration.ts index 28a5c485fa..6b0573d971 100644 --- a/packages/cli/src/cli/commands/integration.ts +++ b/packages/cli/src/cli/commands/integration.ts @@ -7,7 +7,7 @@ import { import { createHash, pbkdf2Sync, randomBytes } from 'node:crypto'; import { hostname } from 'node:os'; import { getProjectPaths } from '@agent-relay/config'; -import type { AgentRelayAgent } from '@agent-relay/sdk'; +import { normalizeWebhookSubscription, type AgentRelayAgent } from '@agent-relay/sdk'; import { cleanupEntryKey, @@ -636,6 +636,13 @@ type ListedBinding = RelayfileBinding & { lastSuccessAt: string | null; lastError: string | null; lastChannelMessageAt: string | null; + lastDeliverySource: 'control-plane' | 'channel-inferred' | null; + writebackSubscription: { + id: string; + events: string[] | null; + active: boolean | null; + deliveryStatus: null; + } | null; githubPrIdentityAuthorized: boolean | null; }; @@ -686,6 +693,23 @@ async function enrichBindingsForList( ? cloudById.get(binding.webhookSubscriptionId) : undefined; const health = cloudRow?.health; + let writebackSubscription: ListedBinding['writebackSubscription'] = null; + if (binding.subscriptionId) { + try { + const raw = await relay.integrations.subscriptions.get(binding.subscriptionId); + const subscription = normalizeWebhookSubscription(raw); + const active = raw.isActive ?? raw.is_active; + writebackSubscription = { + id: subscription.id, + events: subscription.events ?? null, + active: typeof active === 'boolean' ? active : null, + // The pinned engine contract exposes configuration, not delivery attempts. + deliveryStatus: null, + }; + } catch { + /* Missing or unavailable engine status remains unknown. */ + } + } return { ...binding, to: agent ? `@${agent.name}` : null, @@ -696,6 +720,12 @@ async function enrichBindingsForList( lastSuccessAt: health?.lastSuccessAt ?? null, lastError: health?.lastError ?? null, lastChannelMessageAt, + lastDeliverySource: health?.lastDeliveryAt + ? 'control-plane' + : lastChannelMessageAt + ? 'channel-inferred' + : null, + writebackSubscription, githubPrIdentityAuthorized: cloudRow?.githubPrIdentityAuthorized ?? null, }; }) @@ -1576,6 +1606,13 @@ async function runSubscribeSetup( return; } + const requestedEvents = commaList(opts.events); + const events = [...new Set(requestedEvents.length ? requestedEvents : ['message.created', 'thread.reply'])]; + const unsupported = events.filter((event) => !['message.created', 'thread.reply'].includes(event)); + if (unsupported.length) + throw new Error( + `Unsupported subscription events: ${unsupported.join(', ')}. Supported events: message.created, thread.reply.` + ); const { provider, resource, to } = await promptSubscribeOptions(deps, providerArg, opts); const local = await deps.resolveLocalRelayOptions(); await ensureProviderConnected(deps, provider, opts); @@ -1617,7 +1654,6 @@ async function runSubscribeSetup( baseUrl: resolveInboundTargetTransport(effectiveRelayOptions, relayOptions).baseUrl, }) : targetChannel(to); - const events = commaList(opts.events); const writeback = await resolveWriteback(deps, opts, channel); const inboundTarget = await createRelayfileInboundTarget(opts, local, { channel, @@ -1754,13 +1790,26 @@ async function runSubscribeSetup( }); lease.assertHealthy(`${provider} ${pathGlob}`); subscription = await relay.integrations.subscriptions.create({ - event: events.length === 1 ? events[0]! : 'message.created', - events: events.length ? events : ['message.created', 'thread.reply'], + event: events[0]!, + events, filter: { channel }, url: subscriptionTargetUrl, secret: writeback.secret, }); await upgradeAttempt({ subscriptionId: subscription.id }); + const persistedRaw = await relay.integrations.subscriptions.get(subscription.id); + const persisted = normalizeWebhookSubscription(persistedRaw); + if (persisted.id !== subscription.id || (persistedRaw.isActive ?? persistedRaw.is_active) === false) { + throw new Error( + 'Writeback subscription read-back returned a different or inactive subscription. Aborting and rolling back.' + ); + } + const missingEvents = events.filter((event) => !persisted.events?.includes(event)); + if (missingEvents.length) { + throw new Error( + `Writeback subscription did not confirm requested events: ${missingEvents.join(', ')}. Aborting and rolling back.` + ); + } lease.assertHealthy(`${provider} ${pathGlob}`); bindingInput = { provider, @@ -1873,7 +1922,17 @@ async function runSubscribeSetup( const boundLabel = pathGlob === resource ? resource : `${resource} (${pathGlob})`; deps.log(`✓ ${provider} ${boundLabel} bound -> ${to}`); deps.log('✓ Server-side inbound webhook subscription created.'); - deps.log('✓ Listening. Replies will post back in-thread.'); + const cloudRows = await listWebhookSubscriptionsForBindings(deps.relayfile, [bindingInput]); + const authorized = cloudRows.get(bindingInput.webhookSubscriptionId!)?.githubPrIdentityAuthorized; + if (!events.includes('thread.reply')) { + deps.log('✓ Listening. In-thread writeback is disabled: thread.reply was not requested.'); + } else if (provider === 'github' && authorized !== true) { + deps.log( + `✓ Listening. GitHub in-thread writeback is unverified: GitHub PR identity authorization is ${authorized === false ? 'not granted' : 'not reported'}.` + ); + } else { + deps.log('✓ Listening. In-thread writeback is configured; provider delivery is not yet confirmed.'); + } } async function runUnsubscribeOwnedBy( diff --git a/packages/cli/src/cli/commands/relaycast-groups.test.ts b/packages/cli/src/cli/commands/relaycast-groups.test.ts index 633bcb3c17..6b6e971c94 100644 --- a/packages/cli/src/cli/commands/relaycast-groups.test.ts +++ b/packages/cli/src/cli/commands/relaycast-groups.test.ts @@ -84,7 +84,7 @@ function createRelayMock() { subscriptions: { create: vi.fn(async (i: unknown) => ({ id: 'sub1', ...(i as object) })), list: vi.fn(async () => []), - get: vi.fn(async (id: string) => ({ id })), + get: vi.fn(async (id: string) => ({ id, events: ['message.created', 'thread.reply'] })), delete: vi.fn(async () => undefined), }, }, @@ -706,6 +706,13 @@ describe('SDK-backed CLI groups', () => { lastSuccessAt: null, lastError: null, lastChannelMessageAt: null, + lastDeliverySource: null, + writebackSubscription: { + id: 'sub1', + events: ['message.created', 'thread.reply'], + active: null, + deliveryStatus: null, + }, githubPrIdentityAuthorized: null, }, ], diff --git a/packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts b/packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts index 56c7b075d7..0e5f5f5e01 100644 --- a/packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts +++ b/packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts @@ -313,3 +313,41 @@ describe('messaging delivery receipts over MCP', () => { } }); }); + +describe('channel and thread writes over MCP', () => { + it.each([ + ['post_message', 'send', { channel: 'events', text: 'ACK' }], + ['reply_to_thread', 'reply', { message_id: 'provider-parent', text: 'ACK' }], + ] as const)('coalesces parallel %s calls but permits sequential repeats', async (tool, method, args) => { + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const write = vi.fn(async () => { + await gate; + return { id: 'reply-1' }; + }); + const server = new McpServer({ name: 'write-test', version: '1' }); + registerMessagingTools(server, () => ({ [method]: write }) as never); + const client = new Client({ name: 'write-client', version: '1' }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const first = client.callTool({ name: tool, arguments: args }); + const second = client.callTool({ name: tool, arguments: args }); + await vi.waitFor(() => expect(write).toHaveBeenCalledTimes(1)); + release(); + const results = await Promise.all([first, second]); + expect(results[0].isError).not.toBe(true); + expect(results[0].structuredContent).toEqual(results[1].structuredContent); + expect(write).toHaveBeenCalledTimes(1); + await client.callTool({ name: tool, arguments: args }); + expect(write).toHaveBeenCalledTimes(2); + } finally { + release(); + await client.close(); + await server.close(); + } + }); +}); diff --git a/packages/cli/src/cli/mcp/messaging-tools.ts b/packages/cli/src/cli/mcp/messaging-tools.ts index 22d3235248..19db796995 100644 --- a/packages/cli/src/cli/mcp/messaging-tools.ts +++ b/packages/cli/src/cli/mcp/messaging-tools.ts @@ -1,6 +1,7 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { replayMessageMetadata } from '@agent-relay/sdk'; import { z } from 'zod'; +import { track } from '../telemetry/index.js'; import { compactDirectMessageReceipt, @@ -247,13 +248,22 @@ export function registerMessagingTools( openWorldHint: true, }, }, - async ({ channel, text, attachments, mode, as }) => - jsonContent( - await getAgentClient(as).send(channel, text, { - attachments, - data: replayMessageMetadata(), - mode, - }) + async ({ channel, text, attachments, mode, as }, extra) => + replay.run('post_message', extra, undefined, () => + replay.coalesceWrite( + 'post_message', + extra, + [as ?? null, channel, text, attachments ?? [], mode ?? 'wait'], + async () => + jsonContent( + await getAgentClient(as).send(channel, text, { + attachments, + data: replayMessageMetadata(), + mode, + }) + ), + () => track('agent_relay_write_coalesced', { tool_name: 'post_message' }) + ) ) ); @@ -302,8 +312,17 @@ export function registerMessagingTools( openWorldHint: true, }, }, - async ({ message_id, text, as }) => - jsonContent(await getAgentClient(as).reply(message_id, text, { data: replayMessageMetadata() })) + async ({ message_id, text, as }, extra) => + replay.run('reply_to_thread', extra, undefined, () => + replay.coalesceWrite( + 'reply_to_thread', + extra, + [as ?? null, message_id, text], + async () => + jsonContent(await getAgentClient(as).reply(message_id, text, { data: replayMessageMetadata() })), + () => track('agent_relay_write_coalesced', { tool_name: 'reply_to_thread' }) + ) + ) ); server.registerTool( diff --git a/packages/cli/src/cli/mcp/request-replay.test.ts b/packages/cli/src/cli/mcp/request-replay.test.ts new file mode 100644 index 0000000000..c8a38f8b9a --- /dev/null +++ b/packages/cli/src/cli/mcp/request-replay.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it, vi } from 'vitest'; +import { McpRequestReplay } from './request-replay.js'; + +function deferred() { + let resolve!: (value: string) => void; + let reject!: (error: Error) => void; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + return { promise, resolve, reject }; +} + +describe('pending message write coalescing', () => { + it('joins concurrent aliases, reports coalescing, and permits a sequential repeat', async () => { + const replay = new McpRequestReplay(); + const pending = deferred(); + const operation = vi.fn(() => pending.promise); + const coalesced = vi.fn(); + const send = (requestId: number) => + replay.run('reply_to_thread', { requestId, sessionId: 's' }, undefined, () => + replay.coalesceWrite( + 'reply_to_thread', + { sessionId: 's' }, + ['agent', 'parent', 'ACK'], + operation, + coalesced + ) + ); + const first = send(1); + const alias = send(2); + await Promise.resolve(); + expect(operation).toHaveBeenCalledTimes(1); + expect(coalesced).toHaveBeenCalledTimes(1); + pending.resolve('reply'); + expect(await first).toBe(await alias); + await send(3); + expect(operation).toHaveBeenCalledTimes(2); + }); + + it('separates sessions, identities, targets, tools, and write options', async () => { + const replay = new McpRequestReplay(); + const pending = deferred(); + const operation = vi.fn(() => pending.promise); + const writes = [ + replay.coalesceWrite('post_message', { sessionId: 'a' }, ['alice', 'c', 'ACK', [], 'wait'], operation), + replay.coalesceWrite('post_message', { sessionId: 'b' }, ['alice', 'c', 'ACK', [], 'wait'], operation), + replay.coalesceWrite('post_message', { sessionId: 'a' }, ['bob', 'c', 'ACK', [], 'wait'], operation), + replay.coalesceWrite('post_message', { sessionId: 'a' }, ['alice', 'd', 'ACK', [], 'wait'], operation), + replay.coalesceWrite( + 'reply_to_thread', + { sessionId: 'a' }, + ['alice', 'c', 'ACK', [], 'wait'], + operation + ), + replay.coalesceWrite( + 'post_message', + { sessionId: 'a' }, + ['alice', 'c', 'ACK', ['file'], 'wait'], + operation + ), + replay.coalesceWrite('post_message', { sessionId: 'a' }, ['alice', 'c', 'ACK', [], 'steer'], operation), + ]; + await Promise.resolve(); + expect(operation).toHaveBeenCalledTimes(7); + pending.resolve('ok'); + await Promise.all(writes); + }); + + it('clears rejected writes and transport request IDs for a safe retry', async () => { + const replay = new McpRequestReplay(); + const pending = deferred(); + const operation = vi.fn(() => pending.promise); + const send = () => + replay.run('reply_to_thread', { requestId: 1 }, undefined, () => + replay.coalesceWrite('reply_to_thread', {}, ['a', 'p', 'ACK'], operation) + ); + const first = send(); + expect(send()).toBe(first); + const assertion = expect(first).rejects.toThrow('unavailable'); + pending.reject(new Error('unavailable')); + await assertion; + operation.mockResolvedValue('retry'); + expect(await send()).toBe('retry'); + expect(operation).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/cli/src/cli/mcp/request-replay.ts b/packages/cli/src/cli/mcp/request-replay.ts index 3d6f085903..d96a7cbf96 100644 --- a/packages/cli/src/cli/mcp/request-replay.ts +++ b/packages/cli/src/cli/mcp/request-replay.ts @@ -5,11 +5,38 @@ * JSON-RPC permits IDs to be reused after a response. Clients that need to * retry after a lost response provide an explicit idempotency key; that key * retains the completed result briefly. Arguments deliberately do not - * participate, so intentional identical sends remain separate requests. + * participate in this request key. A separate pending-write key includes the + * session, tool, identity and all write arguments to join parallel tool aliases. + * It is removed on settlement so sequential identical sends remain distinct. */ export class McpRequestReplay { private readonly requests = new Map>(); + private readonly writes = new Map>(); + + coalesceWrite( + tool: string, + extra: unknown, + argumentsKey: unknown[], + operation: () => Promise, + onCoalesce: () => void = () => {} + ): Promise { + const sessionId = (extra as { sessionId?: unknown } | undefined)?.sessionId; + const key = JSON.stringify([sessionId ?? '', tool, ...argumentsKey]); + const existing = this.writes.get(key) as Promise | undefined; + if (existing) { + onCoalesce(); + return existing; + } + const pending = Promise.resolve().then(operation); + this.writes.set(key, pending); + void pending.then( + () => this.writes.delete(key), + () => this.writes.delete(key) + ); + return pending; + } + run( tool: string, extra: unknown, diff --git a/packages/cli/src/cli/telemetry/events.ts b/packages/cli/src/cli/telemetry/events.ts index 4bf91b9222..56a89f4d24 100644 --- a/packages/cli/src/cli/telemetry/events.ts +++ b/packages/cli/src/cli/telemetry/events.ts @@ -487,6 +487,7 @@ export type TelemetryEventName = | 'cli_install' | 'cli_update' | 'agent_relay_tool_call' + | 'agent_relay_write_coalesced' | 'workflow_run' | 'cloud_auth' | 'cloud_workflow_run' @@ -510,6 +511,7 @@ export interface TelemetryEventMap { cli_install: CliInstallEvent; cli_update: CliUpdateEvent; agent_relay_tool_call: AgentRelayToolCallEvent; + agent_relay_write_coalesced: { tool_name: 'post_message' | 'reply_to_thread' }; workflow_run: WorkflowRunEvent; cloud_auth: CloudAuthEvent; cloud_workflow_run: CloudWorkflowRunEvent; diff --git a/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts b/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts index bbdcb42cb6..3c5e3047c0 100644 --- a/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts +++ b/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts @@ -57,3 +57,45 @@ describe('native Relay host tools', () => { expect(NATIVE_RELAY_INSTRUCTIONS).toContain('Do not ask the user how to use Relay'); }); }); + +it.each([ + ['post_message', 'send', { channel: 'events', text: 'ACK' }], + ['reply_to_thread', 'reply', { message_id: 'parent', text: 'ACK' }], +] as const)('joins pending native %s writes within a session only', async (tool, method, args) => { + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const write = vi.fn(async () => { + await gate; + return { id: 'reply' }; + }); + const options = { env: { RELAY_AGENT_TOKEN: 'test' }, agentClient: { [method]: write } as never }; + const execute = createNativeRelayTools(options).find((item) => item.spec.name === tool)!.execute; + const first = execute(args, {}); + const second = execute(args, {}); + await Promise.resolve(); + expect(write).toHaveBeenCalledTimes(1); + const anotherSession = createNativeRelayTools(options) + .find((item) => item.spec.name === tool)! + .execute(args, {}); + await Promise.resolve(); + expect(write).toHaveBeenCalledTimes(2); + release(); + expect(await first).toEqual(await second); + await anotherSession; + await execute(args, {}); + expect(write).toHaveBeenCalledTimes(3); +}); + +it('clears a rejected native reply so it can be retried', async () => { + const reply = vi.fn().mockRejectedValueOnce(new Error('offline')).mockResolvedValue({ id: 'retry' }); + const execute = createNativeRelayTools({ + env: { RELAY_AGENT_TOKEN: 'test' }, + agentClient: { reply } as never, + }).find((item) => item.spec.name === 'reply_to_thread')!.execute; + const args = { message_id: 'parent', text: 'ACK' }; + await expect(execute(args, {})).rejects.toThrow('offline'); + await expect(execute(args, {})).resolves.toEqual({ id: 'retry' }); + expect(reply).toHaveBeenCalledTimes(2); +}); diff --git a/packages/harnesses/src/ai-sdk/native-relay-tools.ts b/packages/harnesses/src/ai-sdk/native-relay-tools.ts index faf9bb06b6..33ebfe5b1a 100644 --- a/packages/harnesses/src/ai-sdk/native-relay-tools.ts +++ b/packages/harnesses/src/ai-sdk/native-relay-tools.ts @@ -53,6 +53,21 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha const workspace = options.workspaceClient ?? (workspaceKey ? createWorkspaceClient({ workspaceKey, baseUrl }) : undefined); + // One map per native session; never retain a completed write. + const pendingWrites = new Map>(); + function write(tool: string, target: string, text: string, operation: () => Promise) { + const key = JSON.stringify([tool, target, text]); + const existing = pendingWrites.get(key); + if (existing) return existing; + const pending = Promise.resolve().then(operation); + pendingWrites.set(key, pending); + void pending.then( + () => pendingWrites.delete(key), + () => pendingWrites.delete(key) + ); + return pending; + } + const tools: HarnessHostTool[] = [ { spec: { @@ -89,7 +104,9 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha }, execute: (value) => { const input = objectInput(value); - return agent.send(requiredString(input, 'channel'), requiredString(input, 'text')); + const target = requiredString(input, 'channel'); + const text = requiredString(input, 'text'); + return write('post_message', target, text, () => agent.send(target, text)); }, }, { @@ -150,7 +167,9 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha }, execute: (value) => { const input = objectInput(value); - return agent.reply(requiredString(input, 'message_id'), requiredString(input, 'text')); + const target = requiredString(input, 'message_id'); + const text = requiredString(input, 'text'); + return write('reply_to_thread', target, text, () => agent.reply(target, text)); }, }, { diff --git a/packages/integration-prompts/src/builders.ts b/packages/integration-prompts/src/builders.ts index a59cf99095..a595c9c93a 100644 --- a/packages/integration-prompts/src/builders.ts +++ b/packages/integration-prompts/src/builders.ts @@ -55,6 +55,9 @@ export function fullInjectInstructions(descriptors: IntegrationDescriptor[]): st lines.push('No integration event subscriptions are active for this project.'); } else { lines.push('Active integration event subscriptions for this project:'); + lines.push( + 'Reply once in the Relay thread to respond to a subscription event; when writeback is configured and authorized, that reply posts back to the provider. Use resource file writes to initiate new integration actions.' + ); for (const subscription of subscriptions) { lines.push(subscriptionLine(subscription)); } diff --git a/packages/sdk/src/__tests__/integrations.test.ts b/packages/sdk/src/__tests__/integrations.test.ts index 2979c0ddfe..b4cfc1da2d 100644 --- a/packages/sdk/src/__tests__/integrations.test.ts +++ b/packages/sdk/src/__tests__/integrations.test.ts @@ -54,6 +54,28 @@ describe('SDK integrations / capabilities / workspace passthrough', () => { expect(subs).toHaveLength(1); }); + it('normalizes subscription event coverage without losing engine configuration', async () => { + const relaycast = createRelaycastMock(); + relaycast.subscriptions.get.mockResolvedValue({ + id: 'sub1', + event_types: ['thread.reply'], + is_active: false, + } as never); + const client = new RelaycastMessagingClient({ relaycast: relaycast as never }); + expect(await client.integrations.subscriptions.get('sub1')).toMatchObject({ + id: 'sub1', + events: ['thread.reply'], + is_active: false, + }); + expect(await client.integrations.subscriptions.create({ event: 'thread.reply' })).toMatchObject({ + id: 'sub1', + events: ['thread.reply'], + }); + expect(await client.integrations.subscriptions.list()).toEqual([ + { id: 'sub1', event: 'message.created', events: ['message.created'] }, + ]); + }); + it('delegates capability (command) operations to relaycast', async () => { const relaycast = createRelaycastMock(); const client = new RelaycastMessagingClient({ relaycast: relaycast as never }); diff --git a/packages/sdk/src/__tests__/relaycast-translate.test.ts b/packages/sdk/src/__tests__/relaycast-translate.test.ts index 6e3d5ce052..f6d888ce0f 100644 --- a/packages/sdk/src/__tests__/relaycast-translate.test.ts +++ b/packages/sdk/src/__tests__/relaycast-translate.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { toRelayNode } from '../messaging/relaycast-translate.js'; +import { normalizeWebhookSubscription, toRelayNode } from '../messaging/relaycast-translate.js'; describe('toRelayNode fleet liveness', () => { it('omits stale active-agent load for an offline node', () => { @@ -18,3 +18,22 @@ describe('toRelayNode fleet liveness', () => { expect(node.activeAgents).toBeUndefined(); }); }); + +describe('subscription event normalization', () => { + it.each([ + { events: ['message.created', 'thread.reply'] }, + { event_types: ['message.created', 'thread.reply'] }, + { eventTypes: ['message.created', 'thread.reply'] }, + ])('normalizes plural events: %j', (raw) => { + expect(normalizeWebhookSubscription({ id: 'sub', ...raw }).events).toEqual([ + 'message.created', + 'thread.reply', + ]); + }); + + it('falls back to a singular event only when plural events are absent', () => { + expect(normalizeWebhookSubscription({ event: 'thread.reply' }).events).toEqual(['thread.reply']); + expect(normalizeWebhookSubscription({ events: [], event: 'thread.reply' }).events).toEqual([]); + expect(normalizeWebhookSubscription({}).events).toBeUndefined(); + }); +}); diff --git a/packages/sdk/src/messaging/index.ts b/packages/sdk/src/messaging/index.ts index e544753021..ad1792015f 100644 --- a/packages/sdk/src/messaging/index.ts +++ b/packages/sdk/src/messaging/index.ts @@ -15,3 +15,5 @@ export { type RelaySpawnPlacementState, } from './relaycast.js'; export * from './thin-client.js'; + +export { normalizeWebhookSubscription } from './relaycast-translate.js'; diff --git a/packages/sdk/src/messaging/relaycast-translate.ts b/packages/sdk/src/messaging/relaycast-translate.ts index ecab4a16d2..a925573c09 100644 --- a/packages/sdk/src/messaging/relaycast-translate.ts +++ b/packages/sdk/src/messaging/relaycast-translate.ts @@ -267,9 +267,12 @@ export function normalizeInboundWebhook(raw: unknown): RelayInboundWebhook { /** Normalize a relaycast event subscription into `RelayWebhookSubscription`. */ export function normalizeWebhookSubscription(raw: unknown): RelayWebhookSubscription { const record = asRecord(raw); - const events = Array.isArray(record.events) - ? record.events.filter((event): event is string => typeof event === 'string') - : undefined; + const rawEvents = record.events ?? record.event_types ?? record.eventTypes; + const events = Array.isArray(rawEvents) + ? rawEvents.filter((event): event is string => typeof event === 'string') + : typeof record.event === 'string' + ? [record.event] + : undefined; return { id: readStr(record, 'id') ?? '', ...(readStr(record, 'url') ? { url: readStr(record, 'url') } : {}), diff --git a/packages/sdk/src/messaging/relaycast.ts b/packages/sdk/src/messaging/relaycast.ts index 74c4f115e5..5ad2ed0e87 100644 --- a/packages/sdk/src/messaging/relaycast.ts +++ b/packages/sdk/src/messaging/relaycast.ts @@ -655,12 +655,19 @@ export class RelaycastMessagingClient implements RelayMessagingClient { this.requireWebhooks().trigger(id, payload ?? {}), }, subscriptions: { - create: async (input: RelayCreateSubscriptionInput): Promise => - (await this.requireSubscriptions().create(input)) as RelayEventSubscription, + create: async (input: RelayCreateSubscriptionInput): Promise => { + const raw = (await this.requireSubscriptions().create(input)) as RelayEventSubscription; + return { ...raw, ...normalizeWebhookSubscription(raw) }; + }, list: async (): Promise => - (await this.requireSubscriptions().list()) as RelayEventSubscription[], - get: async (id: string): Promise => - (await this.requireSubscriptions().get(id)) as RelayEventSubscription, + ((await this.requireSubscriptions().list()) as RelayEventSubscription[]).map((raw) => ({ + ...raw, + ...normalizeWebhookSubscription(raw), + })), + get: async (id: string): Promise => { + const raw = (await this.requireSubscriptions().get(id)) as RelayEventSubscription; + return { ...raw, ...normalizeWebhookSubscription(raw) }; + }, delete: async (id: string): Promise => { await this.requireSubscriptions().delete(id); }, diff --git a/packages/sdk/src/messaging/types.ts b/packages/sdk/src/messaging/types.ts index 683c362964..846ec48b4c 100644 --- a/packages/sdk/src/messaging/types.ts +++ b/packages/sdk/src/messaging/types.ts @@ -480,6 +480,7 @@ export interface RelayCreateWebhookInput { export interface RelayEventSubscription { id: string; + events?: string[]; event?: string; createdAt?: string; [key: string]: unknown; diff --git a/summary.md b/summary.md new file mode 100644 index 0000000000..d5fbe021b1 --- /dev/null +++ b/summary.md @@ -0,0 +1,36 @@ +# Prevent duplicate pending replies and verify subscription provisioning + +Channel posts and thread replies bypassed MCP request replay protection, so +parallel identical calls could create duplicate Relay rows. Both tools now use +request replay and pending-only coalescing; native host tools do the same within +a session. Sequential repeats remain distinct. MCP coalesces emit a telemetry +event containing only the tool name. + +`integration subscribe` now validates and de-duplicates event names, reads back +persisted coverage before binding, and rolls back incomplete, inactive, or +mismatched subscriptions. GitHub setup reports missing/unknown identity +authorization instead of promising writeback. `--list` separates outbound +configuration from unknown delivery status and labels inferred inbound activity. +SDK subscription responses share event normalization, and subscription prompts +clarify replying versus initiating provider actions. + +Validation: + +- Focused CLI/MCP/native/prompt regression suites: 152 passed, four existing skips + (147 passed initially; the changed CLI/native suites then passed all 96 tests, + including five additional regressions). +- SDK integration/webhook/normalization suites: 17 passed. +- `npm run typecheck` and subsequent CLI `tsc --noEmit`: passed. +- `git diff --check`: passed. No workflow files changed. + +**This PR does not close the full ticket.** Incident credentials and the +Relayfile daemon are unavailable. The installed client exposes no writeback +ledger status/retry API, and whether subscription ingress uses that ledger is +unverified. Actual GitHub writeback, durable failed-state/idempotent retry, and +the complete inbound → fleet wake → GitHub seam test remain blocked on those +contracts. Provider scopes and explicit idempotency-key arguments were left +unchanged per the reviewed evidence gates. Existing broker reconnect coverage +is cited, not rerun (Cargo unavailable). + +See [investigation evidence](docs/evidence/provider-subscription-replies.md) for +verified contracts, limitations, and the required companion verification. From 16918141fc55e456493f28eb494bf9894eeb4773 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Mon, 5 Oct 2026 10:21:46 +0000 Subject: [PATCH 10/42] Keep relayflow working files out of the change --- summary.md | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 summary.md diff --git a/summary.md b/summary.md deleted file mode 100644 index d5fbe021b1..0000000000 --- a/summary.md +++ /dev/null @@ -1,36 +0,0 @@ -# Prevent duplicate pending replies and verify subscription provisioning - -Channel posts and thread replies bypassed MCP request replay protection, so -parallel identical calls could create duplicate Relay rows. Both tools now use -request replay and pending-only coalescing; native host tools do the same within -a session. Sequential repeats remain distinct. MCP coalesces emit a telemetry -event containing only the tool name. - -`integration subscribe` now validates and de-duplicates event names, reads back -persisted coverage before binding, and rolls back incomplete, inactive, or -mismatched subscriptions. GitHub setup reports missing/unknown identity -authorization instead of promising writeback. `--list` separates outbound -configuration from unknown delivery status and labels inferred inbound activity. -SDK subscription responses share event normalization, and subscription prompts -clarify replying versus initiating provider actions. - -Validation: - -- Focused CLI/MCP/native/prompt regression suites: 152 passed, four existing skips - (147 passed initially; the changed CLI/native suites then passed all 96 tests, - including five additional regressions). -- SDK integration/webhook/normalization suites: 17 passed. -- `npm run typecheck` and subsequent CLI `tsc --noEmit`: passed. -- `git diff --check`: passed. No workflow files changed. - -**This PR does not close the full ticket.** Incident credentials and the -Relayfile daemon are unavailable. The installed client exposes no writeback -ledger status/retry API, and whether subscription ingress uses that ledger is -unverified. Actual GitHub writeback, durable failed-state/idempotent retry, and -the complete inbound → fleet wake → GitHub seam test remain blocked on those -contracts. Provider scopes and explicit idempotency-key arguments were left -unchanged per the reviewed evidence gates. Existing broker reconnect coverage -is cited, not rerun (Cargo unavailable). - -See [investigation evidence](docs/evidence/provider-subscription-replies.md) for -verified contracts, limitations, and the required companion verification. From d82c251fbaa661386f705c0211bcf781ef366528 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 5 Oct 2026 10:22:59 +0000 Subject: [PATCH 11/42] style: auto-format with Prettier --- docs/evidence/provider-subscription-replies.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/evidence/provider-subscription-replies.md b/docs/evidence/provider-subscription-replies.md index 35be3a0f13..996185dedf 100644 --- a/docs/evidence/provider-subscription-replies.md +++ b/docs/evidence/provider-subscription-replies.md @@ -6,7 +6,7 @@ It does **not** establish that the reported GitHub writeback failure is fixed. ## Available evidence - The incident's Relay workspace is inaccessible in this run: `agent-relay - message inbox check` reports no workspace key. No authenticated message, +message inbox check` reports no workspace key. No authenticated message, transcript, subscription, or binding read could be made (E1–E3). - A read-only `RelayfileControlPlaneClient({ autoStart: false })` binding read reports `DAEMON_UNAVAILABLE`. Stored PR globs therefore remain unverified; From b9162ce76c3cadd98254e573a1ad3c0378115057 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Tue, 6 Oct 2026 13:43:26 +0000 Subject: [PATCH 12/42] fix: fail verified Devin spawns on unresolved directory trust --- .../compact_d6sq3rpl6inr_2026-10-06.json | 48 +++++ .../compact_d6sq3rpl6inr_2026-10-06.md | 21 ++ CHANGELOG.md | 6 +- crates/broker/src/pty_worker.rs | 181 +++++++++++++++++- crates/broker/src/runtime/fleet.rs | 8 + crates/broker/src/runtime/tests.rs | 65 +++++++ crates/broker/src/runtime/worker_events.rs | 7 + crates/broker/tests/devin_trust_cli.rs | 179 +++++++++++++++++ crates/relay-pty/src/terminal.rs | 41 ++++ docs/harnesses/devin.md | 11 ++ .../cli/lib/fleet-spawn-confirmation.test.ts | 6 +- 11 files changed, 564 insertions(+), 9 deletions(-) create mode 100644 .agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.json create mode 100644 .agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.md create mode 100644 crates/broker/tests/devin_trust_cli.rs diff --git a/.agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.json b/.agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.json new file mode 100644 index 0000000000..20c5891db1 --- /dev/null +++ b/.agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.json @@ -0,0 +1,48 @@ +{ + "id": "compact_hbapxe3axu5d", + "version": 1, + "type": "compacted", + "compactedAt": "2026-10-06T13:43:02.276Z", + "sourceTrajectories": [ + "traj_t5rhaj5534rg" + ], + "dateRange": { + "start": "2026-10-06T13:38:30.682Z", + "end": "2026-10-06T13:42:41.615Z" + }, + "summary": { + "totalDecisions": 2, + "totalEvents": 2, + "uniqueAgents": [ + "default" + ] + }, + "decisionGroups": [ + { + "category": "api", + "decisions": [ + { + "question": "Implement reviewed-plan Part 1; defer automatic trust acceptance pending real captures", + "chosen": "Implement reviewed-plan Part 1; defer automatic trust acceptance pending real captures", + "reasoning": "No Devin binary or raw trust captures are available. Preserve the existing Claude planner and generic Devin approval restrictions. Add the Devin veto to both startup gate paths; deadline uses detector and verification ticks. CLI help inspection and advisory path corroboration remain Part 2 prerequisites.", + "fromTrajectory": "traj_t5rhaj5534rg" + } + ] + }, + { + "category": "testing", + "decisions": [ + { + "question": "Run broker validation without inherited Git config injection", + "chosen": "Run broker validation without inherited Git config injection", + "reasoning": "The default suite passed 1374 tests and failed four hook-chain tests because the runner injects core.hooksPath. With GIT_CONFIG_COUNT unset only in the test process, all four hook tests pass and all 1378 broker unit tests pass.", + "fromTrajectory": "traj_t5rhaj5534rg" + } + ] + } + ], + "keyLearnings": [], + "keyFindings": [], + "filesAffected": [], + "commits": [] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.md b/.agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.md new file mode 100644 index 0000000000..ba037722b6 --- /dev/null +++ b/.agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.md @@ -0,0 +1,21 @@ +# Trajectory Compaction: Oct 6, 2026 - Oct 6, 2026 + +## Summary +- Sessions: 1 +- Decisions: 2 +- Events: 2 +- Agents: default +- Files: 0 +- Commits: 0 + +## Api +- Implement reviewed-plan Part 1; defer automatic trust acceptance pending real captures -> Implement reviewed-plan Part 1; defer automatic trust acceptance pending real captures (traj_t5rhaj5534rg) + +## Testing +- Run broker validation without inherited Git config injection -> Run broker validation without inherited Git config injection (traj_t5rhaj5534rg) + +## Key Learnings +- None + +## Key Findings +- None \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c64e6119b..1116e78c6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Patch] + +### Fixed + +- Verified Fleet Devin spawns fail with `directory_trust_required` when the directory-trust prompt persists, releasing the worker through spawn cleanup instead of leaving a live but blocked worker. ## [13.1.2] - 2026-10-06 diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index 6ee5866da7..ce995e6cd4 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -38,12 +38,13 @@ use crate::broker::{ }; use crate::cli::command_parse::parse_cli_command; use crate::cli::PtyCommand; -use crate::readiness::{detect_cli_ready, is_muse_cli, GridReadinessSnapshot}; +use crate::readiness::{detect_cli_ready, is_devin_cli, is_muse_cli, GridReadinessSnapshot}; use crate::runtime::{get_terminal_size, send_frame}; use crate::snapshot::Snapshot; use crate::util::ansi::{floor_char_boundary, strip_ansi, AnsiStripper}; use crate::util::terminal::{ - detect_claude_trust_prompt, detect_codex_trust_prompt, detect_muse_device_auth_prompt, + detect_claude_trust_prompt, detect_codex_trust_prompt, detect_devin_trust_prompt, + detect_muse_device_auth_prompt, }; use crate::util::utf8_stream::Utf8StreamDecoder; use crate::worker::detection::ActivityDetector; @@ -510,6 +511,8 @@ struct StartupReadinessState { fallback_sent: bool, wait_warned: bool, auth_error_sent: bool, + trust_first_seen: Option, + trust_error_sent: bool, } fn append_bounded(buf: &mut String, text: &str, max: usize, keep: usize) { @@ -584,7 +587,8 @@ fn evaluate_startup_gate( "codex" | "codex.exe" ); let trust_blocked = detect_codex_trust_prompt(grid.screen) - || detect_claude_trust_prompt(grid.screen) == (true, true); + || detect_claude_trust_prompt(grid.screen) == (true, true) + || (is_devin_cli(resolved_cli) && detect_devin_trust_prompt(grid.screen)); let composer_ready = is_codex && codex_composer_ready(grid); tracing::debug!(target: "relay_broker::startup_gate", cli = resolved_cli, is_codex, composer_ready, trust_blocked, @@ -602,7 +606,7 @@ fn evaluate_startup_gate( /// The startup gate's verdict. /// -/// Three states, not two booleans: `Ready && Blocked` is not representable, +/// Explicit verdicts rather than booleans: `Ready && Blocked` is not representable, /// and the difference between "we could not recognise the prompt" and "the /// harness is deliberately refusing work" decides whether the deadline may /// release queued work at all. @@ -621,14 +625,20 @@ pub(crate) enum StartupGate { /// node will ever answer a device prompt, so the worker reports /// `provider_auth_required` rather than waiting out the deadline. ProviderAuthRequired, + /// Devin requires directory trust. Block immediately and report a terminal + /// spawn error if the screen persists for DIRECTORY_TRUST_TIMEOUT. + DirectoryTrustRequired, } impl StartupGate { /// Whether the harness is deliberately refusing input, as opposed to /// showing a prompt we failed to recognise. No deadline may release - /// queued work past either blocking verdict. + /// queued work past any blocking verdict. fn is_blocking(self) -> bool { - matches!(self, Self::Blocked | Self::ProviderAuthRequired) + matches!( + self, + Self::Blocked | Self::ProviderAuthRequired | Self::DirectoryTrustRequired + ) } } @@ -646,6 +656,9 @@ impl StartupGate { /// on that screen; this names it as deliberate so the deadline cannot /// release the brief into a login form. fn startup_gate_block_reason(resolved_cli: &str, screen: &str) -> Option { + if is_devin_cli(resolved_cli) && detect_devin_trust_prompt(screen) { + return Some(StartupGate::DirectoryTrustRequired); + } if is_muse_cli(resolved_cli) && detect_muse_device_auth_prompt(screen) { return Some(StartupGate::ProviderAuthRequired); } @@ -797,6 +810,8 @@ fn should_block_pending_injection( && pending.queued_at.elapsed() < AUTO_SUGGESTION_BLOCK_TIMEOUT } +const DIRECTORY_TRUST_TIMEOUT: Duration = Duration::from_secs(30); + #[allow(clippy::too_many_arguments)] async fn try_emit_worker_ready( out_tx: &mpsc::Sender>, @@ -810,10 +825,33 @@ async fn try_emit_worker_ready( // init_received_at is Some only after init_worker has been received. // We use it (not init_request_id) as the gate because the broker sends // init_worker without a request_id. - if readiness.ready_sent || init_received_at.is_none() { + if readiness.ready_sent || readiness.trust_error_sent || init_received_at.is_none() { return; } + // Anchor the deadline on the gate's own detector, not a parsed menu row or + // response attempt. The 200 ms verification tick drives this even when the + // child stops producing output. Blocking suppresses the generic fail-open. + if gate == StartupGate::DirectoryTrustRequired && !readiness.fallback_sent { + let first_seen = readiness.trust_first_seen.get_or_insert_with(Instant::now); + if first_seen.elapsed() >= DIRECTORY_TRUST_TIMEOUT { + readiness.trust_error_sent = true; + tracing::warn!(target: "agent_relay::worker::pty", worker = %worker_name, + "Devin directory trust is unresolved; reporting directory_trust_required"); + let _ = send_frame( + out_tx, + "worker_error", + None, + json!({ + "code": "directory_trust_required", + "message": "Devin requires directory trust; run `devin` in the explicit spawn working directory on this node, trust that directory, then retry", + "retryable": false + }), + ).await; + return; + } + } + let startup_ready = gate == StartupGate::Ready; // An interactive provider login cannot be waited out: the approval happens @@ -3338,6 +3376,135 @@ mod tests { ); } + #[test] + fn devin_trust_gate_is_scoped_to_devin_and_vetoes_composer() { + let screen = "Do you trust the authors of this directory?\n❭ 1 Yes, trust\n2 No, exit"; + for cli in ["devin", "/usr/local/bin/devin", "devin.exe"] { + assert_eq!( + startup_gate_block_reason(cli, screen), + Some(StartupGate::DirectoryTrustRequired) + ); + assert!(!evaluate_startup_gate( + cli, + screen, + 1000, + Duration::from_secs(90), + GridReadinessSnapshot { + screen, + cursor: Some((2, 1)) + } + )); + } + for cli in ["claude", "codex", "muse"] { + assert_eq!(startup_gate_block_reason(cli, screen), None); + } + assert_eq!( + startup_gate_block_reason( + "devin", + "Sign in to continue\nVisit https://www.facebook.com/device\n›" + ), + None + ); + } + + #[tokio::test] + async fn devin_trust_deadline_starts_at_detection_and_fails_once() { + let (tx, mut rx) = mpsc::channel(4); + let mut request_id = None; + let started = Instant::now() - STARTUP_READY_TIMEOUT - Duration::from_secs(60); + let mut readiness = StartupReadinessState::default(); + try_emit_worker_ready( + &tx, + "devin-trust", + Some(42), + &mut request_id, + Some(started), + &mut readiness, + StartupGate::DirectoryTrustRequired, + ) + .await; + assert!(readiness.trust_first_seen.is_some()); + assert!( + rx.try_recv().is_err(), + "init age must not advance the trust deadline" + ); + readiness.trust_first_seen = Some(Instant::now() - DIRECTORY_TRUST_TIMEOUT); + for gate in [ + StartupGate::DirectoryTrustRequired, + StartupGate::DirectoryTrustRequired, + StartupGate::Unrecognised, + StartupGate::Ready, + ] { + try_emit_worker_ready( + &tx, + "devin-trust", + Some(42), + &mut request_id, + Some(started), + &mut readiness, + gate, + ) + .await; + } + let frame = rx.try_recv().expect("trust error"); + assert_eq!(frame.msg_type, "worker_error"); + assert_eq!(frame.payload["code"], "directory_trust_required"); + assert_eq!(frame.payload["retryable"], false); + assert!( + rx.try_recv().is_err(), + "terminal error cannot become ready later" + ); + assert!(!readiness.ready_sent && !readiness.fallback_sent); + } + + #[tokio::test] + async fn devin_trust_cannot_retroactively_fail_released_workers() { + for proven in [true, false] { + let (tx, mut rx) = mpsc::channel(4); + let mut request_id = None; + let mut readiness = StartupReadinessState { + ready_sent: proven, + fallback_sent: !proven, + trust_first_seen: Some(Instant::now() - DIRECTORY_TRUST_TIMEOUT), + ..Default::default() + }; + try_emit_worker_ready( + &tx, + "devin-trust", + Some(42), + &mut request_id, + Some(Instant::now()), + &mut readiness, + StartupGate::DirectoryTrustRequired, + ) + .await; + assert!(rx.try_recv().is_err()); + } + } + + #[tokio::test] + async fn devin_trust_cleared_before_deadline_allows_readiness() { + let (tx, mut rx) = mpsc::channel(4); + let mut request_id = None; + let mut readiness = StartupReadinessState::default(); + for gate in [StartupGate::DirectoryTrustRequired, StartupGate::Ready] { + try_emit_worker_ready( + &tx, + "devin-trust", + Some(42), + &mut request_id, + Some(Instant::now()), + &mut readiness, + gate, + ) + .await; + } + let frame = rx.try_recv().expect("ready after trust resolved"); + assert_eq!(frame.msg_type, "worker_ready"); + assert_eq!(frame.payload["readiness_proven"], true); + assert!(rx.try_recv().is_err()); + } + #[test] fn provider_login_and_trust_dialogs_are_classified_apart() { let device_auth = "Sign in to continue\nVisit https://www.facebook.com/device\n›\n"; diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index 94910d78f9..09b36f4274 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -157,6 +157,14 @@ pub(super) struct PendingVerifiedSpawn { } impl PendingVerifiedSpawn { + pub(super) fn directory_trust_failed(&mut self, generation: Uuid) { + if self.generation == generation { + self.deadline = Instant::now(); + // Fixed remediation only: never propagate private terminal output. + self.failure_reason = Some("spawn_directory_trust_required: Devin requires directory trust; run `devin` in the explicit spawn working directory on the selected node, trust that directory, then retry".into()); + } + } + pub(super) fn provider_auth_failed(&mut self, generation: Uuid) { if self.generation == generation { self.deadline = Instant::now(); diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index c8d55338b7..17b8d7667f 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -9353,6 +9353,71 @@ async fn muse_provider_auth_error_expires_verified_spawn_and_releases_capacity() assert!(found, "maintenance must return the specific fleet failure"); } +#[tokio::test] +async fn devin_directory_trust_error_expires_verified_spawn_and_releases_capacity() { + let name = WorkerName::from("devin-trust-test"); + let workers = make_worker_registry_with_worker(name.as_str()).await; + let generation = workers.workers.get(&name).unwrap().generation; + let mut fixture = worker_event_runtime_fixture(workers, HashMap::new()); + fixture.runtime.pending_verified_spawns.insert( + name.clone(), + super::fleet::PendingVerifiedSpawn { + invocation_id: "auth-invocation".into(), + deadline: Instant::now() + Duration::from_secs(90), + started: Instant::now(), + generation, + failure_reason: None, + }, + ); + for event_generation in [Uuid::new_v4(), generation] { + fixture.runtime.handle_worker_event(WorkerEvent::Message { + name: name.clone(), generation: event_generation, + value: json!({"type":"worker_error", "payload":{"code":"directory_trust_required", "message":"private-directory-path"}}), + }).await; + let pending = fixture.runtime.pending_verified_spawns.get(&name).unwrap(); + if event_generation != generation { + assert!(pending.failure_reason.is_none()); + assert!(pending.deadline > Instant::now()); + } else { + assert!(pending + .failure_reason + .as_deref() + .unwrap() + .contains("directory_trust_required")); + assert!(!pending + .failure_reason + .as_deref() + .unwrap() + .contains("private-directory-path")); + assert!(pending.deadline <= Instant::now()); + } + } + fixture + .runtime + .handle_worker_event(WorkerEvent::Message { + name: name.clone(), + generation, + value: json!({"type":"worker_ready", "payload":{"readiness_proven":true}}), + }) + .await; + assert!(fixture.runtime.pending_verified_spawns.contains_key(&name)); + fixture.runtime.handle_maintenance_tick().await; + assert!(!fixture.runtime.pending_verified_spawns.contains_key(&name)); + assert!(!fixture.runtime.workers.has_worker(&name)); + let mut found = false; + while let Ok(command) = fixture.fleet_control_rx.try_recv() { + if let FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) = command { + assert!(match &result.result { + crate::fleet_wire::ActionResultPayload::Error(error) => error.error.as_str(), + _ => panic!("expected fleet error: {result:?}"), + } + .contains("directory_trust_required")); + found = true; + } + } + assert!(found, "maintenance must return the specific fleet failure"); +} + #[tokio::test] async fn muse_fleet_missing_auth_fails_before_registration_and_dedup() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/broker/src/runtime/worker_events.rs b/crates/broker/src/runtime/worker_events.rs index 2134304d15..2eee899f25 100644 --- a/crates/broker/src/runtime/worker_events.rs +++ b/crates/broker/src/runtime/worker_events.rs @@ -1016,6 +1016,13 @@ impl BrokerRuntime { pending.provider_auth_failed(generation); } } + if value.pointer("/payload/code").and_then(Value::as_str) + == Some("directory_trust_required") + { + if let Some(pending) = pending_verified_spawns.get_mut(&name) { + pending.directory_trust_failed(generation); + } + } let is_pty = workers .workers .get(&name) diff --git a/crates/broker/tests/devin_trust_cli.rs b/crates/broker/tests/devin_trust_cli.rs new file mode 100644 index 0000000000..7535b2cc81 --- /dev/null +++ b/crates/broker/tests/devin_trust_cli.rs @@ -0,0 +1,179 @@ +//! Real broker PTY tests with a synthetic Devin fixture (python3 required). +//! These are NOT captured Devin menus and do not authorize automatic acceptance. +//! Each test has a fresh HOME/XDG tree; the trusted arm uses a fixture-only trust +//! record scoped to cwd. Cursor placement and raw input model the Devin composer. +#![cfg(unix)] + +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::fs::PermissionsExt; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +use serde_json::{json, Value}; + +struct Worker(Child); +impl Drop for Worker { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn run_fixture(mode: &str) { + let directory = tempfile::tempdir().expect("fixture directory"); + let home = directory.path().join("home"); + let cwd = directory.path().join("worktree"); + std::fs::create_dir_all(&home).unwrap(); + std::fs::create_dir_all(&cwd).unwrap(); + let trusted = mode == "trusted"; + if trusted { + std::fs::write(home.join("fixture-trust"), cwd.to_str().unwrap()).unwrap(); + } + let fake_devin = directory.path().join("devin"); + let input_marker = directory.path().join("input"); + let task_marker = directory.path().join("task"); + std::fs::write( + &fake_devin, + r#"#!/usr/bin/env python3 +import os, pathlib, sys, tty +tty.setraw(0) +trust = pathlib.Path(os.environ['HOME']) / 'fixture-trust' +trusted = trust.exists() and trust.read_text() == os.getcwd() +if trusted: + sys.stdout.write('❭ Ask Devin to build features, fix bugs, or work on your code') +else: + sys.stdout.write('Do you trust the authors of this directory?\r\nprivate-directory-marker\r\n') + if os.environ['FIXTURE_MODE'] == 'widening': + sys.stdout.write('❭ 1 Yes, trust the parent directory and all subdirectories\r\n2 No, exit') + elif os.environ['FIXTURE_MODE'] == 'wrapped': + sys.stdout.write('❭ 1 Yes, trust\r\n this workspace\r\n2 No, exit') + else: + sys.stdout.write('❭ 1. Yes, I trust the authors\r\n2. No, exit') +sys.stdout.flush() +data = b'' +while True: + part = os.read(0, 4096) + if not part: + break + with open(os.environ['INPUT_MARKER'], 'ab') as log: + log.write(part) + data += part + if trusted and b'\r' in data: + pathlib.Path(os.environ['TASK_MARKER']).write_bytes(data) +"#, + ) + .unwrap(); + std::fs::set_permissions(&fake_devin, std::fs::Permissions::from_mode(0o755)).unwrap(); + let mut worker = Worker( + Command::new(env!("CARGO_BIN_EXE_agent-relay-broker")) + .args([ + "pty", + "--agent-name", + "devin-trust-proof", + fake_devin.to_str().unwrap(), + ]) + .current_dir(&cwd) + .env("HOME", &home) + .env("XDG_CONFIG_HOME", home.join(".config")) + .env("XDG_DATA_HOME", home.join(".local/share")) + .env("AGENT_RELAY_LOCAL_ONLY", "1") + .env("FIXTURE_MODE", mode) + .env("INPUT_MARKER", &input_marker) + .env("TASK_MARKER", &task_marker) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn broker PTY"), + ); + let stdout = worker.0.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + if tx.send(line).is_err() { + break; + } + } + }); + for frame in [ + json!({"v":2, "type":"init_worker", "payload":{"agent":{"name":"devin-trust-proof"}}}), + json!({"v":2, "type":"deliver_relay", "payload":{ + "delivery_id":"fixture-delivery", "event_id":"fixture-event", "from":"operator", + "target":"devin-trust-proof", "body":"Begin fixture task" + }}), + ] { + writeln!(worker.0.stdin.as_mut().unwrap(), "{frame}").unwrap(); + } + let deadline = Instant::now() + Duration::from_secs(if trusted { 10 } else { 40 }); + let mut ready = false; + let mut error = None; + while Instant::now() < deadline { + if trusted && ready && task_marker.exists() { + break; + } + let line = match rx.recv_timeout(Duration::from_millis(100)) { + Ok(Ok(line)) => line, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + other => panic!("broker stdout closed: {other:?}"), + }; + let Ok(frame) = serde_json::from_str::(&line) else { + continue; + }; + match frame["type"].as_str() { + Some("worker_ready") => { + assert!(trusted, "trust dialog reported ready: {frame}"); + assert_eq!(frame["payload"]["readiness_proven"], true); + ready = true; + } + Some("worker_error") => { + error = Some(frame); + break; + } + _ => {} + } + } + if trusted { + assert!(error.is_none(), "{error:?}"); + assert!(ready, "already-trusted fixture must become ready"); + let task = std::fs::read_to_string(&task_marker).expect("task must be submitted"); + assert!(task.contains("Begin fixture task"), "{task}"); + assert_eq!( + std::fs::read_to_string(home.join("fixture-trust")).unwrap(), + cwd.to_str().unwrap() + ); + } else { + let error = error.expect("frozen trust prompt must produce worker_error within 40s"); + assert_eq!(error["payload"]["code"], "directory_trust_required"); + assert_eq!(error["payload"]["retryable"], false); + let message = error["payload"]["message"].as_str().unwrap(); + assert!(message.contains("spawn working directory")); + assert!(!message.contains("private-directory-marker")); + assert!( + !input_marker.exists(), + "no trust response or task may enter the dialog" + ); + assert!(!home.join("fixture-trust").exists()); + assert!(!ready); + } +} + +#[test] +fn fresh_home_frozen_trust_prompt_fails_without_readiness() { + run_fixture("unanswerable"); +} + +#[test] +fn widening_only_trust_menu_is_never_accepted() { + run_fixture("widening"); +} + +#[test] +fn wrapped_workspace_trust_prompt_fails_without_readiness() { + run_fixture("wrapped"); +} + +#[test] +fn already_trusted_directory_becomes_ready_and_begins_task() { + run_fixture("trusted"); +} diff --git a/crates/relay-pty/src/terminal.rs b/crates/relay-pty/src/terminal.rs index fa5df2f694..cb2f3963c8 100644 --- a/crates/relay-pty/src/terminal.rs +++ b/crates/relay-pty/src/terminal.rs @@ -98,6 +98,26 @@ pub fn detect_codex_trust_prompt(clean_output: &str) -> bool { && lower.contains("no, quit") } +/// Detect Devin's startup directory-trust interstitial, including wrapped text. +/// This is a blocking-screen detector, not authority to select a menu option. +/// Corroborate the question with a choice so ordinary trust-related prose does +/// not block startup. Callers must restrict this detector to Devin startup. +pub fn detect_devin_trust_prompt(screen: &str) -> bool { + let compact: String = screen + .chars() + .filter(|c| !c.is_whitespace()) + .flat_map(char::to_lowercase) + .collect(); + let question = compact.contains("doyoutrust") + && ["directory", "authors", "workspace"] + .iter() + .any(|word| compact.contains(word)); + let choice = ["yes,itrust", "yes,trust", "no,exit", "no,quit"] + .iter() + .any(|word| compact.contains(word)); + question && choice +} + /// Detect Muse's interactive provider (device) login screen. /// /// Muse authenticates out of band: the TUI prints a verification URL and a @@ -524,6 +544,27 @@ pub fn is_auto_suggestion(output: &str) -> bool { mod tests { use super::*; + #[test] + fn devin_trust_requires_question_and_choice_including_wrapped_variants() { + for screen in [ + "Do you trust the authors of this directory?\n1. Yes, I trust the authors\n2. No, exit", + "Do you trust this workspace?\n❭ 1 Yes, trust\n this workspace", + "Do you trust the authors of this\ndirectory?\n❯ 2) No, exit", + "Do you trust this directory?\n❭ 1 Yes, trust the parent directory\n2 No, quit", + ] { + assert!(detect_devin_trust_prompt(screen), "{screen}"); + } + for screen in [ + "❭ Ask Devin to build features, fix bugs, or work on your code", + "Do you trust this directory?", + "❭ 1 Yes, trust\n this workspace", + "Documentation: directory trust requires an operator", + "Do you trust this tool?\nYes, trust", + ] { + assert!(!detect_devin_trust_prompt(screen), "{screen}"); + } + } + #[test] fn trust_question_header_is_not_an_affirmative_menu_option() { assert_eq!( diff --git a/docs/harnesses/devin.md b/docs/harnesses/devin.md index 505baa63c6..16dbfc26ed 100644 --- a/docs/harnesses/devin.md +++ b/docs/harnesses/devin.md @@ -23,6 +23,17 @@ PTY auto-responders for Devin. Trust and tool approvals require an operator in an attached terminal. Authenticate and trust the intended worktree before unattended spawning. Trust is scoped to that directory. +During broker-managed PTY startup, a recognised directory-trust prompt blocks +readiness and task injection. If still present 30 seconds after first detection, +the worker reports `directory_trust_required` with instructions to trust the +explicit spawn working directory. Verified fleet spawns return +`spawn_directory_trust_required` and release the worker through spawn cleanup. +An unverified (`confirm: false`) spawn reports the worker error but does not reap +the harness; stop it explicitly or attach to resolve trust. Already-trusted +installations retain the existing startup path. Relay does not yet answer this +menu automatically: that requires captured evidence of Devin's actual menu and +trust scope. + Readiness requires the live idle `❭ Ask Devin to build features, fix bugs, or work on your code` composer at the cursor. Trust choices, busy composers and output byte counts do not establish readiness. Messages remain queued while diff --git a/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts b/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts index a32f44558a..0c1ccfc12e 100644 --- a/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts +++ b/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts @@ -139,7 +139,11 @@ describe('fleet spawn confirmation is observable from the requester (#1430)', () // MUST-FIRE — a node that reports its failure honestly still surfaced as // success before this change, because nothing read the action result. The // broker's detail (startup exit status and worker log path) must survive. - it.each(['spawn_failed: provider_auth_required', 'spawn_provider_auth_required'])( + it.each([ + 'spawn_failed: provider_auth_required', + 'spawn_provider_auth_required', + 'spawn_directory_trust_required', + ])( 'preserves %s from the node', async (reason) => { const { client } = createClient(async (name, id) => ({ From c960f5c29a590b341a0d9f28718ee03b173cce14 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 6 Oct 2026 14:07:10 +0000 Subject: [PATCH 13/42] style: auto-format with Prettier --- .../cli/lib/fleet-spawn-confirmation.test.ts | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts b/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts index 0c1ccfc12e..7edd1a2f1a 100644 --- a/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts +++ b/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts @@ -143,23 +143,20 @@ describe('fleet spawn confirmation is observable from the requester (#1430)', () 'spawn_failed: provider_auth_required', 'spawn_provider_auth_required', 'spawn_directory_trust_required', - ])( - 'preserves %s from the node', - async (reason) => { - const { client } = createClient(async (name, id) => ({ - invocation_id: id, - action_name: name, - status: 'failed', - error: `${reason}: run muse on the selected node to log in`, - })); - const error = await client.placement - .spawn(spawnInput({ confirm: true, confirmTimeoutMs: 60, confirmPollIntervalMs: 10 })) - .catch((caught: unknown) => caught); - expect(error).toBeInstanceOf(RelayPlacementError); - expect((error as RelayPlacementError).code).toBe('spawn_failed'); - expect((error as Error).message).toContain(reason); - } - ); + ])('preserves %s from the node', async (reason) => { + const { client } = createClient(async (name, id) => ({ + invocation_id: id, + action_name: name, + status: 'failed', + error: `${reason}: run muse on the selected node to log in`, + })); + const error = await client.placement + .spawn(spawnInput({ confirm: true, confirmTimeoutMs: 60, confirmPollIntervalMs: 10 })) + .catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(RelayPlacementError); + expect((error as RelayPlacementError).code).toBe('spawn_failed'); + expect((error as Error).message).toContain(reason); + }); it('fails with spawn_failed and preserves the node-reported detail', async () => { const { client } = createClient(async (name, invocationId) => ({ From f12922f19b1fa99af8b7e96e17ddfccbf2000ecc Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Wed, 7 Oct 2026 23:26:46 -0700 Subject: [PATCH 14/42] fix(broker): treat visible Devin trust history as inactive and reset the trust deadline - relay-pty: add devin_trust_prompt_active(grid), true only when the trust question and choice are visible AND Devin's exact idle composer does not hold the cursor. A resumed screen that keeps an answered trust menu above the live composer no longer blocks readiness. - broker: the startup gate and block-reason classifier use the grid-aware check; startup_gate_block_reason now takes the grid snapshot. - broker: clear trust_first_seen whenever the gate is not DirectoryTrustRequired, so a returning prompt gets a full 30 s instead of inheriting a cleared prompt's deadline. Regression tests fail before and pass after: devin_trust_history_above_live_composer_does_not_block_readiness, devin_trust_prompt_that_returns_gets_a_fresh_deadline, devin_trust_history_above_live_composer_is_not_active. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: f9b2ae4e-847e-47c7-9996-b6545ca4353c --- crates/broker/src/pty_worker.rs | 124 +++++++++++++++++++++++++----- crates/relay-pty/src/readiness.rs | 31 +++++++- 2 files changed, 136 insertions(+), 19 deletions(-) diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index ce995e6cd4..677bff7b74 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -38,13 +38,14 @@ use crate::broker::{ }; use crate::cli::command_parse::parse_cli_command; use crate::cli::PtyCommand; -use crate::readiness::{detect_cli_ready, is_devin_cli, is_muse_cli, GridReadinessSnapshot}; +use crate::readiness::{ + detect_cli_ready, devin_trust_prompt_active, is_devin_cli, is_muse_cli, GridReadinessSnapshot, +}; use crate::runtime::{get_terminal_size, send_frame}; use crate::snapshot::Snapshot; use crate::util::ansi::{floor_char_boundary, strip_ansi, AnsiStripper}; use crate::util::terminal::{ - detect_claude_trust_prompt, detect_codex_trust_prompt, detect_devin_trust_prompt, - detect_muse_device_auth_prompt, + detect_claude_trust_prompt, detect_codex_trust_prompt, detect_muse_device_auth_prompt, }; use crate::util::utf8_stream::Utf8StreamDecoder; use crate::worker::detection::ActivityDetector; @@ -588,7 +589,7 @@ fn evaluate_startup_gate( ); let trust_blocked = detect_codex_trust_prompt(grid.screen) || detect_claude_trust_prompt(grid.screen) == (true, true) - || (is_devin_cli(resolved_cli) && detect_devin_trust_prompt(grid.screen)); + || (is_devin_cli(resolved_cli) && devin_trust_prompt_active(grid)); let composer_ready = is_codex && codex_composer_ready(grid); tracing::debug!(target: "relay_broker::startup_gate", cli = resolved_cli, is_codex, composer_ready, trust_blocked, @@ -655,8 +656,12 @@ impl StartupGate { /// browser elsewhere. `detect_cli_ready` already refuses to prove readiness /// on that screen; this names it as deliberate so the deadline cannot /// release the brief into a login form. -fn startup_gate_block_reason(resolved_cli: &str, screen: &str) -> Option { - if is_devin_cli(resolved_cli) && detect_devin_trust_prompt(screen) { +fn startup_gate_block_reason( + resolved_cli: &str, + grid: GridReadinessSnapshot<'_>, +) -> Option { + let screen = grid.screen; + if is_devin_cli(resolved_cli) && devin_trust_prompt_active(grid) { return Some(StartupGate::DirectoryTrustRequired); } if is_muse_cli(resolved_cli) && detect_muse_device_auth_prompt(screen) { @@ -678,19 +683,20 @@ fn startup_gate( pty: &PtySession, ) -> StartupGate { let screen = pty.screen_text(); + let grid = GridReadinessSnapshot { + screen: &screen, + cursor: Some(pty.cursor_position()), + }; if evaluate_startup_gate( resolved_cli, startup_output, startup_total_bytes, output_quiet, - GridReadinessSnapshot { - screen: &screen, - cursor: Some(pty.cursor_position()), - }, + grid, ) { return StartupGate::Ready; } - startup_gate_block_reason(resolved_cli, &screen).unwrap_or(StartupGate::Unrecognised) + startup_gate_block_reason(resolved_cli, grid).unwrap_or(StartupGate::Unrecognised) } /// Whether the loop may pop and start the next pending injection this tick. @@ -832,6 +838,11 @@ async fn try_emit_worker_ready( // Anchor the deadline on the gate's own detector, not a parsed menu row or // response attempt. The 200 ms verification tick drives this even when the // child stops producing output. Blocking suppresses the generic fail-open. + // A prompt that cleared before readiness must not hand its deadline to a + // later one; each appearance gets the full timeout. + if gate != StartupGate::DirectoryTrustRequired { + readiness.trust_first_seen = None; + } if gate == StartupGate::DirectoryTrustRequired && !readiness.fallback_sent { let first_seen = readiness.trust_first_seen.get_or_insert_with(Instant::now); if first_seen.elapsed() >= DIRECTORY_TRUST_TIMEOUT { @@ -3376,12 +3387,19 @@ mod tests { ); } + fn screen_only(screen: &str) -> GridReadinessSnapshot<'_> { + GridReadinessSnapshot { + screen, + cursor: None, + } + } + #[test] fn devin_trust_gate_is_scoped_to_devin_and_vetoes_composer() { let screen = "Do you trust the authors of this directory?\n❭ 1 Yes, trust\n2 No, exit"; for cli in ["devin", "/usr/local/bin/devin", "devin.exe"] { assert_eq!( - startup_gate_block_reason(cli, screen), + startup_gate_block_reason(cli, screen_only(screen)), Some(StartupGate::DirectoryTrustRequired) ); assert!(!evaluate_startup_gate( @@ -3396,12 +3414,12 @@ mod tests { )); } for cli in ["claude", "codex", "muse"] { - assert_eq!(startup_gate_block_reason(cli, screen), None); + assert_eq!(startup_gate_block_reason(cli, screen_only(screen)), None); } assert_eq!( startup_gate_block_reason( "devin", - "Sign in to continue\nVisit https://www.facebook.com/device\n›" + screen_only("Sign in to continue\nVisit https://www.facebook.com/device\n›") ), None ); @@ -3482,6 +3500,68 @@ mod tests { } } + #[test] + fn devin_trust_history_above_live_composer_does_not_block_readiness() { + // A resumed or uncleared Devin screen can keep the answered trust + // question visible above the idle composer that holds the cursor. + let screen = "Do you trust this directory?\n1 Yes, trust\n2 No, exit\n❭ Ask Devin to build features, fix bugs, or work on your code\nSWE-2 High"; + let grid = GridReadinessSnapshot { + screen, + cursor: Some((4, 3)), + }; + assert!(evaluate_startup_gate( + "devin", + screen, + 1000, + Duration::from_secs(90), + grid + )); + assert_eq!(startup_gate_block_reason("devin", grid), None); + // The same text with the cursor on the menu is still the live dialog. + let menu = GridReadinessSnapshot { + screen, + cursor: Some((2, 1)), + }; + assert_eq!( + startup_gate_block_reason("devin", menu), + Some(StartupGate::DirectoryTrustRequired) + ); + } + + #[tokio::test] + async fn devin_trust_prompt_that_returns_gets_a_fresh_deadline() { + let (tx, mut rx) = mpsc::channel(4); + let mut request_id = None; + let mut readiness = StartupReadinessState { + trust_first_seen: Some(Instant::now() - DIRECTORY_TRUST_TIMEOUT), + ..Default::default() + }; + // The earlier prompt cleared before readiness, then trust reappears. + for gate in [ + StartupGate::Unrecognised, + StartupGate::DirectoryTrustRequired, + ] { + try_emit_worker_ready( + &tx, + "devin-trust", + Some(42), + &mut request_id, + Some(Instant::now()), + &mut readiness, + gate, + ) + .await; + } + assert!( + rx.try_recv().is_err(), + "a returning trust prompt must not inherit the cleared prompt's deadline" + ); + assert!(!readiness.trust_error_sent); + assert!(readiness + .trust_first_seen + .is_some_and(|seen| seen.elapsed() < DIRECTORY_TRUST_TIMEOUT)); + } + #[tokio::test] async fn devin_trust_cleared_before_deadline_allows_readiness() { let (tx, mut rx) = mpsc::channel(4); @@ -3509,22 +3589,30 @@ mod tests { fn provider_login_and_trust_dialogs_are_classified_apart() { let device_auth = "Sign in to continue\nVisit https://www.facebook.com/device\n›\n"; assert_eq!( - startup_gate_block_reason("/usr/local/bin/muse", device_auth), + startup_gate_block_reason("/usr/local/bin/muse", screen_only(device_auth)), Some(StartupGate::ProviderAuthRequired) ); // Only Muse carries the device-login arm; another harness rendering // the same words keeps its existing unrecognised-prompt behaviour. - assert_eq!(startup_gate_block_reason("codex", device_auth), None); + assert_eq!( + startup_gate_block_reason("codex", screen_only(device_auth)), + None + ); assert_eq!( startup_gate_block_reason( "codex", - "Do you trust the contents of this directory?\n> Yes, continue\n No, quit\n" + screen_only( + "Do you trust the contents of this directory?\n> Yes, continue\n No, quit\n" + ) ), Some(StartupGate::Blocked) ); // An authenticated composer is neither. assert_eq!( - startup_gate_block_reason("muse", "muse signed in as relay-node\n/login\n›\n"), + startup_gate_block_reason( + "muse", + screen_only("muse signed in as relay-node\n/login\n›\n") + ), None ); } diff --git a/crates/relay-pty/src/readiness.rs b/crates/relay-pty/src/readiness.rs index 93a3e77236..8edfe026c8 100644 --- a/crates/relay-pty/src/readiness.rs +++ b/crates/relay-pty/src/readiness.rs @@ -1,6 +1,6 @@ use crate::{ ansi::{floor_char_boundary, strip_ansi}, - terminal::detect_muse_device_auth_prompt, + terminal::{detect_devin_trust_prompt, detect_muse_device_auth_prompt}, wait::{for_cli, WaitSnapshot}, }; @@ -146,6 +146,16 @@ pub fn is_devin_cli(cli: &str) -> bool { matches!(base.as_str(), "devin" | "devin.exe") } +/// Whether Devin's directory-trust dialog is the active screen. +/// +/// Trust text alone is not enough: a resumed or uncleared screen can keep an +/// answered trust question visible above the live idle composer. The exact +/// idle placeholder holding the cursor proves the dialog is history, since a +/// trust menu never renders that placeholder. +pub fn devin_trust_prompt_active(grid: GridReadinessSnapshot<'_>) -> bool { + detect_devin_trust_prompt(grid.screen) && !devin_prompt_ready(grid) +} + fn devin_prompt_ready(grid: GridReadinessSnapshot<'_>) -> bool { let Some((row, _)) = grid.cursor else { return false; @@ -277,6 +287,25 @@ mod tests { assert!(!is_devin_cli("devin.bat")); } + #[test] + fn devin_trust_history_above_live_composer_is_not_active() { + let screen = "Do you trust this directory?\n1 Yes, trust\n2 No, exit\n❭ Ask Devin to build features, fix bugs, or work on your code\nSWE-2 High"; + assert!(!devin_trust_prompt_active(GridReadinessSnapshot { + screen, + cursor: Some((4, 3)), + })); + for cursor in [Some((2, 1)), None] { + assert!(devin_trust_prompt_active(GridReadinessSnapshot { + screen, + cursor + })); + } + assert!(devin_trust_prompt_active(GridReadinessSnapshot { + screen: "Do you trust the authors of this directory?\n❭ 1 Yes, trust\n2 No, exit", + cursor: Some((2, 3)), + })); + } + #[test] fn devin_wrapped_idle_composer_requires_cursor_in_exact_placeholder() { // Captured from Devin 3000.10.31 at 40 columns, including indentation. From e6bd9d64df5f098f2076704cf7766fb3692b254c Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Wed, 7 Oct 2026 23:38:13 -0700 Subject: [PATCH 15/42] fix(mcp): key channel/thread write replay on the request, not the text; accept engine events Devin review on #1910: - Independent identical replies no longer collapse. Drop content-keyed coalesceWrite: MCP post_message and reply_to_thread go through McpRequestReplay.run keyed on the JSON-RPC request ID, accept an idempotency_key (forwarded to Relaycast, like send_dm), and bind the acting agent client before scheduling. Native host tools join only a replay of the same model tool call ID, which HarnessHost now passes to execute. - integration subscribe validates --events against the engine's SubscribableEventTypeSchema (re-exported by the SDK as SUBSCRIBABLE_EVENT_TYPES) instead of a two-event allowlist, so action.completed, message.updated, etc. provision again. Fail-before/pass-after tests: messaging-tools.protocol (parallel identical writes stay separate; keyed retries join and forward the key; identity bound at call time), request-replay, native-relay-tools (toolCallId keyed), harness-host (toolCallId passed), integration-subscribe (engine events). Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: f9b2ae4e-847e-47c7-9996-b6545ca4353c --- .../evidence/provider-subscription-replies.md | 13 +-- .../commands/integration-subscribe.test.ts | 15 +++ packages/cli/src/cli/commands/integration.ts | 11 ++- .../cli/mcp/messaging-tools.protocol.test.ts | 98 ++++++++++++++++--- packages/cli/src/cli/mcp/messaging-tools.ts | 79 +++++++-------- .../cli/src/cli/mcp/request-replay.test.ts | 64 +++--------- packages/cli/src/cli/mcp/request-replay.ts | 29 +----- packages/cli/src/cli/telemetry/events.ts | 2 - .../harnesses/src/ai-sdk/harness-host.test.ts | 2 +- packages/harnesses/src/ai-sdk/harness-host.ts | 14 ++- .../src/ai-sdk/native-relay-tools.test.ts | 30 +++--- .../src/ai-sdk/native-relay-tools.ts | 18 ++-- packages/sdk/src/messaging/index.ts | 1 + .../sdk/src/messaging/subscribable-events.ts | 7 ++ packages/sdk/src/messaging/thin-client.ts | 8 +- 15 files changed, 221 insertions(+), 170 deletions(-) create mode 100644 packages/sdk/src/messaging/subscribable-events.ts diff --git a/docs/evidence/provider-subscription-replies.md b/docs/evidence/provider-subscription-replies.md index 996185dedf..e1f4141e18 100644 --- a/docs/evidence/provider-subscription-replies.md +++ b/docs/evidence/provider-subscription-replies.md @@ -29,12 +29,13 @@ message inbox check` reports no workspace key. No authenticated message, ## Changes and boundaries -MCP posts/replies use request replay and pending-only argument coalescing. -Native tools use a per-session pending map. Concurrent identical writes join; -settled writes, different identities, targets, options, and sessions remain -separate. MCP joins emit `agent_relay_write_coalesced` with only the tool name. -This cannot deduplicate distinct processes or sequential unkeyed retries after a -lost response. +MCP posts/replies use request replay keyed on the JSON-RPC request ID (or an +explicit `idempotency_key`, also forwarded to Relaycast), matching `send_dm`. +Native tools join only a replay of the same model tool call ID. Message text +never forms a key: independent writes with identical text, including +concurrent ones, remain separate messages. Unkeyed retries after a lost +response, or from a distinct process, are not deduplicated; callers that retry +must pass `idempotency_key`. Subscription setup reads persisted event coverage before binding, rejects unsupported events and inactive/mismatched subscriptions, and uses the existing diff --git a/packages/cli/src/cli/commands/integration-subscribe.test.ts b/packages/cli/src/cli/commands/integration-subscribe.test.ts index d669acd9b9..dc33feddf5 100644 --- a/packages/cli/src/cli/commands/integration-subscribe.test.ts +++ b/packages/cli/src/cli/commands/integration-subscribe.test.ts @@ -2132,6 +2132,21 @@ describe('subscription writeback verification', () => { expect(relayfile.createWebhookSubscription).not.toHaveBeenCalled(); }); + it.each(['action.completed', 'message.updated'])( + 'provisions engine-supported %s subscriptions', + async (event) => { + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockImplementation(async (id) => ({ id, events: [event] })); + const { program, relayfile, error } = harness({ relay }); + await program.parseAsync(ARGS(['--events', event]), { from: 'user' }); + expect(error.mock.calls.flat().join(' ')).not.toContain('Unsupported subscription events'); + expect(relay.integrations.subscriptions.create).toHaveBeenCalledWith( + expect.objectContaining({ event, events: [event] }) + ); + expect(relayfile.bind).toHaveBeenCalledOnce(); + } + ); + it('accepts singular thread.reply coverage', async () => { const relay = createRelayMock(); relay.integrations.subscriptions.get.mockImplementation(async (id) => ({ id, event: 'thread.reply' })); diff --git a/packages/cli/src/cli/commands/integration.ts b/packages/cli/src/cli/commands/integration.ts index 6b0573d971..1b3720686c 100644 --- a/packages/cli/src/cli/commands/integration.ts +++ b/packages/cli/src/cli/commands/integration.ts @@ -7,7 +7,11 @@ import { import { createHash, pbkdf2Sync, randomBytes } from 'node:crypto'; import { hostname } from 'node:os'; import { getProjectPaths } from '@agent-relay/config'; -import { normalizeWebhookSubscription, type AgentRelayAgent } from '@agent-relay/sdk'; +import { + SUBSCRIBABLE_EVENT_TYPES, + normalizeWebhookSubscription, + type AgentRelayAgent, +} from '@agent-relay/sdk'; import { cleanupEntryKey, @@ -1608,10 +1612,11 @@ async function runSubscribeSetup( const requestedEvents = commaList(opts.events); const events = [...new Set(requestedEvents.length ? requestedEvents : ['message.created', 'thread.reply'])]; - const unsupported = events.filter((event) => !['message.created', 'thread.reply'].includes(event)); + const supported: readonly string[] = SUBSCRIBABLE_EVENT_TYPES; + const unsupported = events.filter((event) => !supported.includes(event)); if (unsupported.length) throw new Error( - `Unsupported subscription events: ${unsupported.join(', ')}. Supported events: message.created, thread.reply.` + `Unsupported subscription events: ${unsupported.join(', ')}. Supported events: ${supported.join(', ')}.` ); const { provider, resource, to } = await promptSubscribeOptions(deps, providerArg, opts); const local = await deps.resolveLocalRelayOptions(); diff --git a/packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts b/packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts index 0e5f5f5e01..93df18ccef 100644 --- a/packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts +++ b/packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts @@ -315,39 +315,105 @@ describe('messaging delivery receipts over MCP', () => { }); describe('channel and thread writes over MCP', () => { - it.each([ - ['post_message', 'send', { channel: 'events', text: 'ACK' }], - ['reply_to_thread', 'reply', { message_id: 'provider-parent', text: 'ACK' }], - ] as const)('coalesces parallel %s calls but permits sequential repeats', async (tool, method, args) => { + function gatedWrite() { let release!: () => void; const gate = new Promise((resolve) => { release = resolve; }); + let n = 0; const write = vi.fn(async () => { await gate; - return { id: 'reply-1' }; + n += 1; + return { id: `message-${n}` }; }); + return { write, release }; + } + + async function connect(getAgentClient: Parameters[1]) { const server = new McpServer({ name: 'write-test', version: '1' }); - registerMessagingTools(server, () => ({ [method]: write }) as never); + registerMessagingTools(server, getAgentClient); const client = new Client({ name: 'write-client', version: '1' }); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + return { + client, + close: async () => { + await client.close(); + await server.close(); + }, + }; + } + + it.each([ + ['post_message', 'send', { channel: 'events', text: 'ACK' }], + ['reply_to_thread', 'reply', { message_id: 'provider-parent', text: 'ACK' }], + ] as const)( + 'keeps independent parallel %s calls with identical text separate', + async (tool, method, args) => { + const { write, release } = gatedWrite(); + const { client, close } = await connect(() => ({ [method]: write }) as never); + try { + const first = client.callTool({ name: tool, arguments: args }); + const second = client.callTool({ name: tool, arguments: args }); + await vi.waitFor(() => expect(write).toHaveBeenCalledTimes(2)); + release(); + const results = await Promise.all([first, second]); + expect(results.map((result) => result.isError)).toEqual([undefined, undefined]); + expect(results[0].structuredContent).not.toEqual(results[1].structuredContent); + } finally { + release(); + await close(); + } + } + ); + + it.each([ + ['post_message', 'send', { channel: 'events', text: 'ACK' }], + ['reply_to_thread', 'reply', { message_id: 'provider-parent', text: 'ACK' }], + ] as const)('joins %s calls that share an idempotency key and forwards it', async (tool, method, args) => { + const { write, release } = gatedWrite(); + const { client, close } = await connect(() => ({ [method]: write }) as never); try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const first = client.callTool({ name: tool, arguments: args }); - const second = client.callTool({ name: tool, arguments: args }); + const keyed = { ...args, idempotency_key: ' retry-1 ' }; + const first = client.callTool({ name: tool, arguments: keyed }); + const retry = client.callTool({ name: tool, arguments: keyed }); await vi.waitFor(() => expect(write).toHaveBeenCalledTimes(1)); release(); - const results = await Promise.all([first, second]); - expect(results[0].isError).not.toBe(true); + const results = await Promise.all([first, retry]); expect(results[0].structuredContent).toEqual(results[1].structuredContent); expect(write).toHaveBeenCalledTimes(1); - await client.callTool({ name: tool, arguments: args }); - expect(write).toHaveBeenCalledTimes(2); + expect(write.mock.calls[0]).toContainEqual(expect.objectContaining({ idempotencyKey: 'retry-1' })); } finally { release(); - await client.close(); - await server.close(); + await close(); + } + }); + + it('keeps an in-flight write on the identity that was active when it was called', async () => { + const alice = gatedWrite(); + const bob = gatedWrite(); + let current = 'alice'; + const { client, close } = await connect( + () => ({ reply: current === 'alice' ? alice.write : bob.write }) as never + ); + try { + const pending = client.callTool({ + name: 'reply_to_thread', + arguments: { message_id: 'parent', text: 'ACK' }, + }); + await vi.waitFor(() => expect(alice.write).toHaveBeenCalledTimes(1)); + current = 'bob'; + alice.release(); + bob.release(); + await pending; + await client.callTool({ name: 'reply_to_thread', arguments: { message_id: 'parent', text: 'ACK' } }); + expect(alice.write).toHaveBeenCalledTimes(1); + expect(bob.write).toHaveBeenCalledTimes(1); + } finally { + alice.release(); + bob.release(); + await close(); } }); }); diff --git a/packages/cli/src/cli/mcp/messaging-tools.ts b/packages/cli/src/cli/mcp/messaging-tools.ts index 19db796995..a28a3e4665 100644 --- a/packages/cli/src/cli/mcp/messaging-tools.ts +++ b/packages/cli/src/cli/mcp/messaging-tools.ts @@ -1,7 +1,6 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { replayMessageMetadata } from '@agent-relay/sdk'; import { z } from 'zod'; -import { track } from '../telemetry/index.js'; import { compactDirectMessageReceipt, @@ -68,6 +67,19 @@ function resolveEmoji(input: string): string { * tools. These all act through a single agent client resolved per-call from the * optional `as` identity override. */ +const idempotencyKeyInput = z + .string() + // Relaycast trims this key upstream, so trim before both the local replay + // cache and the forwarded call to keep one logical send on one key; a + // whitespace-only key would otherwise become an unkeyed send. + .trim() + .min(1) + .max(255) + .optional() + .describe( + 'Stable key for retrying this same message after a lost response; use a new key for a new message. Surrounding whitespace is trimmed and a whitespace-only key is rejected.' + ); + export function registerMessagingTools( server: McpServer, getAgentClient: (asIdentity?: string) => AgentClientLike, @@ -238,6 +250,7 @@ export function registerMessagingTools( .describe( 'wait (default): queue delivery until each recipient reaches a safe idle boundary; steer: request immediate injection, which may interrupt active work.' ), + idempotency_key: idempotencyKeyInput, ...identityOverrideInputShape, }, outputSchema: jsonResult, @@ -248,23 +261,21 @@ export function registerMessagingTools( openWorldHint: true, }, }, - async ({ channel, text, attachments, mode, as }, extra) => - replay.run('post_message', extra, undefined, () => - replay.coalesceWrite( - 'post_message', - extra, - [as ?? null, channel, text, attachments ?? [], mode ?? 'wait'], - async () => - jsonContent( - await getAgentClient(as).send(channel, text, { - attachments, - data: replayMessageMetadata(), - mode, - }) - ), - () => track('agent_relay_write_coalesced', { tool_name: 'post_message' }) + async ({ channel, text, attachments, mode, idempotency_key, as }, extra) => { + // Bind the acting identity now: a register_agent that moves the session + // default must not reroute a write that is already in flight. + const client = getAgentClient(as); + return replay.run('post_message', extra, idempotency_key, async () => + jsonContent( + await client.send(channel, text, { + attachments, + data: replayMessageMetadata(), + mode, + idempotencyKey: idempotency_key, + }) ) - ) + ); + } ); server.registerTool( @@ -302,6 +313,7 @@ export function registerMessagingTools( inputSchema: { message_id: z.string().describe('Parent message ID'), text: z.string().describe('Reply text'), + idempotency_key: idempotencyKeyInput, ...identityOverrideInputShape, }, outputSchema: jsonResult, @@ -312,17 +324,17 @@ export function registerMessagingTools( openWorldHint: true, }, }, - async ({ message_id, text, as }, extra) => - replay.run('reply_to_thread', extra, undefined, () => - replay.coalesceWrite( - 'reply_to_thread', - extra, - [as ?? null, message_id, text], - async () => - jsonContent(await getAgentClient(as).reply(message_id, text, { data: replayMessageMetadata() })), - () => track('agent_relay_write_coalesced', { tool_name: 'reply_to_thread' }) + async ({ message_id, text, idempotency_key, as }, extra) => { + const client = getAgentClient(as); + return replay.run('reply_to_thread', extra, idempotency_key, async () => + jsonContent( + await client.reply(message_id, text, { + data: replayMessageMetadata(), + idempotencyKey: idempotency_key, + }) ) - ) + ); + } ); server.registerTool( @@ -367,18 +379,7 @@ export function registerMessagingTools( 'wait (default): queue until the recipient reaches a safe idle boundary; steer: request immediate injection, which may interrupt active work. Both modes return before reading is confirmed.' ), attachments: z.array(z.string()).optional().describe('File attachment IDs'), - idempotency_key: z - .string() - // Relaycast trims this key upstream, so trim before both the local - // replay cache and the forwarded call to keep one logical send on one - // key; a whitespace-only key would otherwise become an unkeyed send. - .trim() - .min(1) - .max(255) - .optional() - .describe( - 'Stable key for retrying this same message after a lost response; use a new key for a new message. Surrounding whitespace is trimmed and a whitespace-only key is rejected.' - ), + idempotency_key: idempotencyKeyInput, ...identityOverrideInputShape, }, outputSchema: directMessageResult, diff --git a/packages/cli/src/cli/mcp/request-replay.test.ts b/packages/cli/src/cli/mcp/request-replay.test.ts index c8a38f8b9a..764470ac75 100644 --- a/packages/cli/src/cli/mcp/request-replay.test.ts +++ b/packages/cli/src/cli/mcp/request-replay.test.ts @@ -11,70 +11,36 @@ function deferred() { return { promise, resolve, reject }; } -describe('pending message write coalescing', () => { - it('joins concurrent aliases, reports coalescing, and permits a sequential repeat', async () => { +describe('MCP request replay', () => { + it('runs distinct request IDs separately even while an identical write is pending', async () => { const replay = new McpRequestReplay(); const pending = deferred(); const operation = vi.fn(() => pending.promise); - const coalesced = vi.fn(); - const send = (requestId: number) => - replay.run('reply_to_thread', { requestId, sessionId: 's' }, undefined, () => - replay.coalesceWrite( - 'reply_to_thread', - { sessionId: 's' }, - ['agent', 'parent', 'ACK'], - operation, - coalesced - ) - ); - const first = send(1); - const alias = send(2); - await Promise.resolve(); - expect(operation).toHaveBeenCalledTimes(1); - expect(coalesced).toHaveBeenCalledTimes(1); - pending.resolve('reply'); - expect(await first).toBe(await alias); - await send(3); + const first = replay.run('reply_to_thread', { requestId: 1, sessionId: 's' }, undefined, operation); + const second = replay.run('reply_to_thread', { requestId: 2, sessionId: 's' }, undefined, operation); expect(operation).toHaveBeenCalledTimes(2); + pending.resolve('reply'); + await Promise.all([first, second]); }); - it('separates sessions, identities, targets, tools, and write options', async () => { + it('joins a transport replay of one request ID within a session and tool', async () => { const replay = new McpRequestReplay(); const pending = deferred(); const operation = vi.fn(() => pending.promise); - const writes = [ - replay.coalesceWrite('post_message', { sessionId: 'a' }, ['alice', 'c', 'ACK', [], 'wait'], operation), - replay.coalesceWrite('post_message', { sessionId: 'b' }, ['alice', 'c', 'ACK', [], 'wait'], operation), - replay.coalesceWrite('post_message', { sessionId: 'a' }, ['bob', 'c', 'ACK', [], 'wait'], operation), - replay.coalesceWrite('post_message', { sessionId: 'a' }, ['alice', 'd', 'ACK', [], 'wait'], operation), - replay.coalesceWrite( - 'reply_to_thread', - { sessionId: 'a' }, - ['alice', 'c', 'ACK', [], 'wait'], - operation - ), - replay.coalesceWrite( - 'post_message', - { sessionId: 'a' }, - ['alice', 'c', 'ACK', ['file'], 'wait'], - operation - ), - replay.coalesceWrite('post_message', { sessionId: 'a' }, ['alice', 'c', 'ACK', [], 'steer'], operation), - ]; - await Promise.resolve(); - expect(operation).toHaveBeenCalledTimes(7); + const first = replay.run('post_message', { requestId: 7, sessionId: 'a' }, undefined, operation); + expect(replay.run('post_message', { requestId: 7, sessionId: 'a' }, undefined, operation)).toBe(first); + void replay.run('post_message', { requestId: 7, sessionId: 'b' }, undefined, operation); + void replay.run('reply_to_thread', { requestId: 7, sessionId: 'a' }, undefined, operation); + expect(operation).toHaveBeenCalledTimes(3); pending.resolve('ok'); - await Promise.all(writes); + await first; }); - it('clears rejected writes and transport request IDs for a safe retry', async () => { + it('clears a rejected request ID for a safe retry', async () => { const replay = new McpRequestReplay(); const pending = deferred(); const operation = vi.fn(() => pending.promise); - const send = () => - replay.run('reply_to_thread', { requestId: 1 }, undefined, () => - replay.coalesceWrite('reply_to_thread', {}, ['a', 'p', 'ACK'], operation) - ); + const send = () => replay.run('reply_to_thread', { requestId: 1 }, undefined, operation); const first = send(); expect(send()).toBe(first); const assertion = expect(first).rejects.toThrow('unavailable'); diff --git a/packages/cli/src/cli/mcp/request-replay.ts b/packages/cli/src/cli/mcp/request-replay.ts index d96a7cbf96..3d6f085903 100644 --- a/packages/cli/src/cli/mcp/request-replay.ts +++ b/packages/cli/src/cli/mcp/request-replay.ts @@ -5,38 +5,11 @@ * JSON-RPC permits IDs to be reused after a response. Clients that need to * retry after a lost response provide an explicit idempotency key; that key * retains the completed result briefly. Arguments deliberately do not - * participate in this request key. A separate pending-write key includes the - * session, tool, identity and all write arguments to join parallel tool aliases. - * It is removed on settlement so sequential identical sends remain distinct. + * participate, so intentional identical sends remain separate requests. */ export class McpRequestReplay { private readonly requests = new Map>(); - private readonly writes = new Map>(); - - coalesceWrite( - tool: string, - extra: unknown, - argumentsKey: unknown[], - operation: () => Promise, - onCoalesce: () => void = () => {} - ): Promise { - const sessionId = (extra as { sessionId?: unknown } | undefined)?.sessionId; - const key = JSON.stringify([sessionId ?? '', tool, ...argumentsKey]); - const existing = this.writes.get(key) as Promise | undefined; - if (existing) { - onCoalesce(); - return existing; - } - const pending = Promise.resolve().then(operation); - this.writes.set(key, pending); - void pending.then( - () => this.writes.delete(key), - () => this.writes.delete(key) - ); - return pending; - } - run( tool: string, extra: unknown, diff --git a/packages/cli/src/cli/telemetry/events.ts b/packages/cli/src/cli/telemetry/events.ts index 56a89f4d24..4bf91b9222 100644 --- a/packages/cli/src/cli/telemetry/events.ts +++ b/packages/cli/src/cli/telemetry/events.ts @@ -487,7 +487,6 @@ export type TelemetryEventName = | 'cli_install' | 'cli_update' | 'agent_relay_tool_call' - | 'agent_relay_write_coalesced' | 'workflow_run' | 'cloud_auth' | 'cloud_workflow_run' @@ -511,7 +510,6 @@ export interface TelemetryEventMap { cli_install: CliInstallEvent; cli_update: CliUpdateEvent; agent_relay_tool_call: AgentRelayToolCallEvent; - agent_relay_write_coalesced: { tool_name: 'post_message' | 'reply_to_thread' }; workflow_run: WorkflowRunEvent; cloud_auth: CloudAuthEvent; cloud_workflow_run: CloudWorkflowRunEvent; diff --git a/packages/harnesses/src/ai-sdk/harness-host.test.ts b/packages/harnesses/src/ai-sdk/harness-host.test.ts index cfde4d64e7..58b57b25e9 100644 --- a/packages/harnesses/src/ai-sdk/harness-host.test.ts +++ b/packages/harnesses/src/ai-sdk/harness-host.test.ts @@ -162,7 +162,7 @@ describe('HarnessHost', () => { await vi.waitFor(() => expect(turns[0].submitToolResult).toHaveBeenCalledOnce()); expect(execute).toHaveBeenCalledWith( { to: 'Worker', text: 'hello' }, - expect.objectContaining({ abortSignal: expect.any(AbortSignal) }) + expect.objectContaining({ abortSignal: expect.any(AbortSignal), toolCallId: 'call-1' }) ); expect(turns[0].submitToolResult).toHaveBeenCalledWith({ toolCallId: 'call-1', diff --git a/packages/harnesses/src/ai-sdk/harness-host.ts b/packages/harnesses/src/ai-sdk/harness-host.ts index 6ee8c617c5..ef58fd5fdc 100644 --- a/packages/harnesses/src/ai-sdk/harness-host.ts +++ b/packages/harnesses/src/ai-sdk/harness-host.ts @@ -66,7 +66,14 @@ export interface HarnessHostOptions { export interface HarnessHostTool { spec: HarnessV1ToolSpec; - execute(input: unknown, options: { abortSignal?: AbortSignal }): Promise | unknown; + /** + * `toolCallId` is the model's ID for this invocation. A replay of one call + * carries the same ID; independent calls never share one. + */ + execute( + input: unknown, + options: { abortSignal?: AbortSignal; toolCallId?: string } + ): Promise | unknown; } export interface HarnessTurnHandle { @@ -348,7 +355,10 @@ export class HarnessHost { const tool = this.#tools.get(call.toolName); if (!tool) return; try { - const output = await tool.execute(parseToolInput(call.input), { abortSignal }); + const output = await tool.execute(parseToolInput(call.input), { + abortSignal, + toolCallId: call.toolCallId, + }); const control = await controlReady; await control.submitToolResult({ toolCallId: call.toolCallId, output }); } catch (error) { diff --git a/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts b/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts index 3c5e3047c0..c9a17f2059 100644 --- a/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts +++ b/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts @@ -61,41 +61,39 @@ describe('native Relay host tools', () => { it.each([ ['post_message', 'send', { channel: 'events', text: 'ACK' }], ['reply_to_thread', 'reply', { message_id: 'parent', text: 'ACK' }], -] as const)('joins pending native %s writes within a session only', async (tool, method, args) => { +] as const)('joins only a replay of the same native %s tool call', async (tool, method, args) => { let release!: () => void; const gate = new Promise((resolve) => { release = resolve; }); const write = vi.fn(async () => { await gate; - return { id: 'reply' }; + return { id: `reply-${write.mock.calls.length}` }; }); const options = { env: { RELAY_AGENT_TOKEN: 'test' }, agentClient: { [method]: write } as never }; const execute = createNativeRelayTools(options).find((item) => item.spec.name === tool)!.execute; - const first = execute(args, {}); - const second = execute(args, {}); + const first = execute(args, { toolCallId: 'call-1' }); + const replay = execute(args, { toolCallId: 'call-1' }); + const independent = execute(args, { toolCallId: 'call-2' }); + const unidentified = [execute(args, {}), execute(args, {})]; await Promise.resolve(); - expect(write).toHaveBeenCalledTimes(1); - const anotherSession = createNativeRelayTools(options) - .find((item) => item.spec.name === tool)! - .execute(args, {}); await Promise.resolve(); - expect(write).toHaveBeenCalledTimes(2); + expect(write).toHaveBeenCalledTimes(4); release(); - expect(await first).toEqual(await second); - await anotherSession; - await execute(args, {}); - expect(write).toHaveBeenCalledTimes(3); + expect(await first).toBe(await replay); + await Promise.all([independent, ...unidentified]); + await execute(args, { toolCallId: 'call-1' }); + expect(write).toHaveBeenCalledTimes(5); }); -it('clears a rejected native reply so it can be retried', async () => { +it('clears a rejected native reply so the same tool call can be retried', async () => { const reply = vi.fn().mockRejectedValueOnce(new Error('offline')).mockResolvedValue({ id: 'retry' }); const execute = createNativeRelayTools({ env: { RELAY_AGENT_TOKEN: 'test' }, agentClient: { reply } as never, }).find((item) => item.spec.name === 'reply_to_thread')!.execute; const args = { message_id: 'parent', text: 'ACK' }; - await expect(execute(args, {})).rejects.toThrow('offline'); - await expect(execute(args, {})).resolves.toEqual({ id: 'retry' }); + await expect(execute(args, { toolCallId: 'call-1' })).rejects.toThrow('offline'); + await expect(execute(args, { toolCallId: 'call-1' })).resolves.toEqual({ id: 'retry' }); expect(reply).toHaveBeenCalledTimes(2); }); diff --git a/packages/harnesses/src/ai-sdk/native-relay-tools.ts b/packages/harnesses/src/ai-sdk/native-relay-tools.ts index 33ebfe5b1a..446f251df2 100644 --- a/packages/harnesses/src/ai-sdk/native-relay-tools.ts +++ b/packages/harnesses/src/ai-sdk/native-relay-tools.ts @@ -53,10 +53,14 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha const workspace = options.workspaceClient ?? (workspaceKey ? createWorkspaceClient({ workspaceKey, baseUrl }) : undefined); - // One map per native session; never retain a completed write. + // One map per native session, keyed by the model's tool call ID so only a + // replay of the same invocation joins a pending write. Text never forms the + // key: independent identical replies are separate messages. A completed + // write is never retained. const pendingWrites = new Map>(); - function write(tool: string, target: string, text: string, operation: () => Promise) { - const key = JSON.stringify([tool, target, text]); + function write(tool: string, toolCallId: string | undefined, operation: () => Promise) { + if (!toolCallId) return Promise.resolve().then(operation); + const key = JSON.stringify([tool, toolCallId]); const existing = pendingWrites.get(key); if (existing) return existing; const pending = Promise.resolve().then(operation); @@ -102,11 +106,11 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha additionalProperties: false, }, }, - execute: (value) => { + execute: (value, { toolCallId } = {}) => { const input = objectInput(value); const target = requiredString(input, 'channel'); const text = requiredString(input, 'text'); - return write('post_message', target, text, () => agent.send(target, text)); + return write('post_message', toolCallId, () => agent.send(target, text)); }, }, { @@ -165,11 +169,11 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha additionalProperties: false, }, }, - execute: (value) => { + execute: (value, { toolCallId } = {}) => { const input = objectInput(value); const target = requiredString(input, 'message_id'); const text = requiredString(input, 'text'); - return write('reply_to_thread', target, text, () => agent.reply(target, text)); + return write('reply_to_thread', toolCallId, () => agent.reply(target, text)); }, }, { diff --git a/packages/sdk/src/messaging/index.ts b/packages/sdk/src/messaging/index.ts index ad1792015f..87bd478de8 100644 --- a/packages/sdk/src/messaging/index.ts +++ b/packages/sdk/src/messaging/index.ts @@ -17,3 +17,4 @@ export { export * from './thin-client.js'; export { normalizeWebhookSubscription } from './relaycast-translate.js'; +export { SUBSCRIBABLE_EVENT_TYPES } from './subscribable-events.js'; diff --git a/packages/sdk/src/messaging/subscribable-events.ts b/packages/sdk/src/messaging/subscribable-events.ts new file mode 100644 index 0000000000..2b2495dcf0 --- /dev/null +++ b/packages/sdk/src/messaging/subscribable-events.ts @@ -0,0 +1,7 @@ +import { SubscribableEventTypeSchema, type SubscribableEventType } from '@relaycast/types'; + +/** + * Every event a Relaycast subscription accepts, taken from the engine's own + * schema so a CLI allowlist cannot drift narrower than the server. + */ +export const SUBSCRIBABLE_EVENT_TYPES: readonly SubscribableEventType[] = SubscribableEventTypeSchema.options; diff --git a/packages/sdk/src/messaging/thin-client.ts b/packages/sdk/src/messaging/thin-client.ts index 97e225e13b..15daced8a2 100644 --- a/packages/sdk/src/messaging/thin-client.ts +++ b/packages/sdk/src/messaging/thin-client.ts @@ -103,6 +103,8 @@ export interface RelayAgentThinClient { channel: string, text: string, options?: { + /** Stable key for retries of one logical send, including across processes. */ + idempotencyKey?: string; attachments?: string[]; data?: Record | null; mode?: RelayMessageMode; @@ -112,7 +114,11 @@ export interface RelayAgentThinClient { reply( messageId: string, text: string, - options?: { data?: Record | null } + options?: { + /** Stable key for retries of one logical reply, including across processes. */ + idempotencyKey?: string; + data?: Record | null; + } ): Promise; thread(messageId: string, options?: { limit?: number }): Promise; dm( From 747b6ed9a0aefa77452f60bd3f425b05095a9ee0 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Wed, 7 Oct 2026 23:45:33 -0700 Subject: [PATCH 16/42] fix(fleet): never confirm a timed-out spawn from a name-only heartbeat; don't report absent after a failed roster read Review on #1900: - Devin: a node heartbeat lists worker names, not the invocation that started them, so a pre-existing worker with the requested name made a timed-out (possibly rejected) spawn exit 0. The CLI and MCP recovery paths no longer convert heartbeat presence into success: the spawn stays spawn_pending (exit 8) with the liveness evidence and invocation ID, and the message says the listed worker may be an earlier one. - Cursor: when the nodes.get/list scan throws, a later agents.get 404 no longer yields `absent` (which claims nothing was observed); it reports `unknown` with the roster read error. README and CHANGELOG updated to match. Fail-before/pass-after tests in spawn-liveness, fleet, and agent-relay-mcp.startup. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: f9b2ae4e-847e-47c7-9996-b6545ca4353c --- CHANGELOG.md | 2 +- packages/cli/README.md | 9 +++---- .../src/cli/agent-relay-mcp.startup.test.ts | 21 ++++++---------- packages/cli/src/cli/agent-relay-mcp.ts | 14 ----------- packages/cli/src/cli/commands/fleet.test.ts | 17 +++++++++---- packages/cli/src/cli/commands/fleet.ts | 15 ------------ .../cli/src/cli/lib/spawn-liveness.test.ts | 24 +++++++++++++++++++ packages/cli/src/cli/lib/spawn-liveness.ts | 16 +++++++++++-- 8 files changed, 64 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index def4c7bba6..fa6dd2e109 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Unconfirmed `fleet spawn` results report `spawn_pending` with invocation diagnostics and exit 8. A fresh target-node heartbeat can return exit 0 with `confirmed:false`, extending the existing accepted-placement shape without claiming harness readiness. +- Unconfirmed `fleet spawn` results report `spawn_pending` with invocation diagnostics and exit 8. Node heartbeat presence is reported as liveness evidence but never confirms the spawn, because heartbeats cannot tell this invocation's worker from an earlier one with the same name. - Removal waits exit 8 if the registration remains present; scripts must handle nonzero exits beyond `$? -eq 1`. Waits remain opt-in, and unavailable verification reads preserve the asynchronous acknowledgement. ### Fixed diff --git a/packages/cli/README.md b/packages/cli/README.md index b64c9f8915..89e257cb0c 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -591,10 +591,11 @@ Hosted equivalents live under `agent-relay cloud …`. An accepted spawn whose result is still unknown returns `spawn_pending` (exit 8), with its invocation ID, dispatch state, and available liveness evidence. Check -`agent-relay fleet agent list --node ` before retrying. A fresh heartbeat -showing the worker on the requested node can return `confirmed:false` with exit 0; -this proves observed worker presence, not harness readiness. Heartbeat snapshots -can change between listings; JSON output includes their timestamps and ages. +`agent-relay fleet agent list --node ` before retrying. A heartbeat that +lists a worker with the requested name is reported as evidence but keeps the +spawn pending: heartbeats carry worker names, not invocation IDs, so that worker +may have been running before this spawn. Heartbeat snapshots can change between +listings; JSON output includes their timestamps and ages. If no worker is running and you intend to reclaim the name, run `agent-relay agent remove --wait` before respawning. Waiting is opt-in; diff --git a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts index 7cc39900d9..f226aef167 100644 --- a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts +++ b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts @@ -1292,7 +1292,7 @@ describe('createAgentRelayMcpServer', () => { }); it.each(['live', 'live_elsewhere', 'stale', 'unknown'] as const)( - 'reports %s evidence without manufacturing readiness', + 'keeps the spawn pending on %s evidence without manufacturing success', async (evidence) => { const { mod, mocks } = await loadAgentRelayMcpModule(); mocks.agentRelayMessagingCommands.invoke.mockResolvedValueOnce({} as never); @@ -1311,19 +1311,12 @@ describe('createAgentRelayMcpServer', () => { const result = await mocks.serverInstances[0].tools .get('spawn')! .handler({ name: 'worker', cli: 'codex', target_node: 'node-a' }); - if (evidence === 'live') { - expect(result.isError).not.toBe(true); - expect(result.structuredContent.placement).toMatchObject({ - state: 'accepted', - confirmed: false, - liveness: { evidence }, - }); - } else { - expect(result).toMatchObject({ - isError: true, - structuredContent: { error: { code: 'spawn_pending', state: 'pending', liveness: { evidence } } }, - }); - } + // A heartbeat names a worker but not the invocation that created it, + // so even fresh target-node presence leaves the spawn pending. + expect(result).toMatchObject({ + isError: true, + structuredContent: { error: { code: 'spawn_pending', state: 'pending', liveness: { evidence } } }, + }); } ); diff --git a/packages/cli/src/cli/agent-relay-mcp.ts b/packages/cli/src/cli/agent-relay-mcp.ts index 876ab067cd..d257d9aa5b 100644 --- a/packages/cli/src/cli/agent-relay-mcp.ts +++ b/packages/cli/src/cli/agent-relay-mcp.ts @@ -950,20 +950,6 @@ async function invokeVerifiedSpawn( }); } } - if (liveness.evidence === 'live') { - return { - invocationId: error.invocationId, - status: 'accepted', - placement: { - state: 'accepted', - confirmed: false, - dispatchState: error.dispatchState, - invocationId: error.invocationId, - liveness, - warning: `A worker named ${JSON.stringify(name)} is running on ${JSON.stringify(liveness.node)} according to its fresh heartbeat; harness readiness was not confirmed within the budget.`, - }, - }; - } throw pendingSpawnError(name, context, liveness, error.message); } } diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index c8f65fbc72..e5bc190527 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1180,15 +1180,24 @@ describe('fleet command support', () => { expect(errors.join('')).toContain( evidence === 'collision' ? '"code":"spawn_name_taken"' : '"code":"spawn_failed"' ); + } else if (evidence === 'live') { + // Heartbeats carry worker names only: an existing worker with this + // name is indistinguishable from this invocation's worker, so the + // observation must not convert the timed-out spawn into success. + expect(exit).toHaveBeenCalledWith(8); + expect(logs).toEqual([]); + const rendered = errors.join(''); + expect(rendered).toContain('"code":"spawn_pending"'); + expect(rendered).toContain(`"invocationId":"${invocationId}"`); + expect(rendered).toContain('"evidence":"live"'); + expect(rendered).toContain('may be an earlier worker'); } else { expect(exit).not.toHaveBeenCalled(); expect(JSON.parse(logs[0]!).invocation.placement).toMatchObject({ - state: evidence === 'terminal' ? 'ready' : 'accepted', - confirmed: evidence === 'terminal', + state: 'ready', + confirmed: true, dispatchState: 'dispatched', }); - if (evidence === 'live') - expect(warn).toHaveBeenCalledWith(expect.stringContaining('harness readiness was not confirmed')); } } ); diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index b6a07f2e65..7b7c914e6f 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -1152,21 +1152,6 @@ export function registerFleetCommands( throw nameTakenSpawnError(name, terminalError); throw terminalError; } - } else if (liveness.evidence === 'live') { - deps.warn( - `A worker named ${JSON.stringify(name)} is running on ${JSON.stringify(liveness.node)} according to its fresh heartbeat; harness readiness was not confirmed within the budget.` - ); - invocation = { - status: 'accepted', - invocationId: error.invocationId, - placement: { - state: 'accepted', - confirmed: false, - invocationId: error.invocationId, - dispatchState: error.dispatchState, - liveness, - }, - }; } else { throw pendingSpawnError(name, error, liveness, error.message); } diff --git a/packages/cli/src/cli/lib/spawn-liveness.test.ts b/packages/cli/src/cli/lib/spawn-liveness.test.ts index 73ac8a214a..d71b920dd3 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.test.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.test.ts @@ -106,6 +106,30 @@ describe('spawn liveness resolution', () => { evidence: 'unknown', }); }); + it.each(['get', 'list'] as const)( + 'does not report absent when the node %s read failed', + async (failing) => { + const workspace = client([node()]); + workspace.nodes[failing].mockRejectedValue(new Error('roster unavailable')); + const result = await probeSpawnLiveness({ + name: 'worker', + ...(failing === 'get' ? { targetNode: 'target' } : {}), + createClient: () => workspace, + now: () => now, + }); + expect(result).toEqual({ evidence: 'unknown', readError: 'roster unavailable' }); + } + ); + it('tells the operator a heartbeat cannot attribute a same-named worker to this spawn', () => { + const pending = pendingSpawnError( + 'worker', + { invocationId: 'inv', node: 'target', dispatchState: 'dispatched' }, + { evidence: 'live', node: 'target', heartbeatAgeMs: 0 } + ); + expect(pending.code).toBe('spawn_pending'); + expect(pending.message).toContain('may be an earlier worker'); + expect(pending.message).toContain('Invocation: inv'); + }); it('bounds a hung invocation read and still probes worker evidence', async () => { vi.useFakeTimers(); try { diff --git a/packages/cli/src/cli/lib/spawn-liveness.ts b/packages/cli/src/cli/lib/spawn-liveness.ts index 74f1a7a428..1062d5debd 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.ts @@ -61,10 +61,17 @@ export function pendingSpawnError( diagnostic?: string ): FleetSpawnError { const nodeOption = context.node ? ` --node ${shellQuote(context.node)}` : ''; + // Heartbeats carry worker names, not invocation IDs, so presence cannot be + // attributed to this spawn: it may be a worker that was already running. + const attribution = + liveness.evidence === 'live' || liveness.evidence === 'live_elsewhere' || liveness.evidence === 'stale' + ? `A worker with this name is listed by a node heartbeat, but heartbeats do not identify the invocation that started it, so it may be an earlier worker rather than this spawn. ` + : ''; return new FleetSpawnError( 'spawn_pending', `Spawn of ${JSON.stringify(name)} was accepted; its outcome is pending. ` + `Evidence: ${liveness.evidence}${liveness.node ? ` on ${JSON.stringify(liveness.node)}` : ''}. ` + + attribution + `Invocation: ${context.invocationId ?? 'unavailable'}; dispatch: ${context.dispatchState ?? 'unknown'}. ` + `Check \`agent-relay fleet agent list${nodeOption}\` before retrying. ` + `Only if no worker is running and you intend to reclaim the name, run \`agent-relay agent remove ${shellQuote(name)} --wait\` and wait for clearance before respawning.`, @@ -103,6 +110,7 @@ export async function probeSpawnLiveness(options: { now?: () => number; }): Promise { let readError: string | undefined; + let rosterUnread = false; const read = (fn: () => Promise) => withDeadline(fn, () => new Error('Spawn evidence read timed out.'), 2_000); if (options.getInvocation) { @@ -156,13 +164,17 @@ export async function probeSpawnLiveness(options: { if (weaker) return weaker; } catch (error) { readError = safeRelayErrorMessage(error); + rosterUnread = true; } try { await read(() => client.agents.get(options.name)); return { evidence: 'registered', ...(readError ? { readError } : {}) }; } catch (error) { - if (isNotFoundError(error)) return { evidence: 'absent', ...(readError ? { readError } : {}) }; - readError = safeRelayErrorMessage(error); + // `absent` claims no worker was observed. A missing registration only + // proves that when the node roster was actually read. + if (isNotFoundError(error) && !rosterUnread) + return { evidence: 'absent', ...(readError ? { readError } : {}) }; + if (!isNotFoundError(error)) readError = safeRelayErrorMessage(error); } } catch (error) { readError = safeRelayErrorMessage(error); From 6ffef381d0e6e77d94dc0b5383b5ab8d0f323154 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 00:28:40 -0700 Subject: [PATCH 17/42] fix(broker): resolve verified spawns in either frame order, release on task failure, never drop oversized wrap messages Review on #1893: - Cursor (high): a task verdict that arrives before proven readiness (startup-fallback worker_ready releases the task first) is now recorded on the pending spawn, and the later proven worker_ready resolves the action from it, as success (whole-payload echo) or spawn_task_unconfirmed. Previously the verdict was ignored and the spawn timed out and was released after a successful delivery. - Cursor (medium): a worker already ready when the launch returns had its PTY task queued unbound to the action; it now resolves as spawn_task_unconfirmed instead of plain success. - Devin: a failed initial task (delivery_failed, or an immediate queue error) now expires the pending spawn with spawn_task_failed, so maintenance releases the worker and its fleet identity before reporting. A corrected retry can reuse the name. - Devin: an oversized wrap message is delivered as a bounded notice naming the sender, size, limit and message ID instead of being silently dropped after it left the queue. - Devin: `message post|reply|dm send` no longer apply the PTY injection cap. The limit stays at the PTY boundary (and on fleet task input). Also merges origin/main, completing the PendingVerifiedSpawn literals both sides extended, and fixes the two red checks this head caused: - Fleet E2E resume: a verified spawn with a task now completes after task verification, and node-a's stub delays readiness 27 s, so the 30 s settle waits become 60 s. - RelayFlow 1615: the fake Claude now redraws its prompt after Enter, as the real composer does, so the pre-write composer check can prove readiness for the second message. Each new test fails with its fix reverted (mutation-checked) and passes with it. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: f9b2ae4e-847e-47c7-9996-b6545ca4353c --- CHANGELOG.md | 4 +- crates/broker/src/runtime/fleet.rs | 86 ++++++++- crates/broker/src/runtime/tests.rs | 182 ++++++++++++++++++ crates/broker/src/runtime/worker_events.rs | 125 ++++++------ crates/broker/src/wrap.rs | 97 ++++++++-- docs/harnesses/injection.md | 11 +- packages/cli/src/cli/commands/message.ts | 4 - .../src/cli/commands/relaycast-groups.test.ts | 17 ++ tests/e2e/fleet/fleet-e2e.test.ts | 7 +- .../run.mjs | 8 +- 10 files changed, 441 insertions(+), 100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ec75b33c6..d371639dd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,12 +59,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Task and relay-message bodies are capped at 16 KiB. `message send|reply|dm`, `fleet spawn`, and the broker reject larger input naming the limit, and the paced fallback wire rejects above 1,536 bytes; write the brief to a file and send a pointer instead. +- Task and PTY-injected relay-message bodies are capped at 16 KiB. `fleet spawn` and the broker's PTY delivery reject larger input naming the limit, and the paced fallback wire rejects above 1,536 bytes; write the brief to a file and send a pointer instead. `message post|reply|dm send` are not capped, and a wrapped agent receives an oversized message as a notice pointing to it in Relay. ### Fixed - PTY tasks and relay messages use atomic bracketed paste when supported, wait for the composer, and prevent embedded carriage returns from submitting partial input. Oversized bodies and tail-only echoes fail explicitly without replay. -- `fleet spawn --task` reports success only when the agent's terminal echoed the whole task. A task acked without that proof fails with `spawn_task_unconfirmed`, which names the live agent and says not to retry, and `delivery_verified` now reports what was observed (`echo_incomplete`, `paste_summary`, `timeout_fallback`) instead of treating head/tail anchors or a collapsed-paste marker as receipt. +- `fleet spawn --task` reports success only when the agent's terminal echoed the whole task; a task that fails to deliver releases the worker before the spawn fails with `spawn_task_failed`. A task acked without that proof fails with `spawn_task_unconfirmed`, which names the live agent and says not to retry, and `delivery_verified` now reports what was observed (`echo_incomplete`, `paste_summary`, `timeout_fallback`) instead of treating head/tail anchors or a collapsed-paste marker as receipt. - Broker fleet control keeps a healthy node WebSocket open when Relaycast returns retryable `d1_pressure` for `node.register` or `inventory.sync`, retrying the same frame with bounded exponential backoff while the existing registration and application-liveness deadlines remain fail-closed. - `--state-dir` on `node agent` commands, `node status`, and `node down` also accepts a fleet node directory whose broker state lives in `state/`. - An explicit `--state-dir` on `node agent` commands is no longer overridden by `RELAY_BROKER_URL` / `RELAY_BROKER_API_KEY`. diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index 37945ad258..a23e98a1e1 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -156,6 +156,20 @@ pub(super) struct PendingVerifiedSpawn { pub(super) generation: Uuid, pub(super) readiness_proven: bool, pub(super) task_event_id: Option, + /// The initial task's verdict, recorded when it arrives before harness + /// readiness is proven (for example after a startup-fallback + /// `worker_ready`). The proven `worker_ready` that follows resolves the + /// action from it, so the order of the two frames cannot strand a spawn + /// whose task was already delivered. + pub(super) task_verification: Option, +} + +/// What the worker reported for a verified spawn's initial task. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct TaskVerification { + /// Whether the label is whole-payload evidence of receipt. + pub(super) confirmed: bool, + pub(super) label: Option, } impl PendingVerifiedSpawn { @@ -163,7 +177,42 @@ impl PendingVerifiedSpawn { self.generation == generation && self.task_event_id.as_deref() == Some(event_id) } pub(super) fn can_report_ready(&self, generation: Uuid) -> bool { - self.generation == generation && self.readiness_proven && self.task_event_id.is_none() + self.generation == generation + && self.readiness_proven + && self.task_event_id.is_none() + && self.failure_reason.is_none() + } + /// Record the initial task's verdict; the task is no longer outstanding. + pub(super) fn record_task_verification(&mut self, confirmed: bool, label: Option<&str>) { + self.task_event_id = None; + self.task_verification = Some(TaskVerification { + confirmed, + label: label.map(str::to_string), + }); + } + /// The initial task failed. Expire the entry with a specific reason so + /// maintenance releases the worker and cleans up its fleet identity before + /// reporting, exactly as for a readiness timeout. Replying here instead + /// would leave a registered, running worker holding the name. + pub(super) fn initial_task_failed(&mut self, generation: Uuid, reason: &str) { + if self.generation == generation && self.failure_reason.is_none() { + self.deadline = Instant::now(); + self.failure_reason = Some(format!( + "spawn_task_failed: initial task was not delivered ({reason}); the worker was released" + )); + } + } + /// The action result once readiness is proven and the task is resolved. + pub(super) fn completion(self, name: &WorkerName) -> ActionResult { + match self.task_verification { + Some(TaskVerification { + confirmed: false, + label, + }) => { + verified_spawn_task_unconfirmed_result(self.invocation_id, name, label.as_deref()) + } + _ => verified_spawn_ready_result(self.invocation_id, name), + } } } @@ -219,6 +268,26 @@ pub(super) fn verified_spawn_task_unconfirmed_result( ) } +/// Result for a verified spawn whose worker had already reported +/// `worker_ready` before this action's pending entry existed. +/// +/// The runtime loop is serial, so for a fresh worker that cannot happen; it is +/// reached only when the launch resolved to a worker that was already up. Its +/// initial PTY task, if any, was then queued without being bound to this +/// action, so no task verdict can reach it: report the live agent as +/// unconfirmed rather than claim `spawned:true, ready:true` (relay#1893 review). +pub(super) fn already_ready_spawn_result( + invocation_id: String, + name: &WorkerName, + unbound_pty_task: bool, +) -> ActionResult { + if unbound_pty_task { + verified_spawn_task_unconfirmed_result(invocation_id, name, Some("unbound")) + } else { + verified_spawn_ready_result(invocation_id, name) + } +} + pub(super) fn verified_spawn_failed_result(invocation_id: String, error: &str) -> ActionResult { ActionResult { task: None, @@ -1587,6 +1656,7 @@ impl BrokerRuntime { } }; let task = action_invoke_string(&invoke.input, &["task", "initial_task", "prompt"]); + let carries_task = task.is_some(); let channel = action_invoke_string(&invoke.input, &["channel"]); let model = action_invoke_string(&invoke.input, &["model"]); @@ -1685,21 +1755,26 @@ impl BrokerRuntime { // while maintenance fails it after an early exit/readiness timeout // and performs cleanup. if verify_ready { - let (already_ready, generation) = { + let (already_ready, generation, is_pty) = { let worker = self .workers .workers .get(&name) .expect("verified spawn worker must still exist"); - (worker.ready_at.is_some(), worker.generation) + ( + worker.ready_at.is_some(), + worker.generation, + worker.spec.runtime == AgentRuntime::Pty, + ) }; if already_ready { tracing::info!(invocation_id = %invoke.invocation_id, worker = %name, verify_ready, elapsed_ms = started.elapsed().as_millis() as u64, "sending verified fleet spawn result"); - self.send_fleet_action_result(verified_spawn_ready_result( + self.send_fleet_action_result(already_ready_spawn_result( invoke.invocation_id, &name, + is_pty && carries_task, )) .await; } else { @@ -1713,6 +1788,7 @@ impl BrokerRuntime { generation, readiness_proven: false, task_event_id: None, + task_verification: None, }, ); } @@ -3409,9 +3485,11 @@ mod tests { invocation_id: "test".into(), deadline: Instant::now(), started: Instant::now(), + failure_reason: None, generation, readiness_proven: false, task_event_id: Some("init_test".into()), + task_verification: None, }; assert!(!pending.can_report_ready(generation)); pending.readiness_proven = true; diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index efaba03a0a..f4f40fccbb 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -901,8 +901,10 @@ async fn a_spawn_task_acked_without_proof_of_receipt_never_reports_success() { deadline: Instant::now() + Duration::from_secs(90), started: Instant::now(), generation, + failure_reason: None, readiness_proven: true, task_event_id: Some(event_id.to_string()), + task_verification: None, }, ); @@ -973,8 +975,10 @@ async fn a_spawn_task_proven_received_reports_spawned_and_ready() { deadline: Instant::now() + Duration::from_secs(90), started: Instant::now(), generation, + failure_reason: None, readiness_proven: true, task_event_id: Some(event_id.to_string()), + task_verification: None, }, ); @@ -1005,6 +1009,181 @@ async fn a_spawn_task_proven_received_reports_spawned_and_ready() { } } +fn pending_spawn_with_task( + generation: Uuid, + event_id: &str, + readiness_proven: bool, +) -> super::fleet::PendingVerifiedSpawn { + super::fleet::PendingVerifiedSpawn { + invocation_id: "inv-order".to_string(), + deadline: Instant::now() + Duration::from_secs(90), + started: Instant::now(), + failure_reason: None, + generation, + readiness_proven, + task_event_id: Some(event_id.to_string()), + task_verification: None, + } +} + +fn drain_action_results( + rx: &mut mpsc::Receiver, +) -> Vec { + let mut results = Vec::new(); + while let Ok(command) = rx.try_recv() { + if let FleetControlCommand::Send(crate::fleet_wire::BrokerToRelaycast::ActionResult( + result, + )) = command + { + results.push(result); + } + } + results +} + +/// relay#1893 review (Cursor, high): a startup-fallback `worker_ready` releases +/// the initial task before readiness is proven. Its verdict used to be ignored +/// because the spawn required proven readiness first, and the later proven +/// `worker_ready` could not resolve it either, so a delivered task timed out +/// and the worker was released. Either order must resolve exactly once. +#[tokio::test] +async fn a_task_verdict_before_proven_readiness_resolves_on_the_proven_ready() { + use crate::fleet_wire::ActionResultPayload; + for (verification, expect_success) in [("echo", true), ("timeout_fallback", false)] { + let worker_name = "spawn-order"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let mut fixture = worker_event_runtime_fixture(registry, HashMap::new()); + let name = WorkerName::from(worker_name); + fixture.runtime.pending_verified_spawns.insert( + name.clone(), + pending_spawn_with_task(generation, "init_order", false), + ); + + fixture + .runtime + .handle_worker_event(delivery_verified_worker_event( + worker_name, + generation, + "init_order", + Some(verification), + )) + .await; + assert!( + drain_action_results(&mut fixture.fleet_control_rx).is_empty(), + "readiness is not proven yet, so the action stays open" + ); + assert!(fixture.runtime.pending_verified_spawns.contains_key(&name)); + + fixture + .runtime + .handle_worker_event(WorkerEvent::Message { + name: name.clone(), + generation, + value: json!({"type":"worker_ready", "payload":{"readiness_proven":true}}), + }) + .await; + let results = drain_action_results(&mut fixture.fleet_control_rx); + assert_eq!(results.len(), 1, "{verification}: resolved exactly once"); + match (&results[0].result, expect_success) { + (ActionResultPayload::Output(output), true) => assert_eq!( + output.output, + json!({"spawned": true, "ready": true, "name": worker_name}) + ), + (ActionResultPayload::Error(error), false) => { + assert!( + error.error.starts_with("spawn_task_unconfirmed: "), + "{}", + error.error + ); + assert!(error.error.contains(verification), "{}", error.error); + } + (other, _) => panic!("{verification}: unexpected result {other:?}"), + } + assert!(!fixture.runtime.pending_verified_spawns.contains_key(&name)); + } +} + +/// relay#1893 review (Devin): a verified spawn whose initial task fails used to +/// get its error while the worker stayed registered and running, so a +/// corrected retry collided with the name. The failure must expire the spawn +/// so maintenance releases the worker before reporting the specific reason. +#[tokio::test] +async fn a_failed_initial_task_releases_the_worker_before_failing_the_spawn() { + use crate::fleet_wire::ActionResultPayload; + let worker_name = "spawn-task-failed"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let mut fixture = worker_event_runtime_fixture(registry, HashMap::new()); + let name = WorkerName::from(worker_name); + fixture.runtime.pending_verified_spawns.insert( + name.clone(), + pending_spawn_with_task(generation, "init_failed", true), + ); + + fixture + .runtime + .handle_worker_event(WorkerEvent::Message { + name: name.clone(), + generation, + value: json!({"type":"delivery_failed", "payload":{ + "delivery_id":"del_init_failed", "event_id":"init_failed", + "reason":"injection_too_large"}}), + }) + .await; + assert!( + drain_action_results(&mut fixture.fleet_control_rx).is_empty(), + "the failure is reported only after the worker is released" + ); + let pending = &fixture.runtime.pending_verified_spawns[&name]; + assert!(pending.deadline <= Instant::now()); + + fixture.runtime.handle_maintenance_tick().await; + assert!( + !fixture.runtime.workers.has_worker(&name), + "worker released" + ); + assert!(!fixture.runtime.pending_verified_spawns.contains_key(&name)); + let results = drain_action_results(&mut fixture.fleet_control_rx); + assert_eq!(results.len(), 1); + let ActionResultPayload::Error(error) = &results[0].result else { + panic!("a failed task must fail the spawn: {:?}", results[0]); + }; + assert!( + error.error.starts_with("spawn_task_failed: "), + "{}", + error.error + ); + assert!( + error.error.contains("injection_too_large"), + "{}", + error.error + ); +} + +/// relay#1893 review (Cursor, medium): a worker already ready when the launch +/// returns has its PTY task queued without a binding to this action, so the +/// action cannot learn the task's verdict and must not report plain success. +#[test] +fn an_already_ready_worker_with_an_unbound_pty_task_is_not_reported_ready() { + use crate::fleet_wire::ActionResultPayload; + let name = WorkerName::from("already-ready"); + let ActionResultPayload::Error(error) = + super::fleet::already_ready_spawn_result("inv".into(), &name, true).result + else { + panic!("an unbound PTY task must not report success"); + }; + assert!( + error.error.starts_with("spawn_task_unconfirmed: "), + "{}", + error.error + ); + assert!(matches!( + super::fleet::already_ready_spawn_result("inv".into(), &name, false).result, + ActionResultPayload::Output(_) + )); +} + fn inbound_ctx<'a>(event_id: &'a str) -> InboundContext<'a> { InboundContext { from: "Alice", @@ -9473,6 +9652,9 @@ async fn muse_provider_auth_error_expires_verified_spawn_and_releases_capacity() started: Instant::now(), generation, failure_reason: None, + readiness_proven: false, + task_event_id: None, + task_verification: None, }, ); for event_generation in [Uuid::new_v4(), generation] { diff --git a/crates/broker/src/runtime/worker_events.rs b/crates/broker/src/runtime/worker_events.rs index 80018f4ce4..76cdd9d8c4 100644 --- a/crates/broker/src/runtime/worker_events.rs +++ b/crates/broker/src/runtime/worker_events.rs @@ -1,6 +1,6 @@ use super::fleet::{ confirm_pending_delivery_and_resolve_fleet_ack, fail_terminal_session, - refresh_fleet_inventory_session_ref, try_send_terminal, verified_spawn_ready_result, + refresh_fleet_inventory_session_ref, try_send_terminal, }; use super::*; use crate::terminal_control::{TerminalControlCommand, TerminalToCloud}; @@ -880,44 +880,47 @@ impl BrokerRuntime { let receipt_confirmed = verification.is_some_and( crate::broker::delivery_verification::verification_label_confirms_receipt, ); - if pending_verified_spawns.get(&name).is_some_and(|pending| { - pending.readiness_proven - && pending.matches_task(generation, event_id) - }) { - if let Some(pending) = pending_verified_spawns.remove(&name) { - // A spawn carrying a task succeeds only when - // that task is proven received. An ack the - // worker could not verify leaves the agent - // live, so the action resolves as explicitly - // unconfirmed rather than as either a success - // or a retryable failure. - let result = if receipt_confirmed { - verified_spawn_ready_result(pending.invocation_id, &name) - } else { - tracing::warn!( - target = "agent_relay::broker", - worker = %name, - delivery_id = %delivery_id, - event_id = %event_id, - verification = verification.unwrap_or("none"), - "verified spawn's initial task was acked without \ - proof of full receipt; reporting it unconfirmed" - ); - super::fleet::verified_spawn_task_unconfirmed_result( - pending.invocation_id, - &name, - verification, - ) - }; - let _ = fleet_control_tx - .send(FleetControlCommand::Send( - crate::fleet_wire::BrokerToRelaycast::ActionResult( - result, - ), - )) - .await; + // Record the verdict whenever it belongs to this + // spawn's task, even before readiness is proven: a + // startup-fallback `worker_ready` releases the task + // first, and the proven frame that follows resolves + // the action from what was recorded here. + if let Some(pending) = pending_verified_spawns + .get_mut(&name) + .filter(|pending| pending.matches_task(generation, event_id)) + { + pending.record_task_verification(receipt_confirmed, verification); + if !receipt_confirmed { + tracing::warn!( + target = "agent_relay::broker", + worker = %name, + delivery_id = %delivery_id, + event_id = %event_id, + verification = verification.unwrap_or("none"), + "verified spawn's initial task was acked without \ + proof of full receipt; reporting it unconfirmed" + ); } } + // A spawn carrying a task succeeds only when that + // task is proven received. An ack the worker could + // not verify leaves the agent live, so the action + // resolves as explicitly unconfirmed rather than as + // either a success or a retryable failure. + let resolved = pending_verified_spawns + .get(&name) + .is_some_and(|pending| pending.can_report_ready(generation)) + .then(|| pending_verified_spawns.remove(&name)) + .flatten(); + if let Some(pending) = resolved { + let _ = fleet_control_tx + .send(FleetControlCommand::Send( + crate::fleet_wire::BrokerToRelaycast::ActionResult( + pending.completion(&name), + ), + )) + .await; + } if !receipt_confirmed { tracing::info!( @@ -1004,26 +1007,20 @@ impl BrokerRuntime { .get("event_id") .and_then(Value::as_str) .unwrap_or(""); - if pending_verified_spawns - .get(&name) - .is_some_and(|pending| pending.matches_task(generation, event_id)) + // Expire rather than reply: maintenance releases the + // worker and its fleet identity, then reports this + // reason, so a corrected retry can reuse the name. + if let Some(pending) = pending_verified_spawns + .get_mut(&name) + .filter(|pending| pending.matches_task(generation, event_id)) { - if let Some(pending) = pending_verified_spawns.remove(&name) { - let result = super::fleet::verified_spawn_failed_result( - pending.invocation_id, - payload - .get("reason") - .and_then(Value::as_str) - .unwrap_or("initial_task_failed"), - ); - let _ = fleet_control_tx - .send(FleetControlCommand::Send( - crate::fleet_wire::BrokerToRelaycast::ActionResult( - result, - ), - )) - .await; - } + pending.initial_task_failed( + generation, + payload + .get("reason") + .and_then(Value::as_str) + .unwrap_or("initial_task_failed"), + ); } let reason = payload @@ -1521,7 +1518,7 @@ impl BrokerRuntime { let _ = fleet_control_tx .send(FleetControlCommand::Send( crate::fleet_wire::BrokerToRelaycast::ActionResult( - verified_spawn_ready_result(pending.invocation_id, &name), + pending.completion(&name), ), )) .await; @@ -1570,18 +1567,8 @@ impl BrokerRuntime { .await { tracing::warn!(worker = %name, error = %e, "failed to deliver initial_task"); - if let Some(pending) = pending_verified_spawns.remove(&name) { - let result = super::fleet::verified_spawn_failed_result( - pending.invocation_id, - &e.to_string(), - ); - let _ = fleet_control_tx - .send(FleetControlCommand::Send( - crate::fleet_wire::BrokerToRelaycast::ActionResult( - result, - ), - )) - .await; + if let Some(pending) = pending_verified_spawns.get_mut(&name) { + pending.initial_task_failed(generation, &e.to_string()); } } // The initial task bypasses the delivery-mode queue, but the diff --git a/crates/broker/src/wrap.rs b/crates/broker/src/wrap.rs index c71441f71e..69776a87af 100644 --- a/crates/broker/src/wrap.rs +++ b/crates/broker/src/wrap.rs @@ -280,6 +280,55 @@ fn prepare_wrap_retry( // the one unconfirmed verdict that is not evidence of anything going wrong — // the harness simply never echoes content — so it must not drive the injection // delay up on every delivery to a paste-collapsing TUI. +/// Format a wrap injection, replacing a body whose envelope would exceed the +/// PTY injection limit with a bounded pointer to the message. +/// +/// The entry has already left `pending_wrap_injections`, and wrap has no +/// channel back to the sender, so skipping the write would make the message +/// vanish silently. The notice tells the agent the message exists and where to +/// read it; the full text stays in Relay (relay#1893 review). +#[allow(clippy::too_many_arguments)] +fn format_wrap_injection( + from: &str, + event_id: &str, + body: &str, + target: &str, + include_reminder: bool, + workspace_id: Option<&str>, + workspace_alias: Option<&str>, +) -> String { + let format = |body: &str| { + format_injection_for_worker_with_workspace( + from, + event_id, + body, + target, + include_reminder, + true, // pre_registered + None, // assigned_name + workspace_id, + workspace_alias, + ) + }; + let injection = format(body); + if injection.len() <= crate::injection_wire::MAX_INJECTION_BODY_BYTES { + return injection; + } + tracing::warn!( + event_id, + bytes = injection.len(), + limit = crate::injection_wire::MAX_INJECTION_BODY_BYTES, + "wrap: injection_too_large; delivering a pointer instead of the body" + ); + format(&format!( + "[Message body not shown: {} bytes exceeds the {}-byte terminal delivery limit. \ + The full message is stored in Relay as message {event_id}; read it with your Relay \ + message tools before replying.]", + body.len(), + crate::injection_wire::MAX_INJECTION_BODY_BYTES + )) +} + fn wrap_timeout_outcome( verdict: crate::broker::delivery_verification::EchoVerdict, ) -> DeliveryOutcome { @@ -2162,22 +2211,15 @@ pub(crate) async fn run_wrap( tracing::debug!("relay from {} → {}", pending.from, pending.target); let include_reminder = !skip_prompt && mcp_reminder_throttle.should_include(Instant::now()); - let injection = format_injection_for_worker_with_workspace( + let injection = format_wrap_injection( &pending.from, &pending.event_id, &pending.body, &pending.target, include_reminder, - true, // pre_registered - None, // assigned_name pending.workspace_id.as_deref(), pending.workspace_alias.as_deref(), ); - if injection.len() > crate::injection_wire::MAX_INJECTION_BODY_BYTES { - tracing::warn!(limit = crate::injection_wire::MAX_INJECTION_BODY_BYTES, "wrap: injection_too_large; body was not written and will not be replayed"); - throttle.record(DeliveryOutcome::Failed); - continue; - } let bytes = crate::injection_wire::injection_bytes(crate::injection_wire::injection_wire(&resolved_cli, &pty), &injection); let write = submit_injection_body(&pty, &resolved_cli, bytes, Duration::ZERO); match write { @@ -2377,22 +2419,15 @@ pub(crate) async fn run_wrap( // a fresh one within the cooldown is redundant. let include_reminder = !skip_prompt && mcp_reminder_throttle.should_include(Instant::now()); - let injection = format_injection_for_worker_with_workspace( + let injection = format_wrap_injection( &pv.from, &pv.event_id, &pv.body, &pv.target, include_reminder, - true, - None, pv.workspace_id.as_deref(), pv.workspace_alias.as_deref(), ); - if injection.len() > crate::injection_wire::MAX_INJECTION_BODY_BYTES { - tracing::warn!(limit = crate::injection_wire::MAX_INJECTION_BODY_BYTES, "wrap: injection_too_large; body was not written and will not be replayed"); - throttle.record(DeliveryOutcome::Failed); - continue; - } let bytes = crate::injection_wire::injection_bytes(crate::injection_wire::injection_wire(&resolved_cli, &pty), &injection); let write = submit_injection_body(&pty, &resolved_cli, bytes, Duration::ZERO); match write { @@ -2748,6 +2783,36 @@ sys.stdout.flush()"#; assert_eq!(verification.attempts, 2); } + #[test] + fn oversized_wrap_messages_are_delivered_as_a_bounded_pointer() { + let limit = crate::injection_wire::MAX_INJECTION_BODY_BYTES; + let normal = + super::format_wrap_injection("alice", "evt_ok", "hello", "bob", true, None, None); + assert!(normal.contains("Relay message from alice [evt_ok]: hello")); + // A body that fits on its own but whose reminder and attribution push + // the envelope past the limit (the reviewer's 16,000-byte DM), and one + // far over it. + for len in [limit - 384, limit * 2] { + let body = format!("UNIQUE-BODY-MARKER{}", "x".repeat(len)); + let injection = + super::format_wrap_injection("alice", "evt_big", &body, "bob", true, None, None); + assert!(injection.len() <= limit, "{len}: {} bytes", injection.len()); + assert!( + !injection.contains("UNIQUE-BODY-MARKER"), + "{len}: body leaked" + ); + assert!( + injection.contains("Relay message from alice [evt_big]"), + "{injection}" + ); + assert!(injection.contains("message evt_big"), "{injection}"); + assert!( + injection.contains(&format!("{} bytes exceeds", body.len())), + "{injection}" + ); + } + } + #[test] fn wrap_timeout_stays_failed_without_confirming_evidence() { use crate::broker::delivery_verification::{DeliveryOutcome, EchoVerdict}; diff --git a/docs/harnesses/injection.md b/docs/harnesses/injection.md index bb7a6fa108..7494449a89 100644 --- a/docs/harnesses/injection.md +++ b/docs/harnesses/injection.md @@ -18,6 +18,11 @@ the default 5 ms pace. A slower `RELAY_INJECT_RATE_MS` lowers that limit to keep writing within 7.68 seconds. Unpaced fallback uses the global ceiling. Codex uses the smaller of the global ceiling and its deadline-derived limit. Rejections name the harness and effective limit; oversized bodies are never partially written. +Wrap sessions have no failure channel back to the sender, so an oversized relay +message is delivered as a short notice naming the sender, size, limit and message +ID instead of its body; the full message stays readable in Relay. The limit is +enforced only at the PTY boundary: `message post|reply|dm send` publish any size +to Relaycast, since recipients may be native agents or history only. For longer briefs, create a file on the node and send a short instruction to read it. The input wire normalizes CRLF to LF and strips bare carriage returns and ESC @@ -48,8 +53,10 @@ between them), `paste_summary` (the harness collapsed the paste, so no content was echoed at all) and `timeout_fallback` (no echo), the last with a warning and a process-local fallback counter. -Verified fleet PTY spawns with a task wait for its delivery verdict after proven -startup readiness. A failed task fails the spawn action. A task acked without +Verified fleet PTY spawns with a task wait for its delivery verdict and for +proven startup readiness, in either order. A failed task releases the worker and +its fleet identity, then fails the spawn action with `spawn_task_failed`, so a +corrected retry can reuse the name. A task acked without confirmed receipt resolves the action as `spawn_task_unconfirmed`, naming the live agent: the spawn must not be retried (that duplicates the agent), so resend the task or point the agent at a brief file. Wrap uses the same echo evidence and diff --git a/packages/cli/src/cli/commands/message.ts b/packages/cli/src/cli/commands/message.ts index 5a923890b4..7fc9d9fa45 100644 --- a/packages/cli/src/cli/commands/message.ts +++ b/packages/cli/src/cli/commands/message.ts @@ -1,4 +1,3 @@ -import { validateInjectionSize } from '../lib/task-input.js'; import { InvalidArgumentError, type Command } from 'commander'; import { @@ -54,7 +53,6 @@ export function registerMessageCommands( .argument('', 'Message text') ).action(async (channel: string, text: string, o: Record) => { await runSdk(deps, async () => { - validateInjectionSize(text); printJson(deps, await deps.createAgentRelay(opts(o)).messages.send({ channel, text })); }); }); @@ -82,7 +80,6 @@ export function registerMessageCommands( .argument('', 'Reply text') ).action(async (messageId: string, text: string, o: Record) => { await runSdk(deps, async () => { - validateInjectionSize(text); printJson(deps, await deps.createAgentRelay(opts(o)).messages.reply({ messageId, text })); }); }); @@ -135,7 +132,6 @@ export function registerMessageCommands( ) ).action(async (agent: string, text: string, o: Record) => { await runSdk(deps, async () => { - validateInjectionSize(text); const mode = o.mode as 'wait' | 'steer' | undefined; const options = opts(o); const relay = deps.createAgentRelay(options); diff --git a/packages/cli/src/cli/commands/relaycast-groups.test.ts b/packages/cli/src/cli/commands/relaycast-groups.test.ts index 633bcb3c17..5742708b49 100644 --- a/packages/cli/src/cli/commands/relaycast-groups.test.ts +++ b/packages/cli/src/cli/commands/relaycast-groups.test.ts @@ -170,6 +170,23 @@ describe('SDK-backed CLI groups', () => { expect(relay.channels.update).toHaveBeenCalledWith('ops', { topic: 'New topic' }); }); + // The 16 KiB ceiling is a PTY injection limit enforced where the broker + // types into a terminal. Relaycast messages may target native agents or be + // stored for history only, so the generic message commands must not apply it. + it('message post, reply and dm send publish bodies larger than the PTY injection limit', async () => { + const text = 'x'.repeat(20 * 1024); + const { program, relay, error } = harness(registerMessageCommands); + const reply = vi.fn(async (i: unknown) => ({ id: 'r1', ...(i as object) })); + (relay.messages as Record).reply = reply; + await program.parseAsync(['message', 'post', 'ops', text], { from: 'user' }); + await program.parseAsync(['message', 'reply', 'm1', text], { from: 'user' }); + await program.parseAsync(['message', 'dm', 'send', 'lead', text], { from: 'user' }); + expect(error).not.toHaveBeenCalled(); + expect(relay.messages.send).toHaveBeenCalledWith({ channel: 'ops', text }); + expect(reply).toHaveBeenCalledWith({ messageId: 'm1', text }); + expect(relay.messages.direct).toHaveBeenCalledWith({ to: 'lead', text }); + }); + it('message post routes to messages.send with the channel', async () => { const { program, relay } = harness(registerMessageCommands); await program.parseAsync(['message', 'post', 'ops', 'hello'], { from: 'user' }); diff --git a/tests/e2e/fleet/fleet-e2e.test.ts b/tests/e2e/fleet/fleet-e2e.test.ts index acc5e449f1..2f80194092 100644 --- a/tests/e2e/fleet/fleet-e2e.test.ts +++ b/tests/e2e/fleet/fleet-e2e.test.ts @@ -556,7 +556,10 @@ describe.skipIf(!pre.ok)('two-node fleet scenario matrix', () => { const inv = await getInvocation(engine, driverToken, 'spawn', first.invocationId!); return inv.status === 'completed' || inv.status === 'failed' ? inv : null; }, - { label: 'resumable spawn settled', timeoutMs: 30_000, intervalMs: 300 } + // A verified spawn carrying a task now completes only once that task's + // receipt is verified, after readiness. node-a's stub delays readiness + // by 27 s, so allow for readiness plus delivery verification. + { label: 'resumable spawn settled', timeoutMs: 60_000, intervalMs: 300 } ); expect(firstDone.status).toBe('completed'); // resumable spawn carried session_ref through token authority expect(firstDone.output).toMatchObject({ spawned: true, ready: true, name }); @@ -620,7 +623,7 @@ describe.skipIf(!pre.ok)('two-node fleet scenario matrix', () => { const inv = await getInvocation(engine, driverToken, 'spawn', resume.invocationId!); return ['completed', 'failed'].includes(inv.status) ? inv : null; }, - { label: 'resumed worker confirmed ready', timeoutMs: 30_000 } + { label: 'resumed worker confirmed ready', timeoutMs: 60_000 } ); expect(resumed.status).toBe('completed'); expect(resumed.output).toMatchObject({ spawned: true, ready: true, name }); diff --git a/tests/relayflows/cases/1615-api-send-recipient-reachability/run.mjs b/tests/relayflows/cases/1615-api-send-recipient-reachability/run.mjs index 97366bb490..6739f14655 100644 --- a/tests/relayflows/cases/1615-api-send-recipient-reachability/run.mjs +++ b/tests/relayflows/cases/1615-api-send-recipient-reachability/run.mjs @@ -51,7 +51,13 @@ const fakeClaudeSource = String.raw`#!/usr/bin/env node process.stdin.setRawMode?.(true); process.stdin.resume(); process.stdout.write('RELAYFLOW_RECIPIENT_READY\r\n❯'); -process.stdin.on('data', (chunk) => process.stdout.write(chunk)); +// Like the real composer, redraw the prompt after Enter submits a line, so +// the broker's pre-write composer check can prove readiness for the next +// message instead of seeing only the previous message's echo at the cursor. +process.stdin.on('data', (chunk) => { + process.stdout.write(chunk); + if (chunk.includes(13)) process.stdout.write('\r\n❯'); +}); `; let broker; From 9dc1a7574dcfa9167f797a03f839b5af10646d54 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 00:52:05 -0700 Subject: [PATCH 18/42] fix(broker): leave room for the envelope in body caps; reject oversized tasks before launch; keep the reminder on rejection Cursor review on #1893 (6ffef38): - Body caps ignored the envelope. The PTY worker caps the formatted envelope at 16 KiB, but readTaskInput and queue_and_try_delivery_raw accepted a 16 KiB raw body, so a brief at the advertised limit launched a worker that then failed injection_too_large and was released. Bodies are now capped at MAX_BODY_BYTES (16 KiB less a 2 KiB ENVELOPE_RESERVE_BYTES) in the CLI and in raw delivery, and the fleet spawn action rejects an oversized task with spawn_task_too_large before registering or launching anything. - The MCP reminder throttle was marked sent before the size gate, so a rejected envelope stole the reminder from the next delivery. It is now noted only after the envelope passes every pre-write gate. Regressions (each fails with its fix reverted): the worst-case envelope for a body at the cap fits 16 KiB (128-char names, workspace label, full reminder); an oversized spawn task is rejected before registration; a rejected injection leaves the next delivery its full reminder (real broker integration test); the CLI rejects a 16 KiB task naming the envelope. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: f9b2ae4e-847e-47c7-9996-b6545ca4353c --- CHANGELOG.md | 2 +- crates/broker/src/injection_wire.rs | 28 +++++++++++ crates/broker/src/pty_worker.rs | 8 +-- crates/broker/src/runtime/delivery.rs | 5 +- crates/broker/src/runtime/fleet.rs | 18 +++++++ crates/broker/src/runtime/tests.rs | 50 +++++++++++++++++++ .../broker/tests/pty_injection_integrity.rs | 20 ++++++++ docs/harnesses/injection.md | 6 ++- packages/cli/src/cli/lib/task-input.test.ts | 15 ++++-- packages/cli/src/cli/lib/task-input.ts | 10 ++-- 10 files changed, 146 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d371639dd4..8cc701de42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,7 +59,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Task and PTY-injected relay-message bodies are capped at 16 KiB. `fleet spawn` and the broker's PTY delivery reject larger input naming the limit, and the paced fallback wire rejects above 1,536 bytes; write the brief to a file and send a pointer instead. `message post|reply|dm send` are not capped, and a wrapped agent receives an oversized message as a notice pointing to it in Relay. +- PTY injections are capped at 16 KiB including their envelope, and task and PTY-delivered message bodies at 14 KiB so the envelope always fits. `fleet spawn`, the fleet spawn action (before launching any worker) and the broker's PTY delivery reject larger input naming the limit, and the paced fallback wire rejects above 1,536 bytes; write the brief to a file and send a pointer instead. `message post|reply|dm send` are not capped, and a wrapped agent receives an oversized message as a notice pointing to it in Relay. ### Fixed diff --git a/crates/broker/src/injection_wire.rs b/crates/broker/src/injection_wire.rs index d213527ec2..b86734f6a0 100644 --- a/crates/broker/src/injection_wire.rs +++ b/crates/broker/src/injection_wire.rs @@ -3,6 +3,11 @@ use crate::pty::PtySession; use std::time::Duration; pub(crate) const MAX_INJECTION_BODY_BYTES: usize = 16 * 1024; +/// Room kept for the attribution line and MCP reminder the worker wraps around +/// a body. Body-level caps use `MAX_BODY_BYTES` so a body accepted up front +/// still fits `MAX_INJECTION_BODY_BYTES` once formatted. +pub(crate) const ENVELOPE_RESERVE_BYTES: usize = 2 * 1024; +pub(crate) const MAX_BODY_BYTES: usize = MAX_INJECTION_BODY_BYTES - ENVELOPE_RESERVE_BYTES; // At the default 5 ms/byte, leaves room for readiness, prompt recheck and verification. pub(crate) const MAX_TYPED_WRITE_TIME: Duration = Duration::from_millis(7680); pub(crate) const MAX_TYPED_BYTES: usize = MAX_TYPED_WRITE_TIME.as_millis() as usize / 5; @@ -72,6 +77,29 @@ mod tests { } } } + /// A body accepted at `MAX_BODY_BYTES` must still fit the PTY limit once the + /// worker adds the attribution line and the full MCP reminder (relay#1893 + /// review: the body cap used to ignore the envelope). + #[test] + fn a_body_at_the_body_cap_fits_the_pty_limit_with_its_envelope() { + let long = "n".repeat(128); + let envelope = crate::broker::injection_format::format_injection_for_worker_with_workspace( + &long, + &format!("init_{long}"), + &"x".repeat(MAX_BODY_BYTES), + &format!("#{long}"), + true, + false, + Some(&long), + Some(&format!("rw_{long}")), + Some(&long), + ); + assert!( + envelope.len() <= MAX_INJECTION_BODY_BYTES, + "{} > {MAX_INJECTION_BODY_BYTES}", + envelope.len() + ); + } #[test] fn capability_and_override() { assert_eq!(select_wire("claude", true, false), InjectionWire::Paste); diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index 86e2ee0e3d..c37dce84df 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -2144,9 +2144,6 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { inj.pending.delivery.workspace_alias.as_deref(), ); let injection = String::from_utf8(crate::injection_wire::injection_bytes(crate::injection_wire::InjectionWire::Typed, &injection))?; - if include_mcp_reminder { - mcp_reminder_throttle.note_sent(Instant::now()); - } let wire = crate::injection_wire::injection_wire(&resolved_cli, &pty); let limit = if initial_codex_delivery(&resolved_cli, &inj.pending.delivery) { initial_codex_max_body_bytes().min(crate::injection_wire::MAX_INJECTION_BODY_BYTES) @@ -2159,6 +2156,11 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { // Retain the pending id; terminal failure is never a successful replay. continue; } + // Only an envelope that passed the size gate is written, + // so only it may consume the reminder throttle. + if include_mcp_reminder { + mcp_reminder_throttle.note_sent(Instant::now()); + } if initial_codex_delivery(&resolved_cli, &inj.pending.delivery) { // Reject before any byte is written rather than after // guaranteeing a partial-write timeout: a body needing diff --git a/crates/broker/src/runtime/delivery.rs b/crates/broker/src/runtime/delivery.rs index e597e40280..b85c2d5c30 100644 --- a/crates/broker/src/runtime/delivery.rs +++ b/crates/broker/src/runtime/delivery.rs @@ -862,8 +862,9 @@ pub(crate) async fn queue_and_try_delivery_raw( withheld_fleet_ack_floor: Option, ) -> Result { anyhow::ensure!( - body.len() <= crate::injection_wire::MAX_INJECTION_BODY_BYTES, - "injection_too_large: global body limit is {} bytes; use a brief file pointer", + body.len() <= crate::injection_wire::MAX_BODY_BYTES, + "injection_too_large: body limit is {} bytes so the formatted envelope fits the {}-byte PTY limit; use a brief file pointer", + crate::injection_wire::MAX_BODY_BYTES, crate::injection_wire::MAX_INJECTION_BODY_BYTES ); // Fleet delivery IDs are stable across Relaycast retries. Preserve that diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index a23e98a1e1..5550575058 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -288,6 +288,18 @@ pub(super) fn already_ready_spawn_result( } } +/// The spawn error for an initial task whose body cannot fit the PTY envelope. +pub(super) fn spawn_task_too_large(task: &str) -> Option { + (task.len() > crate::injection_wire::MAX_BODY_BYTES).then(|| { + format!( + "spawn_task_too_large: task is {} bytes; the limit is {} bytes so its envelope fits the {}-byte PTY limit. Write the brief to a file on the node and send a short pointer.", + task.len(), + crate::injection_wire::MAX_BODY_BYTES, + crate::injection_wire::MAX_INJECTION_BODY_BYTES + ) + }) +} + pub(super) fn verified_spawn_failed_result(invocation_id: String, error: &str) -> ActionResult { ActionResult { task: None, @@ -1657,6 +1669,12 @@ impl BrokerRuntime { }; let task = action_invoke_string(&invoke.input, &["task", "initial_task", "prompt"]); let carries_task = task.is_some(); + // Reject before launching: an oversized task can only fail after the + // worker exists, costing a launch and a release for a known outcome. + if let Some(error) = task.as_deref().and_then(spawn_task_too_large) { + self.reply_action_error(&invoke.invocation_id, &error).await; + return; + } let channel = action_invoke_string(&invoke.input, &["channel"]); let model = action_invoke_string(&invoke.input, &["model"]); diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index f4f40fccbb..ce34ccafbf 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -1184,6 +1184,56 @@ fn an_already_ready_worker_with_an_unbound_pty_task_is_not_reported_ready() { )); } +/// relay#1893 review (Cursor): a task that cannot fit the PTY envelope used to +/// launch a worker that then failed `injection_too_large` and was released. +/// It must be rejected before any registration or launch. +#[tokio::test] +async fn an_oversized_spawn_task_is_rejected_before_launch() { + let temp = tempfile::tempdir().unwrap(); + let (tx, _rx) = mpsc::channel(16); + let workers = WorkerRegistry::new(tx, vec![], temp.path().into(), Instant::now()); + let mut fixture = worker_event_runtime_fixture(workers, HashMap::new()); + assert!( + super::fleet::spawn_task_too_large(&"x".repeat(crate::injection_wire::MAX_BODY_BYTES)) + .is_none() + ); + fixture + .runtime + .handle_fleet_control_event(crate::node_control::FleetControlEvent::Message( + crate::fleet_wire::RelaycastToBroker::ActionInvoke(crate::fleet_wire::ActionInvoke { + task_execution: None, + v: FLEET_WIRE_VERSION, + invocation_id: "oversized-task".into(), + action: "spawn".into(), + input: json!({"name":"oversized", "cli":"codex", "verify_ready":true, + "cwd":temp.path(), "task":"x".repeat(crate::injection_wire::MAX_BODY_BYTES + 1)}), + agent_name: Some("oversized".into()), + agent_id: None, + }), + )) + .await; + let mut found = false; + while let Ok(command) = fixture.fleet_control_rx.try_recv() { + match command { + FleetControlCommand::RegisterAgent { .. } => panic!("must reject before registration"), + FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) => { + let crate::fleet_wire::ActionResultPayload::Error(error) = &result.result else { + panic!("expected fleet error: {result:?}"); + }; + assert!( + error.error.starts_with("spawn_task_too_large: "), + "{}", + error.error + ); + found = true; + } + _ => {} + } + } + assert!(found); + assert!(fixture.runtime.workers.workers.is_empty()); +} + fn inbound_ctx<'a>(event_id: &'a str) -> InboundContext<'a> { InboundContext { from: "Alice", diff --git a/crates/broker/tests/pty_injection_integrity.rs b/crates/broker/tests/pty_injection_integrity.rs index a046e43168..36cd9e0541 100644 --- a/crates/broker/tests/pty_injection_integrity.rs +++ b/crates/broker/tests/pty_injection_integrity.rs @@ -178,6 +178,26 @@ fn over_limit_and_typed_fallback_reject_before_writing() { assert!(f.transcript().is_empty()); } } +/// relay#1893 review (Cursor): the reminder throttle was marked sent before the +/// size gate, so a rejected envelope stole the reminder from the next delivery. +#[test] +fn rejected_injection_does_not_consume_the_reminder() { + let mut f = Fixture::new("echo"); + f.deliver("relay_big", &"x".repeat(17_000)); + assert!(f.wait("delivery_failed")["payload"]["reason"] + .as_str() + .unwrap() + .contains("injection_too_large")); + f.deliver("relay_next", "small follow-up"); + f.wait("delivery_verified"); + let transcript = String::from_utf8_lossy(&f.transcript()).into_owned(); + assert!(transcript.contains("small follow-up"), "{transcript}"); + // The full reminder, not the short hint sent while it is throttled. + assert!( + transcript.contains("Self-termination is not automatic"), + "{transcript}" + ); +} #[test] fn absent_echo_preserves_timeout_fallback() { let mut f = Fixture::new("silent"); diff --git a/docs/harnesses/injection.md b/docs/harnesses/injection.md index 7494449a89..782f9a82e5 100644 --- a/docs/harnesses/injection.md +++ b/docs/harnesses/injection.md @@ -11,8 +11,10 @@ for the session, independently of composer readiness. Devin also uses this wire. `RELAY_INJECT_PASTE=0` forces the typed fallback. Wrap sessions retain bulk writes. Codex initial tasks retain their existing chunked delivery path. -The broker-wide ceiling is **16,384 UTF-8 bytes**. The formatted envelope must fit -this ceiling as well, so leave room for attribution and broker reminders. Paced +The broker-wide ceiling is **16,384 UTF-8 bytes** for the formatted envelope. Bodies +are capped at **14,336 bytes** (`fleet spawn` task input, the fleet spawn action +before any launch, and broker PTY delivery), leaving 2 KiB for attribution and +broker reminders so an accepted body still fits once formatted. Paced fallback input has an effective limit of **1,536 bytes including the envelope** at the default 5 ms pace. A slower `RELAY_INJECT_RATE_MS` lowers that limit to keep writing within 7.68 seconds. Unpaced fallback uses the global ceiling. Codex uses diff --git a/packages/cli/src/cli/lib/task-input.test.ts b/packages/cli/src/cli/lib/task-input.test.ts index 149f05b12e..2eccfb9db6 100644 --- a/packages/cli/src/cli/lib/task-input.test.ts +++ b/packages/cli/src/cli/lib/task-input.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, it, expect } from 'vitest'; -import { readTaskInput, MAX_INJECTION_BODY_BYTES } from './task-input.js'; +import { readTaskInput, MAX_INJECTION_BODY_BYTES, MAX_TASK_BODY_BYTES } from './task-input.js'; describe('task input', () => { it('requires exactly one source for fleet, allows neither locally', async () => { await expect(readTaskInput(undefined, undefined, true)).rejects.toThrow('exactly one'); @@ -23,10 +23,15 @@ describe('task input', () => { } }); it('counts UTF-8 bytes', async () => { - expect(await readTaskInput('a'.repeat(MAX_INJECTION_BODY_BYTES), undefined)).toHaveLength( - MAX_INJECTION_BODY_BYTES - ); - await expect(readTaskInput('é'.repeat(MAX_INJECTION_BODY_BYTES), undefined)).rejects.toThrow( + expect(await readTaskInput('a'.repeat(MAX_TASK_BODY_BYTES), undefined)).toHaveLength(MAX_TASK_BODY_BYTES); + await expect(readTaskInput('é'.repeat(MAX_TASK_BODY_BYTES), undefined)).rejects.toThrow('UTF-8 bytes'); + }); + // The broker caps the formatted envelope at 16 KiB, so a body at that size + // would launch a worker and only then fail; leave room for the envelope. + it('rejects a body that only fits the PTY limit before its envelope is added', async () => { + expect(MAX_TASK_BODY_BYTES).toBeLessThan(MAX_INJECTION_BODY_BYTES); + await expect(readTaskInput('a'.repeat(MAX_INJECTION_BODY_BYTES), undefined)).rejects.toThrow('envelope'); + await expect(readTaskInput('a'.repeat(MAX_TASK_BODY_BYTES + 1), undefined)).rejects.toThrow( 'UTF-8 bytes' ); }); diff --git a/packages/cli/src/cli/lib/task-input.ts b/packages/cli/src/cli/lib/task-input.ts index e1c759650f..a77d08f801 100644 --- a/packages/cli/src/cli/lib/task-input.ts +++ b/packages/cli/src/cli/lib/task-input.ts @@ -1,7 +1,11 @@ import { readFile } from 'node:fs/promises'; -/** Broker-wide ceiling; a harness may reject a lower effective wire limit. */ +/** Broker-wide PTY ceiling for the formatted envelope; a harness may reject a lower effective wire limit. */ export const MAX_INJECTION_BODY_BYTES = 16 * 1024; +/** Room the broker keeps for attribution and its MCP reminder around a body. */ +export const ENVELOPE_RESERVE_BYTES = 2 * 1024; +/** Largest body that still fits {@link MAX_INJECTION_BODY_BYTES} once formatted. */ +export const MAX_TASK_BODY_BYTES = MAX_INJECTION_BODY_BYTES - ENVELOPE_RESERVE_BYTES; export async function readTaskInput( task: unknown, taskFile: unknown, @@ -20,9 +24,9 @@ export async function readTaskInput( } export function validateInjectionSize(text: string): void { - if (Buffer.byteLength(text, 'utf8') > MAX_INJECTION_BODY_BYTES) { + if (Buffer.byteLength(text, 'utf8') > MAX_TASK_BODY_BYTES) { throw new Error( - `Injection exceeds ${MAX_INJECTION_BODY_BYTES} UTF-8 bytes; use a brief file on the node and send a short pointer` + `Task exceeds ${MAX_TASK_BODY_BYTES} UTF-8 bytes (the ${MAX_INJECTION_BODY_BYTES}-byte PTY limit less room for the message envelope); use a brief file on the node and send a short pointer` ); } } From e9ef78fc352484c6efcda89f95e74321473468b0 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 01:07:52 -0700 Subject: [PATCH 19/42] fix(broker): measure the decorated initial task against the body cap before launch Cursor review on #1893 (9dc1a75): the pre-launch check measured the raw task, but spawn paths then append the exit-after-task contract, the relay skill prefix and continuity context, so an accepted brief could still fail at worker_ready, after launch. validate_pty_initial_task_size measures the final task as it will be injected and rejects it with spawn_task_too_large. It runs in spawn_worker_from_request right after decoration (before any Relaycast registration) and in WorkerRegistry::spawn_with_generation, the gate every spawn path (HTTP, WS, fleet) funnels through, before anything is launched. Muse takes its task in argv and non-PTY runtimes are not injected, so neither is measured. One message builder (injection_wire::task_too_large_error) backs every rejection. Regressions: a fleet spawn whose raw task fits but whose exit-after-task decoration overflows fails before RegisterAgent (fails with the new call removed); the validator measures only injected PTY tasks. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: f9b2ae4e-847e-47c7-9996-b6545ca4353c --- crates/broker/src/injection_wire.rs | 10 +++ crates/broker/src/runtime/fleet.rs | 9 +-- crates/broker/src/runtime/relaycast_events.rs | 3 + crates/broker/src/runtime/tests.rs | 42 +++++++++++ crates/broker/src/worker.rs | 75 +++++++++++++++++-- 5 files changed, 124 insertions(+), 15 deletions(-) diff --git a/crates/broker/src/injection_wire.rs b/crates/broker/src/injection_wire.rs index b86734f6a0..5d6dccf460 100644 --- a/crates/broker/src/injection_wire.rs +++ b/crates/broker/src/injection_wire.rs @@ -8,6 +8,16 @@ pub(crate) const MAX_INJECTION_BODY_BYTES: usize = 16 * 1024; /// still fits `MAX_INJECTION_BODY_BYTES` once formatted. pub(crate) const ENVELOPE_RESERVE_BYTES: usize = 2 * 1024; pub(crate) const MAX_BODY_BYTES: usize = MAX_INJECTION_BODY_BYTES - ENVELOPE_RESERVE_BYTES; + +/// The spawn error for an initial PTY task that cannot fit its envelope. +pub(crate) fn task_too_large_error(task: &str) -> Option { + (task.len() > MAX_BODY_BYTES).then(|| { + format!( + "spawn_task_too_large: task is {} bytes; the limit is {MAX_BODY_BYTES} bytes so its envelope fits the {MAX_INJECTION_BODY_BYTES}-byte PTY limit. Write the brief to a file on the node and send a short pointer.", + task.len() + ) + }) +} // At the default 5 ms/byte, leaves room for readiness, prompt recheck and verification. pub(crate) const MAX_TYPED_WRITE_TIME: Duration = Duration::from_millis(7680); pub(crate) const MAX_TYPED_BYTES: usize = MAX_TYPED_WRITE_TIME.as_millis() as usize / 5; diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index 5550575058..ca1fa652b9 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -290,14 +290,7 @@ pub(super) fn already_ready_spawn_result( /// The spawn error for an initial task whose body cannot fit the PTY envelope. pub(super) fn spawn_task_too_large(task: &str) -> Option { - (task.len() > crate::injection_wire::MAX_BODY_BYTES).then(|| { - format!( - "spawn_task_too_large: task is {} bytes; the limit is {} bytes so its envelope fits the {}-byte PTY limit. Write the brief to a file on the node and send a short pointer.", - task.len(), - crate::injection_wire::MAX_BODY_BYTES, - crate::injection_wire::MAX_INJECTION_BODY_BYTES - ) - }) + crate::injection_wire::task_too_large_error(task) } pub(super) fn verified_spawn_failed_result(invocation_id: String, error: &str) -> ActionResult { diff --git a/crates/broker/src/runtime/relaycast_events.rs b/crates/broker/src/runtime/relaycast_events.rs index cb1c002b2a..4c98a1cdb1 100644 --- a/crates/broker/src/runtime/relaycast_events.rs +++ b/crates/broker/src/runtime/relaycast_events.rs @@ -798,6 +798,9 @@ pub(super) async fn spawn_worker_from_request( ); } crate::worker::validate_muse_startup_prompt_for_spec(&spec, effective_task.as_deref())?; + // Measure the task as it will be injected, after the exit contract and + // skill prefix, and before registering or launching anything. + crate::worker::validate_pty_initial_task_size(&spec, effective_task.as_deref())?; // Pre-register an agent token for every spawned worker. // The Agent Relay MCP server needs RELAY_AGENT_TOKEN + diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index ce34ccafbf..4a27ef9677 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -1232,6 +1232,48 @@ async fn an_oversized_spawn_task_is_rejected_before_launch() { } assert!(found); assert!(fixture.runtime.workers.workers.is_empty()); + + // relay#1893 review (Cursor): the raw task fits, but the exit-after-task + // contract appended by the spawn path pushes the injected task past the + // cap. That must also fail before registration, not after launch. + fixture + .runtime + .handle_fleet_control_event(crate::node_control::FleetControlEvent::Message( + crate::fleet_wire::RelaycastToBroker::ActionInvoke(crate::fleet_wire::ActionInvoke { + task_execution: None, + v: FLEET_WIRE_VERSION, + invocation_id: "decorated-task".into(), + action: "spawn".into(), + input: json!({"name":"decorated", "cli":"codex", "verify_ready":true, + "exit_after_task": true, "cwd":temp.path(), + "task":"x".repeat(crate::injection_wire::MAX_BODY_BYTES)}), + agent_name: Some("decorated".into()), + agent_id: None, + }), + )) + .await; + let mut found = false; + while let Ok(command) = fixture.fleet_control_rx.try_recv() { + match command { + FleetControlCommand::RegisterAgent { .. } => { + panic!("a decorated task over the cap must fail before registration") + } + FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) => { + let crate::fleet_wire::ActionResultPayload::Error(error) = &result.result else { + panic!("expected fleet error: {result:?}"); + }; + assert!( + error.error.contains("spawn_task_too_large: "), + "{}", + error.error + ); + found = true; + } + _ => {} + } + } + assert!(found); + assert!(fixture.runtime.workers.workers.is_empty()); } fn inbound_ctx<'a>(event_id: &'a str) -> InboundContext<'a> { diff --git a/crates/broker/src/worker.rs b/crates/broker/src/worker.rs index ee2d45a630..53a40cd440 100644 --- a/crates/broker/src/worker.rs +++ b/crates/broker/src/worker.rs @@ -907,6 +907,7 @@ impl WorkerRegistry { anyhow::bail!("agent '{}' already exists", spec.name); } validate_muse_startup_prompt_for_spec(&spec, initial_task.as_deref())?; + validate_pty_initial_task_size(&spec, initial_task.as_deref())?; tracing::info!( target = "broker::spawn", @@ -2487,11 +2488,9 @@ pub(crate) fn validate_muse_startup_prompt(cli: &str, task: Option<&str>) -> Res /// Resolve the command that a PTY spec will actually launch before validating /// its Muse prompt. Explicit PTY harness configs own the executable; ordinary /// PTY specs use `spec.cli`. -pub(crate) fn validate_muse_startup_prompt_for_spec( - spec: &AgentSpec, - task: Option<&str>, -) -> Result<()> { - let cli = match spec.harness_config.as_ref() { +/// The PTY command a spec will launch, if it is a PTY spawn. +fn pty_cli_for_spec(spec: &AgentSpec) -> Result> { + Ok(match spec.harness_config.as_ref() { Some(ResolvedHarnessConfig::Pty(config)) => Some( parse_cli_command(&config.command) .with_context(|| format!("invalid harness command '{}'", config.command))? @@ -2506,13 +2505,35 @@ pub(crate) fn validate_muse_startup_prompt_for_spec( None => None, }, _ => None, - }; - match cli { + }) +} + +pub(crate) fn validate_muse_startup_prompt_for_spec( + spec: &AgentSpec, + task: Option<&str>, +) -> Result<()> { + match pty_cli_for_spec(spec)? { Some(cli) => validate_muse_startup_prompt(&cli, task), None => Ok(()), } } +/// Reject, before launch, a PTY initial task whose final (decorated) body +/// cannot fit the injection envelope. Muse takes its task in argv, so only +/// injected tasks are measured (relay#1893 review). +pub(crate) fn validate_pty_initial_task_size(spec: &AgentSpec, task: Option<&str>) -> Result<()> { + let (Some(cli), Some(task)) = (pty_cli_for_spec(spec)?, task) else { + return Ok(()); + }; + if muse_startup_prompt(&cli.to_ascii_lowercase(), Some(task)).is_some() { + return Ok(()); + } + match crate::injection_wire::task_too_large_error(task) { + Some(error) => anyhow::bail!(error), + None => Ok(()), + } +} + /// Clean Muse config home for a worker spawn, when Relay MCP injection is /// active for Muse: an isolated `XDG_CONFIG_HOME` scope provisioned by the /// MCP configurator. Returns `None` for other CLIs and when injection is @@ -4813,6 +4834,46 @@ sleep 30 ); } + #[test] + fn pty_initial_task_size_is_measured_only_for_injected_tasks() { + let spec = |cli: &str, runtime: AgentRuntime| AgentSpec { + name: WorkerName::from("size-check"), + runtime, + provider: None, + cli: Some(cli.to_string()), + session_id: None, + harness_config: None, + model: None, + cwd: None, + team: None, + shadow_of: None, + shadow_mode: None, + args: Vec::new(), + channels: Vec::new(), + restart_policy: None, + }; + let fits = "x".repeat(crate::injection_wire::MAX_BODY_BYTES); + let over = "x".repeat(crate::injection_wire::MAX_BODY_BYTES + 1); + assert!( + validate_pty_initial_task_size(&spec("codex", AgentRuntime::Pty), Some(&fits)).is_ok() + ); + let error = + validate_pty_initial_task_size(&spec("codex --yolo", AgentRuntime::Pty), Some(&over)) + .expect_err("an injected task over the body cap is rejected") + .to_string(); + assert!(error.starts_with("spawn_task_too_large: "), "{error}"); + // Muse receives its task in argv, and native runtimes are not injected. + assert!( + validate_pty_initial_task_size(&spec("muse", AgentRuntime::Pty), Some(&over)).is_ok() + ); + assert!(validate_pty_initial_task_size( + &spec("codex", AgentRuntime::Headless), + Some(&over) + ) + .is_ok()); + assert!(validate_pty_initial_task_size(&spec("codex", AgentRuntime::Pty), None).is_ok()); + } + #[test] fn argv_initial_task_blocks_followups_but_is_not_injected_twice() { let mut registry = make_registry(Vec::new()); From efafa4de377c089265af1bade9e6f3add6b9c179 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 01:46:06 -0700 Subject: [PATCH 20/42] test(broker): canonicalize the Devin trust fixture's cwd so the trusted arm works on macOS The fixture compares its trust record with os.getcwd(), which reports the resolved path. On macOS the temp dir sits behind /var -> /private/var, so the record never matched, the trusted arm showed the trust dialog, and already_trusted_directory_becomes_ready_and_begins_task failed every time on Rust Tests (macos-latest). Linux was unaffected. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: f9b2ae4e-847e-47c7-9996-b6545ca4353c --- crates/broker/tests/devin_trust_cli.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/broker/tests/devin_trust_cli.rs b/crates/broker/tests/devin_trust_cli.rs index 7535b2cc81..db6e8e9171 100644 --- a/crates/broker/tests/devin_trust_cli.rs +++ b/crates/broker/tests/devin_trust_cli.rs @@ -26,6 +26,9 @@ fn run_fixture(mode: &str) { let cwd = directory.path().join("worktree"); std::fs::create_dir_all(&home).unwrap(); std::fs::create_dir_all(&cwd).unwrap(); + // The fixture compares its trust record with os.getcwd(), which reports + // the resolved path; on macOS the temp dir sits behind /var -> /private/var. + let cwd = std::fs::canonicalize(&cwd).unwrap(); let trusted = mode == "trusted"; if trusted { std::fs::write(home.join("fixture-trust"), cwd.to_str().unwrap()).unwrap(); From b8b639489ac6f76dd3771cd73b8e7d0483c644e3 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 02:29:41 -0700 Subject: [PATCH 21/42] test(fleet-e2e): model Codex's composer in the stub so the chunked initial task completes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The resume scenario's `pool` harness runs the stub as `codex`, which sends the initial task through the broker's chunked Codex path. That path confirms delivery by watching the composer: each chunk rendered after the prompt, then the composer cleared once Enter submits the turn. The stub only had the kernel's cooked-mode echo and never cleared its composer, so after Enter the task still looked parked; the broker retried the submit, exhausted its retries and dead-lettered `initial_injection_incomplete`. On main that dead letter was invisible because a verified spawn resolved at readiness. This PR fails a spawn whose task was not delivered, so the scenario failed. As `codex`, the stub now takes raw input, renders typed text after `›`, and redraws an empty composer on Enter. Pre-ready input is still discarded and, with raw mode, no longer echoed by the kernel either. Other stub uses are unchanged. Verified locally against the CI-pinned engine: the resume scenario now completes and resumes on its origin node. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: f9b2ae4e-847e-47c7-9996-b6545ca4353c --- tests/e2e/fleet/nodes/stub-agent.cjs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/e2e/fleet/nodes/stub-agent.cjs b/tests/e2e/fleet/nodes/stub-agent.cjs index f753fc14d2..02e59e0218 100644 --- a/tests/e2e/fleet/nodes/stub-agent.cjs +++ b/tests/e2e/fleet/nodes/stub-agent.cjs @@ -35,10 +35,28 @@ function recordBriefNonce(nonce) { ); } +// Invoked as `codex`, model Codex's composer closely enough for the broker's +// chunked initial-task path: raw input (no kernel echo, so discarded +// pre-ready keystrokes never appear), typed text rendered after the `›` +// prompt, and Enter submitting the turn and redrawing an empty composer. A +// composer that keeps the submitted text visible looks like a parked task, so +// the broker would retry the submit and report the delivery incomplete. +const codexComposer = path.basename(process.argv[1]) === 'codex'; +if (codexComposer && process.stdin.isTTY) process.stdin.setRawMode(true); + +function renderCodexComposer(text) { + for (const ch of text) { + if (ch === '\r') process.stdout.write('\r\n› '); + else if (ch === '\n') process.stdout.write('\r\n '); + else process.stdout.write(ch); + } +} + try { process.stdin.resume(); process.stdin.on('data', (chunk) => { if (!ready) return; + if (codexComposer) renderCodexComposer(chunk.toString()); input += chunk.toString(); // Require a delimiter after the nonce. PTY chunks can split anywhere, so // treating the current buffer end as a complete token could record a From 24002fb7c02d98489dc92b564c9d64a898792f66 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 02:47:53 -0700 Subject: [PATCH 22/42] fix(fleet): keep the public spawn_unconfirmed code for pending spawns RelayFlow regression case 1563-spawn-lifecycle-truth pins the MCP contract for an accepted spawn whose outcome is unknown: code `spawn_unconfirmed` and "Do not retry blindly". This PR had renamed that code to `spawn_pending`, breaking that contract and any agent or tool keyed on the existing code (Flows v2 shard 17). The code is `spawn_unconfirmed` again on the MCP and CLI JSON. The new behaviour stays: `state: "pending"`, exit 8, liveness evidence, inspection and reclaim guidance. The message keeps "Do not retry blindly". Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: f9b2ae4e-847e-47c7-9996-b6545ca4353c --- CHANGELOG.md | 2 +- packages/cli/README.md | 3 ++- .../cli/src/cli/agent-relay-mcp.startup.test.ts | 12 ++++++------ packages/cli/src/cli/commands/fleet.test.ts | 6 +++--- packages/cli/src/cli/lib/sdk-command.test.ts | 2 +- packages/cli/src/cli/lib/spawn-liveness.test.ts | 2 +- packages/cli/src/cli/lib/spawn-liveness.ts | 15 +++++++++------ 7 files changed, 23 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa6dd2e109..bee076b6f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Unconfirmed `fleet spawn` results report `spawn_pending` with invocation diagnostics and exit 8. Node heartbeat presence is reported as liveness evidence but never confirms the spawn, because heartbeats cannot tell this invocation's worker from an earlier one with the same name. +- Unconfirmed `fleet spawn` results keep the `spawn_unconfirmed` code, now with `state: "pending"`, invocation diagnostics, liveness evidence and exit 8. Node heartbeat presence is reported as liveness evidence but never confirms the spawn, because heartbeats cannot tell this invocation's worker from an earlier one with the same name. - Removal waits exit 8 if the registration remains present; scripts must handle nonzero exits beyond `$? -eq 1`. Waits remain opt-in, and unavailable verification reads preserve the asynchronous acknowledgement. ### Fixed diff --git a/packages/cli/README.md b/packages/cli/README.md index 89e257cb0c..c1fb4f5a05 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -589,7 +589,8 @@ Hosted equivalents live under `agent-relay cloud …`. ### Pending fleet spawns and name reuse -An accepted spawn whose result is still unknown returns `spawn_pending` (exit 8), +An accepted spawn whose result is still unknown returns `spawn_unconfirmed` with +`state: "pending"` (exit 8), with its invocation ID, dispatch state, and available liveness evidence. Check `agent-relay fleet agent list --node ` before retrying. A heartbeat that lists a worker with the requested name is reported as evidence but keeps the diff --git a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts index f226aef167..edc1938013 100644 --- a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts +++ b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts @@ -1172,7 +1172,7 @@ describe('createAgentRelayMcpServer', () => { isError: true, structuredContent: { error: { - code: 'spawn_pending', + code: 'spawn_unconfirmed', invocationId: 'inv_nested', dispatchState: 'dispatched', node: 'nested-node', @@ -1217,7 +1217,7 @@ describe('createAgentRelayMcpServer', () => { isError: true, structuredContent: { error: { - code: 'spawn_pending', + code: 'spawn_unconfirmed', state: 'pending', dispatchState: 'unknown', message: expect.stringContaining('outcome is pending'), @@ -1246,7 +1246,7 @@ describe('createAgentRelayMcpServer', () => { isError: true, structuredContent: { error: { - code: 'spawn_pending', + code: 'spawn_unconfirmed', state: 'pending', dispatchState: 'unknown', invocationId: 'inv_1', @@ -1287,7 +1287,7 @@ describe('createAgentRelayMcpServer', () => { .handler({ name: 'worker', persona: 'reviewer' }); expect(result).toMatchObject({ isError: true, - structuredContent: { error: { code: 'spawn_pending', invocationId: 'inv_child', node: 'child-node' } }, + structuredContent: { error: { code: 'spawn_unconfirmed', invocationId: 'inv_child', node: 'child-node' } }, }); }); @@ -1315,7 +1315,7 @@ describe('createAgentRelayMcpServer', () => { // so even fresh target-node presence leaves the spawn pending. expect(result).toMatchObject({ isError: true, - structuredContent: { error: { code: 'spawn_pending', state: 'pending', liveness: { evidence } } }, + structuredContent: { error: { code: 'spawn_unconfirmed', state: 'pending', liveness: { evidence } } }, }); } ); @@ -1402,7 +1402,7 @@ describe('createAgentRelayMcpServer', () => { isError: true, structuredContent: { error: { - code: 'spawn_pending', + code: 'spawn_unconfirmed', invocationId: 'inv_nested', dispatchState: 'dispatched', node: 'nested-node', diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index e5bc190527..5b625fe18a 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1092,7 +1092,7 @@ describe('fleet command support', () => { const rendered = errors.join('\n'); expect(rendered).toContain('do not retry blindly'); - expect(rendered).toContain('"code":"spawn_pending"'); + expect(rendered).toContain('"code":"spawn_unconfirmed"'); expect(rendered).toContain('"state":"pending"'); expect(rendered).toContain('"dispatchState":"dispatched"'); expect(rendered).toContain(`"invocationId":"${invocationId}"`); @@ -1187,7 +1187,7 @@ describe('fleet command support', () => { expect(exit).toHaveBeenCalledWith(8); expect(logs).toEqual([]); const rendered = errors.join(''); - expect(rendered).toContain('"code":"spawn_pending"'); + expect(rendered).toContain('"code":"spawn_unconfirmed"'); expect(rendered).toContain(`"invocationId":"${invocationId}"`); expect(rendered).toContain('"evidence":"live"'); expect(rendered).toContain('may be an earlier worker'); @@ -3374,7 +3374,7 @@ describe('fleet command support', () => { ).rejects.toThrow('__exit__'); expect(errors.join('\n')).toContain('dispatch may still be running'); - expect(errors.join('\n')).toContain('"code":"spawn_pending"'); + expect(errors.join('\n')).toContain('"code":"spawn_unconfirmed"'); expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( expect.objectContaining({ workloadProfile: 'long-running-agent', diff --git a/packages/cli/src/cli/lib/sdk-command.test.ts b/packages/cli/src/cli/lib/sdk-command.test.ts index b56b906482..847941f466 100644 --- a/packages/cli/src/cli/lib/sdk-command.test.ts +++ b/packages/cli/src/cli/lib/sdk-command.test.ts @@ -98,7 +98,7 @@ describe('runSdk structured error output', () => { ); }); expect(sdkDeps.exit).toHaveBeenCalledWith(8); - expect(errors.join('')).toContain('"code":"spawn_pending"'); + expect(errors.join('')).toContain('"code":"spawn_unconfirmed"'); expect(errors.join('')).toContain('"liveness":{"evidence":"unknown"}'); await runSdk(sdkDeps, async () => { throw Object.assign(new Error('plain'), { exitCode: 0 }); diff --git a/packages/cli/src/cli/lib/spawn-liveness.test.ts b/packages/cli/src/cli/lib/spawn-liveness.test.ts index d71b920dd3..57ef6bdc2b 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.test.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.test.ts @@ -126,7 +126,7 @@ describe('spawn liveness resolution', () => { { invocationId: 'inv', node: 'target', dispatchState: 'dispatched' }, { evidence: 'live', node: 'target', heartbeatAgeMs: 0 } ); - expect(pending.code).toBe('spawn_pending'); + expect(pending.code).toBe('spawn_unconfirmed'); expect(pending.message).toContain('may be an earlier worker'); expect(pending.message).toContain('Invocation: inv'); }); diff --git a/packages/cli/src/cli/lib/spawn-liveness.ts b/packages/cli/src/cli/lib/spawn-liveness.ts index 1062d5debd..97f3a0b2ce 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.ts @@ -31,15 +31,18 @@ export class FleetSpawnError extends Error { readonly dispatchState?: string; readonly receipt?: Record; constructor( - readonly code: 'spawn_pending' | 'spawn_name_taken', + // `spawn_unconfirmed` is the established public code for an accepted spawn + // whose outcome is unknown (relay#1563); keep it so callers keyed on it keep + // working, and carry the pending outcome in `state` and the exit code. + readonly code: 'spawn_unconfirmed' | 'spawn_name_taken', message: string, context: SpawnContext, readonly liveness?: SpawnLiveness, readonly diagnostic?: string ) { super(message); - this.state = code === 'spawn_pending' ? 'pending' : 'failed'; - this.exitCode = code === 'spawn_pending' ? 8 : 1; + this.state = code === 'spawn_unconfirmed' ? 'pending' : 'failed'; + this.exitCode = code === 'spawn_unconfirmed' ? 8 : 1; this.invocationId = context.invocationId; this.node = context.node; this.dispatchState = context.dispatchState; @@ -50,7 +53,7 @@ export class FleetSpawnError extends Error { export function mayStillBeRunning(error: unknown): boolean { return ( (error instanceof RelayPlacementError && error.state === 'unconfirmed_may_be_running') || - (error instanceof FleetSpawnError && error.code === 'spawn_pending') + (error instanceof FleetSpawnError && error.code === 'spawn_unconfirmed') ); } @@ -68,12 +71,12 @@ export function pendingSpawnError( ? `A worker with this name is listed by a node heartbeat, but heartbeats do not identify the invocation that started it, so it may be an earlier worker rather than this spawn. ` : ''; return new FleetSpawnError( - 'spawn_pending', + 'spawn_unconfirmed', `Spawn of ${JSON.stringify(name)} was accepted; its outcome is pending. ` + `Evidence: ${liveness.evidence}${liveness.node ? ` on ${JSON.stringify(liveness.node)}` : ''}. ` + attribution + `Invocation: ${context.invocationId ?? 'unavailable'}; dispatch: ${context.dispatchState ?? 'unknown'}. ` + - `Check \`agent-relay fleet agent list${nodeOption}\` before retrying. ` + + `Do not retry blindly: check \`agent-relay fleet agent list${nodeOption}\` first. ` + `Only if no worker is running and you intend to reclaim the name, run \`agent-relay agent remove ${shellQuote(name)} --wait\` and wait for clearance before respawning.`, context, liveness, From 7b9de91d3535f822f0468d28834ba1f82d55138d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 8 Oct 2026 10:41:07 +0000 Subject: [PATCH 23/42] style: auto-format with Prettier --- packages/cli/src/cli/agent-relay-mcp.startup.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts index edc1938013..1e6e53d53c 100644 --- a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts +++ b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts @@ -1287,7 +1287,9 @@ describe('createAgentRelayMcpServer', () => { .handler({ name: 'worker', persona: 'reviewer' }); expect(result).toMatchObject({ isError: true, - structuredContent: { error: { code: 'spawn_unconfirmed', invocationId: 'inv_child', node: 'child-node' } }, + structuredContent: { + error: { code: 'spawn_unconfirmed', invocationId: 'inv_child', node: 'child-node' }, + }, }); }); From d5f616ee756892715caf78f1721989535d516a2e Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 04:33:42 -0700 Subject: [PATCH 24/42] ci: run checks on the auto-formatted head The Prettier bot's commit (7b9de91) cancelled the in-flight run for 24002fb through the branch concurrency group and, as a GITHUB_TOKEN push, started no workflows of its own, leaving the head without CI. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: f9b2ae4e-847e-47c7-9996-b6545ca4353c From 3901e426706937a2c2a24f97b8e9a67c7dbc1482 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 10:40:47 -0700 Subject: [PATCH 25/42] docs(changelog): order bundled Unreleased sections Added/Changed/Fixed Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e40a08d090..f04c257df9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `agent remove --wait` and `fleet release --delete-agent --wait` verify registration clearance before name reuse, with `--wait-timeout` and `--no-wait` controls. Fleet release adds `removal` evidence when waiting. - `fleet agent list` JSON includes node heartbeat timestamps and ages, with warnings for stale snapshots. +### Changed + +- Unconfirmed `fleet spawn` results keep the `spawn_unconfirmed` code, now with `state: "pending"`, invocation diagnostics, liveness evidence and exit 8. Node heartbeat presence is reported as liveness evidence but never confirms the spawn, because heartbeats cannot tell this invocation's worker from an earlier one with the same name. +- Removal waits exit 8 if the registration remains present; scripts must handle nonzero exits beyond `$? -eq 1`. Waits remain opt-in, and unavailable verification reads preserve the asynchronous acknowledgement. + ### Fixed - When Cloud ends a sandbox preparation (for example `relayfile_mount_failed` / `initial_sync_deadline`), `fleet spawn --sandbox` names the cause and confirms no sandbox was left running, without a redundant cleanup call or unknown-outcome guidance. @@ -31,11 +36,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fleet registration collisions use readable `spawn_name_taken` guidance covering asynchronous removal and failed-spawn cleanup, instead of Rust `Fatal(AlreadyExists {...})` text. - Pending fleet spawns preserve provisioned sandboxes and distinguish live, stale, elsewhere, registered, absent, and unknown evidence before advising a retry. -### Changed - -- Unconfirmed `fleet spawn` results keep the `spawn_unconfirmed` code, now with `state: "pending"`, invocation diagnostics, liveness evidence and exit 8. Node heartbeat presence is reported as liveness evidence but never confirms the spawn, because heartbeats cannot tell this invocation's worker from an earlier one with the same name. -- Removal waits exit 8 if the registration remains present; scripts must handle nonzero exits beyond `$? -eq 1`. Waits remain opt-in, and unavailable verification reads preserve the asynchronous acknowledgement. - ## [13.1.5] - 2026-10-08 ### Fixed From c3f5a7638bd8934c3dc37d36b728c5d30a2fcb7e Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 10:50:38 -0700 Subject: [PATCH 26/42] test(broker): build #1913's Devin trust fixture with #1893's PendingVerifiedSpawn fields #1893 added readiness_proven, task_event_id and task_verification to PendingVerifiedSpawn; #1913's new test predates them, so the bundled broker lib tests did not compile. The trust prompt blocks before readiness, so the fixture is an unready spawn with no task. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- crates/broker/src/runtime/tests.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index 4e53acabb0..b9da169e41 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -9812,6 +9812,9 @@ async fn devin_directory_trust_error_expires_verified_spawn_and_releases_capacit started: Instant::now(), generation, failure_reason: None, + readiness_proven: false, + task_event_id: None, + task_verification: None, }, ); for event_generation in [Uuid::new_v4(), generation] { From 7a59ddcb35eedffd4a614c1ba3f12e1ae6547d8c Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 11:22:30 -0700 Subject: [PATCH 27/42] fix(relay-pty): latch bracketed paste when DECSET 2004 spans reads The reader only stepped the parser byte by byte when a single read held the whole `2004` parameter. A split `2004h` cleared in the next read without spelling 2004 (for example by RIS) never survived to a chunk end, so the capability was missed and the broker fell back to the typed cap. Carry the previous read's last three bytes into the scan. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- crates/relay-pty/src/pty.rs | 94 ++++++++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 6 deletions(-) diff --git a/crates/relay-pty/src/pty.rs b/crates/relay-pty/src/pty.rs index 620dbe76e7..1da61aa89c 100644 --- a/crates/relay-pty/src/pty.rs +++ b/crates/relay-pty/src/pty.rs @@ -929,6 +929,36 @@ fn read_was_cancelled(error: &io::Error) -> bool { } /// Whether the grid currently has bracketed-paste mode (DECSET 2004) set. +/// Finds the DECSET 2004 parameter in PTY output, including one split across +/// reads, by carrying the previous read's last three bytes. +#[derive(Default)] +struct Decset2004Scan { + tail: [u8; 3], + tail_len: usize, +} + +impl Decset2004Scan { + fn may_carry(&mut self, bytes: &[u8]) -> bool { + let head = bytes.len().min(3); + let mut seam = [0u8; 6]; + seam[..self.tail_len].copy_from_slice(&self.tail[..self.tail_len]); + seam[self.tail_len..self.tail_len + head].copy_from_slice(&bytes[..head]); + let found = seam[..self.tail_len + head] + .windows(4) + .chain(bytes.windows(4)) + .any(|window| window == b"2004"); + let joined = &seam[..self.tail_len + head]; + let carried = if bytes.len() >= 3 { + &bytes[bytes.len() - 3..] + } else { + &joined[joined.len().saturating_sub(3)..] + }; + self.tail_len = carried.len(); + self.tail[..self.tail_len].copy_from_slice(carried); + found + } +} + fn paste_mode_enabled(term: &Term) -> bool { term.mode() .contains(alacritty_terminal::term::TermMode::BRACKETED_PASTE) @@ -1084,6 +1114,7 @@ impl PtySession { let write_admission_pending_reader = write_admission_pending.clone(); let reader_thread = thread::spawn(move || { let mut buf = [0u8; 4096]; + let mut decset_2004_scan = Decset2004Scan::default(); loop { #[cfg(unix)] if wait_for_pty_readable(&poll_fd_reader).is_err() { @@ -1144,13 +1175,13 @@ impl PtySession { // time is the only way to see a transition that does not // survive to the end of the chunk, but it costs the reader // thread a parser call per byte, so it is reserved for chunks - // that can actually carry the DECSET 2004 parameter. Every - // other chunk is parsed in bulk and then checked once, which - // still latches a `2004h` split across reads. + // that can actually carry the DECSET 2004 parameter, counting + // one split across the previous read. Every other chunk is + // parsed in bulk and then checked once. if paste_capable_reader.load(Ordering::Acquire) { processor_guard.advance(&mut *term_guard, bytes); } else { - if bytes.windows(4).any(|window| window == b"2004") { + if decset_2004_scan.may_carry(bytes) { for byte in bytes { processor_guard .advance(&mut *term_guard, std::slice::from_ref(byte)); @@ -1714,8 +1745,8 @@ mod tests { UnixOutputReservations, }; use super::{ - read_and_tag_output, GridSize, Mutex, PtyOutput, PtySession, PtyWriteSubmitError, - DEFAULT_NO_PID_EXIT_THRESHOLD, + read_and_tag_output, Decset2004Scan, GridSize, Mutex, PtyOutput, PtySession, + PtyWriteSubmitError, DEFAULT_NO_PID_EXIT_THRESHOLD, }; use crate::snapshot::Snapshot; use alacritty_terminal::event::VoidListener; @@ -1911,6 +1942,57 @@ mod tests { drain.abort(); } + /// A split `2004h` that a full reset (RIS, which never spells `2004`) + /// clears in the same second read never survives to the end of either + /// chunk, so the read boundary must still select the byte-by-byte path. + #[cfg(unix)] + #[tokio::test] + async fn paste_capability_latches_split_enable_reset_in_the_next_read() { + let (pty, mut rx) = PtySession::spawn( + "sh", + &[ + "-c".into(), + "printf '\\033[?20'; sleep 0.3; printf '04h\\033cREADY'; sleep 2".into(), + ], + 24, + 80, + ) + .unwrap(); + let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); + tokio::time::timeout(Duration::from_secs(3), async { + while !pty.screen_text().contains("READY") { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!(pty.bracketed_paste_enabled()); + pty.shutdown().unwrap(); + drain.abort(); + } + + #[test] + fn decset_2004_scan_sees_the_parameter_across_every_split() { + let stream = b"ab\x1b[?2004hcd"; + for split in 0..=stream.len() { + let mut scan = Decset2004Scan::default(); + let first = scan.may_carry(&stream[..split]); + let second = scan.may_carry(&stream[split..]); + assert!(first || second, "split at {split} hid the parameter"); + } + let mut scan = Decset2004Scan::default(); + assert!(!scan.may_carry(b"200")); + assert!( + scan.may_carry(b"4"), + "a one-byte read completes the parameter" + ); + assert!(!scan.may_carry(b"x")); + assert!(!scan.may_carry(b"2")); + assert!(!scan.may_carry(b"00")); + assert!(scan.may_carry(b"4h"), "three short reads carry the prefix"); + assert!(!scan.may_carry(b"2005")); + } + #[tokio::test] async fn consumed_offset_tracks_total_bytes_parsed_into_grid() { let (pty, mut rx) = PtySession::spawn("echo", &["offset-line".into()], 24, 80).unwrap(); From a2a20ef0ae0ef1fac8e982c2ed1e08a79b317209 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 11:22:30 -0700 Subject: [PATCH 28/42] fix: apply the PTY body cap only where the PTY envelope is used - Broker delivery rejected bodies over 14 KiB for every recipient; headless recipients never type into a terminal, so only PTY (or unknown) workers keep the cap. The PTY worker still enforces its exact wire limit. - Muse takes its startup task as one argv entry. The CLI (agent spawn/new, fleet spawn) and the node's fleet spawn check now apply Muse's 16 KiB argv limit instead of the PTY body limit, matching validate_pty_initial_task_size. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- crates/broker/src/runtime/delivery.rs | 9 ++- crates/broker/src/runtime/fleet.rs | 16 ++++- crates/broker/src/runtime/tests.rs | 62 ++++++++++++++++++-- packages/cli/src/cli/commands/fleet.ts | 2 +- packages/cli/src/cli/commands/local-agent.ts | 4 +- packages/cli/src/cli/lib/task-input.test.ts | 20 ++++++- packages/cli/src/cli/lib/task-input.ts | 24 +++++++- 7 files changed, 123 insertions(+), 14 deletions(-) diff --git a/crates/broker/src/runtime/delivery.rs b/crates/broker/src/runtime/delivery.rs index b85c2d5c30..be46fb61a6 100644 --- a/crates/broker/src/runtime/delivery.rs +++ b/crates/broker/src/runtime/delivery.rs @@ -861,8 +861,15 @@ pub(crate) async fn queue_and_try_delivery_raw( withheld_fleet_ack: Option, withheld_fleet_ack_floor: Option, ) -> Result { + // The envelope-sized cap exists because PTY delivery types the message into + // a terminal. Headless recipients never do, so only a known non-PTY worker + // skips it; an unknown recipient keeps the conservative check. + let pty_recipient = workers + .workers + .get(worker_name) + .is_none_or(|handle| handle.spec.runtime == AgentRuntime::Pty); anyhow::ensure!( - body.len() <= crate::injection_wire::MAX_BODY_BYTES, + !pty_recipient || body.len() <= crate::injection_wire::MAX_BODY_BYTES, "injection_too_large: body limit is {} bytes so the formatted envelope fits the {}-byte PTY limit; use a brief file pointer", crate::injection_wire::MAX_BODY_BYTES, crate::injection_wire::MAX_INJECTION_BODY_BYTES diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index 71ac159dbc..9030ebd8e6 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -296,8 +296,15 @@ pub(super) fn already_ready_spawn_result( } } -/// The spawn error for an initial task whose body cannot fit the PTY envelope. -pub(super) fn spawn_task_too_large(task: &str) -> Option { +/// The spawn error for an initial task that cannot reach the worker: a PTY +/// task must fit the injection envelope, while Muse takes its startup task as +/// one argv entry and is bounded by its portable argv limit instead. +pub(super) fn spawn_task_too_large(cli: &str, task: &str) -> Option { + if crate::snippets::is_muse_executable(cli) { + return crate::worker::validate_muse_startup_prompt(cli, Some(task)) + .err() + .map(|error| format!("spawn_task_too_large: {error}")); + } crate::injection_wire::task_too_large_error(task) } @@ -1672,7 +1679,10 @@ impl BrokerRuntime { let carries_task = task.is_some(); // Reject before launching: an oversized task can only fail after the // worker exists, costing a launch and a release for a known outcome. - if let Some(error) = task.as_deref().and_then(spawn_task_too_large) { + if let Some(error) = task + .as_deref() + .and_then(|task| spawn_task_too_large(&cli, task)) + { self.reply_action_error(&invoke.invocation_id, &error).await; return; } diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index b9da169e41..82fe85499d 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -1193,10 +1193,21 @@ async fn an_oversized_spawn_task_is_rejected_before_launch() { let (tx, _rx) = mpsc::channel(16); let workers = WorkerRegistry::new(tx, vec![], temp.path().into(), Instant::now()); let mut fixture = worker_event_runtime_fixture(workers, HashMap::new()); - assert!( - super::fleet::spawn_task_too_large(&"x".repeat(crate::injection_wire::MAX_BODY_BYTES)) - .is_none() - ); + assert!(super::fleet::spawn_task_too_large( + "codex", + &"x".repeat(crate::injection_wire::MAX_BODY_BYTES) + ) + .is_none()); + // Muse takes its startup task as one argv entry, not through the PTY + // envelope, so only its portable argv limit applies. + let over_pty = "x".repeat(crate::injection_wire::MAX_BODY_BYTES + 1); + assert!(super::fleet::spawn_task_too_large("codex", &over_pty).is_some()); + assert!(super::fleet::spawn_task_too_large("/usr/local/bin/Muse.exe", &over_pty).is_none()); + assert!(super::fleet::spawn_task_too_large( + "muse", + &"x".repeat(crate::worker::MUSE_STARTUP_PROMPT_MAX_BYTES + 1) + ) + .is_some_and(|error| error.starts_with("spawn_task_too_large: "))); fixture .runtime .handle_fleet_control_event(crate::node_control::FleetControlEvent::Message( @@ -4167,6 +4178,49 @@ async fn initial_delivery_failure_stays_owned_until_dead_lettered() { assert_eq!(pending.delivery.body, "must remain auditable"); } +#[tokio::test] +async fn pty_body_cap_applies_only_to_pty_recipients() { + let over_cap = "x".repeat(crate::injection_wire::MAX_BODY_BYTES + 1); + for runtime in [AgentRuntime::Pty, AgentRuntime::Headless] { + let mut workers = make_worker_registry_with_worker("Worker").await; + workers + .workers + .get_mut(&WorkerName::from("Worker")) + .unwrap() + .spec + .runtime = runtime.clone(); + let mut pending_deliveries = HashMap::new(); + let result = super::queue_and_try_delivery_raw( + &mut workers, + &mut pending_deliveries, + "Worker", + "evt_large", + "orchestrator", + "Worker", + &over_cap, + None, + Some(WorkspaceId::new("ws_demo")), + None, + 2, + MessageInjectionMode::Wait, + Duration::from_secs(1), + None, + None, + ) + .await; + match runtime { + AgentRuntime::Pty => assert!(result + .expect_err("PTY recipients keep the envelope-sized body cap") + .to_string() + .contains("injection_too_large")), + AgentRuntime::Headless => { + result.expect("headless recipients never type into a PTY, so no PTY cap"); + assert_eq!(pending_deliveries.len(), 1); + } + } + } +} + #[tokio::test] async fn delivery_retry_transient_blip_emits_failed_event_for_present_worker() { let worker_name = "worker-blip"; diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 642573b966..439b1a77d1 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -507,7 +507,7 @@ export function registerFleetCommands( await runSdk(deps.sdk, async () => { const clientOptions = sdkOptionsFromOpts(options); const name = requiredText(options.name, 'Worker name'); - const task = (await readTaskInput(options.task, options.taskFile, true))!; + const task = (await readTaskInput(options.task, options.taskFile, true, cli))!; let targetNode = optionalText(options.targetNode, 'Target node') ?? optionalText(options.node, 'Node'); const useSandbox = options.sandbox === true; // Explicit hosted credentials/transport and personas retain their legacy diff --git a/packages/cli/src/cli/commands/local-agent.ts b/packages/cli/src/cli/commands/local-agent.ts index 48fa1a62ce..ef018d5bd5 100644 --- a/packages/cli/src/cli/commands/local-agent.ts +++ b/packages/cli/src/cli/commands/local-agent.ts @@ -875,7 +875,7 @@ export function registerLocalAgentCommands( const resolved = resolveAutoSpawn( provider, baseName, - await readTaskInput(opts.task, opts.taskFile), + await readTaskInput(opts.task, opts.taskFile, false, provider), opts.model as string | undefined ); await spawnAgentWithClient(client, { @@ -936,7 +936,7 @@ export function registerLocalAgentCommands( const resolved = resolveAutoSpawn( provider, baseName, - await readTaskInput(options.task, options.taskFile), + await readTaskInput(options.task, options.taskFile, false, provider), options.model as string | undefined ); await run( diff --git a/packages/cli/src/cli/lib/task-input.test.ts b/packages/cli/src/cli/lib/task-input.test.ts index 2eccfb9db6..08a7c6b104 100644 --- a/packages/cli/src/cli/lib/task-input.test.ts +++ b/packages/cli/src/cli/lib/task-input.test.ts @@ -2,7 +2,12 @@ import { mkdtemp, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, it, expect } from 'vitest'; -import { readTaskInput, MAX_INJECTION_BODY_BYTES, MAX_TASK_BODY_BYTES } from './task-input.js'; +import { + readTaskInput, + MAX_INJECTION_BODY_BYTES, + MAX_TASK_BODY_BYTES, + MUSE_STARTUP_PROMPT_MAX_BYTES, +} from './task-input.js'; describe('task input', () => { it('requires exactly one source for fleet, allows neither locally', async () => { await expect(readTaskInput(undefined, undefined, true)).rejects.toThrow('exactly one'); @@ -35,4 +40,17 @@ describe('task input', () => { 'UTF-8 bytes' ); }); + // Muse takes its startup task as one argv entry, never through the PTY + // envelope, so the broker's 16 KiB argv limit applies instead. + it('applies the Muse argv limit, not the PTY body limit, to Muse tasks', async () => { + const between = 'a'.repeat(MAX_TASK_BODY_BYTES + 1); + for (const cli of ['muse', 'MUSE.exe', '/usr/local/bin/muse', 'C:\\Tools\\Muse.CMD']) { + expect(await readTaskInput(between, undefined, false, cli)).toBe(between); + await expect( + readTaskInput('a'.repeat(MUSE_STARTUP_PROMPT_MAX_BYTES + 1), undefined, false, cli) + ).rejects.toThrow('argv'); + } + await expect(readTaskInput(between, undefined, false, 'musey')).rejects.toThrow('envelope'); + await expect(readTaskInput(between, undefined, false, 'claude')).rejects.toThrow('envelope'); + }); }); diff --git a/packages/cli/src/cli/lib/task-input.ts b/packages/cli/src/cli/lib/task-input.ts index a77d08f801..1ac2cf52b1 100644 --- a/packages/cli/src/cli/lib/task-input.ts +++ b/packages/cli/src/cli/lib/task-input.ts @@ -6,10 +6,20 @@ export const MAX_INJECTION_BODY_BYTES = 16 * 1024; export const ENVELOPE_RESERVE_BYTES = 2 * 1024; /** Largest body that still fits {@link MAX_INJECTION_BODY_BYTES} once formatted. */ export const MAX_TASK_BODY_BYTES = MAX_INJECTION_BODY_BYTES - ENVELOPE_RESERVE_BYTES; +/** The broker's portable argv ceiling for Muse's single-argument startup prompt. */ +export const MUSE_STARTUP_PROMPT_MAX_BYTES = 16 * 1024; + +/** Mirrors the broker's `is_muse_executable`: the basename, case-insensitive, minus a Windows launcher suffix. */ +function isMuseExecutable(cli: string): boolean { + const basename = cli.split(/[\\/]/).pop() || cli; + return basename.toLowerCase().replace(/\.(exe|cmd|bat)$/, '') === 'muse'; +} + export async function readTaskInput( task: unknown, taskFile: unknown, - required = false + required = false, + cli?: string ): Promise { if (task !== undefined && taskFile !== undefined) throw new Error('Specify exactly one of --task or --task-file'); @@ -19,10 +29,20 @@ export async function readTaskInput( } const text = taskFile === undefined ? task : await readFile(String(taskFile), 'utf8'); if (typeof text !== 'string' || !text.trim()) throw new Error('Task must not be empty'); - validateInjectionSize(text); + // Muse receives its startup task through argv, not the PTY envelope. + if (cli !== undefined && isMuseExecutable(cli)) validateMuseStartupPromptSize(text); + else validateInjectionSize(text); return text; } +function validateMuseStartupPromptSize(text: string): void { + if (Buffer.byteLength(text, 'utf8') > MUSE_STARTUP_PROMPT_MAX_BYTES) { + throw new Error( + `Muse startup task exceeds the ${MUSE_STARTUP_PROMPT_MAX_BYTES}-byte portable argv limit; use a brief file on the node and send a short pointer` + ); + } +} + export function validateInjectionSize(text: string): void { if (Buffer.byteLength(text, 'utf8') > MAX_TASK_BODY_BYTES) { throw new Error( From f516fb15107bbbd94eb41cced8ea213dd0ea634d Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 11:22:30 -0700 Subject: [PATCH 29/42] fix(mcp): scope message replay to the acting agent token A retained idempotency key was scoped by MCP session and tool only, so a session that switched identity (`as`, or register_agent moving the default) and reused a key within five minutes got the earlier identity's receipt without sending. Clients are tagged with a SHA-256 of their agent token, which fixes both identity and target workspace, and post_message, reply_to_thread and send_dm include it in the replay key. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- packages/cli/src/cli/agent-relay-mcp.ts | 4 +- packages/cli/src/cli/mcp/messaging-tools.ts | 84 +++++++++++-------- .../cli/src/cli/mcp/request-replay.test.ts | 25 +++++- packages/cli/src/cli/mcp/request-replay.ts | 30 ++++++- 4 files changed, 105 insertions(+), 38 deletions(-) diff --git a/packages/cli/src/cli/agent-relay-mcp.ts b/packages/cli/src/cli/agent-relay-mcp.ts index d257d9aa5b..b6480246b8 100644 --- a/packages/cli/src/cli/agent-relay-mcp.ts +++ b/packages/cli/src/cli/agent-relay-mcp.ts @@ -59,7 +59,7 @@ import { import { enableInboxPiggyback } from './mcp/telemetry.js'; import { registerAgentRelayActionTools } from './mcp/action-tools.js'; import { registerMessagingTools } from './mcp/messaging-tools.js'; -import { McpRequestReplay } from './mcp/request-replay.js'; +import { McpRequestReplay, withReplayScope } from './mcp/request-replay.js'; import { identityOverrideInputShape, messageResult } from './mcp/tool-shapes.js'; import { registerSharedSessionTools, @@ -1787,7 +1787,7 @@ export function createAgentRelayMcpServer(options: AgentRelayMcpServerOptions): const getAgentClient = (asIdentity?: string): AgentClientLike => { const agentToken = resolveAgentToken(asIdentity); - return createAgentClient({ agentToken, baseUrl: options.baseUrl }); + return withReplayScope(createAgentClient({ agentToken, baseUrl: options.baseUrl }), agentToken); }; enableInboxPiggyback( diff --git a/packages/cli/src/cli/mcp/messaging-tools.ts b/packages/cli/src/cli/mcp/messaging-tools.ts index a28a3e4665..6cd6df49b5 100644 --- a/packages/cli/src/cli/mcp/messaging-tools.ts +++ b/packages/cli/src/cli/mcp/messaging-tools.ts @@ -11,7 +11,7 @@ import { } from '../lib/message-delivery-receipts.js'; import { jsonContent, jsonResult, textContent } from './tool-results.js'; import { identityOverrideInputShape, messageResult } from './tool-shapes.js'; -import { McpRequestReplay } from './request-replay.js'; +import { McpRequestReplay, replayScopeOf } from './request-replay.js'; import type { AgentClientLike } from './types.js'; const directMessageResult = z.looseObject({ @@ -265,15 +265,20 @@ export function registerMessagingTools( // Bind the acting identity now: a register_agent that moves the session // default must not reroute a write that is already in flight. const client = getAgentClient(as); - return replay.run('post_message', extra, idempotency_key, async () => - jsonContent( - await client.send(channel, text, { - attachments, - data: replayMessageMetadata(), - mode, - idempotencyKey: idempotency_key, - }) - ) + return replay.run( + 'post_message', + extra, + idempotency_key, + async () => + jsonContent( + await client.send(channel, text, { + attachments, + data: replayMessageMetadata(), + mode, + idempotencyKey: idempotency_key, + }) + ), + replayScopeOf(client) ); } ); @@ -326,13 +331,18 @@ export function registerMessagingTools( }, async ({ message_id, text, idempotency_key, as }, extra) => { const client = getAgentClient(as); - return replay.run('reply_to_thread', extra, idempotency_key, async () => - jsonContent( - await client.reply(message_id, text, { - data: replayMessageMetadata(), - idempotencyKey: idempotency_key, - }) - ) + return replay.run( + 'reply_to_thread', + extra, + idempotency_key, + async () => + jsonContent( + await client.reply(message_id, text, { + data: replayMessageMetadata(), + idempotencyKey: idempotency_key, + }) + ), + replayScopeOf(client) ); } ); @@ -390,22 +400,30 @@ export function registerMessagingTools( openWorldHint: true, }, }, - async ({ to, text, mode, attachments, idempotency_key, as }, extra) => - replay.run('send_dm', extra, idempotency_key, async () => { - const agents = await listAgentsForRecipientResolution?.(); - const resolvedRecipient = agents ? resolveExactAgentName(agents, to) : undefined; - const message = await getAgentClient(as).dm(to, text, { - idempotencyKey: idempotency_key, - mode, - attachments, - data: replayMessageMetadata(), - }); - const receipt = compactDirectMessageReceipt( - directMessageReceipt(message, to, mode, resolvedRecipient) - ); - const result = jsonContent(receipt); - return directMessageDeliveryFailure(receipt) ? { ...result, isError: true as const } : result; - }) + async ({ to, text, mode, attachments, idempotency_key, as }, extra) => { + const client = getAgentClient(as); + return replay.run( + 'send_dm', + extra, + idempotency_key, + async () => { + const agents = await listAgentsForRecipientResolution?.(); + const resolvedRecipient = agents ? resolveExactAgentName(agents, to) : undefined; + const message = await client.dm(to, text, { + idempotencyKey: idempotency_key, + mode, + attachments, + data: replayMessageMetadata(), + }); + const receipt = compactDirectMessageReceipt( + directMessageReceipt(message, to, mode, resolvedRecipient) + ); + const result = jsonContent(receipt); + return directMessageDeliveryFailure(receipt) ? { ...result, isError: true as const } : result; + }, + replayScopeOf(client) + ); + } ); server.registerTool( diff --git a/packages/cli/src/cli/mcp/request-replay.test.ts b/packages/cli/src/cli/mcp/request-replay.test.ts index 764470ac75..82df64e855 100644 --- a/packages/cli/src/cli/mcp/request-replay.test.ts +++ b/packages/cli/src/cli/mcp/request-replay.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { McpRequestReplay } from './request-replay.js'; +import { McpRequestReplay, replayScopeOf, withReplayScope } from './request-replay.js'; function deferred() { let resolve!: (value: string) => void; @@ -36,6 +36,29 @@ describe('MCP request replay', () => { await first; }); + // A session can switch its acting identity (`as`, or register_agent moving + // the default) and reuse a key; that must send, not replay another + // identity's receipt. + it('scopes a retained idempotency key to the acting identity', async () => { + const replay = new McpRequestReplay(); + const operation = vi.fn(async () => 'receipt'); + const extra = { requestId: 1, sessionId: 's' }; + await replay.run('post_message', extra, 'k', operation, 'agent-a'); + await replay.run('post_message', { ...extra, requestId: 2 }, 'k', operation, 'agent-a'); + expect(operation).toHaveBeenCalledTimes(1); + await replay.run('post_message', { ...extra, requestId: 3 }, 'k', operation, 'agent-b'); + expect(operation).toHaveBeenCalledTimes(2); + }); + + it('derives a stable token-free scope from a tagged client', () => { + const a = withReplayScope({}, 'token-a'); + expect(replayScopeOf(a)).toBe(replayScopeOf(withReplayScope({}, 'token-a'))); + expect(replayScopeOf(a)).not.toBe(replayScopeOf(withReplayScope({}, 'token-b'))); + expect(replayScopeOf(a)).not.toContain('token-a'); + expect(JSON.stringify(a)).toBe('{}'); + expect(replayScopeOf({})).toBe(''); + }); + it('clears a rejected request ID for a safe retry', async () => { const replay = new McpRequestReplay(); const pending = deferred(); diff --git a/packages/cli/src/cli/mcp/request-replay.ts b/packages/cli/src/cli/mcp/request-replay.ts index 3d6f085903..4e09642854 100644 --- a/packages/cli/src/cli/mcp/request-replay.ts +++ b/packages/cli/src/cli/mcp/request-replay.ts @@ -1,7 +1,31 @@ +import { createHash } from 'node:crypto'; + +const REPLAY_SCOPE = Symbol('agentRelay.replayScope'); + +/** + * Tag a Relay client with the replay scope of the agent token it acts with. + * The token decides both the acting identity and the workspace a write reaches, + * so a key reused after either changes cannot replay another context's + * receipt. Only a SHA-256 digest is kept, in a non-enumerable property. + */ +export function withReplayScope(client: T, agentToken: string): T { + Object.defineProperty(client, REPLAY_SCOPE, { + value: createHash('sha256').update(agentToken).digest('hex'), + enumerable: false, + }); + return client; +} + +/** The replay scope a client was tagged with, or '' for an untagged client. */ +export function replayScopeOf(client: object): string { + const scope = (client as { [REPLAY_SCOPE]?: unknown })[REPLAY_SCOPE]; + return typeof scope === 'string' ? scope : ''; +} + /** * Coalesce a transport replay of one MCP request before it can repeat a * state-changing Relay call. A typed JSON-RPC request ID, scoped by MCP - * session and tool, identifies only an in-flight logical request because + * session, acting identity and tool, identifies only an in-flight logical request because * JSON-RPC permits IDs to be reused after a response. Clients that need to * retry after a lost response provide an explicit idempotency key; that key * retains the completed result briefly. Arguments deliberately do not @@ -14,7 +38,8 @@ export class McpRequestReplay { tool: string, extra: unknown, idempotencyKey: string | undefined, - operation: () => Promise + operation: () => Promise, + scope = '' ): Promise { const request = extra as { requestId?: unknown; sessionId?: unknown } | undefined; const requestId = request?.requestId; @@ -26,6 +51,7 @@ export class McpRequestReplay { const key = JSON.stringify([ sessionId, + scope, tool, hasIdempotencyKey ? 'idempotency' : typeof requestId, hasIdempotencyKey ? idempotencyKey : requestId, From 35b6e330427770cd68fcf2fee8aacdae65d68f3c Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 12:36:44 -0700 Subject: [PATCH 30/42] fix(broker): address bundle review findings in delivery, wrap and PTY scan - can_inject gates Muse on its composer like the other known harnesses. - A wrap pointer whose metadata alone overflows the envelope falls back to a clipped last-resort envelope instead of requeueing forever. - The DECSET 2004 scan matches only a private-mode parameter (?2004/;2004), still across reads, so plain output never takes the per-byte path. - delivery_verified omits verification when the worker sent no label instead of claiming an echo. - The early fleet spawn task guard applies only to PTY-injected spawns. - Document both echo_incomplete cases. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- crates/broker/src/injection_wire.rs | 23 ++++- crates/broker/src/protocol.rs | 8 +- crates/broker/src/runtime/fleet.rs | 13 +++ crates/broker/src/runtime/relaycast_events.rs | 4 +- crates/broker/src/runtime/tests.rs | 99 +++++++++++++++++++ crates/broker/src/runtime/worker_events.rs | 11 ++- crates/broker/src/wrap.rs | 58 +++++++++-- crates/relay-pty/src/pty.rs | 51 +++++----- 8 files changed, 229 insertions(+), 38 deletions(-) diff --git a/crates/broker/src/injection_wire.rs b/crates/broker/src/injection_wire.rs index 5d6dccf460..df4da47e28 100644 --- a/crates/broker/src/injection_wire.rs +++ b/crates/broker/src/injection_wire.rs @@ -57,13 +57,18 @@ pub(crate) fn effective_limit(wire: InjectionWire, pace: Duration) -> usize { MAX_TYPED_BYTES.min((MAX_TYPED_WRITE_TIME.as_millis() / pace.as_millis().max(1)) as usize) } } -pub(crate) fn can_inject(cli: &str, pty: &PtySession) -> bool { +/// Harnesses whose composer readiness `cli_prompt_ready` can prove, so an +/// injection waits for it instead of writing into a login or other interstitial. +fn readiness_gated(cli: &str) -> bool { let lower = cli.to_lowercase(); - let known = ["claude", "codex", "gemini"] + ["claude", "codex", "gemini"] .iter() .any(|name| lower.contains(name)) - || crate::readiness::is_devin_cli(cli); - !known + || crate::readiness::is_devin_cli(cli) + || crate::readiness::is_muse_cli(cli) +} +pub(crate) fn can_inject(cli: &str, pty: &PtySession) -> bool { + !readiness_gated(cli) || crate::readiness::cli_prompt_ready( cli, crate::readiness::GridReadinessSnapshot { @@ -76,6 +81,16 @@ pub(crate) fn can_inject(cli: &str, pty: &PtySession) -> bool { mod tests { use super::*; #[test] + fn muse_waits_for_its_composer_like_the_other_known_harnesses() { + for cli in ["muse", "/Users/khaliqgant/.local/bin/muse", "MUSE.exe"] { + assert!(readiness_gated(cli), "{cli}"); + } + for cli in ["claude", "codex", "gemini", "devin"] { + assert!(readiness_gated(cli), "{cli}"); + } + assert!(!readiness_gated("bash")); + } + #[test] fn body_cannot_submit_or_close_paste() { for ch in (0..=0x10ffff).filter_map(char::from_u32) { let body = format!("head{ch}\x1b[201~\r\nend\r"); diff --git a/crates/broker/src/protocol.rs b/crates/broker/src/protocol.rs index 67e88dca50..b0d55708bb 100644 --- a/crates/broker/src/protocol.rs +++ b/crates/broker/src/protocol.rs @@ -419,9 +419,11 @@ pub enum BrokerEvent { /// whole envelope verbatim) and "echo_normalized" (the whole envelope /// once the TUI's wrapping whitespace is removed) confirm full /// receipt. "echo_incomplete" (head and tail without the payload - /// between them), "paste_summary" (the harness collapsed the paste, so - /// no content was echoed) and "timeout_fallback" (no echo at all) are - /// acks without proof of receipt. A tail without its head is a + /// between them, or a tail without its head when the observation + /// window was trimmed so head loss cannot be proven), "paste_summary" + /// (the harness collapsed the paste, so no content was echoed) and + /// "timeout_fallback" (no echo at all) are acks without proof of + /// receipt. A tail without its head in an untrimmed window is a /// `DeliveryFailed` with reason "echo_head_missing", never a /// verification label. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index 9030ebd8e6..b4301ccc5f 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -296,6 +296,18 @@ pub(super) fn already_ready_spawn_result( } } +/// Whether a fleet spawn's initial task will be injected through a PTY, the +/// only path the envelope limit applies to. An explicit headless or native +/// harness config hands the task over without a terminal; an absent or +/// invalid config keeps the conservative PTY answer (the spawn path rejects an +/// invalid config on its own). +pub(super) fn spawn_task_is_pty_injected(input: &Value) -> bool { + super::relaycast_events::relaycast_harness_config(input) + .ok() + .flatten() + .is_none_or(|config| config.runtime() == AgentRuntime::Pty) +} + /// The spawn error for an initial task that cannot reach the worker: a PTY /// task must fit the injection envelope, while Muse takes its startup task as /// one argv entry and is bounded by its portable argv limit instead. @@ -1681,6 +1693,7 @@ impl BrokerRuntime { // worker exists, costing a launch and a release for a known outcome. if let Some(error) = task .as_deref() + .filter(|_| spawn_task_is_pty_injected(&invoke.input)) .and_then(|task| spawn_task_too_large(&cli, task)) { self.reply_action_error(&invoke.invocation_id, &error).await; diff --git a/crates/broker/src/runtime/relaycast_events.rs b/crates/broker/src/runtime/relaycast_events.rs index ff53f7b29a..0e3eaf2889 100644 --- a/crates/broker/src/runtime/relaycast_events.rs +++ b/crates/broker/src/runtime/relaycast_events.rs @@ -171,7 +171,9 @@ pub(super) fn relaycast_spawn_spec_session_id( }) } -fn relaycast_harness_config(value: &Value) -> Result, String> { +pub(super) fn relaycast_harness_config( + value: &Value, +) -> Result, String> { let agent = value.get("agent"); let harness_id = agent .and_then(|agent| { diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index 82fe85499d..a26abe5940 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -1009,6 +1009,40 @@ async fn a_spawn_task_proven_received_reports_spawned_and_ready() { } } +/// relay#1930 review (cubic): a `delivery_verified` frame without a label is +/// already treated as unconfirmed for spawn receipt, so the SDK event must not +/// synthesize the confirming "echo" label for it. +#[tokio::test] +async fn an_unlabelled_delivery_verified_event_does_not_claim_an_echo() { + let worker_name = "unlabelled-verify"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let mut fixture = worker_event_runtime_fixture(registry, HashMap::new()); + for verification in [None, Some("echo_normalized")] { + fixture + .runtime + .handle_worker_event(delivery_verified_worker_event( + worker_name, + generation, + "evt_unlabelled", + verification, + )) + .await; + let mut verified = None; + while let Ok(frame) = fixture._sdk_out_rx.try_recv() { + if frame.payload["kind"] == "delivery_verified" { + verified = Some(frame.payload); + } + } + let verified = verified.expect("the SDK must still see the delivery_verified event"); + assert_eq!( + verified.get("verification").and_then(Value::as_str), + verification, + "the event label must be the worker's label, never a synthesized echo" + ); + } +} + fn pending_spawn_with_task( generation: Uuid, event_id: &str, @@ -1184,6 +1218,71 @@ fn an_already_ready_worker_with_an_unbound_pty_task_is_not_reported_ready() { )); } +/// relay#1930 review (cubic): the early fleet size guard is a PTY envelope +/// limit, so a spawn whose harness config selects a headless or native +/// runtime must not be rejected by it. Absent or invalid configs stay PTY. +#[test] +fn early_spawn_task_guard_applies_only_to_pty_injected_spawns() { + assert!(super::fleet::spawn_task_is_pty_injected( + &json!({"cli": "codex"}) + )); + assert!(super::fleet::spawn_task_is_pty_injected(&json!({ + "harnessConfig": {"runtime": "pty", "command": "codex", "args": []} + }))); + assert!(super::fleet::spawn_task_is_pty_injected( + &json!({"harnessId": "x"}) + )); + assert!(!super::fleet::spawn_task_is_pty_injected(&json!({ + "agent": {"harnessConfig": { + "runtime": "headless", "protocol": "opencode", + "endpoint": "http://127.0.0.1:4096", "sessionId": "ses_1", "release": "abort" + }} + }))); +} + +#[tokio::test] +async fn an_oversized_task_for_a_headless_fleet_spawn_skips_the_pty_guard() { + let temp = tempfile::tempdir().unwrap(); + let (tx, _rx) = mpsc::channel(16); + let workers = WorkerRegistry::new(tx, vec![], temp.path().into(), Instant::now()); + let mut fixture = worker_event_runtime_fixture(workers, HashMap::new()); + // The guard replies at once; a spawn past it may wait on registration, so + // bound the call and inspect only what was replied in that window. + let _ = tokio::time::timeout( + Duration::from_secs(2), + fixture.runtime.handle_fleet_control_event( + crate::node_control::FleetControlEvent::Message( + crate::fleet_wire::RelaycastToBroker::ActionInvoke( + crate::fleet_wire::ActionInvoke { + task_execution: None, + v: FLEET_WIRE_VERSION, + invocation_id: "headless-task".into(), + action: "spawn".into(), + input: json!({"name":"headless", "cli":"opencode", "cwd":temp.path(), + "task":"x".repeat(crate::injection_wire::MAX_BODY_BYTES + 1), + "harnessConfig": {"runtime": "headless", "protocol": "opencode", + "endpoint": "http://127.0.0.1:9", "sessionId": "ses_1"}}), + agent_name: Some("headless".into()), + agent_id: None, + }, + ), + ), + ), + ) + .await; + while let Ok(command) = fixture.fleet_control_rx.try_recv() { + if let FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) = command { + if let crate::fleet_wire::ActionResultPayload::Error(error) = &result.result { + assert!( + !error.error.contains("spawn_task_too_large"), + "a headless spawn never types its task into a PTY: {}", + error.error + ); + } + } + } +} + /// relay#1893 review (Cursor): a task that cannot fit the PTY envelope used to /// launch a worker that then failed `injection_too_large` and was released. /// It must be rejected before any registration or launch. diff --git a/crates/broker/src/runtime/worker_events.rs b/crates/broker/src/runtime/worker_events.rs index 101048cc51..09686a9df9 100644 --- a/crates/broker/src/runtime/worker_events.rs +++ b/crates/broker/src/runtime/worker_events.rs @@ -953,8 +953,17 @@ impl BrokerRuntime { "name": name, "delivery_id": delivery_id, "event_id": event_id, - "verification": verification.unwrap_or("echo"), }); + // An unlabelled frame (a headless exit) proves no + // echo, so its label stays absent rather than "echo". + if let (Some(verification), Some(map)) = + (verification, verified_event.as_object_mut()) + { + map.insert( + "verification".to_string(), + Value::String(verification.to_string()), + ); + } if let (Some(reason), Some(map)) = (reason, verified_event.as_object_mut()) { diff --git a/crates/broker/src/wrap.rs b/crates/broker/src/wrap.rs index 69776a87af..0f6c51fd2f 100644 --- a/crates/broker/src/wrap.rs +++ b/crates/broker/src/wrap.rs @@ -320,13 +320,36 @@ fn format_wrap_injection( limit = crate::injection_wire::MAX_INJECTION_BODY_BYTES, "wrap: injection_too_large; delivering a pointer instead of the body" ); - format(&format!( - "[Message body not shown: {} bytes exceeds the {}-byte terminal delivery limit. \ - The full message is stored in Relay as message {event_id}; read it with your Relay \ - message tools before replying.]", - body.len(), - crate::injection_wire::MAX_INJECTION_BODY_BYTES - )) + let pointer = |event_id: &str| { + format!( + "[Message body not shown: {} bytes exceeds the {}-byte terminal delivery limit. \ + The full message is stored in Relay as message {event_id}; read it with your Relay \ + message tools before replying.]", + body.len(), + crate::injection_wire::MAX_INJECTION_BODY_BYTES + ) + }; + let injection = format(&pointer(event_id)); + if injection.len() <= crate::injection_wire::MAX_INJECTION_BODY_BYTES { + return injection; + } + // Oversized sender, target or workspace metadata. A write that cannot fit + // is re-queued at the head of the queue, so the last resort must always + // fit: clip every field and drop the optional reminder and workspace label. + fn clip(value: &str) -> &str { + &value[..crate::util::ansi::floor_char_boundary(value, 256)] + } + format_injection_for_worker_with_workspace( + clip(from), + clip(event_id), + &pointer(clip(event_id)), + clip(target), + false, + true, + None, + None, + None, + ) } fn wrap_timeout_outcome( @@ -2813,6 +2836,27 @@ sys.stdout.flush()"#; } } + /// relay#1930 review (cubic): the pointer replaces only the body, so + /// oversized sender, target or workspace metadata kept the envelope over + /// the limit, failing submit and re-queuing the delivery at the head of + /// the queue forever. The final fallback must always fit. + #[test] + fn oversized_wrap_metadata_still_yields_a_bounded_pointer() { + let limit = crate::injection_wire::MAX_INJECTION_BODY_BYTES; + let huge = "é".repeat(limit); + let injection = super::format_wrap_injection( + &huge, + "evt_meta", + &"x".repeat(limit), + &huge, + true, + Some(&huge), + Some(&huge), + ); + assert!(injection.len() <= limit, "{} bytes", injection.len()); + assert!(injection.contains("message evt_meta"), "{injection}"); + } + #[test] fn wrap_timeout_stays_failed_without_confirming_evidence() { use crate::broker::delivery_verification::{DeliveryOutcome, EchoVerdict}; diff --git a/crates/relay-pty/src/pty.rs b/crates/relay-pty/src/pty.rs index 1da61aa89c..626023ed28 100644 --- a/crates/relay-pty/src/pty.rs +++ b/crates/relay-pty/src/pty.rs @@ -929,29 +929,28 @@ fn read_was_cancelled(error: &io::Error) -> bool { } /// Whether the grid currently has bracketed-paste mode (DECSET 2004) set. -/// Finds the DECSET 2004 parameter in PTY output, including one split across -/// reads, by carrying the previous read's last three bytes. +/// Finds the DECSET 2004 private-mode parameter (`?2004` or `;2004`) in PTY +/// output, including one split across reads, by carrying the previous read's +/// last four bytes. Plain text that merely contains `2004` does not match. #[derive(Default)] struct Decset2004Scan { - tail: [u8; 3], + tail: [u8; 4], tail_len: usize, } impl Decset2004Scan { fn may_carry(&mut self, bytes: &[u8]) -> bool { - let head = bytes.len().min(3); - let mut seam = [0u8; 6]; + let is_parameter = |window: &[u8]| matches!(window, [b'?' | b';', b'2', b'0', b'0', b'4']); + let head = bytes.len().min(4); + let mut seam = [0u8; 8]; seam[..self.tail_len].copy_from_slice(&self.tail[..self.tail_len]); seam[self.tail_len..self.tail_len + head].copy_from_slice(&bytes[..head]); - let found = seam[..self.tail_len + head] - .windows(4) - .chain(bytes.windows(4)) - .any(|window| window == b"2004"); let joined = &seam[..self.tail_len + head]; - let carried = if bytes.len() >= 3 { - &bytes[bytes.len() - 3..] + let found = joined.windows(5).chain(bytes.windows(5)).any(is_parameter); + let carried = if bytes.len() >= 4 { + &bytes[bytes.len() - 4..] } else { - &joined[joined.len().saturating_sub(3)..] + &joined[joined.len().saturating_sub(4)..] }; self.tail_len = carried.len(); self.tail[..self.tail_len].copy_from_slice(carried); @@ -1973,24 +1972,32 @@ mod tests { #[test] fn decset_2004_scan_sees_the_parameter_across_every_split() { - let stream = b"ab\x1b[?2004hcd"; - for split in 0..=stream.len() { - let mut scan = Decset2004Scan::default(); - let first = scan.may_carry(&stream[..split]); - let second = scan.may_carry(&stream[split..]); - assert!(first || second, "split at {split} hid the parameter"); + for stream in [&b"ab\x1b[?2004hcd"[..], &b"ab\x1b[?1;2004hcd"[..]] { + for split in 0..=stream.len() { + let mut scan = Decset2004Scan::default(); + let first = scan.may_carry(&stream[..split]); + let second = scan.may_carry(&stream[split..]); + assert!(first || second, "split at {split} hid the parameter"); + } } let mut scan = Decset2004Scan::default(); - assert!(!scan.may_carry(b"200")); + assert!(!scan.may_carry(b"?200")); assert!( scan.may_carry(b"4"), "a one-byte read completes the parameter" ); assert!(!scan.may_carry(b"x")); - assert!(!scan.may_carry(b"2")); + assert!(!scan.may_carry(b"?2")); assert!(!scan.may_carry(b"00")); - assert!(scan.may_carry(b"4h"), "three short reads carry the prefix"); - assert!(!scan.may_carry(b"2005")); + assert!(scan.may_carry(b"4h"), "short reads carry the prefix"); + assert!(!scan.may_carry(b"?2005")); + // relay#1930 review (cubic): ordinary text that merely contains 2004 + // (a year, a port) is not a private-mode parameter, so it must keep + // the bulk parse instead of a per-byte parse on every such read. + let mut scan = Decset2004Scan::default(); + assert!(!scan.may_carry(b"released in 2004, port 2004")); + assert!(!scan.may_carry(b" 200")); + assert!(!scan.may_carry(b"4")); } #[tokio::test] From c81a678cf2bc1aa413f3450c5deed653265c40e6 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 12:36:45 -0700 Subject: [PATCH 31/42] test(mcp): stop leaked timers from flaking MCP startup tests The startup tests used a real realtime client, which keeps reconnecting after its test and arms a connect timeout on a later test's fake clock (seen as a stray 10 s timer in the sessions-only and Cloud-discovery tests, ~10% of runs; 0/25 on main, where fewer tests register agents). Stub the realtime client in the SDK mock. Request replay also retained completed keys with a 5-minute timer; it now expires them lazily, so settling a write schedules nothing. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- .../src/cli/agent-relay-mcp.startup.test.ts | 29 ++++++++++++++++++- .../cli/src/cli/mcp/request-replay.test.ts | 21 ++++++++++++++ packages/cli/src/cli/mcp/request-replay.ts | 29 +++++++++++-------- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts index 1e6e53d53c..f8cd1a19ab 100644 --- a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts +++ b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts @@ -348,7 +348,14 @@ async function loadAgentRelayMcpModule(options: LoadOptions = {}) { })); vi.doMock('@agent-relay/sdk', async () => { const actual = await vi.importActual>('@agent-relay/sdk'); - return { ...actual, AgentRelay: AgentRelayMock }; + // A real realtime client keeps reconnecting after its test ends, and a + // reconnect's connect timeout then lands on a later test's fake clock. + const createRealtimeClient = () => ({ + on: () => () => undefined, + connect: () => undefined, + disconnect: () => undefined, + }); + return { ...actual, AgentRelay: AgentRelayMock, createRealtimeClient }; }); vi.doMock('./telemetry/index.js', () => ({ initTelemetry: telemetryInit, @@ -1363,6 +1370,26 @@ describe('createAgentRelayMcpServer', () => { } ); + // relay#1930 review (cubic): an agent-token-only session cannot read the + // roster, so it keeps the accepted removal and says why instead of polling. + it('remove_agent wait without a workspace key keeps the acknowledgement and does not poll', async () => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mod.createAgentRelayMcpServer({ agentToken: 'at_live_fleet', agentName: 'orchestrator' }); + const started = Date.now(); + const result = await mocks.serverInstances[0].tools + .get('remove_agent')! + .handler({ name: 'worker', delete_agent: true, wait: true }); + expect(Date.now() - started).toBeLessThan(5_000); + expect(result.isError).toBeUndefined(); + expect(result.structuredContent.invocation).toBeDefined(); + expect(result.structuredContent.removal).toMatchObject({ + cleared: false, + observedPresent: false, + readError: expect.stringContaining('Workspace key not configured'), + }); + expect(mocks.agentRelayAgentsGet).not.toHaveBeenCalled(); + }, 10_000); + it('correlates nested authorization failures to the nested route', async () => { const { mod, mocks } = await loadAgentRelayMcpModule(); mocks.agentRelayMessagingCommands.invoke.mockResolvedValueOnce({ diff --git a/packages/cli/src/cli/mcp/request-replay.test.ts b/packages/cli/src/cli/mcp/request-replay.test.ts index 82df64e855..98afae0c09 100644 --- a/packages/cli/src/cli/mcp/request-replay.test.ts +++ b/packages/cli/src/cli/mcp/request-replay.test.ts @@ -59,6 +59,27 @@ describe('MCP request replay', () => { expect(replayScopeOf({})).toBe(''); }); + // Retention is checked lazily: a settling write must not leave a timer + // behind (it would land on whatever clock a later caller installed). + it('retains a completed idempotency key for five minutes without scheduling a timer', async () => { + vi.useFakeTimers(); + try { + const replay = new McpRequestReplay(); + const operation = vi.fn(async () => 'receipt'); + await replay.run('post_message', { requestId: 1 }, 'k', operation); + await Promise.resolve(); + expect(vi.getTimerCount()).toBe(0); + vi.setSystemTime(Date.now() + 5 * 60 * 1000 - 1); + await replay.run('post_message', { requestId: 2 }, 'k', operation); + expect(operation).toHaveBeenCalledTimes(1); + vi.setSystemTime(Date.now() + 1); + await replay.run('post_message', { requestId: 3 }, 'k', operation); + expect(operation).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + it('clears a rejected request ID for a safe retry', async () => { const replay = new McpRequestReplay(); const pending = deferred(); diff --git a/packages/cli/src/cli/mcp/request-replay.ts b/packages/cli/src/cli/mcp/request-replay.ts index 4e09642854..2d6614139f 100644 --- a/packages/cli/src/cli/mcp/request-replay.ts +++ b/packages/cli/src/cli/mcp/request-replay.ts @@ -31,8 +31,11 @@ export function replayScopeOf(client: object): string { * retains the completed result briefly. Arguments deliberately do not * participate, so intentional identical sends remain separate requests. */ +const RETAIN_COMPLETED_MS = 5 * 60 * 1000; + export class McpRequestReplay { - private readonly requests = new Map>(); + /** `expiresAt` is set once a keyed request settles; expiry is checked lazily, never by a timer. */ + private readonly requests = new Map; expiresAt?: number }>(); run( tool: string, @@ -56,27 +59,29 @@ export class McpRequestReplay { hasIdempotencyKey ? 'idempotency' : typeof requestId, hasIdempotencyKey ? idempotencyKey : requestId, ]); - const existing = this.requests.get(key) as Promise | undefined; - if (existing) return existing; + this.pruneExpired(); + const existing = this.requests.get(key); + if (existing) return existing.pending as Promise; const pending = operation(); - this.requests.set(key, pending); + const entry: { pending: Promise; expiresAt?: number } = { pending }; + this.requests.set(key, entry); // JSON-RPC permits an ID to be reused after its response. Only a client // supplied idempotency key can safely keep a completed result for a retry. void pending.then( - () => this.clearAfterSettlement(key, hasIdempotencyKey), + () => { + if (hasIdempotencyKey) entry.expiresAt = Date.now() + RETAIN_COMPLETED_MS; + else this.requests.delete(key); + }, () => this.requests.delete(key) ); return pending; } - private clearAfterSettlement(key: string, retainCompletedResult: boolean): void { - if (!retainCompletedResult) { - this.requests.delete(key); - return; + private pruneExpired(): void { + const now = Date.now(); + for (const [key, entry] of this.requests) { + if (entry.expiresAt !== undefined && entry.expiresAt <= now) this.requests.delete(key); } - - const cleanup = setTimeout(() => this.requests.delete(key), 5 * 60 * 1000); - cleanup.unref(); } } From 8c2167af11c9ab88b5c255cf6131f916a96eea4d Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 12:36:45 -0700 Subject: [PATCH 32/42] fix(cli): address bundle review findings in task input, removal and liveness - --task-file reads at most limit+1 bytes; composed sandbox and auto-routed tasks are validated after composition; agent new reports task errors via the CLI error path. - --wait-timeout rejects NaN/non-positive values before releasing; an unverifiable --wait no longer suggests --wait; MCP remove_agent wait without a workspace key returns the acknowledgement instead of polling. - Liveness scans the fleet unless the target entry is fresh, available and holds the name, prefers fresh live_elsewhere, and suggests a fleet-wide check. - integration subscribe --list reads subscriptions once instead of per row. - Native relay tools send a tool-call idempotency key so a late replay cannot write twice. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- packages/cli/README.md | 4 +- packages/cli/src/cli/agent-relay-mcp.ts | 26 +-- packages/cli/src/cli/commands/agent.test.ts | 16 +- packages/cli/src/cli/commands/agent.ts | 17 +- packages/cli/src/cli/commands/fleet.test.ts | 151 ++++++++++++++++++ packages/cli/src/cli/commands/fleet.ts | 38 +++-- .../commands/integration-subscribe.test.ts | 38 ++++- packages/cli/src/cli/commands/integration.ts | 15 +- .../cli/src/cli/commands/local-agent.test.ts | 28 ++++ packages/cli/src/cli/commands/local-agent.ts | 23 ++- .../src/cli/commands/relaycast-groups.test.ts | 3 + packages/cli/src/cli/lib/agent-removal.ts | 11 ++ .../cli/lib/fleet-spawn-confirmation.test.ts | 14 +- .../cli/src/cli/lib/spawn-liveness.test.ts | 36 +++++ packages/cli/src/cli/lib/spawn-liveness.ts | 21 ++- packages/cli/src/cli/lib/task-input.test.ts | 10 ++ packages/cli/src/cli/lib/task-input.ts | 35 +++- packages/cli/src/cli/mcp/messaging-tools.ts | 10 +- packages/harness-driver/src/protocol.ts | 4 +- .../src/ai-sdk/native-relay-tools.test.ts | 20 +++ .../src/ai-sdk/native-relay-tools.ts | 23 ++- 21 files changed, 472 insertions(+), 71 deletions(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index c1fb4f5a05..60add4e553 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -590,8 +590,8 @@ Hosted equivalents live under `agent-relay cloud …`. ### Pending fleet spawns and name reuse An accepted spawn whose result is still unknown returns `spawn_unconfirmed` with -`state: "pending"` (exit 8), -with its invocation ID, dispatch state, and available liveness evidence. Check +`state: "pending"` (exit 8), plus its invocation ID and dispatch state when +known and any available liveness evidence. Check `agent-relay fleet agent list --node ` before retrying. A heartbeat that lists a worker with the requested name is reported as evidence but keeps the spawn pending: heartbeats carry worker names, not invocation IDs, so that worker diff --git a/packages/cli/src/cli/agent-relay-mcp.ts b/packages/cli/src/cli/agent-relay-mcp.ts index b6480246b8..66da945ef0 100644 --- a/packages/cli/src/cli/agent-relay-mcp.ts +++ b/packages/cli/src/cli/agent-relay-mcp.ts @@ -1624,16 +1624,24 @@ function registerAgentRelayTools( } } if (delete_agent && wait) { - const workspace = () => { + // Clearance reads need workspace authority. Without it every read + // would fail, so keep the accepted removal and say why at once. + let workspace: AgentRelay | undefined; + let unavailable: string | undefined; + try { requireWorkspaceKey(session); - return new AgentRelay({ workspaceKey: session.workspaceKey!, baseUrl }); - }; - const removal = await waitForAgentRemoval({ - name, - getAgent: (name) => workspace().agents.get(name), - listAgents: () => workspace().agents.list(), - timeoutMs: wait_timeout_ms, - }); + workspace = new AgentRelay({ workspaceKey: session.workspaceKey!, baseUrl }); + } catch (error) { + unavailable = error instanceof Error ? error.message : String(error); + } + const removal = workspace + ? await waitForAgentRemoval({ + name, + getAgent: (name) => workspace.agents.get(name), + listAgents: () => workspace.agents.list(), + timeoutMs: wait_timeout_ms, + }) + : { cleared: false, waitedMs: 0, observedPresent: false, readError: unavailable }; return { ...jsonContent({ invocation, diff --git a/packages/cli/src/cli/commands/agent.test.ts b/packages/cli/src/cli/commands/agent.test.ts index 0d67f55e97..84b205e0eb 100644 --- a/packages/cli/src/cli/commands/agent.test.ts +++ b/packages/cli/src/cli/commands/agent.test.ts @@ -13,6 +13,7 @@ function createHarness() { const workspaceRelay = { agents: { get: vi.fn(async (name: string) => ({ id: 'agent_existing', name, status: 'online' })), + list: vi.fn(async () => [{ name: 'worker' }]), }, workspace: { register: vi.fn(async ({ name }: { name: string }) => ({ @@ -307,11 +308,24 @@ describe('agent identity lifecycle commands', () => { it('keeps the asynchronous acknowledgement when every verification read is unavailable', async () => { const { program, workspaceRelay, log, error } = createHarness(); workspaceRelay.agents.get.mockRejectedValue(new Error('403 forbidden')); + workspaceRelay.agents.list.mockRejectedValue(new Error('403 roster forbidden')); await program.parseAsync(['agent', 'remove', 'worker', '--wait', '--wait-timeout', '1'], { from: 'user', }); + expect(workspaceRelay.agents.list).toHaveBeenCalled(); expect(log.mock.calls.flat().join('')).toContain('initiated'); - expect(error.mock.calls.flat().join('')).toContain('Could not verify'); + // --wait already ran; suggesting it again would misdescribe the outcome. + expect(log.mock.calls.flat().join('')).not.toContain('Use --wait'); + expect(error.mock.calls.flat().join('')).toContain('Could not verify removal: 403 roster forbidden'); + }); + + it('rejects a non-numeric --wait-timeout before removing anything', async () => { + const { program, workspaceRelay, error } = createHarness(); + await expect( + program.parseAsync(['agent', 'remove', 'worker', '--wait', '--wait-timeout', 'abc'], { from: 'user' }) + ).rejects.toThrow('exit:1'); + expect(error.mock.calls.flat().join('')).toContain('--wait-timeout'); + expect(workspaceRelay.workspace.release).not.toHaveBeenCalled(); }); it('does not claim clearance or poll with --no-wait', async () => { diff --git a/packages/cli/src/cli/commands/agent.ts b/packages/cli/src/cli/commands/agent.ts index d9958ec170..bd08567b03 100644 --- a/packages/cli/src/cli/commands/agent.ts +++ b/packages/cli/src/cli/commands/agent.ts @@ -8,7 +8,12 @@ import { withSdkDefaults, type SdkCommandDeps, } from '../lib/sdk-command.js'; -import { AgentRemovalPendingError, isNotFoundError, waitForAgentRemoval } from '../lib/agent-removal.js'; +import { + AgentRemovalPendingError, + isNotFoundError, + parseRemovalWaitTimeout, + waitForAgentRemoval, +} from '../lib/agent-removal.js'; import { withAgentRegistrationDeadline, withDeadline } from '../lib/agent-registration.js'; import { attributableReleaseReason } from '../lib/release-reason.js'; @@ -154,20 +159,26 @@ export function registerAgentCommands( process.env.RELAY_AGENT_NAME ?? 'agent-relay CLI', 'agent removed' ); + const waitTimeoutMs = opts.wait === true ? parseRemovalWaitTimeout(opts.waitTimeout) : undefined; const result = await relay.workspace.release({ name, reason, deleteAgent: true }); - if (opts.wait === true) { + if (waitTimeoutMs !== undefined) { const removal = await waitForAgentRemoval({ name, getAgent: (name) => relay.agents.get(name), listAgents: () => relay.agents.list(), - timeoutMs: Number(opts.waitTimeout), + timeoutMs: waitTimeoutMs, }); if (removal.cleared) { deps.log(`Removed agent ${name}.`); return; } if (removal.observedPresent) throw new AgentRemovalPendingError(name); + // Unverifiable clearance keeps the asynchronous acknowledgement. deps.error(`Could not verify removal: ${removal.readError ?? 'registration unavailable'}`); + deps.log( + `Removal of agent ${name} was initiated (status: ${result.status ?? 'pending'}) and is processed asynchronously; registration clearance could not be verified.` + ); + return; } deps.log( `Removal of agent ${name} was initiated (status: ${result.status ?? 'pending'}) and is processed asynchronously. Use --wait to verify the name is reusable.` diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 702d9f403d..493f9e5fdd 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -49,6 +49,7 @@ vi.mock('@agent-relay/harness-driver', async (importOriginal) => ({ import { registerFleetCommands } from './fleet.js'; import { writeProjectWorkspaceKey } from '../lib/project-workspace-key.js'; import { spawnPlacementReceipt } from '../lib/spawn-lifecycle.js'; +import { MAX_TASK_BODY_BYTES } from '../lib/task-input.js'; const REPLAY_SANDBOX_ID = 'sbx_123e4567-e89b-42d3-a456-426614174000'; const REPLAY_SANDBOX_NAME = 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000'; @@ -1357,6 +1358,116 @@ describe('fleet command support', () => { }); }); + // relay#1930 review (cubic): the sandbox context is appended after + // readTaskInput accepted the task, so the composed task is what must fit. + it('rejects a sandbox task whose appended context pushes it past the PTY body limit, and cleans up the sandbox', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', undefined); + const revision = '0123456789abcdef0123456789abcdef01234567'; + const repositorySelection = { + repository: 'AgentWorkforce/cloud', + repositoryName: 'cloud', + revision, + projectRoot: '/local/cloud', + repositoryRelativeCwd: 'packages/web', + workerCwd: '/srv/agent-workforce/cloud/packages/web', + }; + const events: string[] = []; + const warnings: string[] = []; + const materializeCloudRelayfileRepository = vi.fn(async () => { + events.push('materialize'); + return { + cloudWorkspaceId: 'cloud-workspace', + repository: 'AgentWorkforce/cloud', + revision, + filesWritten: 4312, + sourceProfile: 'complete-v1' as const, + contentRoot: '/github/repos/AgentWorkforce/cloud/contents', + sentinelPath: '/github/repos/AgentWorkforce/cloud/.relayfile/clone.json', + }; + }); + const ensureCloudFleetSandbox = vi.fn(async () => { + events.push('ensure'); + return { + outcome: 'provisioned' as const, + providerId: 'agent37' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-live', + nodeName: 'live-node', + sandboxId: 'sandbox-live', + providerSandboxId: 'provider-live', + relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, + relayfileMounted: true, + relayfileMountPath: '/workspace', + }; + }); + const placement = { + spawn: vi.fn(async () => { + events.push('spawn'); + return { invocationId: 'inv_live', node: { name: 'live-node' } }; + }), + }; + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { + info: vi.fn(async () => ({ id: 'rw_abc' })), + register: vi.fn(async () => ({ token: 'at_live_launcher' })), + release: vi.fn(async () => ({ released: true, deleted: true })), + }, + })); + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const sdkError = vi.fn(); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + resolveSandboxRepository: vi.fn(() => repositorySelection), + materializeCloudRelayfileRepository, + ensureCloudFleetSandbox, + sdk: { + createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, + createWorkspaceRelay: createWorkspaceRelay as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: sdkError, + exit: vi.fn() as never, + }, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/local/cloud/.agentworkforce/relay/workspace-key.json', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: (message) => warnings.push(message), + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-provider', + 'agent37', + '--name', + 'cloud-live', + '--task', + 'x'.repeat(MAX_TASK_BODY_BYTES), + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + expect(sdkError.mock.calls.flat().join(' ')).toContain('UTF-8 bytes'); + expect(placement.spawn).not.toHaveBeenCalled(); + expect(deleteCloudFleetSandbox).toHaveBeenCalledWith( + expect.objectContaining({ sandboxId: 'sandbox-live' }) + ); + }); + it('plain --sandbox materializes the inferred repository through Relayfile and starts in its live relative cwd', async () => { vi.stubEnv('RELAY_AGENT_TOKEN', undefined); const revision = '0123456789abcdef0123456789abcdef01234567'; @@ -5361,6 +5472,46 @@ describe('fleet command support', () => { }); }); + // relay#1930 review (cubic): Number('abc') is NaN, which the wait helper + // silently normalized to its default instead of reporting invalid input. + it.each(['abc', '0', '-5'])( + 'fleet release --wait-timeout %s is rejected before anything is released', + async (timeout) => { + const release = vi.fn(async () => ({ status: 'dispatched' })); + const sdkError = vi.fn(); + const program = new Command(); + registerFleetCommands(program, { + sdk: { + createWorkspaceRelay: vi.fn() as never, + createAgentRelay: vi.fn() as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: sdkError, + exit: vi.fn() as never, + }, + createFleetWorkspaceClient: vi.fn(() => ({ agents: { release } })) as never, + retireOwnedBindings: vi.fn(async () => undefined), + warn: vi.fn(), + }); + await program.parseAsync( + [ + 'fleet', + 'release', + 'worker', + '--delete-agent', + '--wait', + '--wait-timeout', + timeout, + '--workspace-key', + 'rk_test', + ], + { from: 'user' } + ); + expect(sdkError.mock.calls.flat().join(' ')).toContain('--wait-timeout'); + expect(release).not.toHaveBeenCalled(); + } + ); + it('fleet release --delete-agent aborts without deleting when binding retirement fails', async () => { const release = vi.fn(async () => ({ name: 'api-worker', diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 439b1a77d1..abb79d73c3 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -1,4 +1,4 @@ -import { readTaskInput } from '../lib/task-input.js'; +import { readTaskInput, validateTaskSize } from '../lib/task-input.js'; import { randomUUID } from 'node:crypto'; import path from 'node:path'; @@ -39,7 +39,11 @@ import { readBrokerConnection } from '../lib/broker-lifecycle.js'; import { spawnAgentWithClient } from '../lib/client-factory.js'; import { formatRelativeTime, sanitizeForTerminalLine } from '../lib/formatting.js'; import { connectProjectBrokerClient } from '../lib/project-broker-client.js'; -import { AgentRemovalPendingError, waitForAgentRemoval } from '../lib/agent-removal.js'; +import { + AgentRemovalPendingError, + parseRemovalWaitTimeout, + waitForAgentRemoval, +} from '../lib/agent-removal.js'; import { classifySpawnFailure, mayStillBeRunning, @@ -1128,6 +1132,19 @@ export function registerFleetCommands( 'The Relayfile daemon synchronizes this tree; it intentionally has no .git directory.', ].join(' ') : undefined; + const spawnTask = + sandbox && + checkoutRepository && + sandboxRepository && + mountSandboxRelayfile && + sandbox.outcome === 'provisioned' + ? `${task}\n\nAgent Relay sandbox context: Relayfile records are available at ${sandbox.relayfileMountPath ?? '/workspace'}. The source checkout is separate; use ${workerCwd ?? 'the worker checkout'} for repository files and the mount for Relayfile records.` + : liveSandboxContext + ? `${task}\n\n${liveSandboxContext}` + : task; + // The sandbox context is appended after readTaskInput accepted the + // task, so the composed task is what the worker must receive. + validateTaskSize(spawnTask, cli); let invocation; try { invocation = await relay.messaging.placement.spawn({ @@ -1139,16 +1156,7 @@ export function registerFleetCommands( input: { name, cli, - task: - sandbox && - checkoutRepository && - sandboxRepository && - mountSandboxRelayfile && - sandbox.outcome === 'provisioned' - ? `${task}\n\nAgent Relay sandbox context: Relayfile records are available at ${sandbox.relayfileMountPath ?? '/workspace'}. The source checkout is separate; use ${workerCwd ?? 'the worker checkout'} for repository files and the mount for Relayfile records.` - : liveSandboxContext - ? `${task}\n\n${liveSandboxContext}` - : task, + task: spawnTask, ...(channel ? { channels: [channel] } : {}), ...(model ? { model } : {}), ...(workerCwd ? { worker_cwd: workerCwd } : {}), @@ -1315,6 +1323,8 @@ export function registerFleetCommands( warnIfInferredFromProjectSession(options, deps.warn); const workerName = requiredText(name, 'Worker name'); const deleteAgent = options.deleteAgent === true; + const waitTimeoutMs = + deleteAgent && options.wait === true ? parseRemovalWaitTimeout(options.waitTimeout) : undefined; const sdkOpts = sdkOptionsFromOpts(options); // One transport for both cleanup and release. Cleanup otherwise // substitutes the local broker session whenever --workspace-key is @@ -1363,13 +1373,13 @@ export function registerFleetCommands( reason, deleteAgent: true, }); - if (options.wait === true) { + if (waitTimeoutMs !== undefined) { // Construct/read on the same selected transport; unavailable credentials are uncertainty. const removal = await waitForAgentRemoval({ name: workerName, getAgent: (name) => deps.sdk.createWorkspaceRelay(transport).agents.get(name), listAgents: () => deps.sdk.createWorkspaceRelay(transport).agents.list(), - timeoutMs: Number(options.waitTimeout), + timeoutMs: waitTimeoutMs, }); printJson(deps.sdk, { ...deleted, removal }); if (!removal.cleared && removal.observedPresent) throw new AgentRemovalPendingError(workerName); diff --git a/packages/cli/src/cli/commands/integration-subscribe.test.ts b/packages/cli/src/cli/commands/integration-subscribe.test.ts index 552a876860..2eaf0423bc 100644 --- a/packages/cli/src/cli/commands/integration-subscribe.test.ts +++ b/packages/cli/src/cli/commands/integration-subscribe.test.ts @@ -44,6 +44,7 @@ function createRelayMock(opts: { inboundWebhooks?: InboundWebhook[] } = {}) { return row; }), get: vi.fn(async (id: string) => subscriptions.get(id) ?? { id }), + list: vi.fn(async () => [...subscriptions.values()]), delete: vi.fn(async () => undefined), }, }, @@ -2231,11 +2232,9 @@ describe('subscription writeback verification', () => { it('reports outbound configuration separately from inferred inbound activity', async () => { const relay = createRelayMock(); - relay.integrations.subscriptions.get.mockResolvedValue({ - id: 'sub_existing', - events: ['thread.reply'], - isActive: false, - }); + relay.integrations.subscriptions.list.mockResolvedValue([ + { id: 'sub_existing', events: ['thread.reply'], isActive: false }, + ] as never); relay.messages.list.mockResolvedValue([{ createdAt: '2026-10-05T00:00:00Z' }] as never); const relayfile = createRelayfileMock([ { @@ -2257,12 +2256,39 @@ it('reports outbound configuration separately from inferred inbound activity', a deliveryStatus: null, }, }); - relay.integrations.subscriptions.get.mockRejectedValue(new Error('unavailable')); + relay.integrations.subscriptions.list.mockRejectedValue(new Error('unavailable')); log.mockClear(); await program.parseAsync(['integration', 'subscribe', '--list'], { from: 'user' }); expect(JSON.parse(log.mock.calls[0][0]).bindings[0].writebackSubscription).toBeNull(); }); +// One collection read serves every bound row, so a large workspace cannot +// turn --list into one subscription request per binding. +it('reads writeback subscriptions once for all bound rows', async () => { + const relay = createRelayMock(); + relay.integrations.subscriptions.list.mockResolvedValue([ + { id: 'sub_a', events: ['thread.reply'], isActive: true }, + { id: 'sub_b', events: ['message.created'], isActive: true }, + ] as never); + const relayfile = createRelayfileMock( + ['sub_a', 'sub_b', 'sub_missing'].map((subscriptionId, index) => ({ + provider: 'github', + resource: `/github/repos/AgentWorkforce/relaycast/pulls/${index}/**`, + channel: 'general', + subscriptionId, + webhookId: `in_${index}`, + })) + ); + const { program, log } = harness({ relay, relayfile }); + await program.parseAsync(['integration', 'subscribe', '--list'], { from: 'user' }); + const rows = JSON.parse(log.mock.calls[0][0]).bindings; + expect( + rows.map((row: { writebackSubscription: { id: string } | null }) => row.writebackSubscription?.id ?? null) + ).toEqual(['sub_a', 'sub_b', null]); + expect(relay.integrations.subscriptions.list).toHaveBeenCalledTimes(1); + expect(relay.integrations.subscriptions.get).not.toHaveBeenCalled(); +}); + it.each([ { id: 'wrong', events: ['message.created', 'thread.reply'] }, { isActive: false, events: ['message.created', 'thread.reply'] }, diff --git a/packages/cli/src/cli/commands/integration.ts b/packages/cli/src/cli/commands/integration.ts index 0cdf8b74c0..65d446e1c9 100644 --- a/packages/cli/src/cli/commands/integration.ts +++ b/packages/cli/src/cli/commands/integration.ts @@ -671,6 +671,15 @@ async function enrichBindingsForList( ).catch(() => [] as Array<{ name: string; metadata?: Record }>); const channelByName = new Map(channels.map((channel) => [channel.name, channel])); + // One collection read for every bound row; per-row reads would burst one + // request per binding. Missing or unavailable engine status stays unknown. + const subscriptionById = bindings.some((binding) => binding.subscriptionId) + ? await relay.integrations.subscriptions + .list() + .then((rows) => new Map(rows.map((raw) => [String(raw.id), raw]))) + .catch(() => new Map>>()) + : new Map>>(); + return Promise.all( bindings.map(async (binding) => { const channelMeta = channelByName.get(binding.channel); @@ -700,9 +709,9 @@ async function enrichBindingsForList( : undefined; const health = cloudRow?.health; let writebackSubscription: ListedBinding['writebackSubscription'] = null; - if (binding.subscriptionId) { + const raw = binding.subscriptionId ? subscriptionById.get(binding.subscriptionId) : undefined; + if (raw) { try { - const raw = await relay.integrations.subscriptions.get(binding.subscriptionId); const subscription = normalizeWebhookSubscription(raw); const active = raw.isActive ?? raw.is_active; writebackSubscription = { @@ -713,7 +722,7 @@ async function enrichBindingsForList( deliveryStatus: null, }; } catch { - /* Missing or unavailable engine status remains unknown. */ + /* A malformed engine row remains unknown. */ } } return { diff --git a/packages/cli/src/cli/commands/local-agent.test.ts b/packages/cli/src/cli/commands/local-agent.test.ts index 5faf20bbc5..4ce0a48d72 100644 --- a/packages/cli/src/cli/commands/local-agent.test.ts +++ b/packages/cli/src/cli/commands/local-agent.test.ts @@ -20,6 +20,7 @@ import { withDeliveryStatus, type LocalAgentDependencies, } from './local-agent.js'; +import { MAX_TASK_BODY_BYTES } from '../lib/task-input.js'; function harness(overrides: Partial = {}) { const client = { @@ -996,6 +997,33 @@ describe('local agent subtree', () => { } ); + // relay#1930 review (cubic): auto-routing wraps the task in a Director + // prompt, so the routed task is what must fit the PTY body limit. + it('rejects an auto-routed task whose Director prompt exceeds the PTY body limit', async () => { + const { program, client, error, exit } = harness(); + await program.parseAsync( + ['local', 'agent', 'spawn', 'claude', '--model', 'auto', '--task', 'x'.repeat(MAX_TASK_BODY_BYTES)], + { from: 'user' } + ); + expect(error.mock.calls.flat().join(' ')).toContain('UTF-8 bytes'); + expect(exit).toHaveBeenCalledWith(1); + expect(client.spawnPty).not.toHaveBeenCalled(); + }); + + it.each(['spawn', 'new'])( + '%s reports an unreadable --task-file through the CLI error path', + async (command) => { + const { program, client, error, exit } = harness(); + await program.parseAsync( + ['local', 'agent', command, 'codex', '--task-file', '/nonexistent/relay-brief.md'], + { from: 'user' } + ); + expect(error.mock.calls.flat().join(' ')).toContain('ENOENT'); + expect(exit).toHaveBeenCalledWith(1); + expect(client.spawnPty).not.toHaveBeenCalled(); + } + ); + it('release calls client.release', async () => { const { program, client } = harness(); await program.parseAsync(['local', 'agent', 'release', 'lead'], { from: 'user' }); diff --git a/packages/cli/src/cli/commands/local-agent.ts b/packages/cli/src/cli/commands/local-agent.ts index ef018d5bd5..13aa8e4259 100644 --- a/packages/cli/src/cli/commands/local-agent.ts +++ b/packages/cli/src/cli/commands/local-agent.ts @@ -1,4 +1,4 @@ -import { readTaskInput } from '../lib/task-input.js'; +import { readTaskInput, validateTaskSize } from '../lib/task-input.js'; import type { Command } from 'commander'; import { AGENT37_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; @@ -61,6 +61,8 @@ function resolveAutoSpawn( const assessment = classifyTask(task); const team = composeTeam(assessment, task); const directorPrompt = buildDirectorPrompt(task, team); + // The Director prompt repeats the task, so it is what must fit. + validateTaskSize(directorPrompt, provider); return { name: name === provider ? 'Director' : name, task: directorPrompt, @@ -933,12 +935,19 @@ export function registerLocalAgentCommands( return; const brokerOptions = brokerOptionsFromOpts(options); const baseName = (options.name as string | undefined) ?? provider; - const resolved = resolveAutoSpawn( - provider, - baseName, - await readTaskInput(options.task, options.taskFile, false, provider), - options.model as string | undefined - ); + let resolved: ReturnType; + try { + resolved = resolveAutoSpawn( + provider, + baseName, + await readTaskInput(options.task, options.taskFile, false, provider), + options.model as string | undefined + ); + } catch (err) { + deps.error(err instanceof Error ? err.message : String(err)); + deps.exit(1); + return; + } await run( deps, async (client) => { diff --git a/packages/cli/src/cli/commands/relaycast-groups.test.ts b/packages/cli/src/cli/commands/relaycast-groups.test.ts index ffc2dad9fd..f42596d3ef 100644 --- a/packages/cli/src/cli/commands/relaycast-groups.test.ts +++ b/packages/cli/src/cli/commands/relaycast-groups.test.ts @@ -707,6 +707,9 @@ describe('SDK-backed CLI groups', () => { relayfile, cleanupJournal: memoryJournal(), } satisfies Partial); + relay.integrations.subscriptions.list.mockResolvedValueOnce([ + { id: 'sub1', events: ['message.created', 'thread.reply'] } as never, + ]); await program.parseAsync(['integration', 'subscribe', '--list'], { from: 'user' }); diff --git a/packages/cli/src/cli/lib/agent-removal.ts b/packages/cli/src/cli/lib/agent-removal.ts index 5b367dce68..954c5b5304 100644 --- a/packages/cli/src/cli/lib/agent-removal.ts +++ b/packages/cli/src/cli/lib/agent-removal.ts @@ -13,6 +13,17 @@ export function isNotFoundError(error: unknown): boolean { return Number(status) === 404; } +/** Parse `--wait-timeout`; reject what the wait would otherwise silently replace with its default. */ +export function parseRemovalWaitTimeout(value: unknown): number { + const timeoutMs = Number(value); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error( + `--wait-timeout must be a positive number of milliseconds (got ${JSON.stringify(value)}).` + ); + } + return timeoutMs; +} + export interface RemovalResult { cleared: boolean; waitedMs: number; diff --git a/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts b/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts index 045e3f9b56..ea9f0a0b8f 100644 --- a/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts +++ b/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts @@ -140,15 +140,18 @@ describe('fleet spawn confirmation is observable from the requester (#1430)', () // success before this change, because nothing read the action result. The // broker's detail (startup exit status and worker log path) must survive. it.each([ - 'spawn_failed: provider_auth_required', - 'spawn_provider_auth_required', - 'spawn_directory_trust_required', - ])('preserves %s from the node', async (reason) => { + ['spawn_failed: provider_auth_required', 'run muse on the selected node to log in'], + ['spawn_provider_auth_required', 'run muse on the selected node to log in'], + [ + 'spawn_directory_trust_required', + 'Devin requires directory trust; run `devin` in the explicit spawn working directory on the selected node, trust that directory, then retry', + ], + ])('preserves %s and its remediation from the node', async (reason, detail) => { const { client } = createClient(async (name, id) => ({ invocation_id: id, action_name: name, status: 'failed', - error: `${reason}: run muse on the selected node to log in`, + error: `${reason}: ${detail}`, })); const error = await client.placement .spawn(spawnInput({ confirm: true, confirmTimeoutMs: 60, confirmPollIntervalMs: 10 })) @@ -156,6 +159,7 @@ describe('fleet spawn confirmation is observable from the requester (#1430)', () expect(error).toBeInstanceOf(RelayPlacementError); expect((error as RelayPlacementError).code).toBe('spawn_failed'); expect((error as Error).message).toContain(reason); + expect((error as Error).message).toContain(detail); }); it('fails with spawn_failed and preserves the node-reported detail', async () => { diff --git a/packages/cli/src/cli/lib/spawn-liveness.test.ts b/packages/cli/src/cli/lib/spawn-liveness.test.ts index 57ef6bdc2b..f9b6917b8f 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.test.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.test.ts @@ -48,6 +48,42 @@ describe('spawn liveness resolution', () => { ).toMatchObject({ evidence }); if (evidence === 'live') expect(workspace.nodes.list).not.toHaveBeenCalled(); }); + // relay#1930 review (cubic): a stale or unavailable target entry must not + // hide a fresh same-named worker on another node. + it.each([ + ['stale target', node('target', 36_001)], + ['unavailable target', { ...node(), status: 'offline' }], + ] as const)( + 'scans the fleet past a %s and prefers fresh live_elsewhere evidence', + async (_label, target) => { + const workspace = client([target, node('other')]); + workspace.nodes.get.mockResolvedValue(target); + workspace.nodes.list.mockResolvedValue([node('other')]); + expect( + await probeSpawnLiveness({ + name: 'worker', + targetNode: 'target', + createClient: () => workspace, + now: () => now, + }) + ).toMatchObject({ evidence: 'live_elsewhere', node: 'other' }); + expect(workspace.nodes.list).toHaveBeenCalledOnce(); + } + ); + it('points a live_elsewhere recovery check at the whole fleet, not the requested node', () => { + const pending = pendingSpawnError( + 'worker', + { invocationId: 'inv', node: 'target' }, + { evidence: 'live_elsewhere', node: 'other', heartbeatAgeMs: 0 } + ); + expect(pending.message).toContain('check `agent-relay fleet agent list` first'); + const targeted = pendingSpawnError( + 'worker', + { invocationId: 'inv', node: 'target' }, + { evidence: 'stale' } + ); + expect(targeted.message).toContain('fleet agent list --node'); + }); it('folds construction errors into unknown without replacing accepted dispatch evidence', async () => { const result = await probeSpawnLiveness({ name: 'worker', diff --git a/packages/cli/src/cli/lib/spawn-liveness.ts b/packages/cli/src/cli/lib/spawn-liveness.ts index 97f3a0b2ce..ced83ab76b 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.ts @@ -63,7 +63,9 @@ export function pendingSpawnError( liveness: SpawnLiveness, diagnostic?: string ): FleetSpawnError { - const nodeOption = context.node ? ` --node ${shellQuote(context.node)}` : ''; + // A worker found elsewhere is invisible to a node-scoped check. + const nodeOption = + context.node && liveness.evidence !== 'live_elsewhere' ? ` --node ${shellQuote(context.node)}` : ''; // Heartbeats carry worker names, not invocation IDs, so presence cannot be // attributed to this spawn: it may be a worker that was already running. const attribution = @@ -136,13 +138,21 @@ export async function probeSpawnLiveness(options: { try { const client = options.createClient(); try { + const now = options.now?.() ?? Date.now(); const target = options.targetNode ? await read(() => client.nodes.get(options.targetNode!)) : undefined; const nodes = options.targetNode && target ? [target] : await read(() => client.nodes.list()); - // Prefer the targeted read; consult the fleet only if it doesn't claim this name. + // Prefer the targeted read; consult the fleet unless the target is an + // available node whose fresh heartbeat claims this name. + const targetAge = target ? heartbeatAgeMs(target, now) : null; if ( options.targetNode && target && - !readRemoteLiveAgents(target).agents.some((a) => a.name === options.name) + !( + isAvailableFleetNode(target) && + targetAge !== null && + targetAge <= MAX_LIVE_HEARTBEAT_AGE_MS && + readRemoteLiveAgents(target).agents.some((a) => a.name === options.name) + ) ) { nodes.push(...(await read(() => client.nodes.list()))); } @@ -153,7 +163,7 @@ export async function probeSpawnLiveness(options: { !readRemoteLiveAgents(node).agents.some((agent) => agent.name === options.name) ) continue; - const age = heartbeatAgeMs(node, options.now?.() ?? Date.now()); + const age = heartbeatAgeMs(node, now); const evidence = age === null || age > MAX_LIVE_HEARTBEAT_AGE_MS ? 'stale' @@ -162,7 +172,8 @@ export async function probeSpawnLiveness(options: { : 'live'; const result: SpawnLiveness = { evidence, node: node.name, heartbeatAgeMs: age }; if (evidence === 'live') return result; - weaker ??= result; + // Fresh evidence elsewhere outranks a stale entry. + if (!weaker || (weaker.evidence === 'stale' && evidence === 'live_elsewhere')) weaker = result; } if (weaker) return weaker; } catch (error) { diff --git a/packages/cli/src/cli/lib/task-input.test.ts b/packages/cli/src/cli/lib/task-input.test.ts index 08a7c6b104..54b8aea514 100644 --- a/packages/cli/src/cli/lib/task-input.test.ts +++ b/packages/cli/src/cli/lib/task-input.test.ts @@ -27,6 +27,16 @@ describe('task input', () => { await rm(dir, { recursive: true, force: true }); } }); + // Only one byte past the limit is ever read, so an oversized or endless + // --task-file fails cheaply instead of being materialized in memory. + it.skipIf(process.platform === 'win32')( + 'reads at most one byte past the limit from a task file', + async () => { + await expect(readTaskInput(undefined, '/dev/zero')).rejects.toThrow('UTF-8 bytes'); + await expect(readTaskInput(undefined, '/dev/zero', false, 'muse')).rejects.toThrow('argv'); + }, + 2000 + ); it('counts UTF-8 bytes', async () => { expect(await readTaskInput('a'.repeat(MAX_TASK_BODY_BYTES), undefined)).toHaveLength(MAX_TASK_BODY_BYTES); await expect(readTaskInput('é'.repeat(MAX_TASK_BODY_BYTES), undefined)).rejects.toThrow('UTF-8 bytes'); diff --git a/packages/cli/src/cli/lib/task-input.ts b/packages/cli/src/cli/lib/task-input.ts index 1ac2cf52b1..0a12b2a2a2 100644 --- a/packages/cli/src/cli/lib/task-input.ts +++ b/packages/cli/src/cli/lib/task-input.ts @@ -1,4 +1,4 @@ -import { readFile } from 'node:fs/promises'; +import { open } from 'node:fs/promises'; /** Broker-wide PTY ceiling for the formatted envelope; a harness may reject a lower effective wire limit. */ export const MAX_INJECTION_BODY_BYTES = 16 * 1024; @@ -27,12 +27,39 @@ export async function readTaskInput( if (required) throw new Error('Specify exactly one of --task or --task-file'); return undefined; } - const text = taskFile === undefined ? task : await readFile(String(taskFile), 'utf8'); - if (typeof text !== 'string' || !text.trim()) throw new Error('Task must not be empty'); // Muse receives its startup task through argv, not the PTY envelope. + const muse = cli !== undefined && isMuseExecutable(cli); + const limit = muse ? MUSE_STARTUP_PROMPT_MAX_BYTES : MAX_TASK_BODY_BYTES; + const text = taskFile === undefined ? task : await readFilePrefix(String(taskFile), limit + 1); + if (typeof text !== 'string' || !text.trim()) throw new Error('Task must not be empty'); + validateTaskSize(text, cli); + return text; +} + +/** Validate a task as the given CLI receives it; recheck after anything is appended to it. */ +export function validateTaskSize(text: string, cli?: string): void { if (cli !== undefined && isMuseExecutable(cli)) validateMuseStartupPromptSize(text); else validateInjectionSize(text); - return text; +} + +/** + * Read at most `maxBytes` of a UTF-8 file. Reading one byte past a size limit + * is enough to reject an oversized file without materializing all of it. + */ +async function readFilePrefix(path: string, maxBytes: number): Promise { + const handle = await open(path, 'r'); + try { + const buffer = Buffer.alloc(maxBytes); + let length = 0; + while (length < maxBytes) { + const { bytesRead } = await handle.read(buffer, length, maxBytes - length, null); + if (bytesRead === 0) break; + length += bytesRead; + } + return buffer.subarray(0, length).toString('utf8'); + } finally { + await handle.close(); + } } function validateMuseStartupPromptSize(text: string): void { diff --git a/packages/cli/src/cli/mcp/messaging-tools.ts b/packages/cli/src/cli/mcp/messaging-tools.ts index 6cd6df49b5..0a415fefe0 100644 --- a/packages/cli/src/cli/mcp/messaging-tools.ts +++ b/packages/cli/src/cli/mcp/messaging-tools.ts @@ -62,11 +62,6 @@ function resolveEmoji(input: string): string { return aliases[normalized] ?? input; } -/** - * Register the channel, message, thread, DM, reaction, search, and inbox MCP - * tools. These all act through a single agent client resolved per-call from the - * optional `as` identity override. - */ const idempotencyKeyInput = z .string() // Relaycast trims this key upstream, so trim before both the local replay @@ -80,6 +75,11 @@ const idempotencyKeyInput = z 'Stable key for retrying this same message after a lost response; use a new key for a new message. Surrounding whitespace is trimmed and a whitespace-only key is rejected.' ); +/** + * Register the channel, message, thread, DM, reaction, search, and inbox MCP + * tools. These all act through a single agent client resolved per-call from the + * optional `as` identity override. + */ export function registerMessagingTools( server: McpServer, getAgentClient: (asIdentity?: string) => AgentClientLike, diff --git a/packages/harness-driver/src/protocol.ts b/packages/harness-driver/src/protocol.ts index c27d02c359..ed7ac25342 100644 --- a/packages/harness-driver/src/protocol.ts +++ b/packages/harness-driver/src/protocol.ts @@ -621,7 +621,9 @@ export type BrokerEvent = * 'paste_summary' (the harness collapsed the paste, so no content was * echoed) and 'timeout_fallback' (no echo at all) are acks without proof * of receipt. A tail without its head is a 'delivery_failed' with reason - * 'echo_head_missing'. + * 'echo_head_missing'. Workers that predate this contract report only + * 'echo' (verbatim) and 'timeout_fallback'; treat any value not listed + * here as an ack without proof of receipt. */ verification?: string; reason?: string; diff --git a/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts b/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts index c9a17f2059..22c8568ad1 100644 --- a/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts +++ b/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts @@ -86,6 +86,26 @@ it.each([ expect(write).toHaveBeenCalledTimes(5); }); +// A replay that arrives after the first write settled (for example while its +// tool result is still being submitted) reaches Relay again, so the write +// carries a key derived from the tool call for Relay to deduplicate. +it.each([ + ['post_message', 'send', { channel: 'events', text: 'ACK' }, 'events'], + ['reply_to_thread', 'reply', { message_id: 'parent', text: 'ACK' }, 'parent'], +] as const)('sends native %s with a tool-call idempotency key', async (tool, method, args, target) => { + const write = vi.fn(async () => ({ id: 'm1' })); + const execute = createNativeRelayTools({ + env: { RELAY_AGENT_TOKEN: 'test' }, + agentClient: { [method]: write } as never, + }).find((item) => item.spec.name === tool)!.execute; + await execute(args, { toolCallId: 'call-1' }); + await execute(args, { toolCallId: 'call-1' }); + await execute(args, {}); + expect(write).toHaveBeenNthCalledWith(1, target, 'ACK', { idempotencyKey: `native:${tool}:call-1` }); + expect(write).toHaveBeenNthCalledWith(2, target, 'ACK', { idempotencyKey: `native:${tool}:call-1` }); + expect(write).toHaveBeenNthCalledWith(3, target, 'ACK'); +}); + it('clears a rejected native reply so the same tool call can be retried', async () => { const reply = vi.fn().mockRejectedValueOnce(new Error('offline')).mockResolvedValue({ id: 'retry' }); const execute = createNativeRelayTools({ diff --git a/packages/harnesses/src/ai-sdk/native-relay-tools.ts b/packages/harnesses/src/ai-sdk/native-relay-tools.ts index 446f251df2..d23246d5c8 100644 --- a/packages/harnesses/src/ai-sdk/native-relay-tools.ts +++ b/packages/harnesses/src/ai-sdk/native-relay-tools.ts @@ -56,14 +56,21 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha // One map per native session, keyed by the model's tool call ID so only a // replay of the same invocation joins a pending write. Text never forms the // key: independent identical replies are separate messages. A completed - // write is never retained. + // write is never retained locally; a replay after it settled reaches Relay + // with the same tool-call idempotency key, which Relay deduplicates. const pendingWrites = new Map>(); - function write(tool: string, toolCallId: string | undefined, operation: () => Promise) { - if (!toolCallId) return Promise.resolve().then(operation); + function write( + tool: string, + toolCallId: string | undefined, + operation: (options?: { idempotencyKey: string }) => Promise + ) { + if (!toolCallId) return Promise.resolve().then(() => operation()); const key = JSON.stringify([tool, toolCallId]); const existing = pendingWrites.get(key); if (existing) return existing; - const pending = Promise.resolve().then(operation); + const pending = Promise.resolve().then(() => + operation({ idempotencyKey: `native:${tool}:${toolCallId}` }) + ); pendingWrites.set(key, pending); void pending.then( () => pendingWrites.delete(key), @@ -110,7 +117,9 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha const input = objectInput(value); const target = requiredString(input, 'channel'); const text = requiredString(input, 'text'); - return write('post_message', toolCallId, () => agent.send(target, text)); + return write('post_message', toolCallId, (options) => + options ? agent.send(target, text, options) : agent.send(target, text) + ); }, }, { @@ -173,7 +182,9 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha const input = objectInput(value); const target = requiredString(input, 'message_id'); const text = requiredString(input, 'text'); - return write('reply_to_thread', toolCallId, () => agent.reply(target, text)); + return write('reply_to_thread', toolCallId, (options) => + options ? agent.reply(target, text, options) : agent.reply(target, text) + ); }, }, { From cd303e7b8b13277868e853c7b432800fe811760e Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 12:36:45 -0700 Subject: [PATCH 33/42] docs: correct bundled changelog placement and review-flagged docs Move four #1893 entries that auto-merged under the released 13.1.1 into [Unreleased - Minor] (released sections now match main) and split combined bullets. Fix the Devin trust recovery advice, the local --task/--task-file wording, pending-spawn README fields and the evidence doc's plan reference. Decode stub-agent stdin as a UTF-8 stream and give the resumable-spawn e2e a timeout that covers its own waits. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- CHANGELOG.md | 16 ++++++++-------- .../evidence/provider-subscription-replies.md | 19 +++++++++++++++---- docs/harnesses/devin.md | 4 +++- docs/harnesses/injection.md | 3 ++- tests/e2e/fleet/fleet-e2e.test.ts | 4 +++- tests/e2e/fleet/nodes/stub-agent.cjs | 2 ++ 6 files changed, 33 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f04c257df9..75189d6afb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,11 +14,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - MCP `spawn` reports pending/liveness evidence, and `remove_agent` accepts optional registration-clearance waits. - `agent remove --wait` and `fleet release --delete-agent --wait` verify registration clearance before name reuse, with `--wait-timeout` and `--no-wait` controls. Fleet release adds `removal` evidence when waiting. - `fleet agent list` JSON includes node heartbeat timestamps and ages, with warnings for stale snapshots. +- `fleet spawn`, `agent spawn`, and `node agent spawn` accept `--task-file` to read an initial brief from a local UTF-8 file. Fleet requires exactly one of `--task` and `--task-file`. ### Changed - Unconfirmed `fleet spawn` results keep the `spawn_unconfirmed` code, now with `state: "pending"`, invocation diagnostics, liveness evidence and exit 8. Node heartbeat presence is reported as liveness evidence but never confirms the spawn, because heartbeats cannot tell this invocation's worker from an earlier one with the same name. - Removal waits exit 8 if the registration remains present; scripts must handle nonzero exits beyond `$? -eq 1`. Waits remain opt-in, and unavailable verification reads preserve the asynchronous acknowledgement. +- PTY-delivered tasks and messages are limited to 14 KiB (16 KiB with the message envelope); `fleet spawn`, `agent spawn` and PTY delivery reject larger input and name the limit, so write the brief to a file and send a pointer. Agents whose terminal lacks bracketed paste accept at most 1,536 bytes per injection. Muse startup tasks keep their 16 KiB argument limit, headless agents are not capped, `message post|reply|dm send` are not capped, and a wrapped agent receives an oversized message as a pointer to it in Relay. ### Fixed @@ -32,9 +34,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `agent-relay fleet spawn` reports registration `d1_pressure` as retryable only when the registration is proven to have failed before any change was made. - Verified Fleet Devin spawns fail with `directory_trust_required` when the directory-trust prompt persists, releasing the worker through spawn cleanup instead of leaving a live but blocked worker. - Relay MCP `post_message` and `reply_to_thread` coalesce a transport replay of one request (and accept an `idempotency_key` for retries after a lost response), matching `send_dm`; native host tools key on the tool call ID. Independent writes with identical text remain separate messages. -- `integration subscribe` validates `--events` against the engine's full subscribable event list, verifies persisted writeback events, rolls back incomplete subscriptions, and reports GitHub authorization uncertainty; `--list` distinguishes subscription configuration from delivery confirmation. +- `integration subscribe` rejects `--events` values the engine cannot subscribe to. +- `integration subscribe` checks that the requested writeback events were persisted and rolls back a subscription left incomplete. +- `integration subscribe` says when GitHub authorization for writeback could not be verified, instead of implying it succeeded. +- `integration subscribe --list` separates a subscription's configuration from delivery confirmation, so a configured subscription is no longer read as a delivered one. - Fleet registration collisions use readable `spawn_name_taken` guidance covering asynchronous removal and failed-spawn cleanup, instead of Rust `Fatal(AlreadyExists {...})` text. - Pending fleet spawns preserve provisioned sandboxes and distinguish live, stale, elsewhere, registered, absent, and unknown evidence before advising a retry. +- PTY tasks and relay messages use atomic bracketed paste when supported, wait for the composer, and prevent embedded carriage returns from submitting partial input. Oversized bodies and tail-only echoes fail explicitly without replay. +- `fleet spawn --task` reports success only when the agent's terminal echoed the whole task; a task that fails to deliver releases the worker before the spawn fails with `spawn_task_failed`. A task acked without that proof fails with `spawn_task_unconfirmed`, which names the live agent and says not to retry, and `delivery_verified` now reports what was observed (`echo_incomplete`, `paste_summary`, `timeout_fallback`) instead of treating head/tail anchors or a collapsed-paste marker as receipt. ## [13.1.5] - 2026-10-08 @@ -83,17 +90,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `fleet spawn`, `agent spawn`, and `node agent spawn` accept `--task-file` to read an initial brief from a local UTF-8 file. Fleet requires exactly one of `--task` and `--task-file`. - `node agent list|spawn|new|release|set-model` and `node tail` accept `--state-dir`, `--broker-url`, and `--api-key`, so brokers started with `node up --state-dir` — such as fleet nodes — can be managed from any directory. Without these flags, `RELAY_BROKER_URL` / `RELAY_BROKER_API_KEY` or the enclosing project's broker is used. -### Changed - -- PTY injections are capped at 16 KiB including their envelope, and task and PTY-delivered message bodies at 14 KiB so the envelope always fits. `fleet spawn`, the fleet spawn action (before launching any worker) and the broker's PTY delivery reject larger input naming the limit, and the paced fallback wire rejects above 1,536 bytes; write the brief to a file and send a pointer instead. `message post|reply|dm send` are not capped, and a wrapped agent receives an oversized message as a notice pointing to it in Relay. - ### Fixed -- PTY tasks and relay messages use atomic bracketed paste when supported, wait for the composer, and prevent embedded carriage returns from submitting partial input. Oversized bodies and tail-only echoes fail explicitly without replay. -- `fleet spawn --task` reports success only when the agent's terminal echoed the whole task; a task that fails to deliver releases the worker before the spawn fails with `spawn_task_failed`. A task acked without that proof fails with `spawn_task_unconfirmed`, which names the live agent and says not to retry, and `delivery_verified` now reports what was observed (`echo_incomplete`, `paste_summary`, `timeout_fallback`) instead of treating head/tail anchors or a collapsed-paste marker as receipt. - Broker fleet control keeps a healthy node WebSocket open when Relaycast returns retryable `d1_pressure` for `node.register` or `inventory.sync`, retrying the same frame with bounded exponential backoff while the existing registration and application-liveness deadlines remain fail-closed. - `--state-dir` on `node agent` commands, `node status`, and `node down` also accepts a fleet node directory whose broker state lives in `state/`. - An explicit `--state-dir` on `node agent` commands is no longer overridden by `RELAY_BROKER_URL` / `RELAY_BROKER_API_KEY`. diff --git a/docs/evidence/provider-subscription-replies.md b/docs/evidence/provider-subscription-replies.md index e1f4141e18..22f7279506 100644 --- a/docs/evidence/provider-subscription-replies.md +++ b/docs/evidence/provider-subscription-replies.md @@ -1,7 +1,17 @@ # Provider subscription reply investigation (2026-10-05) -This change implements the locally verifiable portions of `reviewed-plan.md`. -It does **not** establish that the reported GitHub writeback failure is fixed. +This change implements the locally verifiable portions of the reviewed plan +for #1823 (relay#1910). It does **not** establish that the reported GitHub +writeback failure is fixed. + +Evidence labels used below: + +- **E1–E3**: authenticated reads of the incident's message and transcript, its + subscription, and its provider binding. +- **E4**: whether cloud subscription writeback ingress goes through the + file-write ops ledger. +- **E5**: whether the Relaycast engine enforces `Idempotency-Key` on channel + posts and thread replies. ## Available evidence @@ -21,8 +31,9 @@ message inbox check` reports no workspace key. No authenticated message, file-write ops ledger (E4 remains open). - Installed `@relaycast/sdk/dist/agent.js` supplies `Idempotency-Key` headers for channel posts and thread replies, including generated keys. That proves - transport behavior, not engine enforcement. No explicit `idempotency_key` - tool argument is added without E5 confirmation. + transport behavior, not engine enforcement (E5 remains open). The explicit + `idempotency_key` tool argument added later is forwarded to Relaycast and + also deduplicates locally, so it does not depend on E5. - The existing changelog also records a standalone MCP double-dispatch fix in 13.1.0. The incident was on 12.4.0; without its transcript, parallel aliases cannot be asserted as the incident's root cause. diff --git a/docs/harnesses/devin.md b/docs/harnesses/devin.md index 16dbfc26ed..c9c1e5857f 100644 --- a/docs/harnesses/devin.md +++ b/docs/harnesses/devin.md @@ -29,7 +29,9 @@ the worker reports `directory_trust_required` with instructions to trust the explicit spawn working directory. Verified fleet spawns return `spawn_directory_trust_required` and release the worker through spawn cleanup. An unverified (`confirm: false`) spawn reports the worker error but does not reap -the harness; stop it explicitly or attach to resolve trust. Already-trusted +the harness. Once reported, the worker never becomes ready, so trusting the +directory in an attached terminal does not resume it or deliver its initial +task: stop it, trust the directory, then spawn again. Already-trusted installations retain the existing startup path. Relay does not yet answer this menu automatically: that requires captured evidence of Devin's actual menu and trust scope. diff --git a/docs/harnesses/injection.md b/docs/harnesses/injection.md index 782f9a82e5..006d238ecf 100644 --- a/docs/harnesses/injection.md +++ b/docs/harnesses/injection.md @@ -39,7 +39,8 @@ agent-relay node agent spawn claude --name reviewer --task-file ./brief.md `--task-file` reads a UTF-8 file on the **requester's machine**, then sends the contents through the usual task transport. It does not create a file on the node or bypass size limits. Fleet requires exactly one of `--task` and `--task-file`; -local `agent spawn` and `node agent spawn` allow neither. The integration command's +local `agent spawn` and `node agent spawn` accept either but require neither, and +reject both together. The integration command's existing task option remains unchanged. Only a whole-payload echo confirms receipt: `echo` (verbatim) or diff --git a/tests/e2e/fleet/fleet-e2e.test.ts b/tests/e2e/fleet/fleet-e2e.test.ts index 2f80194092..219a7eb6c9 100644 --- a/tests/e2e/fleet/fleet-e2e.test.ts +++ b/tests/e2e/fleet/fleet-e2e.test.ts @@ -633,7 +633,9 @@ describe.skipIf(!pre.ok)('two-node fleet scenario matrix', () => { args: expect.arrayContaining(['resume', sessionRef]), }); }, - 180_000 + // Two 60 s spawn settles, two 15 s brief observations and a 30 s release + // wait alone reach 180 s, so leave room for the actions between them. + 240_000 ); it('propagates unsupported generic session resume as a terminal failure with no retained identity', async () => { diff --git a/tests/e2e/fleet/nodes/stub-agent.cjs b/tests/e2e/fleet/nodes/stub-agent.cjs index 02e59e0218..f8631dd37f 100644 --- a/tests/e2e/fleet/nodes/stub-agent.cjs +++ b/tests/e2e/fleet/nodes/stub-agent.cjs @@ -53,6 +53,8 @@ function renderCodexComposer(text) { } try { + // Decode as a stream: a UTF-8 character can straddle two PTY reads. + process.stdin.setEncoding('utf8'); process.stdin.resume(); process.stdin.on('data', (chunk) => { if (!ready) return; From 31fde2249759b775d4b1e55639938de27d874d69 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 13:03:40 -0700 Subject: [PATCH 34/42] fix(broker): bound headless delivery by the single-argument ceiling Headless providers receive the body as one argv entry, so dropping the PTY envelope cap for them left an exec-time E2BIG instead of a clear error. They now get the same portable 16 KiB ceiling as Muse. Also assert the headless fleet-spawn test reaches registration, and reattach paste_mode_enabled's doc. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- crates/broker/src/runtime/delivery.rs | 25 ++++++++----- crates/broker/src/runtime/tests.rs | 51 +++++++++++++++++++++++---- crates/relay-pty/src/pty.rs | 2 +- 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/crates/broker/src/runtime/delivery.rs b/crates/broker/src/runtime/delivery.rs index be46fb61a6..556e7c16db 100644 --- a/crates/broker/src/runtime/delivery.rs +++ b/crates/broker/src/runtime/delivery.rs @@ -862,18 +862,27 @@ pub(crate) async fn queue_and_try_delivery_raw( withheld_fleet_ack_floor: Option, ) -> Result { // The envelope-sized cap exists because PTY delivery types the message into - // a terminal. Headless recipients never do, so only a known non-PTY worker - // skips it; an unknown recipient keeps the conservative check. + // a terminal, so it applies to PTY and unknown recipients. A headless + // provider instead receives the body as one argv entry, which is bounded + // by the portable single-argument ceiling rather than the PTY envelope. let pty_recipient = workers .workers .get(worker_name) .is_none_or(|handle| handle.spec.runtime == AgentRuntime::Pty); - anyhow::ensure!( - !pty_recipient || body.len() <= crate::injection_wire::MAX_BODY_BYTES, - "injection_too_large: body limit is {} bytes so the formatted envelope fits the {}-byte PTY limit; use a brief file pointer", - crate::injection_wire::MAX_BODY_BYTES, - crate::injection_wire::MAX_INJECTION_BODY_BYTES - ); + if pty_recipient { + anyhow::ensure!( + body.len() <= crate::injection_wire::MAX_BODY_BYTES, + "injection_too_large: body limit is {} bytes so the formatted envelope fits the {}-byte PTY limit; use a brief file pointer", + crate::injection_wire::MAX_BODY_BYTES, + crate::injection_wire::MAX_INJECTION_BODY_BYTES + ); + } else { + anyhow::ensure!( + body.len() <= crate::worker::MUSE_STARTUP_PROMPT_MAX_BYTES, + "injection_too_large: a headless agent receives the body as one process argument, limited to {} bytes; use a brief file pointer", + crate::worker::MUSE_STARTUP_PROMPT_MAX_BYTES + ); + } // Fleet delivery IDs are stable across Relaycast retries. Preserve that // identity all the way into the worker so its completed-delivery cache can // re-ACK a replay without pasting the instruction a second time. Local diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index a26abe5940..2a432a72be 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -1270,17 +1270,25 @@ async fn an_oversized_task_for_a_headless_fleet_spawn_skips_the_pty_guard() { ), ) .await; + let mut registered = false; while let Ok(command) = fixture.fleet_control_rx.try_recv() { - if let FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) = command { - if let crate::fleet_wire::ActionResultPayload::Error(error) = &result.result { - assert!( - !error.error.contains("spawn_task_too_large"), - "a headless spawn never types its task into a PTY: {}", - error.error - ); + match command { + FleetControlCommand::RegisterAgent { .. } => registered = true, + FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) => { + if let crate::fleet_wire::ActionResultPayload::Error(error) = &result.result { + assert!( + !error.error.contains("spawn_task_too_large"), + "a headless spawn never types its task into a PTY: {}", + error.error + ); + } } + _ => {} } } + // Registration is the next awaited step after the guard, so observing it + // proves the oversized headless task passed the guard. + assert!(registered, "the headless spawn must reach registration"); } /// relay#1893 review (Cursor): a task that cannot fit the PTY envelope used to @@ -4318,6 +4326,35 @@ async fn pty_body_cap_applies_only_to_pty_recipients() { } } } + // Headless providers receive the body as one argv entry, so they keep + // the portable single-argument ceiling instead of no bound at all. + let mut workers = make_worker_registry_with_worker("Worker").await; + workers + .workers + .get_mut(&WorkerName::from("Worker")) + .unwrap() + .spec + .runtime = AgentRuntime::Headless; + let error = super::queue_and_try_delivery_raw( + &mut workers, + &mut HashMap::new(), + "Worker", + "evt_argv", + "orchestrator", + "Worker", + &"x".repeat(crate::worker::MUSE_STARTUP_PROMPT_MAX_BYTES + 1), + None, + Some(WorkspaceId::new("ws_demo")), + None, + 2, + MessageInjectionMode::Wait, + Duration::from_secs(1), + None, + None, + ) + .await + .expect_err("a headless body past the argv ceiling fails before delivery"); + assert!(error.to_string().contains("injection_too_large"), "{error}"); } #[tokio::test] diff --git a/crates/relay-pty/src/pty.rs b/crates/relay-pty/src/pty.rs index 626023ed28..b5ecced055 100644 --- a/crates/relay-pty/src/pty.rs +++ b/crates/relay-pty/src/pty.rs @@ -928,7 +928,6 @@ fn read_was_cancelled(error: &io::Error) -> bool { error.raw_os_error() == Some(ERROR_OPERATION_ABORTED as i32) } -/// Whether the grid currently has bracketed-paste mode (DECSET 2004) set. /// Finds the DECSET 2004 private-mode parameter (`?2004` or `;2004`) in PTY /// output, including one split across reads, by carrying the previous read's /// last four bytes. Plain text that merely contains `2004` does not match. @@ -958,6 +957,7 @@ impl Decset2004Scan { } } +/// Whether the grid currently has bracketed-paste mode (DECSET 2004) set. fn paste_mode_enabled(term: &Term) -> bool { term.mode() .contains(alacritty_terminal::term::TermMode::BRACKETED_PASTE) From f04888c1849be3006b6b17051109e9d8d0f9c398 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 13:03:41 -0700 Subject: [PATCH 35/42] fix(cli): size tasks by runtime, reject invalid UTF-8, tighten timeouts - Task size follows the resolved target: PTY keeps the 14 KiB envelope limit; Muse and the native runtime use the 16 KiB single-argument ceiling, including the bounded --task-file read. - --task-file rejects malformed UTF-8 instead of substituting U+FFFD, and an oversized file is reported by size before decoding. - --wait-timeout is floored to whole milliseconds and clamped to the timer limit. - Spawn liveness reads the clock when judging a heartbeat, after the reads. - MCP replay expires retained keys from a settlement-ordered queue instead of scanning every entry per call. - Changelog: user-facing spawn-outcome wording; headless limit corrected. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- CHANGELOG.md | 4 +- packages/cli/src/cli/commands/fleet.ts | 4 +- packages/cli/src/cli/commands/local-agent.ts | 23 ++++-- .../cli/src/cli/lib/agent-removal.test.ts | 14 +++- packages/cli/src/cli/lib/agent-removal.ts | 13 ++-- .../cli/src/cli/lib/spawn-liveness.test.ts | 18 +++++ packages/cli/src/cli/lib/spawn-liveness.ts | 7 +- packages/cli/src/cli/lib/task-input.test.ts | 47 +++++++++-- packages/cli/src/cli/lib/task-input.ts | 78 +++++++++++++------ packages/cli/src/cli/mcp/request-replay.ts | 32 +++++--- 10 files changed, 181 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75189d6afb..e6c8ab6fcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Unconfirmed `fleet spawn` results keep the `spawn_unconfirmed` code, now with `state: "pending"`, invocation diagnostics, liveness evidence and exit 8. Node heartbeat presence is reported as liveness evidence but never confirms the spawn, because heartbeats cannot tell this invocation's worker from an earlier one with the same name. - Removal waits exit 8 if the registration remains present; scripts must handle nonzero exits beyond `$? -eq 1`. Waits remain opt-in, and unavailable verification reads preserve the asynchronous acknowledgement. -- PTY-delivered tasks and messages are limited to 14 KiB (16 KiB with the message envelope); `fleet spawn`, `agent spawn` and PTY delivery reject larger input and name the limit, so write the brief to a file and send a pointer. Agents whose terminal lacks bracketed paste accept at most 1,536 bytes per injection. Muse startup tasks keep their 16 KiB argument limit, headless agents are not capped, `message post|reply|dm send` are not capped, and a wrapped agent receives an oversized message as a pointer to it in Relay. +- PTY-delivered tasks and messages are limited to 14 KiB (16 KiB with the message envelope); `fleet spawn`, `agent spawn` and PTY delivery reject larger input and name the limit, so write the brief to a file and send a pointer. Agents whose terminal lacks bracketed paste accept at most 1,536 bytes per injection. Muse startup tasks, native-runtime tasks and messages to headless agents, which travel as a single process argument, are limited to 16 KiB instead; `message post|reply|dm send` are not capped, and a wrapped agent receives an oversized message as a pointer to it in Relay. ### Fixed @@ -41,7 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fleet registration collisions use readable `spawn_name_taken` guidance covering asynchronous removal and failed-spawn cleanup, instead of Rust `Fatal(AlreadyExists {...})` text. - Pending fleet spawns preserve provisioned sandboxes and distinguish live, stale, elsewhere, registered, absent, and unknown evidence before advising a retry. - PTY tasks and relay messages use atomic bracketed paste when supported, wait for the composer, and prevent embedded carriage returns from submitting partial input. Oversized bodies and tail-only echoes fail explicitly without replay. -- `fleet spawn --task` reports success only when the agent's terminal echoed the whole task; a task that fails to deliver releases the worker before the spawn fails with `spawn_task_failed`. A task acked without that proof fails with `spawn_task_unconfirmed`, which names the live agent and says not to retry, and `delivery_verified` now reports what was observed (`echo_incomplete`, `paste_summary`, `timeout_fallback`) instead of treating head/tail anchors or a collapsed-paste marker as receipt. +- `fleet spawn --task` succeeds only once the agent has visibly received the whole task. If the task cannot be delivered, the worker is released and the spawn fails with `spawn_task_failed`. If delivery cannot be confirmed, the spawn fails with `spawn_task_unconfirmed`, names the agent that is still running, and says not to retry. ## [13.1.5] - 2026-10-08 diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index abb79d73c3..a0b6078e35 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -511,7 +511,7 @@ export function registerFleetCommands( await runSdk(deps.sdk, async () => { const clientOptions = sdkOptionsFromOpts(options); const name = requiredText(options.name, 'Worker name'); - const task = (await readTaskInput(options.task, options.taskFile, true, cli))!; + const task = (await readTaskInput(options.task, options.taskFile, true, { cli }))!; let targetNode = optionalText(options.targetNode, 'Target node') ?? optionalText(options.node, 'Node'); const useSandbox = options.sandbox === true; // Explicit hosted credentials/transport and personas retain their legacy @@ -1144,7 +1144,7 @@ export function registerFleetCommands( : task; // The sandbox context is appended after readTaskInput accepted the // task, so the composed task is what the worker must receive. - validateTaskSize(spawnTask, cli); + validateTaskSize(spawnTask, { cli }); let invocation; try { invocation = await relay.messaging.placement.spawn({ diff --git a/packages/cli/src/cli/commands/local-agent.ts b/packages/cli/src/cli/commands/local-agent.ts index 13aa8e4259..97952a290b 100644 --- a/packages/cli/src/cli/commands/local-agent.ts +++ b/packages/cli/src/cli/commands/local-agent.ts @@ -1,4 +1,4 @@ -import { readTaskInput, validateTaskSize } from '../lib/task-input.js'; +import { readTaskInput, validateTaskSize, type TaskTarget } from '../lib/task-input.js'; import type { Command } from 'commander'; import { AGENT37_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; @@ -53,7 +53,8 @@ function resolveAutoSpawn( provider: string, name: string, task: string | undefined, - model: string | undefined + model: string | undefined, + runtime?: TaskTarget['runtime'] ): { name: string; task: string | undefined; model: string | undefined } { if (model !== 'auto' || provider !== 'claude' || !task) { return { name, task, model }; @@ -62,7 +63,7 @@ function resolveAutoSpawn( const team = composeTeam(assessment, task); const directorPrompt = buildDirectorPrompt(task, team); // The Director prompt repeats the task, so it is what must fit. - validateTaskSize(directorPrompt, provider); + validateTaskSize(directorPrompt, { cli: provider, runtime }); return { name: name === provider ? 'Director' : name, task: directorPrompt, @@ -877,8 +878,12 @@ export function registerLocalAgentCommands( const resolved = resolveAutoSpawn( provider, baseName, - await readTaskInput(opts.task, opts.taskFile, false, provider), - opts.model as string | undefined + await readTaskInput(opts.task, opts.taskFile, false, { + cli: provider, + runtime: runtime.selected, + }), + opts.model as string | undefined, + runtime.selected ); await spawnAgentWithClient(client, { name: resolved.name, @@ -940,8 +945,12 @@ export function registerLocalAgentCommands( resolved = resolveAutoSpawn( provider, baseName, - await readTaskInput(options.task, options.taskFile, false, provider), - options.model as string | undefined + await readTaskInput(options.task, options.taskFile, false, { + cli: provider, + runtime: runtime.selected, + }), + options.model as string | undefined, + runtime.selected ); } catch (err) { deps.error(err instanceof Error ? err.message : String(err)); diff --git a/packages/cli/src/cli/lib/agent-removal.test.ts b/packages/cli/src/cli/lib/agent-removal.test.ts index 30312984b6..4de25aebe1 100644 --- a/packages/cli/src/cli/lib/agent-removal.test.ts +++ b/packages/cli/src/cli/lib/agent-removal.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { waitForAgentRemoval } from './agent-removal.js'; +import { parseRemovalWaitTimeout, waitForAgentRemoval } from './agent-removal.js'; function harness() { let time = 0; @@ -63,3 +63,15 @@ describe('registration removal wait', () => { } }); }); + +describe('parseRemovalWaitTimeout', () => { + // Timers take whole milliseconds; a fractional deadline would expire before + // the scheduled sleep, and a sub-millisecond one would wait not at all. + it('floors to whole milliseconds and clamps to the timer limit', () => { + expect(parseRemovalWaitTimeout('1500.9')).toBe(1500); + expect(parseRemovalWaitTimeout(1e12)).toBe(2_147_483_647); + for (const value of ['0.5', '0', '-1', 'abc', 'Infinity']) { + expect(() => parseRemovalWaitTimeout(value)).toThrow('--wait-timeout'); + } + }); +}); diff --git a/packages/cli/src/cli/lib/agent-removal.ts b/packages/cli/src/cli/lib/agent-removal.ts index 954c5b5304..4ce9c0cc1a 100644 --- a/packages/cli/src/cli/lib/agent-removal.ts +++ b/packages/cli/src/cli/lib/agent-removal.ts @@ -13,15 +13,18 @@ export function isNotFoundError(error: unknown): boolean { return Number(status) === 404; } -/** Parse `--wait-timeout`; reject what the wait would otherwise silently replace with its default. */ +/** + * Parse `--wait-timeout` as whole milliseconds within the timer limit; reject + * what the wait would otherwise silently replace with its default. + */ export function parseRemovalWaitTimeout(value: unknown): number { - const timeoutMs = Number(value); - if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + const timeoutMs = Math.floor(Number(value)); + if (!Number.isFinite(timeoutMs) || timeoutMs < 1) { throw new Error( - `--wait-timeout must be a positive number of milliseconds (got ${JSON.stringify(value)}).` + `--wait-timeout must be a positive whole number of milliseconds (got ${JSON.stringify(value)}).` ); } - return timeoutMs; + return normalizeTimeoutMs(timeoutMs); } export interface RemovalResult { diff --git a/packages/cli/src/cli/lib/spawn-liveness.test.ts b/packages/cli/src/cli/lib/spawn-liveness.test.ts index f9b6917b8f..87af4211ef 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.test.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.test.ts @@ -48,6 +48,24 @@ describe('spawn liveness resolution', () => { ).toMatchObject({ evidence }); if (evidence === 'live') expect(workspace.nodes.list).not.toHaveBeenCalled(); }); + // relay#1930 review (cubic): the clock is read when a heartbeat is judged, + // so read latency cannot make an old heartbeat look fresh. + it('measures heartbeat age after the bounded node reads, not before', async () => { + let clock = now; + const workspace = client([node('target', 30_000)]); + workspace.nodes.get.mockImplementation(async (name: string) => { + clock = now + 10_000; + return name === 'target' ? node('target', 30_000) : null; + }); + expect( + await probeSpawnLiveness({ + name: 'worker', + targetNode: 'target', + createClient: () => workspace, + now: () => clock, + }) + ).toMatchObject({ evidence: 'stale', heartbeatAgeMs: 40_000 }); + }); // relay#1930 review (cubic): a stale or unavailable target entry must not // hide a fresh same-named worker on another node. it.each([ diff --git a/packages/cli/src/cli/lib/spawn-liveness.ts b/packages/cli/src/cli/lib/spawn-liveness.ts index ced83ab76b..61f3035d88 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.ts @@ -138,12 +138,14 @@ export async function probeSpawnLiveness(options: { try { const client = options.createClient(); try { - const now = options.now?.() ?? Date.now(); + // Read the clock when a heartbeat is judged, after the reads: sampling it + // first would let read latency make an old heartbeat look fresh. + const clock = () => options.now?.() ?? Date.now(); const target = options.targetNode ? await read(() => client.nodes.get(options.targetNode!)) : undefined; const nodes = options.targetNode && target ? [target] : await read(() => client.nodes.list()); // Prefer the targeted read; consult the fleet unless the target is an // available node whose fresh heartbeat claims this name. - const targetAge = target ? heartbeatAgeMs(target, now) : null; + const targetAge = target ? heartbeatAgeMs(target, clock()) : null; if ( options.targetNode && target && @@ -157,6 +159,7 @@ export async function probeSpawnLiveness(options: { nodes.push(...(await read(() => client.nodes.list()))); } let weaker: SpawnLiveness | undefined; + const now = clock(); for (const node of nodes) { if ( !isAvailableFleetNode(node) || diff --git a/packages/cli/src/cli/lib/task-input.test.ts b/packages/cli/src/cli/lib/task-input.test.ts index 54b8aea514..827980ea94 100644 --- a/packages/cli/src/cli/lib/task-input.test.ts +++ b/packages/cli/src/cli/lib/task-input.test.ts @@ -6,7 +6,7 @@ import { readTaskInput, MAX_INJECTION_BODY_BYTES, MAX_TASK_BODY_BYTES, - MUSE_STARTUP_PROMPT_MAX_BYTES, + MAX_ARGV_TASK_BYTES, } from './task-input.js'; describe('task input', () => { it('requires exactly one source for fleet, allows neither locally', async () => { @@ -33,7 +33,7 @@ describe('task input', () => { 'reads at most one byte past the limit from a task file', async () => { await expect(readTaskInput(undefined, '/dev/zero')).rejects.toThrow('UTF-8 bytes'); - await expect(readTaskInput(undefined, '/dev/zero', false, 'muse')).rejects.toThrow('argv'); + await expect(readTaskInput(undefined, '/dev/zero', false, { cli: 'muse' })).rejects.toThrow('argv'); }, 2000 ); @@ -55,12 +55,47 @@ describe('task input', () => { it('applies the Muse argv limit, not the PTY body limit, to Muse tasks', async () => { const between = 'a'.repeat(MAX_TASK_BODY_BYTES + 1); for (const cli of ['muse', 'MUSE.exe', '/usr/local/bin/muse', 'C:\\Tools\\Muse.CMD']) { - expect(await readTaskInput(between, undefined, false, cli)).toBe(between); + expect(await readTaskInput(between, undefined, false, { cli })).toBe(between); await expect( - readTaskInput('a'.repeat(MUSE_STARTUP_PROMPT_MAX_BYTES + 1), undefined, false, cli) + readTaskInput('a'.repeat(MAX_ARGV_TASK_BYTES + 1), undefined, false, { cli }) ).rejects.toThrow('argv'); } - await expect(readTaskInput(between, undefined, false, 'musey')).rejects.toThrow('envelope'); - await expect(readTaskInput(between, undefined, false, 'claude')).rejects.toThrow('envelope'); + await expect(readTaskInput(between, undefined, false, { cli: 'musey' })).rejects.toThrow('envelope'); + await expect(readTaskInput(between, undefined, false, { cli: 'claude' })).rejects.toThrow('envelope'); + }); + // A native runtime hands the task over without the PTY envelope, so only + // the portable single-argument ceiling applies, including to file reads. + it('applies the argv ceiling, not the PTY body limit, to native-runtime tasks', async () => { + const between = 'a'.repeat(MAX_TASK_BODY_BYTES + 1); + const native = { cli: 'claude', runtime: 'native' } as const; + expect(await readTaskInput(between, undefined, false, native)).toBe(between); + await expect( + readTaskInput('a'.repeat(MAX_ARGV_TASK_BYTES + 1), undefined, false, native) + ).rejects.toThrow('argument'); + const dir = await mkdtemp(join(tmpdir(), 'relay-task-')); + try { + const file = join(dir, 'brief.md'); + await writeFile(file, between); + expect(await readTaskInput(undefined, file, false, native)).toBe(between); + await expect(readTaskInput(undefined, file, false, { cli: 'claude', runtime: 'pty' })).rejects.toThrow( + 'envelope' + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + it('rejects a task file that is not valid UTF-8 instead of altering its text', async () => { + const dir = await mkdtemp(join(tmpdir(), 'relay-task-')); + try { + const file = join(dir, 'brief.md'); + await writeFile(file, Buffer.from([0x68, 0x69, 0xff, 0x0a])); + await expect(readTaskInput(undefined, file)).rejects.toThrow('not valid UTF-8'); + // An oversized file whose cut lands inside a character is reported by + // size, not as invalid UTF-8. + await writeFile(file, 'é'.repeat(MAX_TASK_BODY_BYTES)); + await expect(readTaskInput(undefined, file)).rejects.toThrow('UTF-8 bytes'); + } finally { + await rm(dir, { recursive: true, force: true }); + } }); }); diff --git a/packages/cli/src/cli/lib/task-input.ts b/packages/cli/src/cli/lib/task-input.ts index 0a12b2a2a2..647bb58476 100644 --- a/packages/cli/src/cli/lib/task-input.ts +++ b/packages/cli/src/cli/lib/task-input.ts @@ -6,8 +6,18 @@ export const MAX_INJECTION_BODY_BYTES = 16 * 1024; export const ENVELOPE_RESERVE_BYTES = 2 * 1024; /** Largest body that still fits {@link MAX_INJECTION_BODY_BYTES} once formatted. */ export const MAX_TASK_BODY_BYTES = MAX_INJECTION_BODY_BYTES - ENVELOPE_RESERVE_BYTES; -/** The broker's portable argv ceiling for Muse's single-argument startup prompt. */ -export const MUSE_STARTUP_PROMPT_MAX_BYTES = 16 * 1024; +/** + * The broker's portable ceiling for a prompt passed as one process argument + * (Muse's startup task, a headless provider's message). + */ +export const MAX_ARGV_TASK_BYTES = 16 * 1024; + +/** Who receives a task: the CLI, and the runtime once it is resolved. */ +export interface TaskTarget { + cli: string; + /** `native` hands the task over without the PTY envelope; omitted means PTY. */ + runtime?: 'native' | 'pty'; +} /** Mirrors the broker's `is_muse_executable`: the basename, case-insensitive, minus a Windows launcher suffix. */ function isMuseExecutable(cli: string): boolean { @@ -15,11 +25,16 @@ function isMuseExecutable(cli: string): boolean { return basename.toLowerCase().replace(/\.(exe|cmd|bat)$/, '') === 'muse'; } +/** Muse and native runtimes take the task as one argument, never through the PTY envelope. */ +function usesPtyEnvelope(target: TaskTarget | undefined): boolean { + return !target || (target.runtime !== 'native' && !isMuseExecutable(target.cli)); +} + export async function readTaskInput( task: unknown, taskFile: unknown, required = false, - cli?: string + target?: TaskTarget ): Promise { if (task !== undefined && taskFile !== undefined) throw new Error('Specify exactly one of --task or --task-file'); @@ -27,28 +42,33 @@ export async function readTaskInput( if (required) throw new Error('Specify exactly one of --task or --task-file'); return undefined; } - // Muse receives its startup task through argv, not the PTY envelope. - const muse = cli !== undefined && isMuseExecutable(cli); - const limit = muse ? MUSE_STARTUP_PROMPT_MAX_BYTES : MAX_TASK_BODY_BYTES; - const text = taskFile === undefined ? task : await readFilePrefix(String(taskFile), limit + 1); + const text = + taskFile === undefined ? task : await readTaskFile(String(taskFile), taskByteLimit(target), target); if (typeof text !== 'string' || !text.trim()) throw new Error('Task must not be empty'); - validateTaskSize(text, cli); + validateTaskSize(text, target); return text; } -/** Validate a task as the given CLI receives it; recheck after anything is appended to it. */ -export function validateTaskSize(text: string, cli?: string): void { - if (cli !== undefined && isMuseExecutable(cli)) validateMuseStartupPromptSize(text); - else validateInjectionSize(text); +function taskByteLimit(target: TaskTarget | undefined): number { + return usesPtyEnvelope(target) ? MAX_TASK_BODY_BYTES : MAX_ARGV_TASK_BYTES; +} + +/** Validate a task as its target receives it; recheck after anything is appended to it. */ +export function validateTaskSize(text: string, target?: TaskTarget): void { + if (usesPtyEnvelope(target)) validateInjectionSize(text); + else validateArgvTaskSize(Buffer.byteLength(text, 'utf8')); } /** - * Read at most `maxBytes` of a UTF-8 file. Reading one byte past a size limit - * is enough to reject an oversized file without materializing all of it. + * Read a UTF-8 task file, never more than one byte past `limit`: that is enough + * to reject an oversized file without materializing all of it. Size is checked + * before decoding, since the cut can split a character, and malformed UTF-8 is + * rejected rather than replaced, so the agent never gets different text. */ -async function readFilePrefix(path: string, maxBytes: number): Promise { +async function readTaskFile(path: string, limit: number, target: TaskTarget | undefined): Promise { const handle = await open(path, 'r'); try { + const maxBytes = limit + 1; const buffer = Buffer.alloc(maxBytes); let length = 0; while (length < maxBytes) { @@ -56,24 +76,34 @@ async function readFilePrefix(path: string, maxBytes: number): Promise { if (bytesRead === 0) break; length += bytesRead; } - return buffer.subarray(0, length).toString('utf8'); + if (length > limit) { + if (usesPtyEnvelope(target)) throwInjectionTooLarge(); + validateArgvTaskSize(length); + } + try { + return new TextDecoder('utf-8', { fatal: true }).decode(buffer.subarray(0, length)); + } catch { + throw new Error(`Task file ${path} is not valid UTF-8`); + } } finally { await handle.close(); } } -function validateMuseStartupPromptSize(text: string): void { - if (Buffer.byteLength(text, 'utf8') > MUSE_STARTUP_PROMPT_MAX_BYTES) { +function validateArgvTaskSize(bytes: number): void { + if (bytes > MAX_ARGV_TASK_BYTES) { throw new Error( - `Muse startup task exceeds the ${MUSE_STARTUP_PROMPT_MAX_BYTES}-byte portable argv limit; use a brief file on the node and send a short pointer` + `Task exceeds the ${MAX_ARGV_TASK_BYTES}-byte portable argv limit for a single process argument; use a brief file on the node and send a short pointer` ); } } +function throwInjectionTooLarge(): never { + throw new Error( + `Task exceeds ${MAX_TASK_BODY_BYTES} UTF-8 bytes (the ${MAX_INJECTION_BODY_BYTES}-byte PTY limit less room for the message envelope); use a brief file on the node and send a short pointer` + ); +} + export function validateInjectionSize(text: string): void { - if (Buffer.byteLength(text, 'utf8') > MAX_TASK_BODY_BYTES) { - throw new Error( - `Task exceeds ${MAX_TASK_BODY_BYTES} UTF-8 bytes (the ${MAX_INJECTION_BODY_BYTES}-byte PTY limit less room for the message envelope); use a brief file on the node and send a short pointer` - ); - } + if (Buffer.byteLength(text, 'utf8') > MAX_TASK_BODY_BYTES) throwInjectionTooLarge(); } diff --git a/packages/cli/src/cli/mcp/request-replay.ts b/packages/cli/src/cli/mcp/request-replay.ts index 2d6614139f..c7a3802771 100644 --- a/packages/cli/src/cli/mcp/request-replay.ts +++ b/packages/cli/src/cli/mcp/request-replay.ts @@ -34,8 +34,14 @@ export function replayScopeOf(client: object): string { const RETAIN_COMPLETED_MS = 5 * 60 * 1000; export class McpRequestReplay { - /** `expiresAt` is set once a keyed request settles; expiry is checked lazily, never by a timer. */ - private readonly requests = new Map; expiresAt?: number }>(); + private readonly requests = new Map>(); + /** + * Retained keys in settlement order. Every key keeps the same retention, so + * this is also expiry order: pruning pops expired keys off the front, with + * no timer and no scan of live entries. + */ + private readonly retained: Array<{ key: string; pending: Promise; expiresAt: number }> = []; + private retainedHead = 0; run( tool: string, @@ -60,18 +66,18 @@ export class McpRequestReplay { hasIdempotencyKey ? idempotencyKey : requestId, ]); this.pruneExpired(); - const existing = this.requests.get(key); - if (existing) return existing.pending as Promise; + const existing = this.requests.get(key) as Promise | undefined; + if (existing) return existing; const pending = operation(); - const entry: { pending: Promise; expiresAt?: number } = { pending }; - this.requests.set(key, entry); + this.requests.set(key, pending); // JSON-RPC permits an ID to be reused after its response. Only a client // supplied idempotency key can safely keep a completed result for a retry. void pending.then( () => { - if (hasIdempotencyKey) entry.expiresAt = Date.now() + RETAIN_COMPLETED_MS; - else this.requests.delete(key); + if (hasIdempotencyKey) { + this.retained.push({ key, pending, expiresAt: Date.now() + RETAIN_COMPLETED_MS }); + } else this.requests.delete(key); }, () => this.requests.delete(key) ); @@ -80,8 +86,14 @@ export class McpRequestReplay { private pruneExpired(): void { const now = Date.now(); - for (const [key, entry] of this.requests) { - if (entry.expiresAt !== undefined && entry.expiresAt <= now) this.requests.delete(key); + while (this.retainedHead < this.retained.length && this.retained[this.retainedHead].expiresAt <= now) { + const { key, pending } = this.retained[this.retainedHead++]; + if (this.requests.get(key) === pending) this.requests.delete(key); + } + // Compact once the consumed prefix dominates, keeping pruning amortized O(1). + if (this.retainedHead > 64 && this.retainedHead * 2 > this.retained.length) { + this.retained.splice(0, this.retainedHead); + this.retainedHead = 0; } } } From 1b14ce9763474b369ae0761eb9c2354935139562 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 13:26:40 -0700 Subject: [PATCH 36/42] fix: apply the argv ceiling only where a body becomes a process argument Only the headless CLI runner (no harness config) passes a delivery to its provider as argv; app-server and native harness workers receive frames, and a native initial task is delivered the same way. The broker now bounds only that runner, and the CLI no longer applies the 16 KiB argv ceiling to native-runtime tasks (a 1 MiB read bound remains for --task-file). Changelog qualifies the fleet spawn delivery guarantee to confirmed spawns and states --no-confirm. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- CHANGELOG.md | 4 +-- crates/broker/src/runtime/delivery.rs | 17 +++++---- crates/broker/src/runtime/tests.rs | 35 ++++++++++++++++++ packages/cli/src/cli/lib/task-input.test.ts | 19 +++++----- packages/cli/src/cli/lib/task-input.ts | 40 +++++++++++++++------ 5 files changed, 84 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6c8ab6fcc..aa6805c763 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Unconfirmed `fleet spawn` results keep the `spawn_unconfirmed` code, now with `state: "pending"`, invocation diagnostics, liveness evidence and exit 8. Node heartbeat presence is reported as liveness evidence but never confirms the spawn, because heartbeats cannot tell this invocation's worker from an earlier one with the same name. - Removal waits exit 8 if the registration remains present; scripts must handle nonzero exits beyond `$? -eq 1`. Waits remain opt-in, and unavailable verification reads preserve the asynchronous acknowledgement. -- PTY-delivered tasks and messages are limited to 14 KiB (16 KiB with the message envelope); `fleet spawn`, `agent spawn` and PTY delivery reject larger input and name the limit, so write the brief to a file and send a pointer. Agents whose terminal lacks bracketed paste accept at most 1,536 bytes per injection. Muse startup tasks, native-runtime tasks and messages to headless agents, which travel as a single process argument, are limited to 16 KiB instead; `message post|reply|dm send` are not capped, and a wrapped agent receives an oversized message as a pointer to it in Relay. +- PTY-delivered tasks and messages are limited to 14 KiB (16 KiB with the message envelope); `fleet spawn`, `agent spawn` and PTY delivery reject larger input and name the limit, so write the brief to a file and send a pointer. Agents whose terminal lacks bracketed paste accept at most 1,536 bytes per injection. Muse startup tasks and messages to headless CLI agents travel as a single process argument and are limited to 16 KiB instead; native-runtime and app-server agents receive tasks and messages as frames and have no such limit; `message post|reply|dm send` are not capped, and a wrapped agent receives an oversized message as a pointer to it in Relay. ### Fixed @@ -41,7 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fleet registration collisions use readable `spawn_name_taken` guidance covering asynchronous removal and failed-spawn cleanup, instead of Rust `Fatal(AlreadyExists {...})` text. - Pending fleet spawns preserve provisioned sandboxes and distinguish live, stale, elsewhere, registered, absent, and unknown evidence before advising a retry. - PTY tasks and relay messages use atomic bracketed paste when supported, wait for the composer, and prevent embedded carriage returns from submitting partial input. Oversized bodies and tail-only echoes fail explicitly without replay. -- `fleet spawn --task` succeeds only once the agent has visibly received the whole task. If the task cannot be delivered, the worker is released and the spawn fails with `spawn_task_failed`. If delivery cannot be confirmed, the spawn fails with `spawn_task_unconfirmed`, names the agent that is still running, and says not to retry. +- A confirmed `fleet spawn --task` (the default) succeeds only once the agent has visibly received the whole task. If the task cannot be delivered, the worker is released and the spawn fails with `spawn_task_failed`. If delivery cannot be confirmed, the spawn fails with `spawn_task_unconfirmed`, names the agent that is still running, and says not to retry. With `--no-confirm` the spawn reports as soon as the node accepts it, without waiting for the task to arrive. ## [13.1.5] - 2026-10-08 diff --git a/crates/broker/src/runtime/delivery.rs b/crates/broker/src/runtime/delivery.rs index 556e7c16db..c15c1f18bd 100644 --- a/crates/broker/src/runtime/delivery.rs +++ b/crates/broker/src/runtime/delivery.rs @@ -862,21 +862,20 @@ pub(crate) async fn queue_and_try_delivery_raw( withheld_fleet_ack_floor: Option, ) -> Result { // The envelope-sized cap exists because PTY delivery types the message into - // a terminal, so it applies to PTY and unknown recipients. A headless - // provider instead receives the body as one argv entry, which is bounded - // by the portable single-argument ceiling rather than the PTY envelope. - let pty_recipient = workers - .workers - .get(worker_name) - .is_none_or(|handle| handle.spec.runtime == AgentRuntime::Pty); - if pty_recipient { + // a terminal, so it applies to PTY and unknown recipients. The headless CLI + // runner (no harness config) hands the body to its provider as one argv + // entry, so it is bounded by the portable single-argument ceiling instead. + // App-server and native harness workers receive deliveries as frames and + // need neither bound. + let recipient = workers.workers.get(worker_name).map(|handle| &handle.spec); + if recipient.is_none_or(|spec| spec.runtime == AgentRuntime::Pty) { anyhow::ensure!( body.len() <= crate::injection_wire::MAX_BODY_BYTES, "injection_too_large: body limit is {} bytes so the formatted envelope fits the {}-byte PTY limit; use a brief file pointer", crate::injection_wire::MAX_BODY_BYTES, crate::injection_wire::MAX_INJECTION_BODY_BYTES ); - } else { + } else if recipient.is_some_and(|spec| spec.harness_config.is_none()) { anyhow::ensure!( body.len() <= crate::worker::MUSE_STARTUP_PROMPT_MAX_BYTES, "injection_too_large: a headless agent receives the body as one process argument, limited to {} bytes; use a brief file pointer", diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index 2a432a72be..5cb747fdea 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -4355,6 +4355,41 @@ async fn pty_body_cap_applies_only_to_pty_recipients() { .await .expect_err("a headless body past the argv ceiling fails before delivery"); assert!(error.to_string().contains("injection_too_large"), "{error}"); + // An app-server headless worker receives deliveries as frames, never as a + // process argument, so neither ceiling applies to it. + let handle = workers + .workers + .get_mut(&WorkerName::from("Worker")) + .unwrap(); + handle.spec.harness_config = Some(ResolvedHarnessConfig::Headless(HeadlessHarnessConfig { + driver: HeadlessHarnessDriver::AppServer, + protocol: "opencode".to_string(), + endpoint: "http://127.0.0.1:4096".to_string(), + session_id: "session-endpoint".to_string(), + auth: None, + host: None, + release: Some(HarnessReleasePolicy::Abort), + metadata: None, + })); + super::queue_and_try_delivery_raw( + &mut workers, + &mut HashMap::new(), + "Worker", + "evt_frame", + "orchestrator", + "Worker", + &"x".repeat(crate::worker::MUSE_STARTUP_PROMPT_MAX_BYTES + 1), + None, + Some(WorkspaceId::new("ws_demo")), + None, + 2, + MessageInjectionMode::Wait, + Duration::from_secs(1), + None, + None, + ) + .await + .expect("frame-delivered headless workers have no argv ceiling"); } #[tokio::test] diff --git a/packages/cli/src/cli/lib/task-input.test.ts b/packages/cli/src/cli/lib/task-input.test.ts index 827980ea94..f6f826613e 100644 --- a/packages/cli/src/cli/lib/task-input.test.ts +++ b/packages/cli/src/cli/lib/task-input.test.ts @@ -7,6 +7,7 @@ import { MAX_INJECTION_BODY_BYTES, MAX_TASK_BODY_BYTES, MAX_ARGV_TASK_BYTES, + MAX_NATIVE_TASK_BYTES, } from './task-input.js'; describe('task input', () => { it('requires exactly one source for fleet, allows neither locally', async () => { @@ -63,20 +64,20 @@ describe('task input', () => { await expect(readTaskInput(between, undefined, false, { cli: 'musey' })).rejects.toThrow('envelope'); await expect(readTaskInput(between, undefined, false, { cli: 'claude' })).rejects.toThrow('envelope'); }); - // A native runtime hands the task over without the PTY envelope, so only - // the portable single-argument ceiling applies, including to file reads. - it('applies the argv ceiling, not the PTY body limit, to native-runtime tasks', async () => { - const between = 'a'.repeat(MAX_TASK_BODY_BYTES + 1); + // A native runtime receives its task as a delivery frame: neither the PTY + // envelope nor the argv ceiling applies, only a bound on what is read. + it('does not apply PTY or argv limits to native-runtime tasks', async () => { + const overArgv = 'a'.repeat(MAX_ARGV_TASK_BYTES + 1); const native = { cli: 'claude', runtime: 'native' } as const; - expect(await readTaskInput(between, undefined, false, native)).toBe(between); + expect(await readTaskInput(overArgv, undefined, false, native)).toBe(overArgv); await expect( - readTaskInput('a'.repeat(MAX_ARGV_TASK_BYTES + 1), undefined, false, native) - ).rejects.toThrow('argument'); + readTaskInput('a'.repeat(MAX_NATIVE_TASK_BYTES + 1), undefined, false, native) + ).rejects.toThrow(`${MAX_NATIVE_TASK_BYTES} UTF-8 bytes`); const dir = await mkdtemp(join(tmpdir(), 'relay-task-')); try { const file = join(dir, 'brief.md'); - await writeFile(file, between); - expect(await readTaskInput(undefined, file, false, native)).toBe(between); + await writeFile(file, overArgv); + expect(await readTaskInput(undefined, file, false, native)).toBe(overArgv); await expect(readTaskInput(undefined, file, false, { cli: 'claude', runtime: 'pty' })).rejects.toThrow( 'envelope' ); diff --git a/packages/cli/src/cli/lib/task-input.ts b/packages/cli/src/cli/lib/task-input.ts index 647bb58476..c7d405b8a0 100644 --- a/packages/cli/src/cli/lib/task-input.ts +++ b/packages/cli/src/cli/lib/task-input.ts @@ -12,10 +12,16 @@ export const MAX_TASK_BODY_BYTES = MAX_INJECTION_BODY_BYTES - ENVELOPE_RESERVE_B */ export const MAX_ARGV_TASK_BYTES = 16 * 1024; +/** + * A native runtime receives its task as a delivery frame, so no wire limit + * applies; this only bounds how much of a `--task-file` the CLI will read. + */ +export const MAX_NATIVE_TASK_BYTES = 1024 * 1024; + /** Who receives a task: the CLI, and the runtime once it is resolved. */ export interface TaskTarget { cli: string; - /** `native` hands the task over without the PTY envelope; omitted means PTY. */ + /** `native` delivers the task as a frame, without the PTY envelope; omitted means PTY. */ runtime?: 'native' | 'pty'; } @@ -25,9 +31,13 @@ function isMuseExecutable(cli: string): boolean { return basename.toLowerCase().replace(/\.(exe|cmd|bat)$/, '') === 'muse'; } -/** Muse and native runtimes take the task as one argument, never through the PTY envelope. */ -function usesPtyEnvelope(target: TaskTarget | undefined): boolean { - return !target || (target.runtime !== 'native' && !isMuseExecutable(target.cli)); +type TaskLimit = 'pty' | 'argv' | 'native'; + +/** Muse takes its task as one argument; a native runtime as a frame; everything else through the PTY envelope. */ +function taskLimitKind(target: TaskTarget | undefined): TaskLimit { + if (target?.runtime === 'native') return 'native'; + if (target && isMuseExecutable(target.cli)) return 'argv'; + return 'pty'; } export async function readTaskInput( @@ -50,13 +60,24 @@ export async function readTaskInput( } function taskByteLimit(target: TaskTarget | undefined): number { - return usesPtyEnvelope(target) ? MAX_TASK_BODY_BYTES : MAX_ARGV_TASK_BYTES; + const kind = taskLimitKind(target); + return kind === 'pty' ? MAX_TASK_BODY_BYTES : kind === 'argv' ? MAX_ARGV_TASK_BYTES : MAX_NATIVE_TASK_BYTES; } /** Validate a task as its target receives it; recheck after anything is appended to it. */ export function validateTaskSize(text: string, target?: TaskTarget): void { - if (usesPtyEnvelope(target)) validateInjectionSize(text); - else validateArgvTaskSize(Buffer.byteLength(text, 'utf8')); + validateTaskBytes(Buffer.byteLength(text, 'utf8'), target); +} + +function validateTaskBytes(bytes: number, target: TaskTarget | undefined): void { + const kind = taskLimitKind(target); + if (kind === 'pty' && bytes > MAX_TASK_BODY_BYTES) throwInjectionTooLarge(); + if (kind === 'argv') validateArgvTaskSize(bytes); + if (kind === 'native' && bytes > MAX_NATIVE_TASK_BYTES) { + throw new Error( + `Task exceeds ${MAX_NATIVE_TASK_BYTES} UTF-8 bytes; use a brief file on the node and send a short pointer` + ); + } } /** @@ -76,10 +97,7 @@ async function readTaskFile(path: string, limit: number, target: TaskTarget | un if (bytesRead === 0) break; length += bytesRead; } - if (length > limit) { - if (usesPtyEnvelope(target)) throwInjectionTooLarge(); - validateArgvTaskSize(length); - } + if (length > limit) validateTaskBytes(length, target); try { return new TextDecoder('utf-8', { fatal: true }).decode(buffer.subarray(0, length)); } catch { From 47958302d109a02928f49e8496550b913edde1f7 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 13:34:11 -0700 Subject: [PATCH 37/42] fix(mcp): expire retained replay keys on a monotonic clock The settlement-ordered queue relies on expiry order matching insertion order; a backward wall-clock step could break that and keep a key cached past its window. Use performance.now() for both creation and pruning. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- packages/cli/src/cli/mcp/request-replay.test.ts | 10 ++++++++-- packages/cli/src/cli/mcp/request-replay.ts | 8 ++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/cli/mcp/request-replay.test.ts b/packages/cli/src/cli/mcp/request-replay.test.ts index 98afae0c09..063e65ef58 100644 --- a/packages/cli/src/cli/mcp/request-replay.test.ts +++ b/packages/cli/src/cli/mcp/request-replay.test.ts @@ -69,12 +69,18 @@ describe('MCP request replay', () => { await replay.run('post_message', { requestId: 1 }, 'k', operation); await Promise.resolve(); expect(vi.getTimerCount()).toBe(0); - vi.setSystemTime(Date.now() + 5 * 60 * 1000 - 1); + vi.advanceTimersByTime(5 * 60 * 1000 - 1); await replay.run('post_message', { requestId: 2 }, 'k', operation); expect(operation).toHaveBeenCalledTimes(1); - vi.setSystemTime(Date.now() + 1); + vi.advanceTimersByTime(1); await replay.run('post_message', { requestId: 3 }, 'k', operation); expect(operation).toHaveBeenCalledTimes(2); + // Retention follows the monotonic clock: stepping the wall clock back + // cannot keep a key alive past its window. + vi.setSystemTime(Date.now() - 60 * 60 * 1000); + vi.advanceTimersByTime(5 * 60 * 1000); + await replay.run('post_message', { requestId: 4 }, 'k', operation); + expect(operation).toHaveBeenCalledTimes(3); } finally { vi.useRealTimers(); } diff --git a/packages/cli/src/cli/mcp/request-replay.ts b/packages/cli/src/cli/mcp/request-replay.ts index c7a3802771..0dcf381e04 100644 --- a/packages/cli/src/cli/mcp/request-replay.ts +++ b/packages/cli/src/cli/mcp/request-replay.ts @@ -36,8 +36,8 @@ const RETAIN_COMPLETED_MS = 5 * 60 * 1000; export class McpRequestReplay { private readonly requests = new Map>(); /** - * Retained keys in settlement order. Every key keeps the same retention, so - * this is also expiry order: pruning pops expired keys off the front, with + * Retained keys in settlement order. Every key keeps the same retention on a + * monotonic clock, so this is also expiry order: pruning pops expired keys off the front, with * no timer and no scan of live entries. */ private readonly retained: Array<{ key: string; pending: Promise; expiresAt: number }> = []; @@ -76,7 +76,7 @@ export class McpRequestReplay { void pending.then( () => { if (hasIdempotencyKey) { - this.retained.push({ key, pending, expiresAt: Date.now() + RETAIN_COMPLETED_MS }); + this.retained.push({ key, pending, expiresAt: performance.now() + RETAIN_COMPLETED_MS }); } else this.requests.delete(key); }, () => this.requests.delete(key) @@ -85,7 +85,7 @@ export class McpRequestReplay { } private pruneExpired(): void { - const now = Date.now(); + const now = performance.now(); while (this.retainedHead < this.retained.length && this.retained[this.retainedHead].expiresAt <= now) { const { key, pending } = this.retained[this.retainedHead++]; if (this.requests.get(key) === pending) this.requests.delete(key); From 290a30dd63bd7eee73a6ccd187411d606ee2e48d Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 14:53:35 -0700 Subject: [PATCH 38/42] fix(mcp): scope spawn and add_agent replay to their acting credentials spawn and add_agent still used the default empty replay scope, so a key reused after register_agent moved the session default (or after switching workspace) replayed another identity's spawn result. spawn is scoped to the workspace key plus the acting agent token, add_agent to the workspace key. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- .../src/cli/agent-relay-mcp.startup.test.ts | 26 ++++ packages/cli/src/cli/agent-relay-mcp.ts | 138 ++++++++++-------- packages/cli/src/cli/mcp/request-replay.ts | 11 +- 3 files changed, 114 insertions(+), 61 deletions(-) diff --git a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts index f8cd1a19ab..edb6bf1e3a 100644 --- a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts +++ b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts @@ -421,6 +421,32 @@ afterEach(() => { }); describe('agent-relay-mcp startup helpers', () => { + // relay#1930 review (CodeRabbit): a retained spawn key is scoped to the + // acting identity, so a key reused after register_agent moves the session + // default starts that identity's own spawn instead of replaying the first. + it('scopes a retained spawn idempotency key to the acting identity', async () => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mod.createAgentRelayMcpServer({ + workspaceKey: 'rk_live_scope', + agentToken: 'at_live_fleet', + agentName: 'orchestrator', + }); + const server = mocks.serverInstances[0]; + const spawn = server.tools.get('spawn')!.handler; + const input = { name: 'ScopedWorker', cli: 'codex', idempotency_key: 'same-key' }; + await spawn(input, { sessionId: 'mcp-session', requestId: 1 }); + await spawn(input, { sessionId: 'mcp-session', requestId: 2 }); + expect(mocks.agentRelayMessagingCommands.invoke).toHaveBeenCalledTimes(1); + + await server.tools.get('register_agent')!.handler({ name: 'WorkerB' }); + mocks.agentRelayMessagingCommands.invoke.mockResolvedValueOnce({ + invocationId: 'inv_scoped', + actionName: 'spawn', + }); + await spawn(input, { sessionId: 'mcp-session', requestId: 3 }); + expect(mocks.agentRelayMessagingCommands.invoke).toHaveBeenCalledTimes(2); + }); + it('coalesces replayed MCP request ids without collapsing separate same-name spawns', async () => { const { mod, mocks } = await loadAgentRelayMcpModule(); mod.createAgentRelayMcpServer({ agentToken: 'at_live_fleet', agentName: 'orchestrator' }); diff --git a/packages/cli/src/cli/agent-relay-mcp.ts b/packages/cli/src/cli/agent-relay-mcp.ts index 66da945ef0..e09850366e 100644 --- a/packages/cli/src/cli/agent-relay-mcp.ts +++ b/packages/cli/src/cli/agent-relay-mcp.ts @@ -59,7 +59,7 @@ import { import { enableInboxPiggyback } from './mcp/telemetry.js'; import { registerAgentRelayActionTools } from './mcp/action-tools.js'; import { registerMessagingTools } from './mcp/messaging-tools.js'; -import { McpRequestReplay, withReplayScope } from './mcp/request-replay.js'; +import { McpRequestReplay, replayScopeForCredentials, withReplayScope } from './mcp/request-replay.js'; import { identityOverrideInputShape, messageResult } from './mcp/tool-shapes.js'; import { registerSharedSessionTools, @@ -1057,6 +1057,12 @@ function registerAgentRelayTools( forcedAgentType: AgentType | undefined, requestReplay: McpRequestReplay ): void { + // A retained spawn key belongs to the workspace and the identity it acts as. + const spawnReplayScope = (as?: string): string => { + const session = getSession(); + const agentToken = (as ? session.agents.get(as)?.agentToken : session.agentToken) ?? ''; + return replayScopeForCredentials(session.workspaceKey ?? '', agentToken); + }; server.registerTool( 'create_workspace', { @@ -1424,28 +1430,34 @@ function registerAgentRelayTools( { name, cli, task, channel, persona, model, spawn_mode, exit_after_task, idempotency_key }, extra ) => - requestReplay.run('add_agent', extra, idempotency_key, async () => { - const invocation = await getRelay().agents.spawn({ - name, - cli, - task: - exit_after_task || - spawn_mode === 'task_exit' || - spawn_mode === 'task-exit' || - spawn_mode === 'single_shot' || - spawn_mode === 'single-shot' - ? withExitAfterTaskInstruction(task) - : task, - channel, - persona, - // SpawnAgentRequest has no top-level model field; pass via metadata - // so the broker can extract it and forward --model to the launched CLI. - metadata: model ? { model } : undefined, - }); - const failure = terminalSpawnFailureResult(invocation); - if (failure) return failure; - return jsonContent({ ...invocation, placement: spawnReceipt(invocation) }); - }) + requestReplay.run( + 'add_agent', + extra, + idempotency_key, + async () => { + const invocation = await getRelay().agents.spawn({ + name, + cli, + task: + exit_after_task || + spawn_mode === 'task_exit' || + spawn_mode === 'task-exit' || + spawn_mode === 'single_shot' || + spawn_mode === 'single-shot' + ? withExitAfterTaskInstruction(task) + : task, + channel, + persona, + // SpawnAgentRequest has no top-level model field; pass via metadata + // so the broker can extract it and forward --model to the launched CLI. + metadata: model ? { model } : undefined, + }); + const failure = terminalSpawnFailureResult(invocation); + if (failure) return failure; + return jsonContent({ ...invocation, placement: spawnReceipt(invocation) }); + }, + replayScopeForCredentials(getSession().workspaceKey ?? '') + ) ); server.registerTool( @@ -1529,43 +1541,49 @@ function registerAgentRelayTools( }, extra ) => - requestReplay.run('spawn', extra, idempotency_key, async () => { - const request = { - name, - cli, - persona, - task, - cwd, - personaCwd: persona_cwd, - workerCwd: worker_cwd, - channel, - channels, - model, - organization, - project, - workstream, - role, - objective, - sessionRef: session_ref, - targetNode: target_node, - }; - validateSpawnRequest(request); - const actionInput = buildSpawnActionInput(request); - try { - const invocation = await invokeVerifiedSpawn(getSession(), as, baseUrl, actionInput); - return jsonContent({ - invocation, - placement: recordValue(invocation).placement ?? { - state: 'ready', - ...spawnReceipt(recordValue(invocation)), - }, - }); - } catch (error) { - if (error instanceof VerifiedSpawnError || error instanceof FleetSpawnError) - return verifiedSpawnErrorResult(error); - throw error; - } - }) + requestReplay.run( + 'spawn', + extra, + idempotency_key, + async () => { + const request = { + name, + cli, + persona, + task, + cwd, + personaCwd: persona_cwd, + workerCwd: worker_cwd, + channel, + channels, + model, + organization, + project, + workstream, + role, + objective, + sessionRef: session_ref, + targetNode: target_node, + }; + validateSpawnRequest(request); + const actionInput = buildSpawnActionInput(request); + try { + const invocation = await invokeVerifiedSpawn(getSession(), as, baseUrl, actionInput); + return jsonContent({ + invocation, + placement: recordValue(invocation).placement ?? { + state: 'ready', + ...spawnReceipt(recordValue(invocation)), + }, + }); + } catch (error) { + if (error instanceof VerifiedSpawnError || error instanceof FleetSpawnError) + return verifiedSpawnErrorResult(error); + throw error; + } + }, + spawnReplayScope(as) + ) ); server.registerTool( diff --git a/packages/cli/src/cli/mcp/request-replay.ts b/packages/cli/src/cli/mcp/request-replay.ts index 0dcf381e04..03b1bd2c9a 100644 --- a/packages/cli/src/cli/mcp/request-replay.ts +++ b/packages/cli/src/cli/mcp/request-replay.ts @@ -10,12 +10,21 @@ const REPLAY_SCOPE = Symbol('agentRelay.replayScope'); */ export function withReplayScope(client: T, agentToken: string): T { Object.defineProperty(client, REPLAY_SCOPE, { - value: createHash('sha256').update(agentToken).digest('hex'), + value: replayScopeForCredentials(agentToken), enumerable: false, }); return client; } +/** + * A replay scope for whatever credentials decide who a write acts as and where + * it lands. Only a SHA-256 digest is kept, never the credentials themselves. + */ +export function replayScopeForCredentials(...credentials: string[]): string { + const material = credentials.length === 1 ? credentials[0] : JSON.stringify(credentials); + return createHash('sha256').update(material).digest('hex'); +} + /** The replay scope a client was tagged with, or '' for an untagged client. */ export function replayScopeOf(client: object): string { const scope = (client as { [REPLAY_SCOPE]?: unknown })[REPLAY_SCOPE]; From 4fcd1df7bbb4e69d4d67dea77a8f9c71d1473dca Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 16:33:43 -0700 Subject: [PATCH 39/42] docs(changelog): keep bundled entries unreleased after the v13.2.0 cut Merging trunk (which carries the v13.2.0 release) folded the bundle's 17 unreleased bullets into the released [13.2.0] section. Restore 13.2.0 verbatim from trunk and move them to [Unreleased - Minor]. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- CHANGELOG.md | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b91ffd068..cdaae5c022 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,14 +5,10 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] - -## [13.2.0] - 2026-10-08 +## [Unreleased - Minor] ### Added -- `agent-relay fleet spawn --sandbox` (unpinned or `--sandbox-provider agent37`) now uses Cloud's durable `async-v1` preparation and prints each preparation phase, so a slow Relayfile initial sync no longer ends as an unknown outcome. Older Cloud deployments keep working through the synchronous response. -- `@agent-relay/cloud` `ensureCloudFleetSandbox` accepts `preparationMode: 'async-v1'` and an `onPreparationProgress` callback; `CloudFleetSandboxProvisionError` exposes `sandboxAbsent` and a typed `preparationFailure` (`code`, `phase`, `causeStage`). - MCP `spawn` reports pending/liveness evidence, and `remove_agent` accepts optional registration-clearance waits. - `agent remove --wait` and `fleet release --delete-agent --wait` verify registration clearance before name reuse, with `--wait-timeout` and `--no-wait` controls. Fleet release adds `removal` evidence when waiting. - `fleet agent list` JSON includes node heartbeat timestamps and ages, with warnings for stale snapshots. @@ -26,14 +22,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- When Cloud ends a sandbox preparation (for example `relayfile_mount_failed` / `initial_sync_deadline`), `fleet spawn --sandbox` names the cause and confirms no sandbox was left running, without a redundant cleanup call or unknown-outcome guidance. -- A lost Cloud response during `fleet spawn --sandbox` preparation no longer causes the spawn or a preparation step to be resubmitted. -- `fleet spawn --sandbox` prints the generated sandbox identity before provisioning; re-running an interrupted spawn with `--sandbox-id ` resumes that sandbox instead of creating a second one. -- Against a Cloud that does not support async preparation, a typed failure such as `relayfile_mount_failed` or a capacity rejection is reported at once with its cause instead of after a status-polling wait. -- Workspace-scoped commands such as `agent-relay fleet release` continue using the workspace's Relaycast server when `--base-url` is omitted; explicit conflicting `--base-url` values are still rejected. -- Fleet spawns now recover from transient Relaycast `d1_pressure`; persistent registration pressure returns a named `d1_pressure` error within 25 seconds, and any success arriving after retry-count or wall-clock exhaustion is explicitly deregistered. -- Acknowledged `agent.deregister`, cumulative `delivery.ack`, and non-task action results now recover from transient Relaycast `d1_pressure`; exhausted non-task action results are replayed after reconnect. -- `agent-relay fleet spawn` reports registration `d1_pressure` as retryable only when the registration is proven to have failed before any change was made. - Verified Fleet Devin spawns fail with `directory_trust_required` when the directory-trust prompt persists, releasing the worker through spawn cleanup instead of leaving a live but blocked worker. - Relay MCP `post_message` and `reply_to_thread` coalesce a transport replay of one request (and accept an `idempotency_key` for retries after a lost response), matching `send_dm`; native host tools key on the tool call ID. Independent writes with identical text remain separate messages. - `integration subscribe` rejects `--events` values the engine cannot subscribe to. @@ -45,6 +33,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - PTY tasks and relay messages use atomic bracketed paste when supported, wait for the composer, and prevent embedded carriage returns from submitting partial input. Oversized bodies and tail-only echoes fail explicitly without replay. - A confirmed `fleet spawn --task` (the default) succeeds only once the agent has visibly received the whole task. If the task cannot be delivered, the worker is released and the spawn fails with `spawn_task_failed`. If delivery cannot be confirmed, the spawn fails with `spawn_task_unconfirmed`, names the agent that is still running, and says not to retry. With `--no-confirm` the spawn reports as soon as the node accepts it, without waiting for the task to arrive. +## [13.2.0] - 2026-10-08 + +### Added + +- `agent-relay fleet spawn --sandbox` (unpinned or `--sandbox-provider agent37`) now uses Cloud's durable `async-v1` preparation and prints each preparation phase, so a slow Relayfile initial sync no longer ends as an unknown outcome. Older Cloud deployments keep working through the synchronous response. +- `@agent-relay/cloud` `ensureCloudFleetSandbox` accepts `preparationMode: 'async-v1'` and an `onPreparationProgress` callback; `CloudFleetSandboxProvisionError` exposes `sandboxAbsent` and a typed `preparationFailure` (`code`, `phase`, `causeStage`). + +### Fixed + +- When Cloud ends a sandbox preparation (for example `relayfile_mount_failed` / `initial_sync_deadline`), `fleet spawn --sandbox` names the cause and confirms no sandbox was left running, without a redundant cleanup call or unknown-outcome guidance. +- A lost Cloud response during `fleet spawn --sandbox` preparation no longer causes the spawn or a preparation step to be resubmitted. +- `fleet spawn --sandbox` prints the generated sandbox identity before provisioning; re-running an interrupted spawn with `--sandbox-id ` resumes that sandbox instead of creating a second one. +- Against a Cloud that does not support async preparation, a typed failure such as `relayfile_mount_failed` or a capacity rejection is reported at once with its cause instead of after a status-polling wait. +- Workspace-scoped commands such as `agent-relay fleet release` continue using the workspace's Relaycast server when `--base-url` is omitted; explicit conflicting `--base-url` values are still rejected. +- Fleet spawns now recover from transient Relaycast `d1_pressure`; persistent registration pressure returns a named `d1_pressure` error within 25 seconds, and any success arriving after retry-count or wall-clock exhaustion is explicitly deregistered. +- Acknowledged `agent.deregister`, cumulative `delivery.ack`, and non-task action results now recover from transient Relaycast `d1_pressure`; exhausted non-task action results are replayed after reconnect. +- `agent-relay fleet spawn` reports registration `d1_pressure` as retryable only when the registration is proven to have failed before any change was made. + ## [13.1.5] - 2026-10-08 ### Fixed From 2197c0e9d157c337cd9ab7015d34e8ea4f6a8d8e Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 8 Oct 2026 17:17:08 -0700 Subject: [PATCH 40/42] fix: address cubic review on the trunk-based bundle head - Roster-derived spawn liveness keeps a failed invocation read (readError). - A late persona acknowledgement names the terminal parent invocation. - Group-DM sendMessage accepts idempotencyKey in the thin-client type. - Remove the now-unused validateInjectionSize. - Docs: LF stays in typed-fallback payloads; e2e timeout comment corrected. - Changelog: user-facing wording for delivery and pending-spawn bullets. - Give the spawn replay-scope test an explicit budget for loaded hosts. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 36e2deb9-531e-419c-ab12-946ffb5d2dd5 --- CHANGELOG.md | 4 ++-- docs/harnesses/injection.md | 7 +++++-- .../src/cli/agent-relay-mcp.startup.test.ts | 5 ++++- packages/cli/src/cli/agent-relay-mcp.ts | 2 +- .../cli/src/cli/lib/spawn-liveness.test.ts | 20 +++++++++++++++++++ packages/cli/src/cli/lib/spawn-liveness.ts | 8 +++++++- packages/cli/src/cli/lib/task-input.ts | 4 ---- packages/sdk/src/messaging/thin-client.ts | 2 ++ tests/e2e/fleet/fleet-e2e.test.ts | 4 ++-- 9 files changed, 43 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdaae5c022..597cb803b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Unconfirmed `fleet spawn` results keep the `spawn_unconfirmed` code, now with `state: "pending"`, invocation diagnostics, liveness evidence and exit 8. Node heartbeat presence is reported as liveness evidence but never confirms the spawn, because heartbeats cannot tell this invocation's worker from an earlier one with the same name. +- Unconfirmed `fleet spawn` results keep the `spawn_unconfirmed` code, now with `state: "pending"`, invocation diagnostics, liveness evidence and exit 8. Node heartbeat presence is reported as liveness evidence but never confirms the spawn. - Removal waits exit 8 if the registration remains present; scripts must handle nonzero exits beyond `$? -eq 1`. Waits remain opt-in, and unavailable verification reads preserve the asynchronous acknowledgement. - PTY-delivered tasks and messages are limited to 14 KiB (16 KiB with the message envelope); `fleet spawn`, `agent spawn` and PTY delivery reject larger input and name the limit, so write the brief to a file and send a pointer. Agents whose terminal lacks bracketed paste accept at most 1,536 bytes per injection. Muse startup tasks and messages to headless CLI agents travel as a single process argument and are limited to 16 KiB instead; native-runtime and app-server agents receive tasks and messages as frames and have no such limit; `message post|reply|dm send` are not capped, and a wrapped agent receives an oversized message as a pointer to it in Relay. @@ -30,7 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `integration subscribe --list` separates a subscription's configuration from delivery confirmation, so a configured subscription is no longer read as a delivered one. - Fleet registration collisions use readable `spawn_name_taken` guidance covering asynchronous removal and failed-spawn cleanup, instead of Rust `Fatal(AlreadyExists {...})` text. - Pending fleet spawns preserve provisioned sandboxes and distinguish live, stale, elsewhere, registered, absent, and unknown evidence before advising a retry. -- PTY tasks and relay messages use atomic bracketed paste when supported, wait for the composer, and prevent embedded carriage returns from submitting partial input. Oversized bodies and tail-only echoes fail explicitly without replay. +- Tasks and relay messages typed into an agent's terminal wait until the agent is ready for input and can no longer be submitted half-way through, and a message that is too large fails with an explicit error instead of being retried. - A confirmed `fleet spawn --task` (the default) succeeds only once the agent has visibly received the whole task. If the task cannot be delivered, the worker is released and the spawn fails with `spawn_task_failed`. If delivery cannot be confirmed, the spawn fails with `spawn_task_unconfirmed`, names the agent that is still running, and says not to retry. With `--no-confirm` the spawn reports as soon as the node accepts it, without waiting for the task to arrive. ## [13.2.0] - 2026-10-08 diff --git a/docs/harnesses/injection.md b/docs/harnesses/injection.md index 006d238ecf..2cb13d4251 100644 --- a/docs/harnesses/injection.md +++ b/docs/harnesses/injection.md @@ -28,8 +28,11 @@ to Relaycast, since recipients may be native agents or history only. For longer briefs, create a file on the node and send a short instruction to read it. The input wire normalizes CRLF to LF and strips bare carriage returns and ESC -characters. These bytes cannot submit the composer early or close a paste from -inside its body. Other UTF-8 content is preserved. +characters, so they cannot submit the composer early or close a paste from +inside its body. LF stays in the payload: inside a bracketed paste it is a +newline, but on the typed fallback wire a harness may treat it as Enter, so +multiline tasks are only safe on paste-capable harnesses. Other UTF-8 content +is preserved. ```sh agent-relay fleet spawn claude --node my-node --name reviewer --task-file ./brief.md diff --git a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts index edb6bf1e3a..c90d8d1f6a 100644 --- a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts +++ b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts @@ -445,7 +445,8 @@ describe('agent-relay-mcp startup helpers', () => { }); await spawn(input, { sessionId: 'mcp-session', requestId: 3 }); expect(mocks.agentRelayMessagingCommands.invoke).toHaveBeenCalledTimes(2); - }); + // Three spawns through the real confirmation path; allow for a loaded host. + }, 20_000); it('coalesces replayed MCP request ids without collapsing separate same-name spawns', async () => { const { mod, mocks } = await loadAgentRelayMcpModule(); @@ -1324,6 +1325,8 @@ describe('createAgentRelayMcpServer', () => { error: { code: 'spawn_unconfirmed', invocationId: 'inv_child', node: 'child-node' }, }, }); + // The terminal parent stays locatable in the activity log. + expect(JSON.stringify(result)).toContain('inv_1'); }); it.each(['live', 'live_elsewhere', 'stale', 'unknown'] as const)( diff --git a/packages/cli/src/cli/agent-relay-mcp.ts b/packages/cli/src/cli/agent-relay-mcp.ts index e09850366e..be53299344 100644 --- a/packages/cli/src/cli/agent-relay-mcp.ts +++ b/packages/cli/src/cli/agent-relay-mcp.ts @@ -933,7 +933,7 @@ async function invokeVerifiedSpawn( dispatchState: nested.dispatchState, }, { evidence: 'unknown' }, - 'The persona handler returned a nested spawn invocation after the confirmation budget.' + `The persona handler's invocation ${context.invocationId ?? '(unknown)'}${context.node ? ` on ${context.node}` : ''} completed with a nested spawn invocation after the confirmation budget.` ); } try { diff --git a/packages/cli/src/cli/lib/spawn-liveness.test.ts b/packages/cli/src/cli/lib/spawn-liveness.test.ts index 87af4211ef..02221a30ca 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.test.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.test.ts @@ -50,6 +50,26 @@ describe('spawn liveness resolution', () => { }); // relay#1930 review (cubic): the clock is read when a heartbeat is judged, // so read latency cannot make an old heartbeat look fresh. + // relay#1930 review (cubic): a failed invocation re-read must survive into + // roster-derived evidence, as it already does for registered/absent. + it('keeps a failed invocation read on live and stale evidence', async () => { + for (const [record, evidence] of [ + [node(), 'live'], + [node('target', 36_001), 'stale'], + ] as const) { + expect( + await probeSpawnLiveness({ + name: 'worker', + targetNode: 'target', + getInvocation: async () => { + throw new Error('invocation read failed'); + }, + createClient: () => client([record]), + now: () => now, + }) + ).toMatchObject({ evidence, readError: expect.stringContaining('invocation read failed') }); + } + }); it('measures heartbeat age after the bounded node reads, not before', async () => { let clock = now; const workspace = client([node('target', 30_000)]); diff --git a/packages/cli/src/cli/lib/spawn-liveness.ts b/packages/cli/src/cli/lib/spawn-liveness.ts index 61f3035d88..b2bdf7e513 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.ts @@ -173,7 +173,13 @@ export async function probeSpawnLiveness(options: { : options.targetNode && node.name !== options.targetNode ? 'live_elsewhere' : 'live'; - const result: SpawnLiveness = { evidence, node: node.name, heartbeatAgeMs: age }; + // Keep a failed invocation read visible next to roster evidence. + const result: SpawnLiveness = { + evidence, + node: node.name, + heartbeatAgeMs: age, + ...(readError ? { readError } : {}), + }; if (evidence === 'live') return result; // Fresh evidence elsewhere outranks a stale entry. if (!weaker || (weaker.evidence === 'stale' && evidence === 'live_elsewhere')) weaker = result; diff --git a/packages/cli/src/cli/lib/task-input.ts b/packages/cli/src/cli/lib/task-input.ts index c7d405b8a0..4575ca5b27 100644 --- a/packages/cli/src/cli/lib/task-input.ts +++ b/packages/cli/src/cli/lib/task-input.ts @@ -121,7 +121,3 @@ function throwInjectionTooLarge(): never { `Task exceeds ${MAX_TASK_BODY_BYTES} UTF-8 bytes (the ${MAX_INJECTION_BODY_BYTES}-byte PTY limit less room for the message envelope); use a brief file on the node and send a short pointer` ); } - -export function validateInjectionSize(text: string): void { - if (Buffer.byteLength(text, 'utf8') > MAX_TASK_BODY_BYTES) throwInjectionTooLarge(); -} diff --git a/packages/sdk/src/messaging/thin-client.ts b/packages/sdk/src/messaging/thin-client.ts index 15daced8a2..144cdac273 100644 --- a/packages/sdk/src/messaging/thin-client.ts +++ b/packages/sdk/src/messaging/thin-client.ts @@ -140,6 +140,8 @@ export interface RelayAgentThinClient { conversationId: string, text: string, options?: { + /** Stable key for retries of one logical send, including across processes. */ + idempotencyKey?: string; attachments?: string[]; data?: Record | null; mode?: RelayMessageMode; diff --git a/tests/e2e/fleet/fleet-e2e.test.ts b/tests/e2e/fleet/fleet-e2e.test.ts index 219a7eb6c9..04acf9574d 100644 --- a/tests/e2e/fleet/fleet-e2e.test.ts +++ b/tests/e2e/fleet/fleet-e2e.test.ts @@ -557,8 +557,8 @@ describe.skipIf(!pre.ok)('two-node fleet scenario matrix', () => { return inv.status === 'completed' || inv.status === 'failed' ? inv : null; }, // A verified spawn carrying a task now completes only once that task's - // receipt is verified, after readiness. node-a's stub delays readiness - // by 27 s, so allow for readiness plus delivery verification. + // receipt is verified after readiness, so allow for readiness plus + // delivery verification rather than readiness alone. { label: 'resumable spawn settled', timeoutMs: 60_000, intervalMs: 300 } ); expect(firstDone.status).toBe('completed'); // resumable spawn carried session_ref through token authority From bed0f3100954c82b74e973b059a1d71fa87bf620 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 10 Oct 2026 06:19:58 +0000 Subject: [PATCH 41/42] style: auto-format with Prettier --- .github/workflows/codegen-models.yml | 3 +-- .github/workflows/security.yml | 2 -- AGENTS.md | 15 ++++++++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/codegen-models.yml b/.github/workflows/codegen-models.yml index 72f21434cb..8568f7f10a 100644 --- a/.github/workflows/codegen-models.yml +++ b/.github/workflows/codegen-models.yml @@ -4,8 +4,7 @@ on: push: paths: - 'packages/utils/cli-registry.yaml' - branches: - [main] + branches: [main] pull_request: paths: - 'packages/utils/cli-registry.yaml' diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 89d55e1c7c..903a6b9074 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -32,7 +32,6 @@ jobs: ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main')) && (needs.changes.outputs.node_changed == 'true') runs-on: ubuntu-latest - steps: - name: Checkout code uses: actions/checkout@v4 @@ -147,7 +146,6 @@ jobs: ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || (github.head_ref == 'trunk' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == 'main')) && (needs.changes.outputs.node_changed == 'true') runs-on: ubuntu-latest - steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/AGENTS.md b/AGENTS.md index b9bbd8cac9..9459ebd043 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -315,6 +315,7 @@ Future agents can query past trajectories to learn from your decisions. + ## Merging: `trunk` + the `mergeable` label CI suites do **not** run automatically on feature branches. They run only on @@ -326,21 +327,24 @@ run on any branch. (Repos whose default branch is not agent batches ready PRs into `trunk`, gets that one PR green, and merges it. **When you open a PR** + 1. Branch from `trunk` and open the PR with **base `trunk`**, not `main`. A PR into `main` from any other branch fails the `Trunk guard` check. 2. No CI runs on your PR, so verify locally before calling it ready: run the typecheck, tests and lint this repo uses, and list the exact commands and results in the PR body. -**When the PR is ready** -3. Add the label **`mergeable`** once all of these are true: - - The change is complete and the local checks above pass. - - Review feedback (human and bot) is addressed or answered. - - It is not a draft and does not depend on an unmerged PR. +**When the PR is ready** 3. Add the label **`mergeable`** once all of these are true: + +- The change is complete and the local checks above pass. +- Review feedback (human and bot) is addressed or answered. +- It is not a draft and does not depend on an unmerged PR. + 4. Remove `mergeable` if the PR stops being ready (new work, a failing check, a blocking question). The label is read live from GitHub on every sweep. **What you must not do** + - Do not merge your own PR, and never merge into or push to `trunk` or `main` directly. - Do not re-enable CI for feature branches or edit the `trunk` gates in @@ -355,4 +359,5 @@ to rebase on `trunk`; do so and keep the label. > Interim: the sweep worker is not deployed yet. Until it is, a human or a > designated agent performs the merge-agent steps manually. Labelling is unchanged. + From 4690354e1ba1a94c8ce94a9405f48f448ded4ece Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 10 Oct 2026 01:27:16 -0700 Subject: [PATCH 42/42] fix(broker,cli,harnesses): address #1930 review threads - PTY injection drops every control character except LF and tab, so a body cannot interrupt (Ctrl-C), kill or edit the composer line on either wire. - Remove the unreachable second initial-Codex size check: the shared size gate already caps an initial Codex body at initial_codex_max_body_bytes. - Spawn liveness: a heartbeat stamped more than 5s in the future is judged unknown rather than fresh, and an available node whose live-agent roster is missing or undecodable keeps a 404 registration read at "unknown" instead of "absent". - Native send_dm joins a replayed tool call and sends a tool-call idempotency key, like post_message and reply_to_thread. - --task-file keeps a leading UTF-8 BOM, as --task would. - MCP spawn description names the pending shape it actually returns. - Docs scope the 14,336-byte cap to PTY agents and describe control-byte stripping. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 1 + crates/broker/src/injection_wire.rs | 23 +++++++++++++- crates/broker/src/pty_worker.rs | 30 +++---------------- docs/harnesses/injection.md | 15 +++++----- .../src/cli/agent-relay-mcp.startup.test.ts | 15 +++++++++- packages/cli/src/cli/agent-relay-mcp.ts | 2 +- .../cli/src/cli/commands/fleet-agent.test.ts | 22 +++++++++++++- packages/cli/src/cli/lib/fleet-live-agents.ts | 23 +++++++++++++- .../cli/src/cli/lib/spawn-liveness.test.ts | 16 ++++++++++ packages/cli/src/cli/lib/spawn-liveness.ts | 14 +++++---- packages/cli/src/cli/lib/task-input.test.ts | 3 ++ packages/cli/src/cli/lib/task-input.ts | 2 +- .../src/ai-sdk/native-relay-tools.test.ts | 2 ++ .../src/ai-sdk/native-relay-tools.ts | 8 +++-- 14 files changed, 129 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a17087270..bcf1fa6ed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fleet registration collisions use readable `spawn_name_taken` guidance covering asynchronous removal and failed-spawn cleanup, instead of Rust `Fatal(AlreadyExists {...})` text. - Pending fleet spawns preserve provisioned sandboxes and distinguish live, stale, elsewhere, registered, absent, and unknown evidence before advising a retry. - Tasks and relay messages typed into an agent's terminal wait until the agent is ready for input and can no longer be submitted half-way through, and a message that is too large fails with an explicit error instead of being retried. +- Control characters other than newline and tab (for example `Ctrl-C`) are removed from tasks and messages typed into an agent's terminal, so they can no longer interrupt or edit the agent's input line. - A confirmed `fleet spawn --task` (the default) succeeds only once the agent's harness has visibly accepted the task (activity or a cleared composer, not merely the task echoed into the composer). If the task cannot be delivered, the worker is released and the spawn fails with `spawn_task_failed`. If delivery cannot be confirmed, the spawn fails with `spawn_task_unconfirmed`, names the agent that is still running, and says not to retry. With `--no-confirm` the spawn reports as soon as the node accepts it, without waiting for the task to arrive. - `agent-relay message file upload` now uploads and attaches the file; it always failed with "Invalid attachments: file ids must exist in workspace and be complete". - Broker-managed messages retry failed delivery up to three times, and later sends wait until delivery succeeds. diff --git a/crates/broker/src/injection_wire.rs b/crates/broker/src/injection_wire.rs index df4da47e28..ab1171b032 100644 --- a/crates/broker/src/injection_wire.rs +++ b/crates/broker/src/injection_wire.rs @@ -44,7 +44,15 @@ fn select_wire(cli: &str, capable: bool, disabled: bool) -> InjectionWire { } } pub(crate) fn injection_bytes(wire: InjectionWire, text: &str) -> Vec { - let text = text.replace("\r\n", "\n").replace(['\r', '\x1b'], ""); + // A body is content, never keystrokes: drop every control character (C0, + // DEL, C1), so `\x03`, `\x15` or `\x7f` cannot interrupt, kill or edit + // the composer line, and `\r`/ESC cannot submit or close the paste. Line + // breaks and tabs are content and stay. + let text: String = text + .replace("\r\n", "\n") + .chars() + .filter(|&c| c == '\n' || c == '\t' || !c.is_control()) + .collect(); match wire { InjectionWire::Paste => format!("\x1b[200~{text}\x1b[201~").into_bytes(), InjectionWire::Typed => text.into_bytes(), @@ -132,4 +140,17 @@ mod tests { assert_eq!(select_wire("other", false, false), InjectionWire::Typed); assert_eq!(injection_bytes(InjectionWire::Typed, "a\r\nb\rc"), b"a\nbc"); } + /// relay#1930 review: control bytes in a task are keypresses to a PTY. + #[test] + fn control_characters_are_dropped_but_line_breaks_and_tabs_stay() { + let body = "a\x03b\x04c\x15d\x7fe\u{9b}f\x00g\n\th"; + assert_eq!( + injection_bytes(InjectionWire::Typed, body), + "abcdefg\n\th".as_bytes() + ); + assert_eq!( + injection_bytes(InjectionWire::Paste, body), + "\x1b[200~abcdefg\n\th\x1b[201~".as_bytes() + ); + } } diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index fcbc9cc57d..717879e52b 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -2351,32 +2351,10 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { mcp_reminder_throttle.note_sent(Instant::now()); } if initial_codex_delivery(&resolved_cli, &inj.pending.delivery) { - // Reject before any byte is written rather than after - // guaranteeing a partial-write timeout: a body needing - // more chunks than initial_codex_deadline's cap can - // afford cannot reach Enter no matter how fast every - // chunk renders (relay#1782 review). Reuses the - // existing was_chunked && !confirmed path below -- - // delivery_failed, retry record removed, id retained - // to reject a duplicate, body never replayed. - let max_bytes = initial_codex_max_body_bytes(); - if injection.len() > max_bytes { - tracing::warn!( - delivery_id = %inj.pending.delivery.delivery_id, - body_bytes = injection.len(), - max_bytes, - "initial Codex task exceeds chunked delivery's deadline-bounded size limit; rejecting before writing any byte" - ); - inj.stage = InjectionStage::Finalize; - initial_injection_cancel = Some(Arc::new(AtomicBool::new(false))); - injection_ack = Some(Box::pin(async move { - Ok(Err(std::io::Error::other( - "initial Codex task exceeds the maximum size chunked delivery can complete within its deadline; body will not be written", - ))) - })); - active_injection = Some(inj); - continue; - } + // The size gate above already caps an initial Codex + // body at `initial_codex_max_body_bytes`, so every body + // reaching here can finish within its deadline + // (relay#1782 review). let before = pty.consumed_offset(); let end = chunk_end(&injection, 0); let (first_ack, boundary) = match pty.submit_write_paced_with_output_boundary( diff --git a/docs/harnesses/injection.md b/docs/harnesses/injection.md index e260274699..840f158ebe 100644 --- a/docs/harnesses/injection.md +++ b/docs/harnesses/injection.md @@ -12,10 +12,10 @@ for the session, independently of composer readiness. Devin also uses this wire. Codex initial tasks retain their existing chunked delivery path. The broker-wide ceiling is **16,384 UTF-8 bytes** for the formatted envelope. Bodies -are capped at **14,336 bytes** (`fleet spawn` task input, the fleet spawn action -before any launch, and broker PTY delivery), leaving 2 KiB for attribution and -broker reminders so an accepted body still fits once formatted. Paced -fallback input has an effective limit of **1,536 bytes including the envelope** at +are capped at **14,336 bytes** for PTY agents (`fleet spawn` task input, the fleet +spawn action before it launches a PTY agent, and broker PTY delivery); native and +headless tasks are not held to it. That leaves 2 KiB for attribution and broker +reminders so an accepted body still fits once formatted. Paced fallback input has an effective limit of **1,536 bytes including the envelope** at the default 5 ms pace. A slower `RELAY_INJECT_RATE_MS` lowers that limit to keep writing within 7.68 seconds. Unpaced fallback uses the global ceiling. Codex uses the smaller of the global ceiling and its deadline-derived limit. Rejections name @@ -27,9 +27,10 @@ enforced only at the PTY boundary: `message post|reply|dm send` publish any size to Relaycast, since recipients may be native agents or history only. For longer briefs, create a file on the node and send a short instruction to read it. -The input wire normalizes CRLF to LF and strips bare carriage returns and ESC -characters, so they cannot submit the composer early or close a paste from -inside its body. LF stays in the payload: inside a bracketed paste it is a +The input wire normalizes CRLF to LF and strips every other control character +except tab (bare carriage returns, ESC, `Ctrl-C` and the like), so a body cannot +submit the composer early, close a paste, or interrupt or edit the composer +line. LF stays in the payload: inside a bracketed paste it is a newline, but on the typed fallback wire a harness may treat it as Enter, so multiline tasks are only safe on paste-capable harnesses. Other UTF-8 content is preserved. diff --git a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts index c90d8d1f6a..e8a7acdac7 100644 --- a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts +++ b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts @@ -1358,7 +1358,20 @@ describe('createAgentRelayMcpServer', () => { } ); - it('classifies MCP registration collisions without exposing Rust debug text', async () => { + // relay#1930 review: the tool description must name the pending shape the + // handler actually returns (asserted above), not a `confirmed` field. + it('describes pending spawns by the shape the spawn tool returns', async () => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mod.createAgentRelayMcpServer({ agentToken: 'at_live_fleet', agentName: 'orchestrator' }); + const { description } = mocks.serverInstances[0].tools.get('spawn')!.config as { description: string }; + expect(description).toContain('spawn_unconfirmed'); + expect(description).toContain('state: "pending"'); + expect(description).not.toContain('confirmed:false'); + }); + + // The broker formats the collision readably; its Rust test asserts no + // `Fatal(...)` debug text, so this MCP test covers classification only. + it('classifies MCP registration collisions as spawn_name_taken', async () => { const { mod, mocks } = await loadAgentRelayMcpModule(); mocks.agentRelayMessagingCommands.getInvocation.mockResolvedValueOnce({ status: 'failed', diff --git a/packages/cli/src/cli/agent-relay-mcp.ts b/packages/cli/src/cli/agent-relay-mcp.ts index be53299344..61ea2fbfa4 100644 --- a/packages/cli/src/cli/agent-relay-mcp.ts +++ b/packages/cli/src/cli/agent-relay-mcp.ts @@ -1466,7 +1466,7 @@ function registerAgentRelayTools( title: 'Spawn Agent', description: 'Invoke the fleet spawn action with either a raw `cli` or an AgentWorkforce `persona` name/path. ' + - 'Persona requests route to a node exposing `spawn:persona` (for example, `defineWorkforcePersonaSpawnNode` from `@agentworkforce/local-surface`). Both forms wait for broker registration and harness readiness. Unconfirmed outcomes report pending evidence; fresh worker presence may return confirmed:false without readiness proof.', + 'Persona requests route to a node exposing `spawn:persona` (for example, `defineWorkforcePersonaSpawnNode` from `@agentworkforce/local-surface`). Both forms wait for broker registration and harness readiness. Unconfirmed outcomes return error code `spawn_unconfirmed` with `state: "pending"` and `liveness` evidence; a fresh worker heartbeat is evidence, not readiness proof, so do not retry a pending spawn.', inputSchema: { name: z.string().describe('Agent name'), cli: z diff --git a/packages/cli/src/cli/commands/fleet-agent.test.ts b/packages/cli/src/cli/commands/fleet-agent.test.ts index 30bf8f9518..abe2b0d605 100644 --- a/packages/cli/src/cli/commands/fleet-agent.test.ts +++ b/packages/cli/src/cli/commands/fleet-agent.test.ts @@ -59,6 +59,7 @@ describe('readRemoteLiveAgents', () => { expect(result).toEqual({ supported: true, + complete: true, agents: [{ name: 'å-worker' }, { name: 'worker-z' }], }); }); @@ -66,6 +67,7 @@ describe('readRemoteLiveAgents', () => { it('treats an absent marker as an unsupported old broker', () => { expect(readRemoteLiveAgents(node({ name: 'sf-mini', capabilities: [] }))).toEqual({ supported: false, + complete: false, agents: [], }); }); @@ -84,7 +86,25 @@ describe('readRemoteLiveAgents', () => { ], }) ) - ).toMatchObject({ supported: true, agents: [], warning: expect.stringContaining('1 malformed') }); + ).toMatchObject({ + supported: true, + complete: false, + agents: [], + warning: expect.stringContaining('1 malformed'), + }); + }); + + it('keeps a roster with only duplicate names complete', () => { + expect( + readRemoteLiveAgents( + node({ + name: 'sf-mini', + capabilities: [ + { name: LIVE_AGENT_CAPABILITY_NAME, kind: 'capacity', metadata: { names: ['w', 'w'] } }, + ], + }) + ) + ).toMatchObject({ supported: true, complete: true, agents: [{ name: 'w' }] }); }); }); diff --git a/packages/cli/src/cli/lib/fleet-live-agents.ts b/packages/cli/src/cli/lib/fleet-live-agents.ts index 94d58c2bfe..e22da67ada 100644 --- a/packages/cli/src/cli/lib/fleet-live-agents.ts +++ b/packages/cli/src/cli/lib/fleet-live-agents.ts @@ -22,6 +22,11 @@ export const LIVE_AGENT_CAPABILITY_NAME = 'relay:live-agents:v1'; export interface RemoteLiveAgentRead { supported: boolean; + /** + * Whether `agents` is the node's whole live set: the capability is present + * and every entry decoded. Duplicates do not make a roster incomplete. + */ + complete: boolean; agents: RemoteLiveAgent[]; warning?: string; } @@ -30,6 +35,7 @@ export interface RemoteLiveAgentRead { export function readRemoteLiveAgents(node: RelayNode): RemoteLiveAgentRead { let supported = false; let malformed = 0; + let undecoded = false; const names = new Set(); for (const capability of node.capabilities) { if (capability.name !== LIVE_AGENT_CAPABILITY_NAME) continue; @@ -37,11 +43,13 @@ export function readRemoteLiveAgents(node: RelayNode): RemoteLiveAgentRead { const rawNames = capability.metadata?.names; if (!Array.isArray(rawNames)) { malformed += 1; + undecoded = true; continue; } for (const rawName of rawNames) { if (typeof rawName !== 'string' || !rawName || names.has(rawName)) { malformed += 1; + if (typeof rawName !== 'string' || !rawName) undecoded = true; continue; } names.add(rawName); @@ -49,6 +57,7 @@ export function readRemoteLiveAgents(node: RelayNode): RemoteLiveAgentRead { } return { supported, + complete: supported && !undecoded, agents: Array.from(names, (name) => ({ name })).sort((a, b) => a.name.localeCompare(b.name)), ...(malformed > 0 ? { @@ -82,7 +91,19 @@ export function isAvailableFleetNode(node: { export const MAX_LIVE_HEARTBEAT_AGE_MS = 36_000; +/** + * Clock skew tolerated before a server-stamped heartbeat in the future stops + * counting as fresh. Within it the age clamps to zero; beyond it the local + * clock cannot be trusted to judge the heartbeat at all, and a lagging clock + * would otherwise make an old snapshot look current. + */ +export const MAX_HEARTBEAT_CLOCK_SKEW_MS = 5_000; + +/** Age of a node's last heartbeat, or null when it cannot be judged. */ export function heartbeatAgeMs(node: { lastHeartbeatAt?: string }, now = Date.now()): number | null { const timestamp = Date.parse(node.lastHeartbeatAt ?? ''); - return Number.isFinite(timestamp) ? Math.max(0, now - timestamp) : null; + if (!Number.isFinite(timestamp)) return null; + const age = now - timestamp; + if (age < -MAX_HEARTBEAT_CLOCK_SKEW_MS) return null; + return Math.max(0, age); } diff --git a/packages/cli/src/cli/lib/spawn-liveness.test.ts b/packages/cli/src/cli/lib/spawn-liveness.test.ts index 02221a30ca..45f7bf7787 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.test.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.test.ts @@ -35,6 +35,10 @@ describe('spawn liveness resolution', () => { [node('other'), 'live_elsewhere'], [node('target', 36_001), 'stale'], [{ ...node(), lastHeartbeatAt: undefined }, 'stale'], + // A heartbeat stamped within the tolerated clock skew is still fresh, but + // one far in the future cannot be judged by this clock (relay#1930 review). + [node('target', -4_000), 'live'], + [node('target', -60_000), 'stale'], [{ ...node(), status: 'offline' }, 'absent'], ] as const)('requires fresh target-node evidence (%j)', async (record, evidence) => { const workspace = client([record]); @@ -180,6 +184,18 @@ describe('spawn liveness resolution', () => { evidence: 'unknown', }); }); + // relay#1930 review (cubic): an available node whose heartbeat roster is + // missing or undecodable may still run the worker, so a 404 on the agent + // registration cannot prove it absent. + it.each([ + ['an unsupported', []], + ['a malformed', [{ name: 'relay:live-agents:v1', metadata: { names: 'worker' } }]], + ] as const)('does not report absent beside %s heartbeat roster', async (_label, capabilities) => { + const workspace = client([{ ...node('other'), capabilities: [...capabilities] } as RelayNode]); + expect( + await probeSpawnLiveness({ name: 'worker', createClient: () => workspace, now: () => now }) + ).toMatchObject({ evidence: 'unknown' }); + }); it.each(['get', 'list'] as const)( 'does not report absent when the node %s read failed', async (failing) => { diff --git a/packages/cli/src/cli/lib/spawn-liveness.ts b/packages/cli/src/cli/lib/spawn-liveness.ts index b2bdf7e513..a506ac9980 100644 --- a/packages/cli/src/cli/lib/spawn-liveness.ts +++ b/packages/cli/src/cli/lib/spawn-liveness.ts @@ -116,6 +116,7 @@ export async function probeSpawnLiveness(options: { }): Promise { let readError: string | undefined; let rosterUnread = false; + let rosterIncomplete = false; const read = (fn: () => Promise) => withDeadline(fn, () => new Error('Spawn evidence read timed out.'), 2_000); if (options.getInvocation) { @@ -161,11 +162,12 @@ export async function probeSpawnLiveness(options: { let weaker: SpawnLiveness | undefined; const now = clock(); for (const node of nodes) { - if ( - !isAvailableFleetNode(node) || - !readRemoteLiveAgents(node).agents.some((agent) => agent.name === options.name) - ) - continue; + if (!isAvailableFleetNode(node)) continue; + const live = readRemoteLiveAgents(node); + // A node whose heartbeat roster is missing or undecodable may still be + // running the worker, so a later missing registration cannot prove absence. + if (!live.complete) rosterIncomplete = true; + if (!live.agents.some((agent) => agent.name === options.name)) continue; const age = heartbeatAgeMs(node, now); const evidence = age === null || age > MAX_LIVE_HEARTBEAT_AGE_MS @@ -195,7 +197,7 @@ export async function probeSpawnLiveness(options: { } catch (error) { // `absent` claims no worker was observed. A missing registration only // proves that when the node roster was actually read. - if (isNotFoundError(error) && !rosterUnread) + if (isNotFoundError(error) && !rosterUnread && !rosterIncomplete) return { evidence: 'absent', ...(readError ? { readError } : {}) }; if (!isNotFoundError(error)) readError = safeRelayErrorMessage(error); } diff --git a/packages/cli/src/cli/lib/task-input.test.ts b/packages/cli/src/cli/lib/task-input.test.ts index f6f826613e..1aa858edac 100644 --- a/packages/cli/src/cli/lib/task-input.test.ts +++ b/packages/cli/src/cli/lib/task-input.test.ts @@ -21,6 +21,9 @@ describe('task input', () => { const file = join(dir, 'brief.md'); await writeFile(file, ' héllo\r\nworld\n'); expect(await readTaskInput(undefined, file, true)).toBe(' héllo\r\nworld\n'); + // A leading BOM is file text like any other, as it would be via --task. + await writeFile(file, 'brief\n'); + expect(await readTaskInput(undefined, file, true)).toBe('brief\n'); await writeFile(file, ' '); await expect(readTaskInput(undefined, file)).rejects.toThrow('empty'); await expect(readTaskInput(undefined, join(dir, 'missing'))).rejects.toThrow(); diff --git a/packages/cli/src/cli/lib/task-input.ts b/packages/cli/src/cli/lib/task-input.ts index 4575ca5b27..4618d58a26 100644 --- a/packages/cli/src/cli/lib/task-input.ts +++ b/packages/cli/src/cli/lib/task-input.ts @@ -99,7 +99,7 @@ async function readTaskFile(path: string, limit: number, target: TaskTarget | un } if (length > limit) validateTaskBytes(length, target); try { - return new TextDecoder('utf-8', { fatal: true }).decode(buffer.subarray(0, length)); + return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(buffer.subarray(0, length)); } catch { throw new Error(`Task file ${path} is not valid UTF-8`); } diff --git a/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts b/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts index 22c8568ad1..aa9c8efc5c 100644 --- a/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts +++ b/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts @@ -59,6 +59,7 @@ describe('native Relay host tools', () => { }); it.each([ + ['send_dm', 'dm', { to: 'nativeClaude', text: 'ACK' }], ['post_message', 'send', { channel: 'events', text: 'ACK' }], ['reply_to_thread', 'reply', { message_id: 'parent', text: 'ACK' }], ] as const)('joins only a replay of the same native %s tool call', async (tool, method, args) => { @@ -90,6 +91,7 @@ it.each([ // tool result is still being submitted) reaches Relay again, so the write // carries a key derived from the tool call for Relay to deduplicate. it.each([ + ['send_dm', 'dm', { to: 'nativeClaude', text: 'ACK' }, 'nativeClaude'], ['post_message', 'send', { channel: 'events', text: 'ACK' }, 'events'], ['reply_to_thread', 'reply', { message_id: 'parent', text: 'ACK' }, 'parent'], ] as const)('sends native %s with a tool-call idempotency key', async (tool, method, args, target) => { diff --git a/packages/harnesses/src/ai-sdk/native-relay-tools.ts b/packages/harnesses/src/ai-sdk/native-relay-tools.ts index d23246d5c8..a1c2ab432b 100644 --- a/packages/harnesses/src/ai-sdk/native-relay-tools.ts +++ b/packages/harnesses/src/ai-sdk/native-relay-tools.ts @@ -94,9 +94,13 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha additionalProperties: false, }, }, - execute: (value) => { + execute: (value, { toolCallId } = {}) => { const input = objectInput(value); - return agent.dm(requiredString(input, 'to'), requiredString(input, 'text')); + const to = requiredString(input, 'to'); + const text = requiredString(input, 'text'); + return write('send_dm', toolCallId, (options) => + options ? agent.dm(to, text, options) : agent.dm(to, text) + ); }, }, {