diff --git a/.agentworkforce/trajectories/active/traj_ynyux9pee7x7/trajectory.json b/.agentworkforce/trajectories/active/traj_ynyux9pee7x7/trajectory.json index ae39159f30..3fc9615228 100644 --- a/.agentworkforce/trajectories/active/traj_ynyux9pee7x7/trajectory.json +++ b/.agentworkforce/trajectories/active/traj_ynyux9pee7x7/trajectory.json @@ -31,6 +31,108 @@ "reasoning": "npm ci pinned SDK 8.0.7 without metadata while Publish Package freshly resolves 8.14.0 with relay.action-caller=v1. Align CI with release resolution without weakening assertions or changing production code." }, "significance": "high" + }, + { + "ts": 1791096850177, + "type": "decision", + "content": "Root-caused tail truncation to paced per-codepoint typing (5ms/atom) with no bracketed-paste framing for Claude; plan fixes it with capability-gated atomic paste + head/tail-anchored confirmation: Root-caused tail truncation to paced per-codepoint typing (5ms/atom) with no bracketed-paste framing for Claude; plan fixes it with capability-gated atomic paste + head/tail-anchored confirmation", + "raw": { + "question": "Root-caused tail truncation to paced per-codepoint typing (5ms/atom) with no bracketed-paste framing for Claude; plan fixes it with capability-gated atomic paste + head/tail-anchored confirmation", + "chosen": "Root-caused tail truncation to paced per-codepoint typing (5ms/atom) with no bracketed-paste framing for Claude; plan fixes it with capability-gated atomic paste + head/tail-anchored confirmation", + "alternatives": [], + "reasoning": "write_paced emits one VT atom per write_all, so a 6.7KB task spends ~33s only in the TUI composer; any composer clear in that window loses the head. devin.rs:12 wraps paste only for Devin. check_echo_in_output cannot detect the loss, so pty_worker.rs:2368 acks it as timeout_fallback success." + }, + "significance": "high" + }, + { + "ts": 1791099654001, + "type": "decision", + "content": "Reserve per-byte VTE parsing for chunks carrying the DECSET 2004 parameter: Reserve per-byte VTE parsing for chunks carrying the DECSET 2004 parameter", + "raw": { + "question": "Reserve per-byte VTE parsing for chunks carrying the DECSET 2004 parameter", + "chosen": "Reserve per-byte VTE parsing for chunks carrying the DECSET 2004 parameter", + "alternatives": [], + "reasoning": "The paste-capability latch parsed every PTY output byte with its own processor.advance() call until the mode was seen, which is permanent for any harness that never enables 2004. Scanning the chunk for the literal 2004 and otherwise bulk-parsing with one post-chunk mode check keeps enable+disable-in-one-read detection (both complete sequences must contain 2004) and still latches a sequence split across reads." + }, + "significance": "high" + }, + { + "ts": 1791099654447, + "type": "decision", + "content": "Treat the two failing CLI suites and four Rust hook-chain tests as sandbox environment, not branch regressions: Treat the two failing CLI suites and four Rust hook-chain tests as sandbox environment, not branch regressions", + "raw": { + "question": "Treat the two failing CLI suites and four Rust hook-chain tests as sandbox environment, not branch regressions", + "chosen": "Treat the two failing CLI suites and four Rust hook-chain tests as sandbox environment, not branch regressions", + "alternatives": [], + "reasoning": "sdk-client.test.ts and fleet-lifecycle-integration.test.ts assert no ambient gateway is inherited and read the real process env; this sandbox exports RELAY_BASE_URL, which no CI workflow sets. The spawner hook-chain tests are defeated by the sandbox's GIT_CONFIG_COUNT core.hooksPath=/dev/null override. Both fixed in .relayflow/check.sh (uncommitted) rather than by changing tests." + }, + "significance": "high" + }, + { + "ts": 1791100397805, + "type": "decision", + "content": "Review current PR and staged repair together; withhold clean signoff for unverified task success: Review current PR and staged repair together; withhold clean signoff for unverified task success", + "raw": { + "question": "Review current PR and staged repair together; withhold clean signoff for unverified task success", + "chosen": "Review current PR and staged repair together; withhold clean signoff for unverified task success", + "alternatives": [], + "reasoning": "The user requires full receipt confirmation; timeout_fallback still completes a verified spawn successfully." + }, + "significance": "high" + }, + { + "ts": 1791100495394, + "type": "decision", + "content": "Review PR 1893 requests changes: timeout success and incomplete-echo confirmations remain: Review PR 1893 requests changes: timeout success and incomplete-echo confirmations remain", + "raw": { + "question": "Review PR 1893 requests changes: timeout success and incomplete-echo confirmations remain", + "chosen": "Review PR 1893 requests changes: timeout success and incomplete-echo confirmations remain", + "alternatives": [], + "reasoning": "188 CLI tests and six PTY tests pass; temporary adversarial PTY integration reproduces echo_anchors for a missing middle and paste_summary for a marker plus 40-byte tail. Findings recorded in review.md; no review.clean." + }, + "significance": "high" + }, + { + "ts": 1791100867791, + "type": "decision", + "content": "Positive delivery confirmation requires whole-payload echo evidence; endpoint anchors and collapsed-paste markers are inconclusive, and an unconfirmed initial task resolves a verified fleet spawn as an explicit non-retryable unconfirmed outcome rather than success: Positive delivery confirmation requires whole-payload echo evidence; endpoint anchors and collapsed-paste markers are inconclusive, and an unconfirmed initial task resolves a verified fleet spawn as an explicit non-retryable unconfirmed outcome rather than success", + "raw": { + "question": "Positive delivery confirmation requires whole-payload echo evidence; endpoint anchors and collapsed-paste markers are inconclusive, and an unconfirmed initial task resolves a verified fleet spawn as an explicit non-retryable unconfirmed outcome rather than success", + "chosen": "Positive delivery confirmation requires whole-payload echo evidence; endpoint anchors and collapsed-paste markers are inconclusive, and an unconfirmed initial task resolves a verified fleet spawn as an explicit non-retryable unconfirmed outcome rather than success", + "alternatives": [], + "reasoning": "review.md P1 x3: timeout_fallback/anchors/paste_summary all reached the spawn success branch, so a swallowed or middle-truncated task was indistinguishable from a received one" + }, + "significance": "high" + }, + { + "ts": 1791101890744, + "type": "decision", + "content": "Report what the terminal actually showed (echo_incomplete / paste_summary) instead of labelling every unconfirmed ack timeout_fallback: Report what the terminal actually showed (echo_incomplete / paste_summary) instead of labelling every unconfirmed ack timeout_fallback", + "raw": { + "question": "Report what the terminal actually showed (echo_incomplete / paste_summary) instead of labelling every unconfirmed ack timeout_fallback", + "chosen": "Report what the terminal actually showed (echo_incomplete / paste_summary) instead of labelling every unconfirmed ack timeout_fallback", + "alternatives": [], + "reasoning": "Operators need to tell a collapsed paste apart from a silent agent; the runtime gates on verification_label_confirms_receipt, so richer labels cost nothing in correctness" + }, + "significance": "high" + }, + { + "ts": 1791101891202, + "type": "reflection", + "content": "Three P1s shared one root cause: confirmed() conflated 'some evidence' with 'full receipt'. Narrowing confirmation to whole-payload echo and gating the spawn result on the label fixed all three; 20 cold starts and the two adversarial fixture modes pass.", + "raw": { + "focalPoints": [ + "verification-policy", + "spawn-confirmation" + ], + "confidence": 0.85 + }, + "significance": "high", + "tags": [ + "focal:verification-policy", + "focal:spawn-confirmation", + "confidence:0.85" + ] } ] } diff --git a/.agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.json b/.agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.json new file mode 100644 index 0000000000..20c5891db1 --- /dev/null +++ b/.agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.json @@ -0,0 +1,48 @@ +{ + "id": "compact_hbapxe3axu5d", + "version": 1, + "type": "compacted", + "compactedAt": "2026-10-06T13:43:02.276Z", + "sourceTrajectories": [ + "traj_t5rhaj5534rg" + ], + "dateRange": { + "start": "2026-10-06T13:38:30.682Z", + "end": "2026-10-06T13:42:41.615Z" + }, + "summary": { + "totalDecisions": 2, + "totalEvents": 2, + "uniqueAgents": [ + "default" + ] + }, + "decisionGroups": [ + { + "category": "api", + "decisions": [ + { + "question": "Implement reviewed-plan Part 1; defer automatic trust acceptance pending real captures", + "chosen": "Implement reviewed-plan Part 1; defer automatic trust acceptance pending real captures", + "reasoning": "No Devin binary or raw trust captures are available. Preserve the existing Claude planner and generic Devin approval restrictions. Add the Devin veto to both startup gate paths; deadline uses detector and verification ticks. CLI help inspection and advisory path corroboration remain Part 2 prerequisites.", + "fromTrajectory": "traj_t5rhaj5534rg" + } + ] + }, + { + "category": "testing", + "decisions": [ + { + "question": "Run broker validation without inherited Git config injection", + "chosen": "Run broker validation without inherited Git config injection", + "reasoning": "The default suite passed 1374 tests and failed four hook-chain tests because the runner injects core.hooksPath. With GIT_CONFIG_COUNT unset only in the test process, all four hook tests pass and all 1378 broker unit tests pass.", + "fromTrajectory": "traj_t5rhaj5534rg" + } + ] + } + ], + "keyLearnings": [], + "keyFindings": [], + "filesAffected": [], + "commits": [] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.md b/.agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.md new file mode 100644 index 0000000000..ba037722b6 --- /dev/null +++ b/.agentworkforce/trajectories/devin-trust-fix/compacted/compact_d6sq3rpl6inr_2026-10-06.md @@ -0,0 +1,21 @@ +# Trajectory Compaction: Oct 6, 2026 - Oct 6, 2026 + +## Summary +- Sessions: 1 +- Decisions: 2 +- Events: 2 +- Agents: default +- Files: 0 +- Commits: 0 + +## Api +- Implement reviewed-plan Part 1; defer automatic trust acceptance pending real captures -> Implement reviewed-plan Part 1; defer automatic trust acceptance pending real captures (traj_t5rhaj5534rg) + +## Testing +- Run broker validation without inherited Git config injection -> Run broker validation without inherited Git config injection (traj_t5rhaj5534rg) + +## Key Learnings +- None + +## Key Findings +- None \ No newline at end of file diff --git a/.agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.json b/.agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.json new file mode 100644 index 0000000000..ac1cfda54a --- /dev/null +++ b/.agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.json @@ -0,0 +1,89 @@ +{ + "id": "compact_o598x260wugy", + "version": 1, + "type": "compacted", + "compactedAt": "2026-10-04T14:54:40.616Z", + "sourceTrajectories": [ + "traj_ro3v7jmydpo5" + ], + "dateRange": { + "start": "2026-10-04T14:42:43.542Z", + "end": "2026-10-04T14:54:28.331Z" + }, + "summary": { + "totalDecisions": 4, + "totalEvents": 4, + "uniqueAgents": [ + "default" + ] + }, + "decisionGroups": [ + { + "category": "other", + "decisions": [ + { + "question": "Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes", + "chosen": "Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes", + "reasoning": "Preserve SDK behavior and sandbox safety; uncertain accepted spawns exit 8 with diagnostics.", + "fromTrajectory": "traj_ro3v7jmydpo5" + } + ] + }, + { + "category": "security", + "decisions": [ + { + "question": "Ship removal waits opt-in; use get not-found and deadline list absence, never released status", + "chosen": "Ship removal waits opt-in; use get not-found and deadline list absence, never released status", + "reasoning": "No workspace credential environment variables or machine-global agent-relay configuration are available for live deletion verification. Reviewed-plan section 3.7 requires opt-in in this case. Exit 8 requires observed presence; all-read failures keep the asynchronous acknowledgement.", + "fromTrajectory": "traj_ro3v7jmydpo5" + } + ] + }, + { + "category": "naming", + "decisions": [ + { + "question": "Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance", + "chosen": "Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance", + "reasoning": "Cleanup is already scheduled for owned failed spawns. A collision never grants ownership of the incumbent identity. Fleet release adds removal evidence to its existing JSON only with --wait.", + "fromTrajectory": "traj_ro3v7jmydpo5" + } + ] + }, + { + "category": "api", + "decisions": [ + { + "question": "Final invocation reads preserve the actual requested readiness contract", + "chosen": "Final invocation reads preserve the actual requested readiness contract", + "reasoning": "SDK confirm:true also requests verify_ready by default. A late terminal result must have ready:true; live heartbeat recovery remains explicitly confirmed:false. Late persona child acknowledgements stay pending and retain their child invocation/node.", + "fromTrajectory": "traj_ro3v7jmydpo5" + } + ] + } + ], + "keyLearnings": [], + "keyFindings": [], + "filesAffected": [ + ".agentworkforce/trajectories/fleet-spawn-fix/active/traj_ro3v7jmydpo5/trajectory.json", + "CHANGELOG.md", + "crates/broker/src/runtime/relaycast_events.rs", + "packages/cli/README.md", + "packages/cli/src/cli/commands/agent.test.ts", + "packages/cli/src/cli/commands/agent.ts", + "packages/cli/src/cli/commands/fleet.test.ts", + "packages/cli/src/cli/commands/fleet.ts", + "packages/cli/src/cli/lib/agent-registration.ts", + "packages/cli/src/cli/lib/agent-removal.test.ts", + "packages/cli/src/cli/lib/agent-removal.ts", + "packages/cli/src/cli/lib/fleet-live-agents.ts", + "packages/cli/src/cli/lib/sdk-command.test.ts", + "packages/cli/src/cli/lib/sdk-command.ts", + "packages/cli/src/cli/lib/spawn-liveness.test.ts", + "packages/cli/src/cli/lib/spawn-liveness.ts" + ], + "commits": [ + "5c8b798" + ] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.md b/.agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.md new file mode 100644 index 0000000000..de15c369d4 --- /dev/null +++ b/.agentworkforce/trajectories/fleet-spawn-fix/compacted/compact_o598x260wugy_2026-10-04.md @@ -0,0 +1,33 @@ +# Trajectory Compaction: Oct 4, 2026 - Oct 4, 2026 + +## Summary +- Sessions: 1 +- Decisions: 4 +- Events: 4 +- Agents: default +- Files: 16 +- Commits: 1 + +## Other +- Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes -> Keep SDK error unions unchanged; use CLI-owned pending and collision errors with known-class exit codes (traj_ro3v7jmydpo5) + +## Security +- Ship removal waits opt-in; use get not-found and deadline list absence, never released status -> Ship removal waits opt-in; use get not-found and deadline list absence, never released status (traj_ro3v7jmydpo5) + +## Naming +- Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance -> Keep existing asynchronous broker identity cleanup; expose safe name-reuse guidance (traj_ro3v7jmydpo5) + +## Api +- Final invocation reads preserve the actual requested readiness contract -> Final invocation reads preserve the actual requested readiness contract (traj_ro3v7jmydpo5) + +## Key Learnings +- None + +## Key Findings +- None + +## Completion evidence + +CLI and MCP pending spawn evidence, fresh target-node heartbeat checks, sandbox preservation, opt-in name-clearance waits, readable registration collisions, and roster heartbeat ages are implemented. SDK behavior and workflow files are unchanged. + +Full CLI suite: 2,021 passed, 17 skipped with inherited RELAY_BASE_URL unset. Typechecking and the latest 72 MCP regression tests passed. Cargo is unavailable, so the Rust formatter regression was not run. No live workspace credentials were available; removal waits remain opt-in under reviewed-plan section 3.7. diff --git a/.agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.json b/.agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.json new file mode 100644 index 0000000000..1788af304f --- /dev/null +++ b/.agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.json @@ -0,0 +1,89 @@ +{ + "id": "compact_tggynuofmzom", + "version": 1, + "type": "compacted", + "compactedAt": "2026-10-05T10:18:47.735Z", + "sourceTrajectories": [ + "traj_kvnuivdfl4un" + ], + "dateRange": { + "start": "2026-10-05T10:17:44.252Z", + "end": "2026-10-05T10:18:47.370Z" + }, + "summary": { + "totalDecisions": 2, + "totalEvents": 2, + "uniqueAgents": [ + "default" + ] + }, + "decisionGroups": [ + { + "category": "security", + "decisions": [ + { + "question": "Implement pending-only local coalescing and verified provisioning; leave provider scope unchanged", + "chosen": "Implement pending-only local coalescing and verified provisioning; leave provider scope unchanged", + "reasoning": "Reviewed plan confirms MCP replay asymmetry. Incident credentials and control-plane daemon are unavailable; do not widen PR globs without E2 evidence.", + "fromTrajectory": "traj_kvnuivdfl4un" + } + ] + }, + { + "category": "tooling", + "decisions": [ + { + "question": "Demote plural-events hypothesis and defer explicit message idempotency keys", + "chosen": "Demote plural-events hypothesis and defer explicit message idempotency keys", + "reasoning": "Installed Relaycast types require plural events and the SDK already sends generated Idempotency-Key headers. Engine enforcement cannot be verified here. Installed Relayfile client has no writeback ledger status or retry methods.", + "fromTrajectory": "traj_kvnuivdfl4un" + } + ] + } + ], + "keyLearnings": [], + "keyFindings": [], + "filesAffected": [ + "CHANGELOG.md", + "packages/cli/src/cli/commands/integration-subscribe.test.ts", + "packages/cli/src/cli/commands/integration.ts", + "packages/cli/src/cli/commands/relaycast-groups.test.ts", + "packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts", + "packages/cli/src/cli/mcp/messaging-tools.ts", + "packages/cli/src/cli/mcp/request-replay.ts", + "packages/cli/src/cli/telemetry/events.ts", + "packages/harnesses/src/ai-sdk/native-relay-tools.test.ts", + "packages/harnesses/src/ai-sdk/native-relay-tools.ts", + "packages/integration-prompts/src/builders.ts", + "packages/sdk/src/__tests__/integrations.test.ts", + "packages/sdk/src/__tests__/relaycast-translate.test.ts", + "packages/sdk/src/messaging/index.ts", + "packages/sdk/src/messaging/relaycast-translate.ts", + "packages/sdk/src/messaging/relaycast.ts", + "packages/sdk/src/messaging/types.ts", + "packages/cli/src/cli/mcp/request-replay.test.ts", + "summary.md", + "docs/evidence/provider-subscription-replies.md" + ], + "commits": [], + "narrative": "Implemented pending-only MCP/native reply coalescing, subscription event read-back with journaled rollback, authorization-aware output, outbound configuration diagnostics, and regression tests on the current branch. Local tests and typechecks passed. Incident workspace access and the Relayfile daemon are unavailable; actual GitHub writeback and durable retry remain unverified. See summary.md and docs/evidence/provider-subscription-replies.md.", + "decisions": [ + { + "question": "Implement pending-only local coalescing and verified provisioning; leave provider scope unchanged", + "chosen": "Implement pending-only local coalescing and verified provisioning; leave provider scope unchanged", + "reasoning": "Reviewed plan confirms MCP replay asymmetry. Incident credentials and control-plane daemon are unavailable; do not widen PR globs without E2 evidence." + }, + { + "question": "Demote plural-events hypothesis and defer explicit message idempotency keys", + "chosen": "Demote plural-events hypothesis and defer explicit message idempotency keys", + "reasoning": "Installed Relaycast types require plural events and the SDK already sends generated Idempotency-Key headers. Engine enforcement cannot be verified here. Installed Relayfile client has no writeback ledger status or retry methods." + } + ], + "conventions": [], + "lessons": [], + "openQuestions": [ + "Does subscription writeback ingress use the existing Relayfile ops ledger, keyed by Relay message ID?", + "What are the incident binding paths and persisted subscription events?", + "Does the engine enforce message/reply idempotency keys across processes?" + ] +} diff --git a/.agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.md b/.agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.md new file mode 100644 index 0000000000..7ae252bd06 --- /dev/null +++ b/.agentworkforce/trajectories/provider-subscription-replies/compacted/compact_tggynuofmzom_2026-10-05.md @@ -0,0 +1,14 @@ +# Provider subscription reply trajectory + +Implemented pending-only MCP/native reply coalescing, subscription event read-back with journaled rollback, authorization-aware output, outbound configuration diagnostics, and regression tests on the current branch. Local tests and typechecks passed. Incident workspace access and the Relayfile daemon are unavailable; actual GitHub writeback and durable retry remain unverified. See summary.md and docs/evidence/provider-subscription-replies.md. + +## Decisions + +- Implement pending-only local coalescing and verified provisioning; leave provider scope unchanged. Reviewed plan confirms MCP replay asymmetry. Incident credentials and control-plane daemon are unavailable; do not widen PR globs without E2 evidence. +- Demote plural-events hypothesis and defer explicit message idempotency keys. Installed Relaycast types require plural events and the SDK already sends generated Idempotency-Key headers. Engine enforcement cannot be verified here. Installed Relayfile client has no writeback ledger status or retry methods. + +## Open questions + +- Does subscription writeback ingress use the existing Relayfile ops ledger, keyed by Relay message ID? +- What are the incident binding paths and persisted subscription events? +- Does the engine enforce message/reply idempotency keys across processes? diff --git a/.agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.json b/.agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.json new file mode 100644 index 0000000000..ac1bfab2cf --- /dev/null +++ b/.agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.json @@ -0,0 +1,51 @@ +{ + "id": "compact_fe43i9idd2e5", + "version": 1, + "type": "compacted", + "compactedAt": "2026-10-04T07:26:54.566Z", + "sourceTrajectories": [ + "traj_mrt7zp5f4emg" + ], + "dateRange": { + "start": "2026-10-04T07:16:26.257Z", + "end": "2026-10-04T07:26:54.151Z" + }, + "summary": { + "totalDecisions": 1, + "totalEvents": 1, + "uniqueAgents": [ + "default" + ] + }, + "decisionGroups": [ + { + "category": "other", + "decisions": [ + { + "question": "Use a latched paste capability, a 16 KiB ceiling and a 1536-byte default typed cap; preserve absent-echo fallback, fail only positive tail-loss evidence", + "chosen": "Use a latched paste capability, a 16 KiB ceiling and a 1536-byte default typed cap; preserve absent-echo fallback, fail only positive tail-loss evidence", + "reasoning": "Reviewed plan requires a bounded 90-second spawn budget and compatibility for TUIs that do not echo. Wrap already bulk-writes. Live Claude enabled DECSET 2004 during startup.", + "fromTrajectory": "traj_mrt7zp5f4emg" + } + ] + } + ], + "keyLearnings": [], + "keyFindings": [], + "filesAffected": [ + "crates/broker/src/broker/delivery_verification.rs", + "crates/broker/src/devin.rs", + "crates/broker/src/injection_wire.rs", + "crates/broker/src/lib.rs", + "crates/broker/src/pty_worker.rs", + "crates/broker/src/runtime/delivery.rs", + "crates/broker/src/wrap.rs", + "crates/broker/tests/pty_injection_integrity.rs", + "crates/relay-pty/src/pty.rs", + "docs/harnesses/injection.md" + ], + "commits": [ + "a973af9", + "3521cb4" + ] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.md b/.agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.md new file mode 100644 index 0000000000..338921b06d --- /dev/null +++ b/.agentworkforce/trajectories/pty-integrity/compacted/compact_fe43i9idd2e5_2026-10-04.md @@ -0,0 +1,18 @@ +# Trajectory Compaction: Oct 4, 2026 - Oct 4, 2026 + +## Summary +- Sessions: 1 +- Decisions: 1 +- Events: 1 +- Agents: default +- Files: 10 +- Commits: 2 + +## Other +- Use a latched paste capability, a 16 KiB ceiling and a 1536-byte default typed cap; preserve absent-echo fallback, fail only positive tail-loss evidence -> Use a latched paste capability, a 16 KiB ceiling and a 1536-byte default typed cap; preserve absent-echo fallback, fail only positive tail-loss evidence (traj_mrt7zp5f4emg) + +## Key Learnings +- None + +## Key Findings +- None \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 44c90c85ba..bcf1fa6ed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- MCP `spawn` reports pending/liveness evidence, and `remove_agent` accepts optional registration-clearance waits. +- `agent remove --wait` and `fleet release --delete-agent --wait` verify registration clearance before name reuse, with `--wait-timeout` and `--no-wait` controls. Fleet release adds `removal` evidence when waiting. +- `fleet agent list` JSON includes node heartbeat timestamps and ages, with warnings for stale snapshots. +- `fleet spawn`, `agent spawn`, and `node agent spawn` accept `--task-file` to read an initial brief from a local UTF-8 file. Fleet requires exactly one of `--task` and `--task-file`. - `agent-relay message dm send`, `message post` and `message dm send_group` take a repeatable `--file ` that uploads and attaches local files (for example screenshots); `message file upload` accepts `--to ` for a DM. - `agent-relay message file download ` saves a received attachment (default `.agent-relay/attachments//`) and prints its path; `message file get ` shows it with a short-lived download URL. - The `agent-relay mcp` server adds `upload_file` (local path or base64) and `download_file`, and `send_group_dm` accepts `attachments`. - Agents spawned by the broker receive message attachments (up to 25 MiB each) as local files: the injected message ends with an `Attachments:` list giving each file's path under `.agent-relay/attachments/` in the agent's working directory, or an `agent-relay message file download ` command when a file could not be fetched. Messages that carry only attachments are now delivered too. - `@agent-relay/sdk` adds `files.upload` / `files.get` / `files.download` on `AgentRelay` and `RelaycastMessagingClient`, plus `uploadRelayFile` / `downloadRelayFile` helpers. +### Changed + +- Unconfirmed `fleet spawn` results keep the `spawn_unconfirmed` code, now with `state: "pending"`, invocation diagnostics, liveness evidence and exit 8. Node heartbeat presence is reported as liveness evidence but never confirms the spawn. +- Removal waits exit 8 if the registration remains present; scripts must handle nonzero exits beyond `$? -eq 1`. Waits remain opt-in, and unavailable verification reads preserve the asynchronous acknowledgement. +- PTY-delivered tasks and messages are limited to 14 KiB (16 KiB with the message envelope); `fleet spawn`, `agent spawn` and PTY delivery reject larger input and name the limit, so write the brief to a file and send a pointer. Agents whose terminal lacks bracketed paste accept at most 1,536 bytes per injection. Muse startup tasks and messages to headless CLI agents travel as a single process argument and are limited to 16 KiB instead; native-runtime and app-server agents receive tasks and messages as frames and have no such limit; `message post|reply|dm send` are not capped, and a wrapped agent receives an oversized message as a pointer to it in Relay. + ### Fixed +- Verified Fleet Devin spawns fail with `directory_trust_required` when the directory-trust prompt persists, releasing the worker through spawn cleanup instead of leaving a live but blocked worker. +- Relay MCP `post_message` and `reply_to_thread` coalesce a transport replay of one request (and accept an `idempotency_key` for retries after a lost response), matching `send_dm`; native host tools key on the tool call ID. Independent writes with identical text remain separate messages. +- `integration subscribe` rejects `--events` values the engine cannot subscribe to. +- `integration subscribe` checks that the requested writeback events were persisted and rolls back a subscription left incomplete. +- `integration subscribe` says when GitHub authorization for writeback could not be verified, instead of implying it succeeded. +- `integration subscribe --list` separates a subscription's configuration from delivery confirmation, so a configured subscription is no longer read as a delivered one. +- Fleet registration collisions use readable `spawn_name_taken` guidance covering asynchronous removal and failed-spawn cleanup, instead of Rust `Fatal(AlreadyExists {...})` text. +- Pending fleet spawns preserve provisioned sandboxes and distinguish live, stale, elsewhere, registered, absent, and unknown evidence before advising a retry. +- Tasks and relay messages typed into an agent's terminal wait until the agent is ready for input and can no longer be submitted half-way through, and a message that is too large fails with an explicit error instead of being retried. +- Control characters other than newline and tab (for example `Ctrl-C`) are removed from tasks and messages typed into an agent's terminal, so they can no longer interrupt or edit the agent's input line. +- A confirmed `fleet spawn --task` (the default) succeeds only once the agent's harness has visibly accepted the task (activity or a cleared composer, not merely the task echoed into the composer). If the task cannot be delivered, the worker is released and the spawn fails with `spawn_task_failed`. If delivery cannot be confirmed, the spawn fails with `spawn_task_unconfirmed`, names the agent that is still running, and says not to retry. With `--no-confirm` the spawn reports as soon as the node accepts it, without waiting for the task to arrive. - `agent-relay message file upload` now uploads and attaches the file; it always failed with "Invalid attachments: file ids must exist in workspace and be complete". - Broker-managed messages retry failed delivery up to three times, and later sends wait until delivery succeeds. diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index e358156f48..5910926185 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -17,8 +17,8 @@ use crate::{ pub(crate) const ACTIVITY_WINDOW: Duration = Duration::from_secs(5); pub(crate) const ACTIVITY_BUFFER_MAX_BYTES: usize = 16_000; pub(crate) const ACTIVITY_BUFFER_KEEP_BYTES: usize = 12_000; -const VERIFICATION_OUTPUT_MAX_BYTES: usize = 16_000; -const VERIFICATION_OUTPUT_KEEP_BYTES: usize = 12_000; +const VERIFICATION_OUTPUT_MAX_BYTES: usize = 64 * 1024; +const VERIFICATION_OUTPUT_KEEP_BYTES: usize = 48 * 1024; #[derive(Debug, Clone, Copy)] pub(crate) enum DeliveryOutcome { @@ -566,6 +566,28 @@ pub(crate) fn queue_or_take_detected_activity( } } +/// The `delivery_verified` label a PTY worker reports once the harness has +/// accepted the turn (activity, a cleared composer, or a raw process echo). +pub(crate) const HARNESS_ACCEPTANCE: &str = "harness_acceptance"; + +/// The `delivery_failed` reason a PTY worker reports when its acceptance window +/// closed without evidence either way: the body is no longer visibly parked, +/// but nothing proved the harness took the turn. Unlike a body still parked +/// after recovery, this is not evidence that the input was lost. +pub(crate) const HARNESS_ACCEPTANCE_UNPROVEN: &str = "harness acceptance could not be proven"; + +/// Whether a `delivery_verified` label proves full receipt. +/// +/// The broker runtime sees deliveries only as the worker's wire label, so this +/// is the one place that decides which labels a caller — notably the verified +/// fleet spawn completion in `runtime::worker_events` — may treat as receipt. +/// Only harness acceptance does. `completed_replay` re-reports an earlier +/// delivery and legacy labels (`echo`, `timeout_fallback`) come from workers +/// whose echo matching could not tell a parked draft from a submitted turn. +pub(crate) fn verification_label_confirms_receipt(label: &str) -> bool { + label == HARNESS_ACCEPTANCE +} + /// Check if the expected echo string appears in PTY output (after stripping ANSI). pub(crate) fn check_echo_in_output(output: &str, expected: &str) -> bool { let clean = strip_ansi(output); @@ -1017,6 +1039,20 @@ mod tests { assert!(!check_echo_in_output(&header_only, &expected)); } + #[test] + fn only_harness_acceptance_confirms_receipt() { + assert!(verification_label_confirms_receipt(HARNESS_ACCEPTANCE)); + for label in [ + "", + "completed_replay", + "worker_confirmation", + "echo", + "timeout_fallback", + ] { + assert!(!verification_label_confirms_receipt(label), "{label}"); + } + } + #[test] fn check_echo_clean_text() { let output = "some preamble\nRelay message from Alice [evt_1]: hello world\nmore output"; diff --git a/crates/broker/src/devin.rs b/crates/broker/src/devin.rs index 7ee027891b..1d78af38ae 100644 --- a/crates/broker/src/devin.rs +++ b/crates/broker/src/devin.rs @@ -9,12 +9,16 @@ use anyhow::{Context, Result}; use serde_json::Value; use std::{fs, path::Path}; -pub(crate) fn injection_bytes(cli: &str, text: &str) -> Vec { - if is_devin_cli(cli) { - format!("\x1b[200~{}\x1b[201~", text.replace('\x1b', "")).into_bytes() - } else { - text.as_bytes().to_vec() - } +#[cfg(test)] +fn injection_bytes(cli: &str, text: &str) -> Vec { + crate::injection_wire::injection_bytes( + if is_devin_cli(cli) { + crate::injection_wire::InjectionWire::Paste + } else { + crate::injection_wire::InjectionWire::Typed + }, + text, + ) } pub(crate) fn can_inject(cli: &str, pty: &PtySession) -> bool { diff --git a/crates/broker/src/injection_wire.rs b/crates/broker/src/injection_wire.rs new file mode 100644 index 0000000000..ab1171b032 --- /dev/null +++ b/crates/broker/src/injection_wire.rs @@ -0,0 +1,156 @@ +//! Shared, bounded PTY injection wire format for fleet and wrap. +use crate::pty::PtySession; +use std::time::Duration; + +pub(crate) const MAX_INJECTION_BODY_BYTES: usize = 16 * 1024; +/// Room kept for the attribution line and MCP reminder the worker wraps around +/// a body. Body-level caps use `MAX_BODY_BYTES` so a body accepted up front +/// still fits `MAX_INJECTION_BODY_BYTES` once formatted. +pub(crate) const ENVELOPE_RESERVE_BYTES: usize = 2 * 1024; +pub(crate) const MAX_BODY_BYTES: usize = MAX_INJECTION_BODY_BYTES - ENVELOPE_RESERVE_BYTES; + +/// The spawn error for an initial PTY task that cannot fit its envelope. +pub(crate) fn task_too_large_error(task: &str) -> Option { + (task.len() > MAX_BODY_BYTES).then(|| { + format!( + "spawn_task_too_large: task is {} bytes; the limit is {MAX_BODY_BYTES} bytes so its envelope fits the {MAX_INJECTION_BODY_BYTES}-byte PTY limit. Write the brief to a file on the node and send a short pointer.", + task.len() + ) + }) +} +// At the default 5 ms/byte, leaves room for readiness, prompt recheck and verification. +pub(crate) const MAX_TYPED_WRITE_TIME: Duration = Duration::from_millis(7680); +pub(crate) const MAX_TYPED_BYTES: usize = MAX_TYPED_WRITE_TIME.as_millis() as usize / 5; +pub(crate) const INJECTION_PROMPT_WAIT: Duration = Duration::from_secs(15); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum InjectionWire { + Paste, + Typed, +} + +pub(crate) fn injection_wire(cli: &str, pty: &PtySession) -> InjectionWire { + select_wire( + cli, + pty.bracketed_paste_enabled(), + std::env::var("RELAY_INJECT_PASTE").as_deref() == Ok("0"), + ) +} +fn select_wire(cli: &str, capable: bool, disabled: bool) -> InjectionWire { + if !disabled && (capable || crate::readiness::is_devin_cli(cli)) { + InjectionWire::Paste + } else { + InjectionWire::Typed + } +} +pub(crate) fn injection_bytes(wire: InjectionWire, text: &str) -> Vec { + // A body is content, never keystrokes: drop every control character (C0, + // DEL, C1), so `\x03`, `\x15` or `\x7f` cannot interrupt, kill or edit + // the composer line, and `\r`/ESC cannot submit or close the paste. Line + // breaks and tabs are content and stay. + let text: String = text + .replace("\r\n", "\n") + .chars() + .filter(|&c| c == '\n' || c == '\t' || !c.is_control()) + .collect(); + match wire { + InjectionWire::Paste => format!("\x1b[200~{text}\x1b[201~").into_bytes(), + InjectionWire::Typed => text.into_bytes(), + } +} +pub(crate) fn effective_limit(wire: InjectionWire, pace: Duration) -> usize { + if wire == InjectionWire::Paste || pace.is_zero() { + MAX_INJECTION_BODY_BYTES + } else { + MAX_TYPED_BYTES.min((MAX_TYPED_WRITE_TIME.as_millis() / pace.as_millis().max(1)) as usize) + } +} +/// Harnesses whose composer readiness `cli_prompt_ready` can prove, so an +/// injection waits for it instead of writing into a login or other interstitial. +fn readiness_gated(cli: &str) -> bool { + let lower = cli.to_lowercase(); + ["claude", "codex", "gemini"] + .iter() + .any(|name| lower.contains(name)) + || crate::readiness::is_devin_cli(cli) + || crate::readiness::is_muse_cli(cli) +} +pub(crate) fn can_inject(cli: &str, pty: &PtySession) -> bool { + !readiness_gated(cli) + || crate::readiness::cli_prompt_ready( + cli, + crate::readiness::GridReadinessSnapshot { + screen: &pty.screen_text(), + cursor: Some(pty.cursor_position()), + }, + ) +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn muse_waits_for_its_composer_like_the_other_known_harnesses() { + for cli in ["muse", "/Users/khaliqgant/.local/bin/muse", "MUSE.exe"] { + assert!(readiness_gated(cli), "{cli}"); + } + for cli in ["claude", "codex", "gemini", "devin"] { + assert!(readiness_gated(cli), "{cli}"); + } + assert!(!readiness_gated("bash")); + } + #[test] + fn body_cannot_submit_or_close_paste() { + for ch in (0..=0x10ffff).filter_map(char::from_u32) { + let body = format!("head{ch}\x1b[201~\r\nend\r"); + for wire in [InjectionWire::Paste, InjectionWire::Typed] { + let bytes = injection_bytes(wire, &body); + assert!(!bytes.contains(&b'\r')); + let expected_escapes = if wire == InjectionWire::Paste { 2 } else { 0 }; + assert_eq!(bytes.iter().filter(|&&b| b == 27).count(), expected_escapes); + } + } + } + /// A body accepted at `MAX_BODY_BYTES` must still fit the PTY limit once the + /// worker adds the attribution line and the full MCP reminder (relay#1893 + /// review: the body cap used to ignore the envelope). + #[test] + fn a_body_at_the_body_cap_fits_the_pty_limit_with_its_envelope() { + let long = "n".repeat(128); + let envelope = crate::broker::injection_format::format_injection_for_worker_with_workspace( + &long, + &format!("init_{long}"), + &"x".repeat(MAX_BODY_BYTES), + &format!("#{long}"), + true, + false, + Some(&long), + Some(&format!("rw_{long}")), + Some(&long), + ); + assert!( + envelope.len() <= MAX_INJECTION_BODY_BYTES, + "{} > {MAX_INJECTION_BODY_BYTES}", + envelope.len() + ); + } + #[test] + fn capability_and_override() { + assert_eq!(select_wire("claude", true, false), InjectionWire::Paste); + assert_eq!(select_wire("claude", true, true), InjectionWire::Typed); + assert_eq!(select_wire("other", false, false), InjectionWire::Typed); + assert_eq!(injection_bytes(InjectionWire::Typed, "a\r\nb\rc"), b"a\nbc"); + } + /// relay#1930 review: control bytes in a task are keypresses to a PTY. + #[test] + fn control_characters_are_dropped_but_line_breaks_and_tabs_stay() { + let body = "a\x03b\x04c\x15d\x7fe\u{9b}f\x00g\n\th"; + assert_eq!( + injection_bytes(InjectionWire::Typed, body), + "abcdefg\n\th".as_bytes() + ); + assert_eq!( + injection_bytes(InjectionWire::Paste, body), + "\x1b[200~abcdefg\n\th\x1b[201~".as_bytes() + ); + } +} diff --git a/crates/broker/src/lib.rs b/crates/broker/src/lib.rs index f2d09f1d24..14193d4605 100644 --- a/crates/broker/src/lib.rs +++ b/crates/broker/src/lib.rs @@ -9,6 +9,7 @@ mod devin; pub mod fleet_wire; pub mod ids; +mod injection_wire; pub mod protocol; pub mod snippets; diff --git a/crates/broker/src/protocol.rs b/crates/broker/src/protocol.rs index 59f2b92865..a9323b2214 100644 --- a/crates/broker/src/protocol.rs +++ b/crates/broker/src/protocol.rs @@ -679,6 +679,12 @@ pub enum WorkerToBroker { DeliveryFailed { delivery_id: DeliveryId, event_id: EventId, + /// Terminal and never replayed. PTY injection adds + /// "prompt_unproven" (no composer was recognized in time), + /// "body remained parked after bounded submit-key recovery", + /// "harness acceptance could not be proven" (neither parked nor + /// accepted, so not evidence of loss) and "injection_too_large: …" + /// (rejected before any byte was written). reason: String, }, WorkerStream { diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index 48ebd39a27..717879e52b 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -37,7 +37,9 @@ use crate::broker::{ }; use crate::cli::command_parse::parse_cli_command; use crate::cli::PtyCommand; -use crate::readiness::{detect_cli_ready, is_muse_cli, GridReadinessSnapshot}; +use crate::readiness::{ + detect_cli_ready, devin_trust_prompt_active, is_devin_cli, is_muse_cli, GridReadinessSnapshot, +}; use crate::runtime::{get_terminal_size, send_frame}; use crate::snapshot::Snapshot; use crate::util::ansi::{floor_char_boundary, strip_ansi, AnsiStripper}; @@ -167,6 +169,7 @@ enum InjectionStage { struct ActiveInjection { pending: PendingWorkerInjection, stage: InjectionStage, + prompt_wait_started: Instant, next_at: tokio::time::Instant, injection_text: Option, /// Receive-time PTY-output sequence captured atomically with submitting @@ -277,7 +280,7 @@ async fn confirm_harness_acceptance( json!({ "delivery_id": delivery_id, "event_id": event_id, - "verification": "harness_acceptance", + "verification": crate::broker::delivery_verification::HARNESS_ACCEPTANCE, "evidence": evidence, "attempts": pv.attempts, }), @@ -604,6 +607,8 @@ struct StartupReadinessState { fallback_sent: bool, wait_warned: bool, auth_error_sent: bool, + trust_first_seen: Option, + trust_error_sent: bool, } fn append_bounded(buf: &mut String, text: &str, max: usize, keep: usize) { @@ -678,7 +683,8 @@ fn evaluate_startup_gate( "codex" | "codex.exe" ); let trust_blocked = detect_codex_trust_prompt(grid.screen) - || detect_claude_trust_prompt(grid.screen) == (true, true); + || detect_claude_trust_prompt(grid.screen) == (true, true) + || (is_devin_cli(resolved_cli) && devin_trust_prompt_active(grid)); let composer_ready = is_codex && codex_composer_ready(grid); tracing::debug!(target: "relay_broker::startup_gate", cli = resolved_cli, is_codex, composer_ready, trust_blocked, @@ -696,7 +702,7 @@ fn evaluate_startup_gate( /// The startup gate's verdict. /// -/// Three states, not two booleans: `Ready && Blocked` is not representable, +/// Explicit verdicts rather than booleans: `Ready && Blocked` is not representable, /// and the difference between "we could not recognise the prompt" and "the /// harness is deliberately refusing work" decides whether the deadline may /// release queued work at all. @@ -715,14 +721,20 @@ pub(crate) enum StartupGate { /// node will ever answer a device prompt, so the worker reports /// `provider_auth_required` rather than waiting out the deadline. ProviderAuthRequired, + /// Devin requires directory trust. Block immediately and report a terminal + /// spawn error if the screen persists for DIRECTORY_TRUST_TIMEOUT. + DirectoryTrustRequired, } impl StartupGate { /// Whether the harness is deliberately refusing input, as opposed to /// showing a prompt we failed to recognise. No deadline may release - /// queued work past either blocking verdict. + /// queued work past any blocking verdict. fn is_blocking(self) -> bool { - matches!(self, Self::Blocked | Self::ProviderAuthRequired) + matches!( + self, + Self::Blocked | Self::ProviderAuthRequired | Self::DirectoryTrustRequired + ) } } @@ -739,7 +751,14 @@ impl StartupGate { /// browser elsewhere. `detect_cli_ready` already refuses to prove readiness /// on that screen; this names it as deliberate so the deadline cannot /// release the brief into a login form. -fn startup_gate_block_reason(resolved_cli: &str, screen: &str) -> Option { +fn startup_gate_block_reason( + resolved_cli: &str, + grid: GridReadinessSnapshot<'_>, +) -> Option { + let screen = grid.screen; + if is_devin_cli(resolved_cli) && devin_trust_prompt_active(grid) { + return Some(StartupGate::DirectoryTrustRequired); + } if is_muse_cli(resolved_cli) && detect_muse_device_auth_prompt(screen) { return Some(StartupGate::ProviderAuthRequired); } @@ -759,19 +778,20 @@ fn startup_gate( pty: &PtySession, ) -> StartupGate { let screen = pty.screen_text(); + let grid = GridReadinessSnapshot { + screen: &screen, + cursor: Some(pty.cursor_position()), + }; if evaluate_startup_gate( resolved_cli, startup_output, startup_total_bytes, output_quiet, - GridReadinessSnapshot { - screen: &screen, - cursor: Some(pty.cursor_position()), - }, + grid, ) { return StartupGate::Ready; } - startup_gate_block_reason(resolved_cli, &screen).unwrap_or(StartupGate::Unrecognised) + startup_gate_block_reason(resolved_cli, grid).unwrap_or(StartupGate::Unrecognised) } /// Whether the loop may pop and start the next pending injection this tick. @@ -900,6 +920,8 @@ fn should_block_pending_injection( && pending.queued_at.elapsed() < AUTO_SUGGESTION_BLOCK_TIMEOUT } +const DIRECTORY_TRUST_TIMEOUT: Duration = Duration::from_secs(30); + #[allow(clippy::too_many_arguments)] async fn try_emit_worker_ready( out_tx: &mpsc::Sender>, @@ -913,10 +935,38 @@ async fn try_emit_worker_ready( // init_received_at is Some only after init_worker has been received. // We use it (not init_request_id) as the gate because the broker sends // init_worker without a request_id. - if readiness.ready_sent || init_received_at.is_none() { + if readiness.ready_sent || readiness.trust_error_sent || init_received_at.is_none() { return; } + // Anchor the deadline on the gate's own detector, not a parsed menu row or + // response attempt. The 200 ms verification tick drives this even when the + // child stops producing output. Blocking suppresses the generic fail-open. + // A prompt that cleared before readiness must not hand its deadline to a + // later one; each appearance gets the full timeout. + if gate != StartupGate::DirectoryTrustRequired { + readiness.trust_first_seen = None; + } + if gate == StartupGate::DirectoryTrustRequired && !readiness.fallback_sent { + let first_seen = readiness.trust_first_seen.get_or_insert_with(Instant::now); + if first_seen.elapsed() >= DIRECTORY_TRUST_TIMEOUT { + readiness.trust_error_sent = true; + tracing::warn!(target: "agent_relay::worker::pty", worker = %worker_name, + "Devin directory trust is unresolved; reporting directory_trust_required"); + let _ = send_frame( + out_tx, + "worker_error", + None, + json!({ + "code": "directory_trust_required", + "message": "Devin requires directory trust; run `devin` in the explicit spawn working directory on this node, trust that directory, then retry", + "retryable": false + }), + ).await; + return; + } + } + let startup_ready = gate == StartupGate::Ready; // An interactive provider login cannot be waited out: the approval happens @@ -2170,6 +2220,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { active_injection = Some(ActiveInjection { pending, stage: InjectionStage::Escape, + prompt_wait_started: Instant::now(), next_at: tokio::time::Instant::now() + throttle.delay(), injection_text: None, output_boundary: None, @@ -2255,7 +2306,14 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { InjectionStage::Body => { // Recheck after throttling/steer delay: never paste // into a dialog that replaced the previously idle UI. - if !crate::devin::can_inject(&resolved_cli, &pty) { + if !crate::injection_wire::can_inject(&resolved_cli, &pty) { + if inj.prompt_wait_started.elapsed() >= crate::injection_wire::INJECTION_PROMPT_WAIT { + let delivery = &inj.pending.delivery; + let _ = send_frame(&out_tx, "delivery_failed", None, json!({"delivery_id": delivery.delivery_id, "event_id": delivery.event_id, "reason": "prompt_unproven"})).await; + let _ = send_frame(&out_tx, "worker_error", inj.pending.request_id, json!({"code": "prompt_unproven", "retryable": false, "message": "Composer readiness could not be proven; body was not written"})).await; + // Retain the pending id: a retry must never turn failure into an ack. + continue; + } inj.next_at = tokio::time::Instant::now() + Duration::from_millis(50); active_injection = Some(inj); continue; @@ -2274,36 +2332,29 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { inj.pending.delivery.workspace_id.as_deref(), inj.pending.delivery.workspace_alias.as_deref(), ); + let injection = String::from_utf8(crate::injection_wire::injection_bytes(crate::injection_wire::InjectionWire::Typed, &injection))?; + let wire = crate::injection_wire::injection_wire(&resolved_cli, &pty); + let limit = if initial_codex_delivery(&resolved_cli, &inj.pending.delivery) { + initial_codex_max_body_bytes().min(crate::injection_wire::MAX_INJECTION_BODY_BYTES) + } else { crate::injection_wire::effective_limit(wire, inject_rate) }; + if injection.len() > limit { + let delivery = &inj.pending.delivery; + let reason = format!("injection_too_large: {resolved_cli} effective limit is {limit} bytes including envelope; use a brief file pointer"); + let _ = send_frame(&out_tx, "delivery_failed", None, json!({"delivery_id": delivery.delivery_id, "event_id": delivery.event_id, "reason": reason})).await; + let _ = send_frame(&out_tx, "worker_error", inj.pending.request_id, json!({"code": "injection_too_large", "retryable": false, "message": reason})).await; + // Retain the pending id; terminal failure is never a successful replay. + continue; + } + // Only an envelope that passed the size gate is written, + // so only it may consume the reminder throttle. if include_mcp_reminder { mcp_reminder_throttle.note_sent(Instant::now()); } if initial_codex_delivery(&resolved_cli, &inj.pending.delivery) { - // Reject before any byte is written rather than after - // guaranteeing a partial-write timeout: a body needing - // more chunks than initial_codex_deadline's cap can - // afford cannot reach Enter no matter how fast every - // chunk renders (relay#1782 review). Reuses the - // existing was_chunked && !confirmed path below -- - // delivery_failed, retry record removed, id retained - // to reject a duplicate, body never replayed. - let max_bytes = initial_codex_max_body_bytes(); - if injection.len() > max_bytes { - tracing::warn!( - delivery_id = %inj.pending.delivery.delivery_id, - body_bytes = injection.len(), - max_bytes, - "initial Codex task exceeds chunked delivery's deadline-bounded size limit; rejecting before writing any byte" - ); - inj.stage = InjectionStage::Finalize; - initial_injection_cancel = Some(Arc::new(AtomicBool::new(false))); - injection_ack = Some(Box::pin(async move { - Ok(Err(std::io::Error::other( - "initial Codex task exceeds the maximum size chunked delivery can complete within its deadline; body will not be written", - ))) - })); - active_injection = Some(inj); - continue; - } + // The size gate above already caps an initial Codex + // body at `initial_codex_max_body_bytes`, so every body + // reaching here can finish within its deadline + // (relay#1782 review). let before = pty.consumed_offset(); let end = chunk_end(&injection, 0); let (first_ack, boundary) = match pty.submit_write_paced_with_output_boundary( @@ -2335,7 +2386,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { // bracketed-paste body shape; the shared helper then // selects the harness-specific delayed Enter. Finalization // still waits for this ack in the injection-ack arm. - let bytes = crate::devin::injection_bytes(&resolved_cli, &injection); + let bytes = crate::injection_wire::injection_bytes(wire, &injection); let write = submit_injection_body(&pty, &resolved_cli, bytes, inject_rate); match write { @@ -2793,7 +2844,7 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { "body remained parked after bounded submit-key recovery" } HarnessAcceptance::Inconclusive => { - "harness acceptance could not be proven" + crate::broker::delivery_verification::HARNESS_ACCEPTANCE_UNPROVEN } HarnessAcceptance::Accepted(_) => unreachable!(), }; @@ -4045,26 +4096,232 @@ mod tests { ); } + fn screen_only(screen: &str) -> GridReadinessSnapshot<'_> { + GridReadinessSnapshot { + screen, + cursor: None, + } + } + + #[test] + fn devin_trust_gate_is_scoped_to_devin_and_vetoes_composer() { + let screen = "Do you trust the authors of this directory?\n❭ 1 Yes, trust\n2 No, exit"; + for cli in ["devin", "/usr/local/bin/devin", "devin.exe"] { + assert_eq!( + startup_gate_block_reason(cli, screen_only(screen)), + Some(StartupGate::DirectoryTrustRequired) + ); + assert!(!evaluate_startup_gate( + cli, + screen, + 1000, + Duration::from_secs(90), + GridReadinessSnapshot { + screen, + cursor: Some((2, 1)) + } + )); + } + for cli in ["claude", "codex", "muse"] { + assert_eq!(startup_gate_block_reason(cli, screen_only(screen)), None); + } + assert_eq!( + startup_gate_block_reason( + "devin", + screen_only("Sign in to continue\nVisit https://www.facebook.com/device\n›") + ), + None + ); + } + + #[tokio::test] + async fn devin_trust_deadline_starts_at_detection_and_fails_once() { + let (tx, mut rx) = mpsc::channel(4); + let mut request_id = None; + let started = Instant::now() - STARTUP_READY_TIMEOUT - Duration::from_secs(60); + let mut readiness = StartupReadinessState::default(); + try_emit_worker_ready( + &tx, + "devin-trust", + Some(42), + &mut request_id, + Some(started), + &mut readiness, + StartupGate::DirectoryTrustRequired, + ) + .await; + assert!(readiness.trust_first_seen.is_some()); + assert!( + rx.try_recv().is_err(), + "init age must not advance the trust deadline" + ); + readiness.trust_first_seen = Some(Instant::now() - DIRECTORY_TRUST_TIMEOUT); + for gate in [ + StartupGate::DirectoryTrustRequired, + StartupGate::DirectoryTrustRequired, + StartupGate::Unrecognised, + StartupGate::Ready, + ] { + try_emit_worker_ready( + &tx, + "devin-trust", + Some(42), + &mut request_id, + Some(started), + &mut readiness, + gate, + ) + .await; + } + let frame = rx.try_recv().expect("trust error"); + assert_eq!(frame.msg_type, "worker_error"); + assert_eq!(frame.payload["code"], "directory_trust_required"); + assert_eq!(frame.payload["retryable"], false); + assert!( + rx.try_recv().is_err(), + "terminal error cannot become ready later" + ); + assert!(!readiness.ready_sent && !readiness.fallback_sent); + } + + #[tokio::test] + async fn devin_trust_cannot_retroactively_fail_released_workers() { + for proven in [true, false] { + let (tx, mut rx) = mpsc::channel(4); + let mut request_id = None; + let mut readiness = StartupReadinessState { + ready_sent: proven, + fallback_sent: !proven, + trust_first_seen: Some(Instant::now() - DIRECTORY_TRUST_TIMEOUT), + ..Default::default() + }; + try_emit_worker_ready( + &tx, + "devin-trust", + Some(42), + &mut request_id, + Some(Instant::now()), + &mut readiness, + StartupGate::DirectoryTrustRequired, + ) + .await; + assert!(rx.try_recv().is_err()); + } + } + + #[test] + fn devin_trust_history_above_live_composer_does_not_block_readiness() { + // A resumed or uncleared Devin screen can keep the answered trust + // question visible above the idle composer that holds the cursor. + let screen = "Do you trust this directory?\n1 Yes, trust\n2 No, exit\n❭ Ask Devin to build features, fix bugs, or work on your code\nSWE-2 High"; + let grid = GridReadinessSnapshot { + screen, + cursor: Some((4, 3)), + }; + assert!(evaluate_startup_gate( + "devin", + screen, + 1000, + Duration::from_secs(90), + grid + )); + assert_eq!(startup_gate_block_reason("devin", grid), None); + // The same text with the cursor on the menu is still the live dialog. + let menu = GridReadinessSnapshot { + screen, + cursor: Some((2, 1)), + }; + assert_eq!( + startup_gate_block_reason("devin", menu), + Some(StartupGate::DirectoryTrustRequired) + ); + } + + #[tokio::test] + async fn devin_trust_prompt_that_returns_gets_a_fresh_deadline() { + let (tx, mut rx) = mpsc::channel(4); + let mut request_id = None; + let mut readiness = StartupReadinessState { + trust_first_seen: Some(Instant::now() - DIRECTORY_TRUST_TIMEOUT), + ..Default::default() + }; + // The earlier prompt cleared before readiness, then trust reappears. + for gate in [ + StartupGate::Unrecognised, + StartupGate::DirectoryTrustRequired, + ] { + try_emit_worker_ready( + &tx, + "devin-trust", + Some(42), + &mut request_id, + Some(Instant::now()), + &mut readiness, + gate, + ) + .await; + } + assert!( + rx.try_recv().is_err(), + "a returning trust prompt must not inherit the cleared prompt's deadline" + ); + assert!(!readiness.trust_error_sent); + assert!(readiness + .trust_first_seen + .is_some_and(|seen| seen.elapsed() < DIRECTORY_TRUST_TIMEOUT)); + } + + #[tokio::test] + async fn devin_trust_cleared_before_deadline_allows_readiness() { + let (tx, mut rx) = mpsc::channel(4); + let mut request_id = None; + let mut readiness = StartupReadinessState::default(); + for gate in [StartupGate::DirectoryTrustRequired, StartupGate::Ready] { + try_emit_worker_ready( + &tx, + "devin-trust", + Some(42), + &mut request_id, + Some(Instant::now()), + &mut readiness, + gate, + ) + .await; + } + let frame = rx.try_recv().expect("ready after trust resolved"); + assert_eq!(frame.msg_type, "worker_ready"); + assert_eq!(frame.payload["readiness_proven"], true); + assert!(rx.try_recv().is_err()); + } + #[test] fn provider_login_and_trust_dialogs_are_classified_apart() { let device_auth = "Sign in to continue\nVisit https://www.facebook.com/device\n›\n"; assert_eq!( - startup_gate_block_reason("/usr/local/bin/muse", device_auth), + startup_gate_block_reason("/usr/local/bin/muse", screen_only(device_auth)), Some(StartupGate::ProviderAuthRequired) ); // Only Muse carries the device-login arm; another harness rendering // the same words keeps its existing unrecognised-prompt behaviour. - assert_eq!(startup_gate_block_reason("codex", device_auth), None); + assert_eq!( + startup_gate_block_reason("codex", screen_only(device_auth)), + None + ); assert_eq!( startup_gate_block_reason( "codex", - "Do you trust the contents of this directory?\n> Yes, continue\n No, quit\n" + screen_only( + "Do you trust the contents of this directory?\n> Yes, continue\n No, quit\n" + ) ), Some(StartupGate::Blocked) ); // An authenticated composer is neither. assert_eq!( - startup_gate_block_reason("muse", "muse signed in as relay-node\n/login\n›\n"), + startup_gate_block_reason( + "muse", + screen_only("muse signed in as relay-node\n/login\n›\n") + ), None ); } @@ -4294,6 +4551,7 @@ mod tests { let targeted = ActiveInjection { pending: test_pending_injection("init_task"), stage: InjectionStage::Escape, + prompt_wait_started: Instant::now(), next_at: tokio::time::Instant::now(), injection_text: None, output_boundary: None, @@ -4334,6 +4592,7 @@ mod tests { let mut injection = ActiveInjection { pending: test_pending_injection("init_task"), stage: InjectionStage::Escape, + prompt_wait_started: Instant::now(), next_at: tokio::time::Instant::now(), injection_text: None, output_boundary: None, diff --git a/crates/broker/src/runtime/delivery.rs b/crates/broker/src/runtime/delivery.rs index c7f52fc364..31b005df5f 100644 --- a/crates/broker/src/runtime/delivery.rs +++ b/crates/broker/src/runtime/delivery.rs @@ -861,6 +861,27 @@ pub(crate) async fn queue_and_try_delivery_raw( withheld_fleet_ack: Option, withheld_fleet_ack_floor: Option, ) -> Result { + // The envelope-sized cap exists because PTY delivery types the message into + // a terminal, so it applies to PTY and unknown recipients. The headless CLI + // runner (no harness config) hands the body to its provider as one argv + // entry, so it is bounded by the portable single-argument ceiling instead. + // App-server and native harness workers receive deliveries as frames and + // need neither bound. + let recipient = workers.workers.get(worker_name).map(|handle| &handle.spec); + if recipient.is_none_or(|spec| spec.runtime == AgentRuntime::Pty) { + anyhow::ensure!( + body.len() <= crate::injection_wire::MAX_BODY_BYTES, + "injection_too_large: body limit is {} bytes so the formatted envelope fits the {}-byte PTY limit; use a brief file pointer", + crate::injection_wire::MAX_BODY_BYTES, + crate::injection_wire::MAX_INJECTION_BODY_BYTES + ); + } else if recipient.is_some_and(|spec| spec.harness_config.is_none()) { + anyhow::ensure!( + body.len() <= crate::worker::MUSE_STARTUP_PROMPT_MAX_BYTES, + "injection_too_large: a headless agent receives the body as one process argument, limited to {} bytes; use a brief file pointer", + crate::worker::MUSE_STARTUP_PROMPT_MAX_BYTES + ); + } // Fleet delivery IDs are stable across Relaycast retries. Preserve that // identity all the way into the worker so its completed-delivery cache can // re-ACK a replay without pasting the instruction a second time. Local @@ -1060,7 +1081,12 @@ pub(crate) fn delivery_ack_timeout( ) -> Duration { let minimum = match injection_mode { MessageInjectionMode::Wait => WAIT_DELIVERY_ACK_TIMEOUT, - MessageInjectionMode::Steer => crate::broker::delivery_verification::VERIFICATION_WINDOW, + MessageInjectionMode::Steer => { + crate::injection_wire::INJECTION_PROMPT_WAIT + + crate::injection_wire::MAX_TYPED_WRITE_TIME + + Duration::from_millis(250) + + crate::broker::delivery_verification::VERIFICATION_WINDOW + } }; std::cmp::max(retry_interval, minimum) } @@ -1311,3 +1337,15 @@ mod reply_target_tests { } } } + +#[cfg(test)] +mod injection_budget_tests { + use super::*; + #[test] + fn steer_ack_covers_prompt_typing_submit_and_verification() { + let required = crate::injection_wire::INJECTION_PROMPT_WAIT + + Duration::from_millis(crate::injection_wire::MAX_TYPED_BYTES as u64 * 5 + 250) + + crate::broker::delivery_verification::VERIFICATION_WINDOW; + assert!(delivery_ack_timeout(&MessageInjectionMode::Steer, Duration::ZERO) >= required); + } +} diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index e63f9e860a..34a0435317 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -154,9 +154,77 @@ pub(super) struct PendingVerifiedSpawn { pub(super) started: Instant, pub(super) failure_reason: Option, pub(super) generation: Uuid, + pub(super) readiness_proven: bool, + pub(super) task_event_id: Option, + /// The initial task's verdict, recorded when it arrives before harness + /// readiness is proven (for example after a startup-fallback + /// `worker_ready`). The proven `worker_ready` that follows resolves the + /// action from it, so the order of the two frames cannot strand a spawn + /// whose task was already delivered. + pub(super) task_verification: Option, +} + +/// What the worker reported for a verified spawn's initial task. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct TaskVerification { + /// Whether the label is whole-payload evidence of receipt. + pub(super) confirmed: bool, + pub(super) label: Option, +} + +impl PendingVerifiedSpawn { + pub(super) fn matches_task(&self, generation: Uuid, event_id: &str) -> bool { + self.generation == generation && self.task_event_id.as_deref() == Some(event_id) + } + pub(super) fn can_report_ready(&self, generation: Uuid) -> bool { + self.generation == generation + && self.readiness_proven + && self.task_event_id.is_none() + && self.failure_reason.is_none() + } + /// Record the initial task's verdict; the task is no longer outstanding. + pub(super) fn record_task_verification(&mut self, confirmed: bool, label: Option<&str>) { + self.task_event_id = None; + self.task_verification = Some(TaskVerification { + confirmed, + label: label.map(str::to_string), + }); + } + /// The initial task failed. Expire the entry with a specific reason so + /// maintenance releases the worker and cleans up its fleet identity before + /// reporting, exactly as for a readiness timeout. Replying here instead + /// would leave a registered, running worker holding the name. + pub(super) fn initial_task_failed(&mut self, generation: Uuid, reason: &str) { + if self.generation == generation && self.failure_reason.is_none() { + self.deadline = Instant::now(); + self.failure_reason = Some(format!( + "spawn_task_failed: initial task was not delivered ({reason}); the worker was released" + )); + } + } + /// The action result once readiness is proven and the task is resolved. + pub(super) fn completion(self, name: &WorkerName) -> ActionResult { + match self.task_verification { + Some(TaskVerification { + confirmed: false, + label, + }) => { + verified_spawn_task_unconfirmed_result(self.invocation_id, name, label.as_deref()) + } + _ => verified_spawn_ready_result(self.invocation_id, name), + } + } } impl PendingVerifiedSpawn { + pub(super) fn directory_trust_failed(&mut self, generation: Uuid) { + if self.generation == generation { + self.deadline = Instant::now(); + // Fixed remediation only: never propagate private terminal output. + self.failure_reason = Some("spawn_directory_trust_required: Devin requires directory trust; run `devin` in the explicit spawn working directory on the selected node, trust that directory, then retry".into()); + } + } + pub(super) fn provider_auth_failed(&mut self, generation: Uuid) { if self.generation == generation { self.deadline = Instant::now(); @@ -181,6 +249,77 @@ pub(super) fn verified_spawn_ready_result( } } +/// A live, ready agent whose initial task was never confirmed as fully +/// received. +/// +/// This is deliberately neither a success nor an ordinary failure. The spawn +/// did happen — the agent is registered, ready and holding the node's only +/// slot for that name — so the caller must not retry, or it duplicates the +/// agent. But the task it was spawned to run cannot be shown to have arrived +/// intact, and `spawned:true, ready:true` is exactly the proof `fleet spawn` +/// uses to report success, so the only honest answer is an error that names +/// the live agent and says what to do instead (relay#1893 review, P1). +pub(super) fn verified_spawn_task_unconfirmed_result( + invocation_id: String, + name: &WorkerName, + verification: Option<&str>, +) -> ActionResult { + verified_spawn_failed_result( + invocation_id, + &format!( + "spawn_task_unconfirmed: agent '{name}' is live and ready, but full receipt of its \ + initial task was not confirmed (verification: {}). Do not retry this spawn — it \ + would duplicate the agent. Resend the task, or write the brief to a file and send a \ + short pointer, then check the agent's transcript.", + verification.unwrap_or("none") + ), + ) +} + +/// Result for a verified spawn whose worker had already reported +/// `worker_ready` before this action's pending entry existed. +/// +/// The runtime loop is serial, so for a fresh worker that cannot happen; it is +/// reached only when the launch resolved to a worker that was already up. Its +/// initial PTY task, if any, was then queued without being bound to this +/// action, so no task verdict can reach it: report the live agent as +/// unconfirmed rather than claim `spawned:true, ready:true` (relay#1893 review). +pub(super) fn already_ready_spawn_result( + invocation_id: String, + name: &WorkerName, + unbound_pty_task: bool, +) -> ActionResult { + if unbound_pty_task { + verified_spawn_task_unconfirmed_result(invocation_id, name, Some("unbound")) + } else { + verified_spawn_ready_result(invocation_id, name) + } +} + +/// Whether a fleet spawn's initial task will be injected through a PTY, the +/// only path the envelope limit applies to. An explicit headless or native +/// harness config hands the task over without a terminal; an absent or +/// invalid config keeps the conservative PTY answer (the spawn path rejects an +/// invalid config on its own). +pub(super) fn spawn_task_is_pty_injected(input: &Value) -> bool { + super::relaycast_events::relaycast_harness_config(input) + .ok() + .flatten() + .is_none_or(|config| config.runtime() == AgentRuntime::Pty) +} + +/// The spawn error for an initial task that cannot reach the worker: a PTY +/// task must fit the injection envelope, while Muse takes its startup task as +/// one argv entry and is bounded by its portable argv limit instead. +pub(super) fn spawn_task_too_large(cli: &str, task: &str) -> Option { + if crate::snippets::is_muse_executable(cli) { + return crate::worker::validate_muse_startup_prompt(cli, Some(task)) + .err() + .map(|error| format!("spawn_task_too_large: {error}")); + } + crate::injection_wire::task_too_large_error(task) +} + pub(super) fn verified_spawn_failed_result(invocation_id: String, error: &str) -> ActionResult { ActionResult { task: None, @@ -1699,6 +1838,17 @@ impl BrokerRuntime { } }; let task = action_invoke_string(&invoke.input, &["task", "initial_task", "prompt"]); + let carries_task = task.is_some(); + // Reject before launching: an oversized task can only fail after the + // worker exists, costing a launch and a release for a known outcome. + if let Some(error) = task + .as_deref() + .filter(|_| spawn_task_is_pty_injected(&invoke.input)) + .and_then(|task| spawn_task_too_large(&cli, task)) + { + self.reply_action_error(&invoke.invocation_id, &error).await; + return; + } let channel = action_invoke_string(&invoke.input, &["channel"]); let model = action_invoke_string(&invoke.input, &["model"]); @@ -1797,21 +1947,26 @@ impl BrokerRuntime { // while maintenance fails it after an early exit/readiness timeout // and performs cleanup. if verify_ready { - let (already_ready, generation) = { + let (already_ready, generation, is_pty) = { let worker = self .workers .workers .get(&name) .expect("verified spawn worker must still exist"); - (worker.ready_at.is_some(), worker.generation) + ( + worker.ready_at.is_some(), + worker.generation, + worker.spec.runtime == AgentRuntime::Pty, + ) }; if already_ready { tracing::info!(invocation_id = %invoke.invocation_id, worker = %name, verify_ready, elapsed_ms = started.elapsed().as_millis() as u64, "sending verified fleet spawn result"); - self.send_fleet_action_result(verified_spawn_ready_result( + self.send_fleet_action_result(already_ready_spawn_result( invoke.invocation_id, &name, + is_pty && carries_task, )) .await; } else { @@ -1823,6 +1978,9 @@ impl BrokerRuntime { started, failure_reason: None, generation, + readiness_proven: false, + task_event_id: None, + task_verification: None, }, ); } @@ -3534,10 +3692,45 @@ mod tests { ); } // CLI/SDK default confirmation budget is 120s (fleet.ts / relaycast.ts). - assert!(crate::pty_worker::STARTUP_READY_TIMEOUT < VERIFIED_SPAWN_READY_TIMEOUT); + assert!( + crate::pty_worker::STARTUP_READY_TIMEOUT + + crate::injection_wire::INJECTION_PROMPT_WAIT + + Duration::from_millis(crate::injection_wire::MAX_TYPED_BYTES as u64 * 5 + 250) + + crate::broker::delivery_verification::VERIFICATION_WINDOW + < VERIFIED_SPAWN_READY_TIMEOUT + ); assert!(VERIFIED_SPAWN_READY_TIMEOUT < Duration::from_secs(120)); } + #[test] + fn verified_task_spawn_waits_for_matching_delivery_and_proven_prompt() { + let generation = Uuid::new_v4(); + let mut pending = PendingVerifiedSpawn { + invocation_id: "test".into(), + deadline: Instant::now(), + started: Instant::now(), + failure_reason: None, + generation, + readiness_proven: false, + task_event_id: Some("init_test".into()), + task_verification: None, + }; + assert!(!pending.can_report_ready(generation)); + pending.readiness_proven = true; + assert!(!pending.can_report_ready(generation)); + assert!(!pending.matches_task(Uuid::new_v4(), "init_test")); + assert!(!pending.matches_task(generation, "another_message")); + assert!(pending.matches_task(generation, "init_test")); + // The matching task retires this entry whatever its verdict; whether + // that resolves the action as success or as `spawn_task_unconfirmed` + // is `worker_events`' receipt gate, covered by + // `a_spawn_task_acked_without_proof_of_receipt_never_reports_success`. + pending.task_event_id = None; + assert!(pending.can_report_ready(generation)); + pending.readiness_proven = false; + assert!(!pending.can_report_ready(generation)); + } + fn test_agent_spec(session_id: Option<&str>, harness_session_id: Option<&str>) -> AgentSpec { AgentSpec { name: WorkerName::from("agent-a"), diff --git a/crates/broker/src/runtime/relaycast_events.rs b/crates/broker/src/runtime/relaycast_events.rs index cb1c002b2a..0e3eaf2889 100644 --- a/crates/broker/src/runtime/relaycast_events.rs +++ b/crates/broker/src/runtime/relaycast_events.rs @@ -171,7 +171,9 @@ pub(super) fn relaycast_spawn_spec_session_id( }) } -fn relaycast_harness_config(value: &Value) -> Result, String> { +pub(super) fn relaycast_harness_config( + value: &Value, +) -> Result, String> { let agent = value.get("agent"); let harness_id = agent .and_then(|agent| { @@ -798,6 +800,9 @@ pub(super) async fn spawn_worker_from_request( ); } crate::worker::validate_muse_startup_prompt_for_spec(&spec, effective_task.as_deref())?; + // Measure the task as it will be injected, after the exit contract and + // skill prefix, and before registering or launching anything. + crate::worker::validate_pty_initial_task_size(&spec, effective_task.as_deref())?; // Pre-register an agent token for every spawned worker. // The Agent Relay MCP server needs RELAY_AGENT_TOKEN + @@ -936,7 +941,12 @@ pub(super) async fn spawn_worker_from_request( } Some(token) } - Err(error) => anyhow::bail!("WS spawn registration failed: {error:?}"), + Err( + RegRetryOutcome::RetryableExhausted(error) + | RegRetryOutcome::Fatal(error), + ) => { + anyhow::bail!("{}", format_worker_preregistration_error(&name, &error)); + } } } } @@ -1126,6 +1136,19 @@ pub(super) async fn spawn_worker_from_request( #[cfg(test)] mod tests { use super::*; + + #[test] + fn duplicate_registration_keeps_already_exists_for_debug_log_classification() { + let error = ::relaycast::AgentRegistrationError::AlreadyExists { + agent_name: "worker-a".to_string(), + }; + let message = format_worker_preregistration_error("worker-a", &error); + assert!(message.contains("already exists"), "{message}"); + assert!(message.contains("worker-a")); + assert!(!message.contains("Fatal(")); + assert!(!message.contains("AlreadyExists")); + } + use crate::terminal_control::TerminalToCloud; use ::relaycast::WsEvent; diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index 6c99a3cfdb..6663d1b56d 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -859,6 +859,560 @@ fn delivery_lifecycle_worker_event( } } +fn delivery_verified_worker_event( + name: &str, + generation: Uuid, + event_id: &str, + verification: Option<&str>, +) -> WorkerEvent { + let mut payload = json!({ + "delivery_id": format!("del_{event_id}"), + "event_id": event_id, + }); + if let Some(verification) = verification { + payload["verification"] = json!(verification); + } + WorkerEvent::Message { + name: WorkerName::from(name), + generation, + value: json!({ "type": "delivery_verified", "payload": payload }), + } +} + +/// relay#1893 review, P1: the spawn-completion branch checked readiness and the +/// task's event id, then reported `spawned:true, ready:true` without ever +/// looking at *what* confirmed the delivery. A worker that acked through the +/// timeout fallback — i.e. an agent that may have swallowed the whole task — +/// produced a result identical to one that echoed every byte. +#[tokio::test] +async fn a_spawn_task_acked_without_proof_of_receipt_never_reports_success() { + use crate::fleet_wire::{ActionResultPayload, BrokerToRelaycast}; + + // Every label a worker can ack with that is not harness acceptance, + // including the echo labels of workers that predate it: a body echoed in + // the composer may still be a parked draft. + for verification in [ + Some("timeout_fallback"), + Some("echo"), + Some("echo_normalized"), + Some("completed_replay"), + // A worker frame that omits the field entirely claims nothing. + None, + ] { + let worker_name = "spawn-receipt"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let mut fixture = worker_event_runtime_fixture(registry, HashMap::new()); + let event_id = "init_receipt"; + fixture.runtime.pending_verified_spawns.insert( + WorkerName::from(worker_name), + super::fleet::PendingVerifiedSpawn { + invocation_id: "inv-receipt".to_string(), + deadline: Instant::now() + Duration::from_secs(90), + started: Instant::now(), + generation, + failure_reason: None, + readiness_proven: true, + task_event_id: Some(event_id.to_string()), + task_verification: None, + }, + ); + + fixture + .runtime + .handle_worker_event(delivery_verified_worker_event( + worker_name, + generation, + event_id, + verification, + )) + .await; + + let frame = fixture + .fleet_control_rx + .try_recv() + .expect("an unconfirmed task must still resolve the open spawn action"); + let FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) = frame else { + panic!("spawn completion must send an action result"); + }; + assert_eq!(result.invocation_id, "inv-receipt"); + let ActionResultPayload::Error(error) = result.result else { + panic!("{verification:?} is not proof of receipt and must not succeed"); + }; + assert!( + error.error.starts_with("spawn_task_unconfirmed: "), + "{}", + error.error + ); + // The agent is live: name it so the caller can reach it, and say not to + // retry, because a retry duplicates it. + assert!(error.error.contains(worker_name), "{}", error.error); + assert!(error.error.contains("Do not retry"), "{}", error.error); + assert!( + error.error.contains(verification.unwrap_or("none")), + "{}", + error.error + ); + // Resolved, so maintenance's readiness deadline can no longer release + // the live worker this spawn produced. + assert!(!fixture + .runtime + .pending_verified_spawns + .contains_key(&WorkerName::from(worker_name))); + } +} + +/// The other half of the gate: harness acceptance succeeds, and it is exactly +/// the label `verification_label_confirms_receipt` accepts. +#[tokio::test] +async fn a_spawn_task_proven_received_reports_spawned_and_ready() { + use crate::fleet_wire::{ActionResultPayload, BrokerToRelaycast}; + + for verification in [crate::broker::delivery_verification::HARNESS_ACCEPTANCE] { + assert!( + crate::broker::delivery_verification::verification_label_confirms_receipt(verification), + "{verification} must be a confirming label" + ); + let worker_name = "spawn-receipt-ok"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let mut fixture = worker_event_runtime_fixture(registry, HashMap::new()); + let event_id = "init_receipt_ok"; + fixture.runtime.pending_verified_spawns.insert( + WorkerName::from(worker_name), + super::fleet::PendingVerifiedSpawn { + invocation_id: "inv-receipt-ok".to_string(), + deadline: Instant::now() + Duration::from_secs(90), + started: Instant::now(), + generation, + failure_reason: None, + readiness_proven: true, + task_event_id: Some(event_id.to_string()), + task_verification: None, + }, + ); + + fixture + .runtime + .handle_worker_event(delivery_verified_worker_event( + worker_name, + generation, + event_id, + Some(verification), + )) + .await; + + let frame = fixture + .fleet_control_rx + .try_recv() + .expect("a confirmed task must resolve the open spawn action"); + let FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) = frame else { + panic!("spawn completion must send an action result"); + }; + let ActionResultPayload::Output(output) = result.result else { + panic!("{verification} proves receipt and must succeed"); + }; + assert_eq!( + output.output, + json!({"spawned": true, "ready": true, "name": worker_name}) + ); + } +} + +fn pending_spawn_with_task( + generation: Uuid, + event_id: &str, + readiness_proven: bool, +) -> super::fleet::PendingVerifiedSpawn { + super::fleet::PendingVerifiedSpawn { + invocation_id: "inv-order".to_string(), + deadline: Instant::now() + Duration::from_secs(90), + started: Instant::now(), + failure_reason: None, + generation, + readiness_proven, + task_event_id: Some(event_id.to_string()), + task_verification: None, + } +} + +fn drain_action_results( + rx: &mut mpsc::Receiver, +) -> Vec { + let mut results = Vec::new(); + while let Ok(command) = rx.try_recv() { + if let FleetControlCommand::Send(crate::fleet_wire::BrokerToRelaycast::ActionResult( + result, + )) = command + { + results.push(result); + } + } + results +} + +/// relay#1893 review (Cursor, high): a startup-fallback `worker_ready` releases +/// the initial task before readiness is proven. Its verdict used to be ignored +/// because the spawn required proven readiness first, and the later proven +/// `worker_ready` could not resolve it either, so a delivered task timed out +/// and the worker was released. Either order must resolve exactly once. +#[tokio::test] +async fn a_task_verdict_before_proven_readiness_resolves_on_the_proven_ready() { + use crate::fleet_wire::ActionResultPayload; + for (verification, expect_success) in [("harness_acceptance", true), ("echo", false)] { + let worker_name = "spawn-order"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let mut fixture = worker_event_runtime_fixture(registry, HashMap::new()); + let name = WorkerName::from(worker_name); + fixture.runtime.pending_verified_spawns.insert( + name.clone(), + pending_spawn_with_task(generation, "init_order", false), + ); + + fixture + .runtime + .handle_worker_event(delivery_verified_worker_event( + worker_name, + generation, + "init_order", + Some(verification), + )) + .await; + assert!( + drain_action_results(&mut fixture.fleet_control_rx).is_empty(), + "readiness is not proven yet, so the action stays open" + ); + assert!(fixture.runtime.pending_verified_spawns.contains_key(&name)); + + fixture + .runtime + .handle_worker_event(WorkerEvent::Message { + name: name.clone(), + generation, + value: json!({"type":"worker_ready", "payload":{"readiness_proven":true}}), + }) + .await; + let results = drain_action_results(&mut fixture.fleet_control_rx); + assert_eq!(results.len(), 1, "{verification}: resolved exactly once"); + match (&results[0].result, expect_success) { + (ActionResultPayload::Output(output), true) => assert_eq!( + output.output, + json!({"spawned": true, "ready": true, "name": worker_name}) + ), + (ActionResultPayload::Error(error), false) => { + assert!( + error.error.starts_with("spawn_task_unconfirmed: "), + "{}", + error.error + ); + assert!(error.error.contains(verification), "{}", error.error); + } + (other, _) => panic!("{verification}: unexpected result {other:?}"), + } + assert!(!fixture.runtime.pending_verified_spawns.contains_key(&name)); + } +} + +/// relay#1893 review (Devin): a verified spawn whose initial task fails used to +/// get its error while the worker stayed registered and running, so a +/// corrected retry collided with the name. The failure must expire the spawn +/// so maintenance releases the worker before reporting the specific reason. +#[tokio::test] +async fn a_failed_initial_task_releases_the_worker_before_failing_the_spawn() { + use crate::fleet_wire::ActionResultPayload; + let worker_name = "spawn-task-failed"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let mut fixture = worker_event_runtime_fixture(registry, HashMap::new()); + let name = WorkerName::from(worker_name); + fixture.runtime.pending_verified_spawns.insert( + name.clone(), + pending_spawn_with_task(generation, "init_failed", true), + ); + + fixture + .runtime + .handle_worker_event(WorkerEvent::Message { + name: name.clone(), + generation, + value: json!({"type":"delivery_failed", "payload":{ + "delivery_id":"del_init_failed", "event_id":"init_failed", + "reason":"injection_too_large"}}), + }) + .await; + assert!( + drain_action_results(&mut fixture.fleet_control_rx).is_empty(), + "the failure is reported only after the worker is released" + ); + let pending = &fixture.runtime.pending_verified_spawns[&name]; + assert!(pending.deadline <= Instant::now()); + + fixture.runtime.handle_maintenance_tick().await; + assert!( + !fixture.runtime.workers.has_worker(&name), + "worker released" + ); + assert!(!fixture.runtime.pending_verified_spawns.contains_key(&name)); + let results = drain_action_results(&mut fixture.fleet_control_rx); + assert_eq!(results.len(), 1); + let ActionResultPayload::Error(error) = &results[0].result else { + panic!("a failed task must fail the spawn: {:?}", results[0]); + }; + assert!( + error.error.starts_with("spawn_task_failed: "), + "{}", + error.error + ); + assert!( + error.error.contains("injection_too_large"), + "{}", + error.error + ); +} + +/// A PTY acceptance window that closes with neither acceptance nor a parked +/// body is not evidence of loss: the agent may be running the task. The spawn +/// must resolve as unconfirmed and keep the live worker, never release it as a +/// failed task (which could kill the run and invite a duplicating retry). +#[tokio::test] +async fn an_unproven_acceptance_reports_the_spawn_unconfirmed_and_keeps_the_worker() { + use crate::fleet_wire::ActionResultPayload; + let worker_name = "spawn-task-unproven"; + let registry = make_worker_registry_with_worker(worker_name).await; + let generation = registry.workers[worker_name].generation; + let mut fixture = worker_event_runtime_fixture(registry, HashMap::new()); + let name = WorkerName::from(worker_name); + fixture.runtime.pending_verified_spawns.insert( + name.clone(), + pending_spawn_with_task(generation, "init_unproven", true), + ); + + fixture + .runtime + .handle_worker_event(WorkerEvent::Message { + name: name.clone(), + generation, + value: json!({"type":"delivery_failed", "payload":{ + "delivery_id":"del_init_unproven", "event_id":"init_unproven", + "reason": crate::broker::delivery_verification::HARNESS_ACCEPTANCE_UNPROVEN}}), + }) + .await; + let results = drain_action_results(&mut fixture.fleet_control_rx); + assert_eq!(results.len(), 1, "resolved exactly once"); + let ActionResultPayload::Error(error) = &results[0].result else { + panic!("an unproven task must not succeed: {:?}", results[0]); + }; + assert!( + error.error.starts_with("spawn_task_unconfirmed: "), + "{}", + error.error + ); + assert!(error.error.contains("Do not retry"), "{}", error.error); + assert!(!fixture.runtime.pending_verified_spawns.contains_key(&name)); + fixture.runtime.handle_maintenance_tick().await; + assert!(fixture.runtime.workers.has_worker(&name), "worker kept"); +} + +/// relay#1893 review (Cursor, medium): a worker already ready when the launch +/// returns has its PTY task queued without a binding to this action, so the +/// action cannot learn the task's verdict and must not report plain success. +#[test] +fn an_already_ready_worker_with_an_unbound_pty_task_is_not_reported_ready() { + use crate::fleet_wire::ActionResultPayload; + let name = WorkerName::from("already-ready"); + let ActionResultPayload::Error(error) = + super::fleet::already_ready_spawn_result("inv".into(), &name, true).result + else { + panic!("an unbound PTY task must not report success"); + }; + assert!( + error.error.starts_with("spawn_task_unconfirmed: "), + "{}", + error.error + ); + assert!(matches!( + super::fleet::already_ready_spawn_result("inv".into(), &name, false).result, + ActionResultPayload::Output(_) + )); +} + +/// relay#1930 review (cubic): the early fleet size guard is a PTY envelope +/// limit, so a spawn whose harness config selects a headless or native +/// runtime must not be rejected by it. Absent or invalid configs stay PTY. +#[test] +fn early_spawn_task_guard_applies_only_to_pty_injected_spawns() { + assert!(super::fleet::spawn_task_is_pty_injected( + &json!({"cli": "codex"}) + )); + assert!(super::fleet::spawn_task_is_pty_injected(&json!({ + "harnessConfig": {"runtime": "pty", "command": "codex", "args": []} + }))); + assert!(super::fleet::spawn_task_is_pty_injected( + &json!({"harnessId": "x"}) + )); + assert!(!super::fleet::spawn_task_is_pty_injected(&json!({ + "agent": {"harnessConfig": { + "runtime": "headless", "protocol": "opencode", + "endpoint": "http://127.0.0.1:4096", "sessionId": "ses_1", "release": "abort" + }} + }))); +} + +#[tokio::test] +async fn an_oversized_task_for_a_headless_fleet_spawn_skips_the_pty_guard() { + let temp = tempfile::tempdir().unwrap(); + let (tx, _rx) = mpsc::channel(16); + let workers = WorkerRegistry::new(tx, vec![], temp.path().into(), Instant::now()); + let mut fixture = worker_event_runtime_fixture(workers, HashMap::new()); + // The guard replies at once; a spawn past it may wait on registration, so + // bound the call and inspect only what was replied in that window. + let _ = tokio::time::timeout( + Duration::from_secs(2), + fixture.runtime.handle_fleet_control_event( + crate::node_control::FleetControlEvent::Message( + crate::fleet_wire::RelaycastToBroker::ActionInvoke( + crate::fleet_wire::ActionInvoke { + task_execution: None, + v: FLEET_WIRE_VERSION, + invocation_id: "headless-task".into(), + action: "spawn".into(), + input: json!({"name":"headless", "cli":"opencode", "cwd":temp.path(), + "task":"x".repeat(crate::injection_wire::MAX_BODY_BYTES + 1), + "harnessConfig": {"runtime": "headless", "protocol": "opencode", + "endpoint": "http://127.0.0.1:9", "sessionId": "ses_1"}}), + agent_name: Some("headless".into()), + agent_id: None, + }, + ), + ), + ), + ) + .await; + let mut registered = false; + while let Ok(command) = fixture.fleet_control_rx.try_recv() { + match command { + FleetControlCommand::RegisterAgent { .. } => registered = true, + FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) => { + if let crate::fleet_wire::ActionResultPayload::Error(error) = &result.result { + assert!( + !error.error.contains("spawn_task_too_large"), + "a headless spawn never types its task into a PTY: {}", + error.error + ); + } + } + _ => {} + } + } + // Registration is the next awaited step after the guard, so observing it + // proves the oversized headless task passed the guard. + assert!(registered, "the headless spawn must reach registration"); +} + +/// relay#1893 review (Cursor): a task that cannot fit the PTY envelope used to +/// launch a worker that then failed `injection_too_large` and was released. +/// It must be rejected before any registration or launch. +#[tokio::test] +async fn an_oversized_spawn_task_is_rejected_before_launch() { + let temp = tempfile::tempdir().unwrap(); + let (tx, _rx) = mpsc::channel(16); + let workers = WorkerRegistry::new(tx, vec![], temp.path().into(), Instant::now()); + let mut fixture = worker_event_runtime_fixture(workers, HashMap::new()); + assert!(super::fleet::spawn_task_too_large( + "codex", + &"x".repeat(crate::injection_wire::MAX_BODY_BYTES) + ) + .is_none()); + // Muse takes its startup task as one argv entry, not through the PTY + // envelope, so only its portable argv limit applies. + let over_pty = "x".repeat(crate::injection_wire::MAX_BODY_BYTES + 1); + assert!(super::fleet::spawn_task_too_large("codex", &over_pty).is_some()); + assert!(super::fleet::spawn_task_too_large("/usr/local/bin/Muse.exe", &over_pty).is_none()); + assert!(super::fleet::spawn_task_too_large( + "muse", + &"x".repeat(crate::worker::MUSE_STARTUP_PROMPT_MAX_BYTES + 1) + ) + .is_some_and(|error| error.starts_with("spawn_task_too_large: "))); + fixture + .runtime + .handle_fleet_control_event(crate::node_control::FleetControlEvent::Message( + crate::fleet_wire::RelaycastToBroker::ActionInvoke(crate::fleet_wire::ActionInvoke { + task_execution: None, + v: FLEET_WIRE_VERSION, + invocation_id: "oversized-task".into(), + action: "spawn".into(), + input: json!({"name":"oversized", "cli":"codex", "verify_ready":true, + "cwd":temp.path(), "task":"x".repeat(crate::injection_wire::MAX_BODY_BYTES + 1)}), + agent_name: Some("oversized".into()), + agent_id: None, + }), + )) + .await; + let mut found = false; + while let Ok(command) = fixture.fleet_control_rx.try_recv() { + match command { + FleetControlCommand::RegisterAgent { .. } => panic!("must reject before registration"), + FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) => { + let crate::fleet_wire::ActionResultPayload::Error(error) = &result.result else { + panic!("expected fleet error: {result:?}"); + }; + assert!( + error.error.starts_with("spawn_task_too_large: "), + "{}", + error.error + ); + found = true; + } + _ => {} + } + } + assert!(found); + assert!(fixture.runtime.workers.workers.is_empty()); + + // relay#1893 review (Cursor): the raw task fits, but the exit-after-task + // contract appended by the spawn path pushes the injected task past the + // cap. That must also fail before registration, not after launch. + fixture + .runtime + .handle_fleet_control_event(crate::node_control::FleetControlEvent::Message( + crate::fleet_wire::RelaycastToBroker::ActionInvoke(crate::fleet_wire::ActionInvoke { + task_execution: None, + v: FLEET_WIRE_VERSION, + invocation_id: "decorated-task".into(), + action: "spawn".into(), + input: json!({"name":"decorated", "cli":"codex", "verify_ready":true, + "exit_after_task": true, "cwd":temp.path(), + "task":"x".repeat(crate::injection_wire::MAX_BODY_BYTES)}), + agent_name: Some("decorated".into()), + agent_id: None, + }), + )) + .await; + let mut found = false; + while let Ok(command) = fixture.fleet_control_rx.try_recv() { + match command { + FleetControlCommand::RegisterAgent { .. } => { + panic!("a decorated task over the cap must fail before registration") + } + FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) => { + let crate::fleet_wire::ActionResultPayload::Error(error) = &result.result else { + panic!("expected fleet error: {result:?}"); + }; + assert!( + error.error.contains("spawn_task_too_large: "), + "{}", + error.error + ); + found = true; + } + _ => {} + } + } + assert!(found); + assert!(fixture.runtime.workers.workers.is_empty()); +} + fn inbound_ctx<'a>(event_id: &'a str) -> InboundContext<'a> { InboundContext { from: "Alice", @@ -3818,6 +4372,113 @@ async fn initial_delivery_failure_stays_owned_until_dead_lettered() { assert_eq!(pending.delivery.body, "must remain auditable"); } +#[tokio::test] +async fn pty_body_cap_applies_only_to_pty_recipients() { + let over_cap = "x".repeat(crate::injection_wire::MAX_BODY_BYTES + 1); + for runtime in [AgentRuntime::Pty, AgentRuntime::Headless] { + let mut workers = make_worker_registry_with_worker("Worker").await; + workers + .workers + .get_mut(&WorkerName::from("Worker")) + .unwrap() + .spec + .runtime = runtime.clone(); + let mut pending_deliveries = HashMap::new(); + let result = super::queue_and_try_delivery_raw( + &mut workers, + &mut pending_deliveries, + "Worker", + "evt_large", + "orchestrator", + "Worker", + &over_cap, + None, + Some(WorkspaceId::new("ws_demo")), + None, + 2, + MessageInjectionMode::Wait, + Duration::from_secs(1), + None, + None, + ) + .await; + match runtime { + AgentRuntime::Pty => assert!(result + .expect_err("PTY recipients keep the envelope-sized body cap") + .to_string() + .contains("injection_too_large")), + AgentRuntime::Headless => { + result.expect("headless recipients never type into a PTY, so no PTY cap"); + assert_eq!(pending_deliveries.len(), 1); + } + } + } + // Headless providers receive the body as one argv entry, so they keep + // the portable single-argument ceiling instead of no bound at all. + let mut workers = make_worker_registry_with_worker("Worker").await; + workers + .workers + .get_mut(&WorkerName::from("Worker")) + .unwrap() + .spec + .runtime = AgentRuntime::Headless; + let error = super::queue_and_try_delivery_raw( + &mut workers, + &mut HashMap::new(), + "Worker", + "evt_argv", + "orchestrator", + "Worker", + &"x".repeat(crate::worker::MUSE_STARTUP_PROMPT_MAX_BYTES + 1), + None, + Some(WorkspaceId::new("ws_demo")), + None, + 2, + MessageInjectionMode::Wait, + Duration::from_secs(1), + None, + None, + ) + .await + .expect_err("a headless body past the argv ceiling fails before delivery"); + assert!(error.to_string().contains("injection_too_large"), "{error}"); + // An app-server headless worker receives deliveries as frames, never as a + // process argument, so neither ceiling applies to it. + let handle = workers + .workers + .get_mut(&WorkerName::from("Worker")) + .unwrap(); + handle.spec.harness_config = Some(ResolvedHarnessConfig::Headless(HeadlessHarnessConfig { + driver: HeadlessHarnessDriver::AppServer, + protocol: "opencode".to_string(), + endpoint: "http://127.0.0.1:4096".to_string(), + session_id: "session-endpoint".to_string(), + auth: None, + host: None, + release: Some(HarnessReleasePolicy::Abort), + metadata: None, + })); + super::queue_and_try_delivery_raw( + &mut workers, + &mut HashMap::new(), + "Worker", + "evt_frame", + "orchestrator", + "Worker", + &"x".repeat(crate::worker::MUSE_STARTUP_PROMPT_MAX_BYTES + 1), + None, + Some(WorkspaceId::new("ws_demo")), + None, + 2, + MessageInjectionMode::Wait, + Duration::from_secs(1), + None, + None, + ) + .await + .expect("frame-delivered headless workers have no argv ceiling"); +} + #[tokio::test] async fn delivery_retry_transient_blip_emits_failed_event_for_present_worker() { let worker_name = "worker-blip"; @@ -9689,6 +10350,9 @@ async fn muse_provider_auth_error_expires_verified_spawn_and_releases_capacity() started: Instant::now(), generation, failure_reason: None, + readiness_proven: false, + task_event_id: None, + task_verification: None, }, ); for event_generation in [Uuid::new_v4(), generation] { @@ -9740,6 +10404,74 @@ async fn muse_provider_auth_error_expires_verified_spawn_and_releases_capacity() assert!(found, "maintenance must return the specific fleet failure"); } +#[tokio::test] +async fn devin_directory_trust_error_expires_verified_spawn_and_releases_capacity() { + let name = WorkerName::from("devin-trust-test"); + let workers = make_worker_registry_with_worker(name.as_str()).await; + let generation = workers.workers.get(&name).unwrap().generation; + let mut fixture = worker_event_runtime_fixture(workers, HashMap::new()); + fixture.runtime.pending_verified_spawns.insert( + name.clone(), + super::fleet::PendingVerifiedSpawn { + invocation_id: "auth-invocation".into(), + deadline: Instant::now() + Duration::from_secs(90), + started: Instant::now(), + generation, + failure_reason: None, + readiness_proven: false, + task_event_id: None, + task_verification: None, + }, + ); + for event_generation in [Uuid::new_v4(), generation] { + fixture.runtime.handle_worker_event(WorkerEvent::Message { + name: name.clone(), generation: event_generation, + value: json!({"type":"worker_error", "payload":{"code":"directory_trust_required", "message":"private-directory-path"}}), + }).await; + let pending = fixture.runtime.pending_verified_spawns.get(&name).unwrap(); + if event_generation != generation { + assert!(pending.failure_reason.is_none()); + assert!(pending.deadline > Instant::now()); + } else { + assert!(pending + .failure_reason + .as_deref() + .unwrap() + .contains("directory_trust_required")); + assert!(!pending + .failure_reason + .as_deref() + .unwrap() + .contains("private-directory-path")); + assert!(pending.deadline <= Instant::now()); + } + } + fixture + .runtime + .handle_worker_event(WorkerEvent::Message { + name: name.clone(), + generation, + value: json!({"type":"worker_ready", "payload":{"readiness_proven":true}}), + }) + .await; + assert!(fixture.runtime.pending_verified_spawns.contains_key(&name)); + fixture.runtime.handle_maintenance_tick().await; + assert!(!fixture.runtime.pending_verified_spawns.contains_key(&name)); + assert!(!fixture.runtime.workers.has_worker(&name)); + let mut found = false; + while let Ok(command) = fixture.fleet_control_rx.try_recv() { + if let FleetControlCommand::Send(BrokerToRelaycast::ActionResult(result)) = command { + assert!(match &result.result { + crate::fleet_wire::ActionResultPayload::Error(error) => error.error.as_str(), + _ => panic!("expected fleet error: {result:?}"), + } + .contains("directory_trust_required")); + found = true; + } + } + assert!(found, "maintenance must return the specific fleet failure"); +} + #[tokio::test] async fn muse_fleet_missing_auth_fails_before_registration_and_dedup() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/broker/src/runtime/worker_events.rs b/crates/broker/src/runtime/worker_events.rs index 117d4d48fd..8e4f4731d1 100644 --- a/crates/broker/src/runtime/worker_events.rs +++ b/crates/broker/src/runtime/worker_events.rs @@ -1,6 +1,6 @@ use super::fleet::{ confirm_pending_delivery_and_resolve_fleet_ack, fail_terminal_session, - refresh_fleet_inventory_session_ref, try_send_terminal, verified_spawn_ready_result, + refresh_fleet_inventory_session_ref, try_send_terminal, }; use super::*; use crate::terminal_control::{TerminalControlCommand, TerminalToCloud}; @@ -945,18 +945,71 @@ impl BrokerRuntime { .get("event_id") .and_then(Value::as_str) .unwrap_or(""); + // The raw worker label, before the PTY default below: + // an unlabelled frame (a headless exit) proves no + // receipt of a verified spawn's task. + let label = payload.get("verification").and_then(Value::as_str); + // `verification_label_confirms_receipt` is the single + // source of truth for which labels prove receipt: + // "harness_acceptance" from a modern PTY worker. + let receipt_confirmed = label.is_some_and( + crate::broker::delivery_verification::verification_label_confirms_receipt, + ); + // Record the verdict whenever it belongs to this + // spawn's task, even before readiness is proven: a + // startup-fallback `worker_ready` releases the task + // first, and the proven frame that follows resolves + // the action from what was recorded here. + if let Some(pending) = pending_verified_spawns + .get_mut(&name) + .filter(|pending| pending.matches_task(generation, event_id)) + { + pending.record_task_verification(receipt_confirmed, label); + if !receipt_confirmed { + tracing::warn!( + target = "agent_relay::broker", + worker = %name, + delivery_id = %delivery_id, + event_id = %event_id, + verification = label.unwrap_or("none"), + "verified spawn's initial task was acked without \ + proof of harness acceptance; reporting it unconfirmed" + ); + } + } + // A spawn carrying a task succeeds only when that + // task is proven received. An ack the worker could + // not verify leaves the agent live, so the action + // resolves as explicitly unconfirmed rather than as + // either a success or a retryable failure. + let resolved = pending_verified_spawns + .get(&name) + .is_some_and(|pending| pending.can_report_ready(generation)) + .then(|| pending_verified_spawns.remove(&name)) + .flatten(); + if let Some(pending) = resolved { + let _ = fleet_control_tx + .send(FleetControlCommand::Send( + crate::fleet_wire::BrokerToRelaycast::ActionResult( + pending.completion(&name), + ), + )) + .await; + } + // Modern PTY workers use "harness_acceptance": // editor echo alone is never delivery confirmation. - let verification = payload - .get("verification") - .and_then(Value::as_str) - .unwrap_or("worker_confirmation"); + let verification = label.unwrap_or("worker_confirmation"); let is_pty = workers .workers .get(&name) .is_some_and(|handle| handle.spec.runtime == AgentRuntime::Pty); let completed_replay = verification == "completed_replay"; - if is_pty && verification != "harness_acceptance" && !completed_replay { + if is_pty + && verification + != crate::broker::delivery_verification::HARNESS_ACCEPTANCE + && !completed_replay + { tracing::warn!( target = "agent_relay::broker", worker = %name, @@ -1071,6 +1124,49 @@ impl BrokerRuntime { .get("event_id") .and_then(Value::as_str) .unwrap_or(""); + // Expire rather than reply: maintenance releases the + // worker and its fleet identity, then reports this + // reason, so a corrected retry can reuse the name. + // + // An acceptance window that closed without evidence + // either way is not a loss: the agent may be working + // on the task, so releasing it could kill a live run + // and a retry could duplicate it. Report the spawn + // unconfirmed instead, like an unproven ack. + let failure_reason = payload + .get("reason") + .and_then(Value::as_str) + .unwrap_or("initial_task_failed"); + if let Some(pending) = pending_verified_spawns + .get_mut(&name) + .filter(|pending| pending.matches_task(generation, event_id)) + { + if failure_reason + == crate::broker::delivery_verification::HARNESS_ACCEPTANCE_UNPROVEN + { + pending.record_task_verification( + false, + Some("harness_acceptance_unproven"), + ); + } else { + pending.initial_task_failed(generation, failure_reason); + } + } + let resolved = pending_verified_spawns + .get(&name) + .is_some_and(|pending| pending.can_report_ready(generation)) + .then(|| pending_verified_spawns.remove(&name)) + .flatten(); + if let Some(pending) = resolved { + let _ = fleet_control_tx + .send(FleetControlCommand::Send( + crate::fleet_wire::BrokerToRelaycast::ActionResult( + pending.completion(&name), + ), + )) + .await; + } + let reason = payload .get("reason") .and_then(Value::as_str) @@ -1160,6 +1256,13 @@ impl BrokerRuntime { pending.provider_auth_failed(generation); } } + if value.pointer("/payload/code").and_then(Value::as_str) + == Some("directory_trust_required") + { + if let Some(pending) = pending_verified_spawns.get_mut(&name) { + pending.directory_trust_failed(generation); + } + } let is_pty = workers .workers .get(&name) @@ -1579,15 +1682,25 @@ impl BrokerRuntime { .get(&name) .map(|handle| handle.spec.runtime == AgentRuntime::Pty) .unwrap_or(false); + let initial_task = workers.take_initial_task_for_injection(&name); + let initial_event_id = initial_task + .as_ref() + .map(|_| format!("init_{}", Uuid::new_v4().simple())); + if let Some(pending) = pending_verified_spawns.get_mut(&name) { + if pending.generation == generation { + pending.readiness_proven |= readiness_proven; + if is_pty_worker && initial_event_id.is_some() { + pending.task_event_id = initial_event_id.clone(); + } + } + } // Resolve the verified Fleet action before optional SDK // notifications and initial-task work. A congested SDK // output queue must not turn a ready worker into an // action timeout. let pending = pending_verified_spawns .get(&name) - .is_some_and(|pending| { - readiness_proven && pending.generation == generation - }) + .is_some_and(|pending| pending.can_report_ready(generation)) .then(|| pending_verified_spawns.remove(&name)) .flatten(); if let Some(pending) = pending { @@ -1597,7 +1710,7 @@ impl BrokerRuntime { let _ = fleet_control_tx .send(FleetControlCommand::Send( crate::fleet_wire::BrokerToRelaycast::ActionResult( - verified_spawn_ready_result(pending.invocation_id, &name), + pending.completion(&name), ), )) .await; @@ -1624,8 +1737,8 @@ impl BrokerRuntime { ); } } - if let Some(task_text) = workers.take_initial_task_for_injection(&name) { - let event_id = format!("init_{}", Uuid::new_v4().simple()); + if let (Some(task_text), Some(event_id)) = (initial_task, initial_event_id) + { if let Err(e) = queue_and_try_delivery_raw( workers, pending_deliveries, @@ -1646,6 +1759,9 @@ impl BrokerRuntime { .await { tracing::warn!(worker = %name, error = %e, "failed to deliver initial_task"); + if let Some(pending) = pending_verified_spawns.get_mut(&name) { + pending.initial_task_failed(generation, &e.to_string()); + } } // The initial task bypasses the delivery-mode queue, but the // hold replayed above freezes injection pops inside the PTY diff --git a/crates/broker/src/worker.rs b/crates/broker/src/worker.rs index ee2d45a630..53a40cd440 100644 --- a/crates/broker/src/worker.rs +++ b/crates/broker/src/worker.rs @@ -907,6 +907,7 @@ impl WorkerRegistry { anyhow::bail!("agent '{}' already exists", spec.name); } validate_muse_startup_prompt_for_spec(&spec, initial_task.as_deref())?; + validate_pty_initial_task_size(&spec, initial_task.as_deref())?; tracing::info!( target = "broker::spawn", @@ -2487,11 +2488,9 @@ pub(crate) fn validate_muse_startup_prompt(cli: &str, task: Option<&str>) -> Res /// Resolve the command that a PTY spec will actually launch before validating /// its Muse prompt. Explicit PTY harness configs own the executable; ordinary /// PTY specs use `spec.cli`. -pub(crate) fn validate_muse_startup_prompt_for_spec( - spec: &AgentSpec, - task: Option<&str>, -) -> Result<()> { - let cli = match spec.harness_config.as_ref() { +/// The PTY command a spec will launch, if it is a PTY spawn. +fn pty_cli_for_spec(spec: &AgentSpec) -> Result> { + Ok(match spec.harness_config.as_ref() { Some(ResolvedHarnessConfig::Pty(config)) => Some( parse_cli_command(&config.command) .with_context(|| format!("invalid harness command '{}'", config.command))? @@ -2506,13 +2505,35 @@ pub(crate) fn validate_muse_startup_prompt_for_spec( None => None, }, _ => None, - }; - match cli { + }) +} + +pub(crate) fn validate_muse_startup_prompt_for_spec( + spec: &AgentSpec, + task: Option<&str>, +) -> Result<()> { + match pty_cli_for_spec(spec)? { Some(cli) => validate_muse_startup_prompt(&cli, task), None => Ok(()), } } +/// Reject, before launch, a PTY initial task whose final (decorated) body +/// cannot fit the injection envelope. Muse takes its task in argv, so only +/// injected tasks are measured (relay#1893 review). +pub(crate) fn validate_pty_initial_task_size(spec: &AgentSpec, task: Option<&str>) -> Result<()> { + let (Some(cli), Some(task)) = (pty_cli_for_spec(spec)?, task) else { + return Ok(()); + }; + if muse_startup_prompt(&cli.to_ascii_lowercase(), Some(task)).is_some() { + return Ok(()); + } + match crate::injection_wire::task_too_large_error(task) { + Some(error) => anyhow::bail!(error), + None => Ok(()), + } +} + /// Clean Muse config home for a worker spawn, when Relay MCP injection is /// active for Muse: an isolated `XDG_CONFIG_HOME` scope provisioned by the /// MCP configurator. Returns `None` for other CLIs and when injection is @@ -4813,6 +4834,46 @@ sleep 30 ); } + #[test] + fn pty_initial_task_size_is_measured_only_for_injected_tasks() { + let spec = |cli: &str, runtime: AgentRuntime| AgentSpec { + name: WorkerName::from("size-check"), + runtime, + provider: None, + cli: Some(cli.to_string()), + session_id: None, + harness_config: None, + model: None, + cwd: None, + team: None, + shadow_of: None, + shadow_mode: None, + args: Vec::new(), + channels: Vec::new(), + restart_policy: None, + }; + let fits = "x".repeat(crate::injection_wire::MAX_BODY_BYTES); + let over = "x".repeat(crate::injection_wire::MAX_BODY_BYTES + 1); + assert!( + validate_pty_initial_task_size(&spec("codex", AgentRuntime::Pty), Some(&fits)).is_ok() + ); + let error = + validate_pty_initial_task_size(&spec("codex --yolo", AgentRuntime::Pty), Some(&over)) + .expect_err("an injected task over the body cap is rejected") + .to_string(); + assert!(error.starts_with("spawn_task_too_large: "), "{error}"); + // Muse receives its task in argv, and native runtimes are not injected. + assert!( + validate_pty_initial_task_size(&spec("muse", AgentRuntime::Pty), Some(&over)).is_ok() + ); + assert!(validate_pty_initial_task_size( + &spec("codex", AgentRuntime::Headless), + Some(&over) + ) + .is_ok()); + assert!(validate_pty_initial_task_size(&spec("codex", AgentRuntime::Pty), None).is_ok()); + } + #[test] fn argv_initial_task_blocks_followups_but_is_not_injected_twice() { let mut registry = make_registry(Vec::new()); diff --git a/crates/broker/src/wrap.rs b/crates/broker/src/wrap.rs index c1d87d18e6..39c768e6a4 100644 --- a/crates/broker/src/wrap.rs +++ b/crates/broker/src/wrap.rs @@ -103,7 +103,20 @@ pub(crate) fn submit_injection_body( bytes: Vec, pace: Duration, ) -> anyhow::Result<(tokio::sync::oneshot::Receiver>, u64)> { - if let Some(delay) = injection_submit_followup_delay(resolved_cli) { + // Follow the already-selected wire. Capability can latch between body + // construction and queue admission; re-probing here could bulk-type raw bytes. + let wire = if bytes.starts_with(b"\x1b[200~") && bytes.ends_with(b"\x1b[201~") { + crate::injection_wire::InjectionWire::Paste + } else { + crate::injection_wire::InjectionWire::Typed + }; + let limit = crate::injection_wire::effective_limit(wire, pace); + anyhow::ensure!(bytes.len().saturating_sub(if wire == crate::injection_wire::InjectionWire::Paste { 12 } else { 0 }) <= limit, "injection_too_large: {resolved_cli} effective limit is {limit} bytes; use a brief file pointer"); + let paste = wire == crate::injection_wire::InjectionWire::Paste; + let pace = if paste { Duration::ZERO } else { pace }; + if let Some(delay) = injection_submit_followup_delay(resolved_cli) + .or_else(|| paste.then_some(PASTE_INJECTION_SUBMIT_DELAY)) + { pty.submit_write_paced_with_followup_and_output_boundary(bytes, pace, delay, b"\r".to_vec()) } else { let mut burst = bytes; @@ -311,6 +324,78 @@ fn prepare_wrap_retry( true } +/// Format a wrap injection, replacing a body whose envelope would exceed the +/// PTY injection limit with a bounded pointer to the message. +/// +/// The entry has already left `pending_wrap_injections`, and wrap has no +/// channel back to the sender, so skipping the write would make the message +/// vanish silently. The notice tells the agent the message exists and where to +/// read it; the full text stays in Relay (relay#1893 review). +#[allow(clippy::too_many_arguments)] +fn format_wrap_injection( + from: &str, + event_id: &str, + body: &str, + target: &str, + include_reminder: bool, + workspace_id: Option<&str>, + workspace_alias: Option<&str>, +) -> String { + let format = |body: &str| { + format_injection_for_worker_with_workspace( + from, + event_id, + body, + target, + include_reminder, + true, // pre_registered + None, // assigned_name + workspace_id, + workspace_alias, + ) + }; + let injection = format(body); + if injection.len() <= crate::injection_wire::MAX_INJECTION_BODY_BYTES { + return injection; + } + tracing::warn!( + event_id, + bytes = injection.len(), + limit = crate::injection_wire::MAX_INJECTION_BODY_BYTES, + "wrap: injection_too_large; delivering a pointer instead of the body" + ); + let pointer = |event_id: &str| { + format!( + "[Message body not shown: {} bytes exceeds the {}-byte terminal delivery limit. \ + The full message is stored in Relay as message {event_id}; read it with your Relay \ + message tools before replying.]", + body.len(), + crate::injection_wire::MAX_INJECTION_BODY_BYTES + ) + }; + let injection = format(&pointer(event_id)); + if injection.len() <= crate::injection_wire::MAX_INJECTION_BODY_BYTES { + return injection; + } + // Oversized sender, target or workspace metadata. A write that cannot fit + // is re-queued at the head of the queue, so the last resort must always + // fit: clip every field and drop the optional reminder and workspace label. + fn clip(value: &str) -> &str { + &value[..crate::util::ansi::floor_char_boundary(value, 256)] + } + format_injection_for_worker_with_workspace( + clip(from), + clip(event_id), + &pointer(clip(event_id)), + clip(target), + false, + true, + None, + None, + None, + ) +} + /// Start echo verification after a PTY write ack without missing output that /// raced ahead of the ack select arm. Returns `true` when the echo was already /// present and the delivery was confirmed immediately. @@ -2201,18 +2286,16 @@ pub(crate) async fn run_wrap( tracing::debug!("relay from {} → {}", pending.from, pending.target); let include_reminder = !skip_prompt && mcp_reminder_throttle.should_include(Instant::now()); - let injection = format_injection_for_worker_with_workspace( + let injection = format_wrap_injection( &pending.from, &pending.event_id, &pending.body, &pending.target, include_reminder, - true, // pre_registered - None, // assigned_name pending.workspace_id.as_deref(), pending.workspace_alias.as_deref(), ); - let bytes = crate::devin::injection_bytes(&resolved_cli, &injection); + let bytes = crate::injection_wire::injection_bytes(crate::injection_wire::injection_wire(&resolved_cli, &pty), &injection); let write = submit_injection_body(&pty, &resolved_cli, bytes, Duration::ZERO); match write { Ok((ack_rx, output_boundary)) => { @@ -2925,6 +3008,57 @@ sys.stdout.flush()"#; assert_eq!(verification.attempts, 2); } + #[test] + fn oversized_wrap_messages_are_delivered_as_a_bounded_pointer() { + let limit = crate::injection_wire::MAX_INJECTION_BODY_BYTES; + let normal = + super::format_wrap_injection("alice", "evt_ok", "hello", "bob", true, None, None); + assert!(normal.contains("Relay message from alice [evt_ok]: hello")); + // A body that fits on its own but whose reminder and attribution push + // the envelope past the limit (the reviewer's 16,000-byte DM), and one + // far over it. + for len in [limit - 384, limit * 2] { + let body = format!("UNIQUE-BODY-MARKER{}", "x".repeat(len)); + let injection = + super::format_wrap_injection("alice", "evt_big", &body, "bob", true, None, None); + assert!(injection.len() <= limit, "{len}: {} bytes", injection.len()); + assert!( + !injection.contains("UNIQUE-BODY-MARKER"), + "{len}: body leaked" + ); + assert!( + injection.contains("Relay message from alice [evt_big]"), + "{injection}" + ); + assert!(injection.contains("message evt_big"), "{injection}"); + assert!( + injection.contains(&format!("{} bytes exceeds", body.len())), + "{injection}" + ); + } + } + + /// relay#1930 review (cubic): the pointer replaces only the body, so + /// oversized sender, target or workspace metadata kept the envelope over + /// the limit, failing submit and re-queuing the delivery at the head of + /// the queue forever. The final fallback must always fit. + #[test] + fn oversized_wrap_metadata_still_yields_a_bounded_pointer() { + let limit = crate::injection_wire::MAX_INJECTION_BODY_BYTES; + let huge = "é".repeat(limit); + let injection = super::format_wrap_injection( + &huge, + "evt_meta", + &"x".repeat(limit), + &huge, + true, + Some(&huge), + Some(&huge), + ); + assert!(injection.len() <= limit, "{} bytes", injection.len()); + assert!(injection.contains("message evt_meta"), "{injection}"); + } + #[test] fn stdin_drains_in_order_when_queue_accepts() { let mut pending: VecDeque> = VecDeque::new(); diff --git a/crates/broker/tests/devin_trust_cli.rs b/crates/broker/tests/devin_trust_cli.rs new file mode 100644 index 0000000000..db6e8e9171 --- /dev/null +++ b/crates/broker/tests/devin_trust_cli.rs @@ -0,0 +1,182 @@ +//! Real broker PTY tests with a synthetic Devin fixture (python3 required). +//! These are NOT captured Devin menus and do not authorize automatic acceptance. +//! Each test has a fresh HOME/XDG tree; the trusted arm uses a fixture-only trust +//! record scoped to cwd. Cursor placement and raw input model the Devin composer. +#![cfg(unix)] + +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::fs::PermissionsExt; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +use serde_json::{json, Value}; + +struct Worker(Child); +impl Drop for Worker { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn run_fixture(mode: &str) { + let directory = tempfile::tempdir().expect("fixture directory"); + let home = directory.path().join("home"); + let cwd = directory.path().join("worktree"); + std::fs::create_dir_all(&home).unwrap(); + std::fs::create_dir_all(&cwd).unwrap(); + // The fixture compares its trust record with os.getcwd(), which reports + // the resolved path; on macOS the temp dir sits behind /var -> /private/var. + let cwd = std::fs::canonicalize(&cwd).unwrap(); + let trusted = mode == "trusted"; + if trusted { + std::fs::write(home.join("fixture-trust"), cwd.to_str().unwrap()).unwrap(); + } + let fake_devin = directory.path().join("devin"); + let input_marker = directory.path().join("input"); + let task_marker = directory.path().join("task"); + std::fs::write( + &fake_devin, + r#"#!/usr/bin/env python3 +import os, pathlib, sys, tty +tty.setraw(0) +trust = pathlib.Path(os.environ['HOME']) / 'fixture-trust' +trusted = trust.exists() and trust.read_text() == os.getcwd() +if trusted: + sys.stdout.write('❭ Ask Devin to build features, fix bugs, or work on your code') +else: + sys.stdout.write('Do you trust the authors of this directory?\r\nprivate-directory-marker\r\n') + if os.environ['FIXTURE_MODE'] == 'widening': + sys.stdout.write('❭ 1 Yes, trust the parent directory and all subdirectories\r\n2 No, exit') + elif os.environ['FIXTURE_MODE'] == 'wrapped': + sys.stdout.write('❭ 1 Yes, trust\r\n this workspace\r\n2 No, exit') + else: + sys.stdout.write('❭ 1. Yes, I trust the authors\r\n2. No, exit') +sys.stdout.flush() +data = b'' +while True: + part = os.read(0, 4096) + if not part: + break + with open(os.environ['INPUT_MARKER'], 'ab') as log: + log.write(part) + data += part + if trusted and b'\r' in data: + pathlib.Path(os.environ['TASK_MARKER']).write_bytes(data) +"#, + ) + .unwrap(); + std::fs::set_permissions(&fake_devin, std::fs::Permissions::from_mode(0o755)).unwrap(); + let mut worker = Worker( + Command::new(env!("CARGO_BIN_EXE_agent-relay-broker")) + .args([ + "pty", + "--agent-name", + "devin-trust-proof", + fake_devin.to_str().unwrap(), + ]) + .current_dir(&cwd) + .env("HOME", &home) + .env("XDG_CONFIG_HOME", home.join(".config")) + .env("XDG_DATA_HOME", home.join(".local/share")) + .env("AGENT_RELAY_LOCAL_ONLY", "1") + .env("FIXTURE_MODE", mode) + .env("INPUT_MARKER", &input_marker) + .env("TASK_MARKER", &task_marker) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn broker PTY"), + ); + let stdout = worker.0.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + if tx.send(line).is_err() { + break; + } + } + }); + for frame in [ + json!({"v":2, "type":"init_worker", "payload":{"agent":{"name":"devin-trust-proof"}}}), + json!({"v":2, "type":"deliver_relay", "payload":{ + "delivery_id":"fixture-delivery", "event_id":"fixture-event", "from":"operator", + "target":"devin-trust-proof", "body":"Begin fixture task" + }}), + ] { + writeln!(worker.0.stdin.as_mut().unwrap(), "{frame}").unwrap(); + } + let deadline = Instant::now() + Duration::from_secs(if trusted { 10 } else { 40 }); + let mut ready = false; + let mut error = None; + while Instant::now() < deadline { + if trusted && ready && task_marker.exists() { + break; + } + let line = match rx.recv_timeout(Duration::from_millis(100)) { + Ok(Ok(line)) => line, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + other => panic!("broker stdout closed: {other:?}"), + }; + let Ok(frame) = serde_json::from_str::(&line) else { + continue; + }; + match frame["type"].as_str() { + Some("worker_ready") => { + assert!(trusted, "trust dialog reported ready: {frame}"); + assert_eq!(frame["payload"]["readiness_proven"], true); + ready = true; + } + Some("worker_error") => { + error = Some(frame); + break; + } + _ => {} + } + } + if trusted { + assert!(error.is_none(), "{error:?}"); + assert!(ready, "already-trusted fixture must become ready"); + let task = std::fs::read_to_string(&task_marker).expect("task must be submitted"); + assert!(task.contains("Begin fixture task"), "{task}"); + assert_eq!( + std::fs::read_to_string(home.join("fixture-trust")).unwrap(), + cwd.to_str().unwrap() + ); + } else { + let error = error.expect("frozen trust prompt must produce worker_error within 40s"); + assert_eq!(error["payload"]["code"], "directory_trust_required"); + assert_eq!(error["payload"]["retryable"], false); + let message = error["payload"]["message"].as_str().unwrap(); + assert!(message.contains("spawn working directory")); + assert!(!message.contains("private-directory-marker")); + assert!( + !input_marker.exists(), + "no trust response or task may enter the dialog" + ); + assert!(!home.join("fixture-trust").exists()); + assert!(!ready); + } +} + +#[test] +fn fresh_home_frozen_trust_prompt_fails_without_readiness() { + run_fixture("unanswerable"); +} + +#[test] +fn widening_only_trust_menu_is_never_accepted() { + run_fixture("widening"); +} + +#[test] +fn wrapped_workspace_trust_prompt_fails_without_readiness() { + run_fixture("wrapped"); +} + +#[test] +fn already_trusted_directory_becomes_ready_and_begins_task() { + run_fixture("trusted"); +} diff --git a/crates/broker/tests/pty_injection_integrity.rs b/crates/broker/tests/pty_injection_integrity.rs new file mode 100644 index 0000000000..57d71c3803 --- /dev/null +++ b/crates/broker/tests/pty_injection_integrity.rs @@ -0,0 +1,189 @@ +//! Real broker, raw-mode fake Claude TUI, byte-level transcript assertions. +#![cfg(unix)] +use serde_json::{json, Value}; +use std::{ + io::{BufRead, BufReader, Write}, + os::unix::fs::PermissionsExt, + process::{Child, Command, Stdio}, + sync::mpsc, + time::{Duration, Instant}, +}; +struct Fixture { + child: Child, + rx: mpsc::Receiver, + dir: tempfile::TempDir, +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} +impl Fixture { + fn new(mode: &str) -> Self { + let dir = tempfile::tempdir().unwrap(); + let script = dir.path().join("claude-fixture"); + std::fs::write( + &script, + r#"#!/usr/bin/env python3 +import os, tty, time, select +# Canonical mode silently discards input after MAX_CANON. Real TUIs use raw mode. +tty.setraw(0) +mode=os.environ['FIXTURE_MODE'] +time.sleep(0.2) +os.write(1,b'\x1b[?2004h\x1b[?2004l' if mode != 'typed' else b'') +os.write(1,'Claude Code v2.1.261\r\n❯ '.encode()) +data=b'' +while True: + chunk=os.read(0,65536) + with open(os.environ['TRANSCRIPT'],'ab') as f: f.write(chunk) + data+=chunk + if mode == 'hostile' and b'\x1b\x1b' in data: + os.write(1,b'\x1b[2J\x1b[HTrust prompt, no composer') + data=b'' + if data.endswith(b'\r'): + body=data.replace(b'\x1b[200~',b'').replace(b'\x1b[201~',b'').rstrip(b'\r') + os.write(1,b'\r\n'+body+b'\r\n') + os.write(1,'\r\n❯ '.encode()) + data=b'' +"#, + ) + .unwrap(); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_agent-relay-broker")) + .args(["pty", "--agent-name", "integrity", script.to_str().unwrap()]) + .env("FIXTURE_MODE", mode) + .env("TRANSCRIPT", dir.path().join("transcript")) + .env("AGENT_RELAY_LOCAL_ONLY", "1") + .env( + "RELAY_INJECT_PASTE", + if mode == "typed" { "0" } else { "1" }, + ) + .current_dir(dir.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + let stdout = child.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + if tx.send(line.unwrap()).is_err() { + break; + } + } + }); + let mut f = Self { child, rx, dir }; + f.send("init_worker", json!({"agent":{"name":"integrity"}})); + assert_eq!(f.wait("worker_ready")["payload"]["readiness_proven"], true); + f + } + fn send(&mut self, kind: &str, payload: Value) { + writeln!( + self.child.stdin.as_mut().unwrap(), + "{}", + json!({"v":2,"type":kind,"payload":payload}) + ) + .unwrap(); + } + fn wait(&self, kind: &str) -> Value { + let deadline = Instant::now() + Duration::from_secs(25); + loop { + let line = self + .rx + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .expect(kind); + if let Ok(frame) = serde_json::from_str::(&line) { + if frame["type"] == kind { + return frame; + } + } + } + } + fn deliver(&mut self, id: &str, body: &str) { + self.send("deliver_relay",json!({"delivery_id":format!("del_{id}"),"event_id":id,"from":"broker","target":"integrity","body":body,"priority":2,"injection_mode":"wait"})); + } + fn transcript(&self) -> Vec { + std::fs::read(self.dir.path().join("transcript")).unwrap_or_default() + } +} +fn long_pair() { + let mut f = Fixture::new("echo"); + for id in ["init_task", "relay_message"] { + let body = format!("HEAD-{id}\n{}\nTAIL-{id}", "éabc xyz\n".repeat(1100)); + assert!(body.len() > 10_000); + let before = f.transcript().len(); + f.deliver(id, &body); + let verified = f.wait("delivery_verified"); + // Harness acceptance, which is the only verdict a verified fleet + // spawn accepts as receipt of its task. + assert_eq!( + verified["payload"]["verification"], "harness_acceptance", + "{verified}" + ); + let transcript = f.transcript(); + let bytes = &transcript[before..]; + assert!(bytes.windows(body.len()).any(|w| w == body.as_bytes())); + assert_eq!(bytes.windows(6).filter(|w| *w == b"\x1b[200~").count(), 1); + assert_eq!(bytes.windows(6).filter(|w| *w == b"\x1b[201~").count(), 1); + assert_eq!(bytes.iter().filter(|&&b| b == b'\r').count(), 1); + } +} +#[test] +fn ten_kib_task_and_relay_message_are_intact() { + long_pair(); +} +#[test] +fn over_limit_and_typed_fallback_reject_before_writing() { + for mode in ["echo", "typed"] { + let mut f = Fixture::new(mode); + f.deliver( + "init_limit", + &"x".repeat(if mode == "typed" { 10_000 } else { 17_000 }), + ); + assert!(f.wait("delivery_failed")["payload"]["reason"] + .as_str() + .unwrap() + .contains("injection_too_large")); + assert!(f.transcript().is_empty()); + } +} +/// relay#1893 review (Cursor): the reminder throttle was marked sent before the +/// size gate, so a rejected envelope stole the reminder from the next delivery. +#[test] +fn rejected_injection_does_not_consume_the_reminder() { + let mut f = Fixture::new("echo"); + f.deliver("relay_big", &"x".repeat(17_000)); + assert!(f.wait("delivery_failed")["payload"]["reason"] + .as_str() + .unwrap() + .contains("injection_too_large")); + f.deliver("relay_next", "small follow-up"); + f.wait("delivery_verified"); + let transcript = String::from_utf8_lossy(&f.transcript()).into_owned(); + assert!(transcript.contains("small follow-up"), "{transcript}"); + // The full reminder, not the short hint sent while it is throttled. + assert!( + transcript.contains("Self-termination is not automatic"), + "{transcript}" + ); +} +#[test] +#[ignore = "20 cold starts; run separately"] +fn twenty_cold_starts() { + for _ in 0..20 { + long_pair(); + } +} + +#[test] +fn disappearing_composer_fails_before_body_injection() { + let mut f = Fixture::new("hostile"); + f.send("deliver_relay", json!({"delivery_id":"del_hostile", "event_id":"init_hostile", "from":"broker", "target":"integrity", "body":"must not type this into a dialog", "priority":2,"injection_mode":"steer"})); + assert_eq!( + f.wait("delivery_failed")["payload"]["reason"], + "prompt_unproven" + ); + assert!(!String::from_utf8_lossy(&f.transcript()).contains("must not type")); +} diff --git a/crates/relay-pty/src/pty.rs b/crates/relay-pty/src/pty.rs index 36e521604d..b5ecced055 100644 --- a/crates/relay-pty/src/pty.rs +++ b/crates/relay-pty/src/pty.rs @@ -431,6 +431,7 @@ pub struct PtySession { /// buffered stream chunks the snapshot already reflects and apply only /// the ones that came after. consumed_offset: Arc, + paste_capable: Arc, /// Sequence assigned as soon as the reader obtains a PTY output chunk, /// before grid parsing or async queue admission. Consumers sample this /// producer watermark when submitting input so output that was already @@ -927,6 +928,41 @@ fn read_was_cancelled(error: &io::Error) -> bool { error.raw_os_error() == Some(ERROR_OPERATION_ABORTED as i32) } +/// Finds the DECSET 2004 private-mode parameter (`?2004` or `;2004`) in PTY +/// output, including one split across reads, by carrying the previous read's +/// last four bytes. Plain text that merely contains `2004` does not match. +#[derive(Default)] +struct Decset2004Scan { + tail: [u8; 4], + tail_len: usize, +} + +impl Decset2004Scan { + fn may_carry(&mut self, bytes: &[u8]) -> bool { + let is_parameter = |window: &[u8]| matches!(window, [b'?' | b';', b'2', b'0', b'0', b'4']); + let head = bytes.len().min(4); + let mut seam = [0u8; 8]; + seam[..self.tail_len].copy_from_slice(&self.tail[..self.tail_len]); + seam[self.tail_len..self.tail_len + head].copy_from_slice(&bytes[..head]); + let joined = &seam[..self.tail_len + head]; + let found = joined.windows(5).chain(bytes.windows(5)).any(is_parameter); + let carried = if bytes.len() >= 4 { + &bytes[bytes.len() - 4..] + } else { + &joined[joined.len().saturating_sub(4)..] + }; + self.tail_len = carried.len(); + self.tail[..self.tail_len].copy_from_slice(carried); + found + } +} + +/// Whether the grid currently has bracketed-paste mode (DECSET 2004) set. +fn paste_mode_enabled(term: &Term) -> bool { + term.mode() + .contains(alacritty_terminal::term::TermMode::BRACKETED_PASTE) +} + impl PtySession { fn enqueue_write_with_output_boundary( &self, @@ -1057,6 +1093,8 @@ impl PtySession { let (tx, rx) = mpsc::channel(256); let term_clone = term.clone(); let processor_clone = processor.clone(); + let paste_capable = Arc::new(AtomicBool::new(false)); + let paste_capable_reader = paste_capable.clone(); let consumed_offset = Arc::new(AtomicU64::new(0)); let consumed_offset_reader = consumed_offset.clone(); let output_sequence = Arc::new(AtomicU64::new(0)); @@ -1075,6 +1113,7 @@ impl PtySession { let write_admission_pending_reader = write_admission_pending.clone(); let reader_thread = thread::spawn(move || { let mut buf = [0u8; 4096]; + let mut decset_2004_scan = Decset2004Scan::default(); loop { #[cfg(unix)] if wait_for_pty_readable(&poll_fd_reader).is_err() { @@ -1130,7 +1169,32 @@ impl PtySession { // writer lock taken by the drainer thread. let mut processor_guard = processor_clone.lock(); let mut term_guard = term_clone.lock(); - processor_guard.advance(&mut *term_guard, bytes); + // Observe every mode transition, including an enable and a + // disable inside one read. Stepping the parser one byte at a + // time is the only way to see a transition that does not + // survive to the end of the chunk, but it costs the reader + // thread a parser call per byte, so it is reserved for chunks + // that can actually carry the DECSET 2004 parameter, counting + // one split across the previous read. Every other chunk is + // parsed in bulk and then checked once. + if paste_capable_reader.load(Ordering::Acquire) { + processor_guard.advance(&mut *term_guard, bytes); + } else { + if decset_2004_scan.may_carry(bytes) { + for byte in bytes { + processor_guard + .advance(&mut *term_guard, std::slice::from_ref(byte)); + if paste_mode_enabled(&term_guard) { + paste_capable_reader.store(true, Ordering::Release); + } + } + } else { + processor_guard.advance(&mut *term_guard, bytes); + } + if paste_mode_enabled(&term_guard) { + paste_capable_reader.store(true, Ordering::Release); + } + } // Publish the grid's consumed byte offset while // still holding the term lock, so a concurrent // snapshot reader (which also locks `term`) reads @@ -1171,6 +1235,7 @@ impl PtySession { term, processor, consumed_offset, + paste_capable, output_sequence, output_order, #[cfg(unix)] @@ -1455,6 +1520,15 @@ impl PtySession { self.consumed_offset.load(Ordering::Acquire) } + /// Whether the child has ever enabled bracketed-paste mode (DECSET 2004). + /// + /// Sticky: a TUI that turns the mode off while a dialog is open still + /// understands `ESC[200~` framing, so the capability is independent of + /// whether the composer is currently ready. + pub fn bracketed_paste_enabled(&self) -> bool { + self.paste_capable.load(Ordering::Acquire) + } + /// Check if the child process has exited without blocking. /// Returns true if the child has exited (or was already reaped). /// @@ -1670,8 +1744,8 @@ mod tests { UnixOutputReservations, }; use super::{ - read_and_tag_output, GridSize, Mutex, PtyOutput, PtySession, PtyWriteSubmitError, - DEFAULT_NO_PID_EXIT_THRESHOLD, + read_and_tag_output, Decset2004Scan, GridSize, Mutex, PtyOutput, PtySession, + PtyWriteSubmitError, DEFAULT_NO_PID_EXIT_THRESHOLD, }; use crate::snapshot::Snapshot; use alacritty_terminal::event::VoidListener; @@ -1807,6 +1881,125 @@ mod tests { let _ = pty.shutdown(); } + #[cfg(unix)] + #[tokio::test] + async fn paste_capability_latches_across_disable_in_same_read() { + let (pty, mut rx) = PtySession::spawn( + "sh", + &[ + "-c".into(), + "printf '\\033[?2004h\\033[?2004lREADY'; sleep 2".into(), + ], + 24, + 80, + ) + .unwrap(); + let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); + tokio::time::timeout(Duration::from_secs(3), async { + while !pty.screen_text().contains("READY") { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!(pty.bracketed_paste_enabled()); + assert!(!pty + .term + .lock() + .mode() + .contains(alacritty_terminal::term::TermMode::BRACKETED_PASTE)); + pty.shutdown().unwrap(); + drain.abort(); + } + + /// Only a chunk carrying the `2004` parameter is parsed byte-by-byte, so a + /// DECSET split across reads must still latch from the bulk-parse path. + #[cfg(unix)] + #[tokio::test] + async fn paste_capability_latches_when_decset_spans_two_reads() { + let (pty, mut rx) = PtySession::spawn( + "sh", + &[ + "-c".into(), + "printf '\\033[?20'; sleep 0.3; printf '04hREADY'; sleep 2".into(), + ], + 24, + 80, + ) + .unwrap(); + let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); + // `READY` trails the DECSET, so the mode was parsed by the time it renders. + tokio::time::timeout(Duration::from_secs(3), async { + while !pty.screen_text().contains("READY") { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!(pty.bracketed_paste_enabled()); + pty.shutdown().unwrap(); + drain.abort(); + } + + /// A split `2004h` that a full reset (RIS, which never spells `2004`) + /// clears in the same second read never survives to the end of either + /// chunk, so the read boundary must still select the byte-by-byte path. + #[cfg(unix)] + #[tokio::test] + async fn paste_capability_latches_split_enable_reset_in_the_next_read() { + let (pty, mut rx) = PtySession::spawn( + "sh", + &[ + "-c".into(), + "printf '\\033[?20'; sleep 0.3; printf '04h\\033cREADY'; sleep 2".into(), + ], + 24, + 80, + ) + .unwrap(); + let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); + tokio::time::timeout(Duration::from_secs(3), async { + while !pty.screen_text().contains("READY") { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!(pty.bracketed_paste_enabled()); + pty.shutdown().unwrap(); + drain.abort(); + } + + #[test] + fn decset_2004_scan_sees_the_parameter_across_every_split() { + for stream in [&b"ab\x1b[?2004hcd"[..], &b"ab\x1b[?1;2004hcd"[..]] { + for split in 0..=stream.len() { + let mut scan = Decset2004Scan::default(); + let first = scan.may_carry(&stream[..split]); + let second = scan.may_carry(&stream[split..]); + assert!(first || second, "split at {split} hid the parameter"); + } + } + let mut scan = Decset2004Scan::default(); + assert!(!scan.may_carry(b"?200")); + assert!( + scan.may_carry(b"4"), + "a one-byte read completes the parameter" + ); + assert!(!scan.may_carry(b"x")); + assert!(!scan.may_carry(b"?2")); + assert!(!scan.may_carry(b"00")); + assert!(scan.may_carry(b"4h"), "short reads carry the prefix"); + assert!(!scan.may_carry(b"?2005")); + // relay#1930 review (cubic): ordinary text that merely contains 2004 + // (a year, a port) is not a private-mode parameter, so it must keep + // the bulk parse instead of a per-byte parse on every such read. + let mut scan = Decset2004Scan::default(); + assert!(!scan.may_carry(b"released in 2004, port 2004")); + assert!(!scan.may_carry(b" 200")); + assert!(!scan.may_carry(b"4")); + } + #[tokio::test] async fn consumed_offset_tracks_total_bytes_parsed_into_grid() { let (pty, mut rx) = PtySession::spawn("echo", &["offset-line".into()], 24, 80).unwrap(); @@ -2964,20 +3157,21 @@ mod tests { ack: ack_tx, }) .expect("queue accepts paced user input"); + tx.send(WriteMsg::Reply(b"terminal-reply".to_vec())) + .unwrap(); let ack = tokio::time::timeout(Duration::from_secs(2), ack_rx) .await .expect("drainer acks paced write") .expect("ack sender not dropped"); assert!(ack.is_ok(), "paced write must succeed"); + drop(tx); + drainer.join().expect("drainer thread joins cleanly"); assert_eq!( out.lock().unwrap().as_slice(), - b"go\x1b[A\r", - "paced write must deliver every byte in order" + b"go\x1b[A\rterminal-reply", + "terminal replies queue behind the entire typed body" ); - - drop(tx); - drainer.join().expect("drainer thread joins cleanly"); } /// A delayed follow-up must be a distinct PTY write while remaining inside diff --git a/crates/relay-pty/src/readiness.rs b/crates/relay-pty/src/readiness.rs index 93a3e77236..8edfe026c8 100644 --- a/crates/relay-pty/src/readiness.rs +++ b/crates/relay-pty/src/readiness.rs @@ -1,6 +1,6 @@ use crate::{ ansi::{floor_char_boundary, strip_ansi}, - terminal::detect_muse_device_auth_prompt, + terminal::{detect_devin_trust_prompt, detect_muse_device_auth_prompt}, wait::{for_cli, WaitSnapshot}, }; @@ -146,6 +146,16 @@ pub fn is_devin_cli(cli: &str) -> bool { matches!(base.as_str(), "devin" | "devin.exe") } +/// Whether Devin's directory-trust dialog is the active screen. +/// +/// Trust text alone is not enough: a resumed or uncleared screen can keep an +/// answered trust question visible above the live idle composer. The exact +/// idle placeholder holding the cursor proves the dialog is history, since a +/// trust menu never renders that placeholder. +pub fn devin_trust_prompt_active(grid: GridReadinessSnapshot<'_>) -> bool { + detect_devin_trust_prompt(grid.screen) && !devin_prompt_ready(grid) +} + fn devin_prompt_ready(grid: GridReadinessSnapshot<'_>) -> bool { let Some((row, _)) = grid.cursor else { return false; @@ -277,6 +287,25 @@ mod tests { assert!(!is_devin_cli("devin.bat")); } + #[test] + fn devin_trust_history_above_live_composer_is_not_active() { + let screen = "Do you trust this directory?\n1 Yes, trust\n2 No, exit\n❭ Ask Devin to build features, fix bugs, or work on your code\nSWE-2 High"; + assert!(!devin_trust_prompt_active(GridReadinessSnapshot { + screen, + cursor: Some((4, 3)), + })); + for cursor in [Some((2, 1)), None] { + assert!(devin_trust_prompt_active(GridReadinessSnapshot { + screen, + cursor + })); + } + assert!(devin_trust_prompt_active(GridReadinessSnapshot { + screen: "Do you trust the authors of this directory?\n❭ 1 Yes, trust\n2 No, exit", + cursor: Some((2, 3)), + })); + } + #[test] fn devin_wrapped_idle_composer_requires_cursor_in_exact_placeholder() { // Captured from Devin 3000.10.31 at 40 columns, including indentation. diff --git a/crates/relay-pty/src/terminal.rs b/crates/relay-pty/src/terminal.rs index fa5df2f694..cb2f3963c8 100644 --- a/crates/relay-pty/src/terminal.rs +++ b/crates/relay-pty/src/terminal.rs @@ -98,6 +98,26 @@ pub fn detect_codex_trust_prompt(clean_output: &str) -> bool { && lower.contains("no, quit") } +/// Detect Devin's startup directory-trust interstitial, including wrapped text. +/// This is a blocking-screen detector, not authority to select a menu option. +/// Corroborate the question with a choice so ordinary trust-related prose does +/// not block startup. Callers must restrict this detector to Devin startup. +pub fn detect_devin_trust_prompt(screen: &str) -> bool { + let compact: String = screen + .chars() + .filter(|c| !c.is_whitespace()) + .flat_map(char::to_lowercase) + .collect(); + let question = compact.contains("doyoutrust") + && ["directory", "authors", "workspace"] + .iter() + .any(|word| compact.contains(word)); + let choice = ["yes,itrust", "yes,trust", "no,exit", "no,quit"] + .iter() + .any(|word| compact.contains(word)); + question && choice +} + /// Detect Muse's interactive provider (device) login screen. /// /// Muse authenticates out of band: the TUI prints a verification URL and a @@ -524,6 +544,27 @@ pub fn is_auto_suggestion(output: &str) -> bool { mod tests { use super::*; + #[test] + fn devin_trust_requires_question_and_choice_including_wrapped_variants() { + for screen in [ + "Do you trust the authors of this directory?\n1. Yes, I trust the authors\n2. No, exit", + "Do you trust this workspace?\n❭ 1 Yes, trust\n this workspace", + "Do you trust the authors of this\ndirectory?\n❯ 2) No, exit", + "Do you trust this directory?\n❭ 1 Yes, trust the parent directory\n2 No, quit", + ] { + assert!(detect_devin_trust_prompt(screen), "{screen}"); + } + for screen in [ + "❭ Ask Devin to build features, fix bugs, or work on your code", + "Do you trust this directory?", + "❭ 1 Yes, trust\n this workspace", + "Documentation: directory trust requires an operator", + "Do you trust this tool?\nYes, trust", + ] { + assert!(!detect_devin_trust_prompt(screen), "{screen}"); + } + } + #[test] fn trust_question_header_is_not_an_affirmative_menu_option() { assert_eq!( diff --git a/docs/evidence/provider-subscription-replies.md b/docs/evidence/provider-subscription-replies.md new file mode 100644 index 0000000000..22f7279506 --- /dev/null +++ b/docs/evidence/provider-subscription-replies.md @@ -0,0 +1,80 @@ +# Provider subscription reply investigation (2026-10-05) + +This change implements the locally verifiable portions of the reviewed plan +for #1823 (relay#1910). It does **not** establish that the reported GitHub +writeback failure is fixed. + +Evidence labels used below: + +- **E1–E3**: authenticated reads of the incident's message and transcript, its + subscription, and its provider binding. +- **E4**: whether cloud subscription writeback ingress goes through the + file-write ops ledger. +- **E5**: whether the Relaycast engine enforces `Idempotency-Key` on channel + posts and thread replies. + +## Available evidence + +- The incident's Relay workspace is inaccessible in this run: `agent-relay +message inbox check` reports no workspace key. No authenticated message, + transcript, subscription, or binding read could be made (E1–E3). +- A read-only `RelayfileControlPlaneClient({ autoStart: false })` binding read + reports `DAEMON_UNAVAILABLE`. Stored PR globs therefore remain unverified; + this change does not add an issue-comment glob or widen provider scope. +- Installed `@relaycast/types/dist/subscription.d.ts` declares plural `events`, + `is_active`, and subscription configuration. It declares no delivery-attempt + status or retry operation. Event-array support is corroborated, but the + incident subscription's stored coverage remains unknown. +- Installed `@relayfile/client/dist/client.d.ts` exposes binding, webhook, and + writeback-secret operations, but no ops-ledger read or retry API. The local + client does not establish whether cloud subscription ingress uses the + file-write ops ledger (E4 remains open). +- Installed `@relaycast/sdk/dist/agent.js` supplies `Idempotency-Key` headers for + channel posts and thread replies, including generated keys. That proves + transport behavior, not engine enforcement (E5 remains open). The explicit + `idempotency_key` tool argument added later is forwarded to Relaycast and + also deduplicates locally, so it does not depend on E5. +- The existing changelog also records a standalone MCP double-dispatch fix in + 13.1.0. The incident was on 12.4.0; without its transcript, parallel aliases + cannot be asserted as the incident's root cause. + +## Changes and boundaries + +MCP posts/replies use request replay keyed on the JSON-RPC request ID (or an +explicit `idempotency_key`, also forwarded to Relaycast), matching `send_dm`. +Native tools join only a replay of the same model tool call ID. Message text +never forms a key: independent writes with identical text, including +concurrent ones, remain separate messages. Unkeyed retries after a lost +response, or from a distinct process, are not deduplicated; callers that retry +must pass `idempotency_key`. + +Subscription setup reads persisted event coverage before binding, rejects +unsupported events and inactive/mismatched subscriptions, and uses the existing +journaled rollback. Missing GitHub identity authorization produces an explicit +unverified message. `--list` exposes outbound configuration with +`deliveryStatus: null`; this is not a successful delivery receipt. Its +`lastDeliverySource` distinguishes inbound control-plane reports from channel +activity inferred locally. + +The deterministic tests exercise real CLI provisioning/rollback and MCP +transport against boundary fakes, plus native-tool and SDK normalization +regressions. They do not substitute a fake provider ledger for a real +GitHub inbound → fleet wake → reply → writeback integration test. + +## Remaining companion verification + +Read the incident binding and subscription, compare issue-comment and PR paths, +and inspect the cloud subscription writeback ingress. If ingress bypasses the +existing ops ledger, route it through that ledger using the originating Relay +message ID as the stable operation key. Expose durable failed/dead-lettered +status and retry through the control-plane client. Verify retry after both a +provider failure and a lost successful response, with one resulting GitHub +comment. Only then build the full seam test against those verified contracts. + +No broker reconnect behavior changed. Existing delivery-book tests cover +cumulative ACK dedupe, seeded resume cursors, and replay of unacknowledged +manual sequences after restart in `crates/broker/src/node_control.rs`. +`duplicate_and_stale_plans_still_ack_to_stop_redelivery` in +`crates/broker/src/runtime/fleet.rs` covers ACK behavior. These are existing +coverage, not a new exact-PR end-to-end proof; Rust tests were not run here +because Cargo is unavailable. diff --git a/docs/harnesses/devin.md b/docs/harnesses/devin.md index 6d59a6c764..c821cef3cf 100644 --- a/docs/harnesses/devin.md +++ b/docs/harnesses/devin.md @@ -23,6 +23,19 @@ PTY auto-responders for Devin. Trust and tool approvals require an operator in an attached terminal. Authenticate and trust the intended worktree before unattended spawning. Trust is scoped to that directory. +During broker-managed PTY startup, a recognised directory-trust prompt blocks +readiness and task injection. If still present 30 seconds after first detection, +the worker reports `directory_trust_required` with instructions to trust the +explicit spawn working directory. Verified fleet spawns return +`spawn_directory_trust_required` and release the worker through spawn cleanup. +An unverified (`confirm: false`) spawn reports the worker error but does not reap +the harness. Once reported, the worker never becomes ready, so trusting the +directory in an attached terminal does not resume it or deliver its initial +task: stop it, trust the directory, then spawn again. Already-trusted +installations retain the existing startup path. Relay does not yet answer this +menu automatically: that requires captured evidence of Devin's actual menu and +trust scope. + Readiness requires the live idle `❭ Ask Devin to build features, fix bugs, or work on your code` composer at the cursor. Trust choices, busy composers and output byte counts do not establish readiness. Messages remain queued while diff --git a/docs/harnesses/injection.md b/docs/harnesses/injection.md new file mode 100644 index 0000000000..840f158ebe --- /dev/null +++ b/docs/harnesses/injection.md @@ -0,0 +1,67 @@ +# PTY task and relay-message delivery + +Fleet workers wait for a recognized composer before injecting. If the composer +cannot be proven within 15 seconds of starting an injection, delivery fails with +`prompt_unproven` without writing the body. Unknown harnesses retain their existing +input behavior. + +A harness that has enabled terminal bracketed-paste mode (DECSET 2004) receives one +bulk bracketed paste followed by a separate, delayed Enter. Capability stays latched +for the session, independently of composer readiness. Devin also uses this wire. +`RELAY_INJECT_PASTE=0` forces the typed fallback. Wrap sessions retain bulk writes. +Codex initial tasks retain their existing chunked delivery path. + +The broker-wide ceiling is **16,384 UTF-8 bytes** for the formatted envelope. Bodies +are capped at **14,336 bytes** for PTY agents (`fleet spawn` task input, the fleet +spawn action before it launches a PTY agent, and broker PTY delivery); native and +headless tasks are not held to it. That leaves 2 KiB for attribution and broker +reminders so an accepted body still fits once formatted. Paced fallback input has an effective limit of **1,536 bytes including the envelope** at +the default 5 ms pace. A slower `RELAY_INJECT_RATE_MS` lowers that limit to keep +writing within 7.68 seconds. Unpaced fallback uses the global ceiling. Codex uses +the smaller of the global ceiling and its deadline-derived limit. Rejections name +the harness and effective limit; oversized bodies are never partially written. +Wrap sessions have no failure channel back to the sender, so an oversized relay +message is delivered as a short notice naming the sender, size, limit and message +ID instead of its body; the full message stays readable in Relay. The limit is +enforced only at the PTY boundary: `message post|reply|dm send` publish any size +to Relaycast, since recipients may be native agents or history only. +For longer briefs, create a file on the node and send a short instruction to read it. + +The input wire normalizes CRLF to LF and strips every other control character +except tab (bare carriage returns, ESC, `Ctrl-C` and the like), so a body cannot +submit the composer early, close a paste, or interrupt or edit the composer +line. LF stays in the payload: inside a bracketed paste it is a +newline, but on the typed fallback wire a harness may treat it as Enter, so +multiline tasks are only safe on paste-capable harnesses. Other UTF-8 content +is preserved. + +```sh +agent-relay fleet spawn claude --node my-node --name reviewer --task-file ./brief.md +agent-relay node agent spawn claude --name reviewer --task-file ./brief.md +``` + +`--task-file` reads a UTF-8 file on the **requester's machine**, then sends the +contents through the usual task transport. It does not create a file on the node +or bypass size limits. Fleet requires exactly one of `--task` and `--task-file`; +local `agent spawn` and `node agent spawn` accept either but require neither, and +reject both together. The integration command's +existing task option remains unchanged. + +Receipt is proven by harness acceptance, never by echo alone: a body visible in +the composer is still a draft. A PTY delivery is `harness_acceptance` once the +harness shows activity, clears its composer, or (for a plain process) echoes the +body raw. A body still parked in the composer gets a bounded submit-key retry +(`delivery_unconfirmed`, then `delivery_resubmitted`); the body itself is never +replayed. If it stays parked, the delivery fails with "body remained parked after +bounded submit-key recovery"; if the window closes with neither acceptance nor a +parked body, it fails with "harness acceptance could not be proven", which is not +evidence that the input was lost. + +Verified fleet PTY spawns with a task wait for its delivery verdict and for +proven startup readiness, in either order. Only `harness_acceptance` confirms +the task. A failed task releases the worker and its fleet identity, then fails +the spawn action with `spawn_task_failed`, so a corrected retry can reuse the +name. A task acked without proof of acceptance, or whose acceptance could not be +proven either way, resolves the action as `spawn_task_unconfirmed`, naming the +live agent: the spawn must not be retried (that duplicates the agent), so resend +the task or point the agent at a brief file. diff --git a/packages/cli/README.md b/packages/cli/README.md index 55dc8763c3..3b28004349 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -591,3 +591,22 @@ Hosted equivalents live under `agent-relay cloud …`. - `@agent-relay/sdk`: messaging, delivery contracts, and actions. - `@agent-relay/harness-driver`: optional managed harness runtime. - `agent-relay`: CLI and MCP entry point. + +### Pending fleet spawns and name reuse + +An accepted spawn whose result is still unknown returns `spawn_unconfirmed` with +`state: "pending"` (exit 8), plus its invocation ID and dispatch state when +known and any available liveness evidence. Check +`agent-relay fleet agent list --node ` before retrying. A heartbeat that +lists a worker with the requested name is reported as evidence but keeps the +spawn pending: heartbeats carry worker names, not invocation IDs, so that worker +may have been running before this spawn. Heartbeat snapshots can change between +listings; JSON output includes their timestamps and ages. + +If no worker is running and you intend to reclaim the name, run +`agent-relay agent remove --wait` before respawning. Waiting is opt-in; +`--wait-timeout ` defaults to 30000, followed by one bounded membership read. +`fleet release --delete-agent --wait` offers the same check and adds +`removal` evidence to its JSON. A remaining registration exits 8; unavailable +verification reads retain the asynchronous acknowledgement. A name collision +can also mean cleanup from a previous unsuccessful spawn is still in flight. diff --git a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts index 96d615bc59..e8a7acdac7 100644 --- a/packages/cli/src/cli/agent-relay-mcp.startup.test.ts +++ b/packages/cli/src/cli/agent-relay-mcp.startup.test.ts @@ -287,9 +287,17 @@ async function loadAgentRelayMcpModule(options: LoadOptions = {}) { output: { spawned: true, ready: true }, })), }; + const agentRelayNodesGet = vi.fn( + async (name: string) => (await agentRelayNodesList()).find((node) => node.name === name) ?? null + ); + const agentRelayAgentsGet = vi.fn(async (): Promise => { + throw { status: 404 }; + }); + const agentRelayAgentsList = vi.fn(async (): Promise> => []); const AgentRelayMock = vi.fn(function (this: unknown) { return { - nodes: { list: agentRelayNodesList }, + nodes: { list: agentRelayNodesList, get: agentRelayNodesGet }, + agents: { get: agentRelayAgentsGet, list: agentRelayAgentsList }, messaging: { commands: agentRelayMessagingCommands, }, @@ -333,13 +341,21 @@ async function loadAgentRelayMcpModule(options: LoadOptions = {}) { }, }; }); - vi.doMock('@relaycast/sdk', () => ({ + vi.doMock('@relaycast/sdk', async () => ({ + ...(await vi.importActual>('@relaycast/sdk')), RelayCast, SDK_VERSION: 'test-sdk-version', })); vi.doMock('@agent-relay/sdk', async () => { const actual = await vi.importActual>('@agent-relay/sdk'); - return { ...actual, AgentRelay: AgentRelayMock }; + // A real realtime client keeps reconnecting after its test ends, and a + // reconnect's connect timeout then lands on a later test's fake clock. + const createRealtimeClient = () => ({ + on: () => () => undefined, + connect: () => undefined, + disconnect: () => undefined, + }); + return { ...actual, AgentRelay: AgentRelayMock, createRealtimeClient }; }); vi.doMock('./telemetry/index.js', () => ({ initTelemetry: telemetryInit, @@ -376,6 +392,10 @@ async function loadAgentRelayMcpModule(options: LoadOptions = {}) { RelayCast, FakeTransport, agentRelayMessagingCommands, + agentRelayNodesList, + agentRelayNodesGet, + agentRelayAgentsGet, + agentRelayAgentsList, agentClients, }, }; @@ -401,6 +421,33 @@ afterEach(() => { }); describe('agent-relay-mcp startup helpers', () => { + // relay#1930 review (CodeRabbit): a retained spawn key is scoped to the + // acting identity, so a key reused after register_agent moves the session + // default starts that identity's own spawn instead of replaying the first. + it('scopes a retained spawn idempotency key to the acting identity', async () => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mod.createAgentRelayMcpServer({ + workspaceKey: 'rk_live_scope', + agentToken: 'at_live_fleet', + agentName: 'orchestrator', + }); + const server = mocks.serverInstances[0]; + const spawn = server.tools.get('spawn')!.handler; + const input = { name: 'ScopedWorker', cli: 'codex', idempotency_key: 'same-key' }; + await spawn(input, { sessionId: 'mcp-session', requestId: 1 }); + await spawn(input, { sessionId: 'mcp-session', requestId: 2 }); + expect(mocks.agentRelayMessagingCommands.invoke).toHaveBeenCalledTimes(1); + + await server.tools.get('register_agent')!.handler({ name: 'WorkerB' }); + mocks.agentRelayMessagingCommands.invoke.mockResolvedValueOnce({ + invocationId: 'inv_scoped', + actionName: 'spawn', + }); + await spawn(input, { sessionId: 'mcp-session', requestId: 3 }); + expect(mocks.agentRelayMessagingCommands.invoke).toHaveBeenCalledTimes(2); + // Three spawns through the real confirmation path; allow for a loaded host. + }, 20_000); + it('coalesces replayed MCP request ids without collapsing separate same-name spawns', async () => { const { mod, mocks } = await loadAgentRelayMcpModule(); mod.createAgentRelayMcpServer({ agentToken: 'at_live_fleet', agentName: 'orchestrator' }); @@ -1149,11 +1196,11 @@ describe('createAgentRelayMcpServer', () => { (error: unknown) => error ), new Promise((resolve) => - setTimeout(() => resolve('nested timeout test did not finish'), 130_001) + setTimeout(() => resolve('nested timeout test did not finish'), 132_001) ), ]); - await vi.advanceTimersByTimeAsync(130_001); + await vi.advanceTimersByTimeAsync(132_001); await expect(outcome).resolves.toMatchObject({ isError: true, @@ -1194,20 +1241,20 @@ describe('createAgentRelayMcpServer', () => { (error: unknown) => error ), new Promise((resolve) => - setTimeout(() => resolve('still pending after the persona spawn deadline'), 130_001) + setTimeout(() => resolve('still pending after the persona spawn deadline'), 132_001) ), ]); - await vi.advanceTimersByTimeAsync(130_001); + await vi.advanceTimersByTimeAsync(132_001); await expect(outcome).resolves.toMatchObject({ isError: true, structuredContent: { error: { code: 'spawn_unconfirmed', - state: 'unconfirmed_may_be_running', + state: 'pending', dispatchState: 'unknown', - message: expect.stringContaining('may still be running'), + message: expect.stringContaining('outcome is pending'), }, }, }); @@ -1218,7 +1265,7 @@ describe('createAgentRelayMcpServer', () => { it('preserves an unconfirmed receipt when authorization is revoked after acceptance', async () => { const { mod, mocks } = await loadAgentRelayMcpModule(); - mocks.agentRelayMessagingCommands.getInvocation.mockRejectedValueOnce(new Error('401 unauthorized')); + mocks.agentRelayMessagingCommands.getInvocation.mockRejectedValue(new Error('401 unauthorized')); mod.createAgentRelayMcpServer({ workspaceKey: 'rk_live_existing', agentToken: 'at_live_fleet', @@ -1234,15 +1281,157 @@ describe('createAgentRelayMcpServer', () => { structuredContent: { error: { code: 'spawn_unconfirmed', - state: 'unconfirmed_may_be_running', + state: 'pending', dispatchState: 'unknown', invocationId: 'inv_1', - message: expect.stringContaining('may still be running'), + message: expect.stringContaining('outcome is pending'), }, }, }); }); + it('recovers a terminal spawn result on the final invocation read', async () => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mocks.agentRelayMessagingCommands.getInvocation.mockRejectedValueOnce(new Error('401 unauthorized')); + mod.createAgentRelayMcpServer({ agentToken: 'at_live_fleet', agentName: 'orchestrator' }); + const result = await mocks.serverInstances[0].tools + .get('spawn')! + .handler({ name: 'worker', cli: 'codex' }); + expect(result.isError).not.toBe(true); + expect(result.structuredContent.placement).toMatchObject({ state: 'ready', confirmed: true }); + expect(mocks.agentRelayMessagingCommands.getInvocation).toHaveBeenCalledTimes(2); + }); + + it('keeps a late persona child acknowledgement pending with its own correlation', async () => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mocks.agentRelayMessagingCommands.getInvocation + .mockRejectedValueOnce(new Error('401 unauthorized')) + .mockResolvedValueOnce({ + status: 'completed', + output: { + invocationId: 'inv_child', + actionName: 'spawn', + status: 'dispatched', + handlerNodeId: 'child-node', + }, + } as never); + mod.createAgentRelayMcpServer({ agentToken: 'at_live_fleet', agentName: 'orchestrator' }); + const result = await mocks.serverInstances[0].tools + .get('spawn')! + .handler({ name: 'worker', persona: 'reviewer' }); + expect(result).toMatchObject({ + isError: true, + structuredContent: { + error: { code: 'spawn_unconfirmed', invocationId: 'inv_child', node: 'child-node' }, + }, + }); + // The terminal parent stays locatable in the activity log. + expect(JSON.stringify(result)).toContain('inv_1'); + }); + + it.each(['live', 'live_elsewhere', 'stale', 'unknown'] as const)( + 'keeps the spawn pending on %s evidence without manufacturing success', + async (evidence) => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mocks.agentRelayMessagingCommands.invoke.mockResolvedValueOnce({} as never); + const record = { + name: evidence === 'live_elsewhere' ? 'other' : 'node-a', + status: 'online', + lastHeartbeatAt: new Date(Date.now() - (evidence === 'stale' ? 40_000 : 0)).toISOString(), + capabilities: [{ name: 'relay:live-agents:v1', metadata: { names: ['worker'] } }], + }; + mocks.agentRelayNodesList.mockResolvedValue([record]); + mod.createAgentRelayMcpServer({ + agentToken: 'at_live_fleet', + agentName: 'orchestrator', + ...(evidence !== 'unknown' ? { workspaceKey: 'rk_test' } : {}), + }); + const result = await mocks.serverInstances[0].tools + .get('spawn')! + .handler({ name: 'worker', cli: 'codex', target_node: 'node-a' }); + // A heartbeat names a worker but not the invocation that created it, + // so even fresh target-node presence leaves the spawn pending. + expect(result).toMatchObject({ + isError: true, + structuredContent: { error: { code: 'spawn_unconfirmed', state: 'pending', liveness: { evidence } } }, + }); + } + ); + + // relay#1930 review: the tool description must name the pending shape the + // handler actually returns (asserted above), not a `confirmed` field. + it('describes pending spawns by the shape the spawn tool returns', async () => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mod.createAgentRelayMcpServer({ agentToken: 'at_live_fleet', agentName: 'orchestrator' }); + const { description } = mocks.serverInstances[0].tools.get('spawn')!.config as { description: string }; + expect(description).toContain('spawn_unconfirmed'); + expect(description).toContain('state: "pending"'); + expect(description).not.toContain('confirmed:false'); + }); + + // The broker formats the collision readably; its Rust test asserts no + // `Fatal(...)` debug text, so this MCP test covers classification only. + it('classifies MCP registration collisions as spawn_name_taken', async () => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mocks.agentRelayMessagingCommands.getInvocation.mockResolvedValueOnce({ + status: 'failed', + error: + "failed to pre-register worker 'worker': agent 'worker' already exists and registration is create-only; use a unique name", + } as never); + mod.createAgentRelayMcpServer({ agentToken: 'at_live_fleet', agentName: 'orchestrator' }); + const result = await mocks.serverInstances[0].tools + .get('spawn')! + .handler({ name: 'worker', cli: 'codex' }); + expect(result).toMatchObject({ + isError: true, + structuredContent: { error: { code: 'spawn_name_taken', message: expect.stringContaining('--wait') } }, + }); + }); + + it.each(['cleared', 'present', 'unavailable'] as const)( + 'remove_agent wait reports registration %s after acknowledgement', + async (evidence) => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + if (evidence === 'present') { + mocks.agentRelayAgentsGet.mockResolvedValue({ name: 'worker', status: 'released' }); + mocks.agentRelayAgentsList.mockResolvedValue([{ name: 'worker' }]); + } else if (evidence === 'unavailable') { + mocks.agentRelayAgentsGet.mockRejectedValue(new Error('403 forbidden')); + mocks.agentRelayAgentsList.mockRejectedValue(new Error('403 forbidden')); + } + mod.createAgentRelayMcpServer({ + workspaceKey: 'rk_test', + agentToken: 'at_live_fleet', + agentName: 'orchestrator', + }); + const result = await mocks.serverInstances[0].tools + .get('remove_agent')! + .handler({ name: 'worker', delete_agent: true, wait: true, wait_timeout_ms: 1 }); + expect(result.structuredContent.removal).toMatchObject({ cleared: evidence === 'cleared' }); + expect(result.isError === true).toBe(evidence === 'present'); + } + ); + + // relay#1930 review (cubic): an agent-token-only session cannot read the + // roster, so it keeps the accepted removal and says why instead of polling. + it('remove_agent wait without a workspace key keeps the acknowledgement and does not poll', async () => { + const { mod, mocks } = await loadAgentRelayMcpModule(); + mod.createAgentRelayMcpServer({ agentToken: 'at_live_fleet', agentName: 'orchestrator' }); + const started = Date.now(); + const result = await mocks.serverInstances[0].tools + .get('remove_agent')! + .handler({ name: 'worker', delete_agent: true, wait: true }); + expect(Date.now() - started).toBeLessThan(5_000); + expect(result.isError).toBeUndefined(); + expect(result.structuredContent.invocation).toBeDefined(); + expect(result.structuredContent.removal).toMatchObject({ + cleared: false, + observedPresent: false, + readError: expect.stringContaining('Workspace key not configured'), + }); + expect(mocks.agentRelayAgentsGet).not.toHaveBeenCalled(); + }, 10_000); + it('correlates nested authorization failures to the nested route', async () => { const { mod, mocks } = await loadAgentRelayMcpModule(); mocks.agentRelayMessagingCommands.invoke.mockResolvedValueOnce({ diff --git a/packages/cli/src/cli/agent-relay-mcp.ts b/packages/cli/src/cli/agent-relay-mcp.ts index ecd4ac37be..61ea2fbfa4 100644 --- a/packages/cli/src/cli/agent-relay-mcp.ts +++ b/packages/cli/src/cli/agent-relay-mcp.ts @@ -1,4 +1,13 @@ #!/usr/bin/env node +import { waitForAgentRemoval } from './lib/agent-removal.js'; +import { + FleetSpawnError, + classifySpawnFailure, + nameTakenSpawnError, + pendingSpawnError, + probeSpawnLiveness, + terminalSpawnOutcome, +} from './lib/spawn-liveness.js'; import fs from 'node:fs'; import path from 'node:path'; @@ -50,7 +59,7 @@ import { import { enableInboxPiggyback } from './mcp/telemetry.js'; import { registerAgentRelayActionTools } from './mcp/action-tools.js'; import { registerMessagingTools } from './mcp/messaging-tools.js'; -import { McpRequestReplay } from './mcp/request-replay.js'; +import { McpRequestReplay, replayScopeForCredentials, withReplayScope } from './mcp/request-replay.js'; import { identityOverrideInputShape, messageResult } from './mcp/tool-shapes.js'; import { registerSharedSessionTools, @@ -97,6 +106,7 @@ const VERIFIED_SPAWN_FAILURE_STATUSES = new Set(['failed', 'error', 'cancelled', class VerifiedSpawnError extends Error { readonly code: 'spawn_unconfirmed' | 'spawn_failed'; + readonly actionName?: string; readonly invocationId?: string; readonly state: SpawnLifecycleState; readonly dispatchState: SpawnDispatchState; @@ -107,6 +117,7 @@ class VerifiedSpawnError extends Error { message: string, context: { code?: 'spawn_unconfirmed' | 'spawn_failed'; + actionName?: string; state: SpawnLifecycleState; dispatchState?: SpawnDispatchState; invocationId?: string; @@ -116,6 +127,7 @@ class VerifiedSpawnError extends Error { ) { super(message); this.name = 'VerifiedSpawnError'; + this.actionName = context.actionName; this.code = context.code ?? (context.state === 'failed' ? 'spawn_failed' : 'spawn_unconfirmed'); this.state = context.state; this.dispatchState = context.dispatchState ?? 'unknown'; @@ -270,6 +282,7 @@ async function pollInvocation( code: 'spawn_unconfirmed', state: 'unconfirmed_may_be_running', invocationId: current.invocationId, + actionName: current.actionName, dispatchState: current.dispatchState, ...(current.node ? { node: current.node } : {}), } @@ -281,6 +294,7 @@ async function pollInvocation( { state: 'unconfirmed_may_be_running', invocationId: current.invocationId, + actionName: current.actionName, dispatchState: current.dispatchState, ...(current.node ? { node: current.node } : {}), receipt: { status: 'unknown' }, @@ -335,6 +349,7 @@ async function waitForVerifiedSpawn( state: 'failed', dispatchState: dispatchStateForRecord(record, current.dispatchState), invocationId: current.invocationId, + actionName: current.actionName, receipt: record, }); } @@ -349,6 +364,7 @@ async function waitForVerifiedSpawn( state: 'failed', dispatchState: dispatchStateForRecord(record, current.dispatchState), invocationId: current.invocationId, + actionName: current.actionName, node: invocationText(record, 'handlerNodeId', 'handler_node_id') ?? invocationText(record, 'dispatchedNodeId', 'dispatched_node_id') ?? @@ -364,6 +380,7 @@ async function waitForVerifiedSpawn( state: 'failed', dispatchState: dispatchStateForRecord(record, current.dispatchState), invocationId: current.invocationId, + actionName: current.actionName, node: invocationText(record, 'handlerNodeId', 'handler_node_id') ?? invocationText(record, 'dispatchedNodeId', 'dispatched_node_id') ?? @@ -379,6 +396,7 @@ async function waitForVerifiedSpawn( state: 'unconfirmed_may_be_running', dispatchState: dispatchStateForRecord(record, current.dispatchState), invocationId: current.invocationId, + actionName: current.actionName, node: invocationText(record, 'handlerNodeId', 'handler_node_id') ?? invocationText(record, 'dispatchedNodeId', 'dispatched_node_id') ?? @@ -876,15 +894,79 @@ async function invokeVerifiedSpawn( } const commands = new AgentRelay({ agentToken, baseUrl }).messaging.commands; const invocation = await commands.invoke('spawn', actionInput); - return waitForVerifiedSpawn(commands, invocation); + try { + return await waitForVerifiedSpawn(commands, invocation); + } catch (error) { + if (!(error instanceof VerifiedSpawnError)) throw error; + const name = String(actionInput.name); + if (error.code === 'spawn_failed' && classifySpawnFailure(error.message)) + throw nameTakenSpawnError(name, error); + if (error.code !== 'spawn_unconfirmed') throw error; + const requestedNode = typeof actionInput.target_node === 'string' ? actionInput.target_node : undefined; + const targetNode = actionInput.persona ? (error.node ?? requestedNode) : (requestedNode ?? error.node); + const context = { ...error, node: targetNode }; + const liveness = await probeSpawnLiveness({ + name, + targetNode, + ...(error.invocationId + ? { getInvocation: () => commands.getInvocation(error.actionName ?? 'spawn', error.invocationId!) } + : {}), + createClient: () => { + requireWorkspaceKey(session); + const workspace = new AgentRelay({ workspaceKey: session.workspaceKey!, baseUrl }); + return { nodes: workspace.nodes, agents: workspace.agents }; + }, + }); + if (liveness.evidence === 'invocation_terminal') { + const nested = VERIFIED_SPAWN_SUCCESS_STATUSES.has(String(liveness.invocation!.status).toLowerCase()) + ? nestedPersonaSpawnRef(liveness.invocation!) + : undefined; + if (nested) { + // A late persona result can be only an acknowledgement of its child. + // It is neither missing readiness proof nor a terminal worker failure. + throw pendingSpawnError( + name, + { + ...context, + invocationId: nested.invocationId, + node: nested.node ?? targetNode, + dispatchState: nested.dispatchState, + }, + { evidence: 'unknown' }, + `The persona handler's invocation ${context.invocationId ?? '(unknown)'}${context.node ? ` on ${context.node}` : ''} completed with a nested spawn invocation after the confirmation budget.` + ); + } + try { + return terminalSpawnOutcome(liveness.invocation!, context, true); + } catch (terminalError) { + const message = terminalError instanceof Error ? terminalError.message : String(terminalError); + if (classifySpawnFailure(message)) + throw nameTakenSpawnError(name, Object.assign(new Error(message), context)); + throw new VerifiedSpawnError(message, { + ...error, + code: 'spawn_failed', + state: 'failed', + receipt: liveness.invocation, + }); + } + } + throw pendingSpawnError(name, context, liveness, error.message); + } } -function verifiedSpawnErrorResult(error: VerifiedSpawnError) { +function verifiedSpawnErrorResult(error: VerifiedSpawnError | FleetSpawnError) { return { ...jsonContent({ ok: false, error: { code: error.code, + ...(error instanceof FleetSpawnError + ? { + exitCode: error.exitCode, + liveness: error.liveness, + diagnostic: error.diagnostic ? safeRelayErrorMessage(new Error(error.diagnostic)) : undefined, + } + : {}), state: error.state, dispatchState: error.dispatchState, ...(error.invocationId ? { invocationId: error.invocationId } : {}), @@ -975,6 +1057,12 @@ function registerAgentRelayTools( forcedAgentType: AgentType | undefined, requestReplay: McpRequestReplay ): void { + // A retained spawn key belongs to the workspace and the identity it acts as. + const spawnReplayScope = (as?: string): string => { + const session = getSession(); + const agentToken = (as ? session.agents.get(as)?.agentToken : session.agentToken) ?? ''; + return replayScopeForCredentials(session.workspaceKey ?? '', agentToken); + }; server.registerTool( 'create_workspace', { @@ -1342,28 +1430,34 @@ function registerAgentRelayTools( { name, cli, task, channel, persona, model, spawn_mode, exit_after_task, idempotency_key }, extra ) => - requestReplay.run('add_agent', extra, idempotency_key, async () => { - const invocation = await getRelay().agents.spawn({ - name, - cli, - task: - exit_after_task || - spawn_mode === 'task_exit' || - spawn_mode === 'task-exit' || - spawn_mode === 'single_shot' || - spawn_mode === 'single-shot' - ? withExitAfterTaskInstruction(task) - : task, - channel, - persona, - // SpawnAgentRequest has no top-level model field; pass via metadata - // so the broker can extract it and forward --model to the launched CLI. - metadata: model ? { model } : undefined, - }); - const failure = terminalSpawnFailureResult(invocation); - if (failure) return failure; - return jsonContent({ ...invocation, placement: spawnReceipt(invocation) }); - }) + requestReplay.run( + 'add_agent', + extra, + idempotency_key, + async () => { + const invocation = await getRelay().agents.spawn({ + name, + cli, + task: + exit_after_task || + spawn_mode === 'task_exit' || + spawn_mode === 'task-exit' || + spawn_mode === 'single_shot' || + spawn_mode === 'single-shot' + ? withExitAfterTaskInstruction(task) + : task, + channel, + persona, + // SpawnAgentRequest has no top-level model field; pass via metadata + // so the broker can extract it and forward --model to the launched CLI. + metadata: model ? { model } : undefined, + }); + const failure = terminalSpawnFailureResult(invocation); + if (failure) return failure; + return jsonContent({ ...invocation, placement: spawnReceipt(invocation) }); + }, + replayScopeForCredentials(getSession().workspaceKey ?? '') + ) ); server.registerTool( @@ -1372,7 +1466,7 @@ function registerAgentRelayTools( title: 'Spawn Agent', description: 'Invoke the fleet spawn action with either a raw `cli` or an AgentWorkforce `persona` name/path. ' + - 'Persona requests route to a node exposing `spawn:persona` (for example, `defineWorkforcePersonaSpawnNode` from `@agentworkforce/local-surface`). Both forms return only after broker registration and harness readiness are verified.', + 'Persona requests route to a node exposing `spawn:persona` (for example, `defineWorkforcePersonaSpawnNode` from `@agentworkforce/local-surface`). Both forms wait for broker registration and harness readiness. Unconfirmed outcomes return error code `spawn_unconfirmed` with `state: "pending"` and `liveness` evidence; a fresh worker heartbeat is evidence, not readiness proof, so do not retry a pending spawn.', inputSchema: { name: z.string().describe('Agent name'), cli: z @@ -1447,39 +1541,49 @@ function registerAgentRelayTools( }, extra ) => - requestReplay.run('spawn', extra, idempotency_key, async () => { - const request = { - name, - cli, - persona, - task, - cwd, - personaCwd: persona_cwd, - workerCwd: worker_cwd, - channel, - channels, - model, - organization, - project, - workstream, - role, - objective, - sessionRef: session_ref, - targetNode: target_node, - }; - validateSpawnRequest(request); - const actionInput = buildSpawnActionInput(request); - try { - const invocation = await invokeVerifiedSpawn(getSession(), as, baseUrl, actionInput); - return jsonContent({ - invocation, - placement: { state: 'ready', ...spawnReceipt(recordValue(invocation)) }, - }); - } catch (error) { - if (error instanceof VerifiedSpawnError) return verifiedSpawnErrorResult(error); - throw error; - } - }) + requestReplay.run( + 'spawn', + extra, + idempotency_key, + async () => { + const request = { + name, + cli, + persona, + task, + cwd, + personaCwd: persona_cwd, + workerCwd: worker_cwd, + channel, + channels, + model, + organization, + project, + workstream, + role, + objective, + sessionRef: session_ref, + targetNode: target_node, + }; + validateSpawnRequest(request); + const actionInput = buildSpawnActionInput(request); + try { + const invocation = await invokeVerifiedSpawn(getSession(), as, baseUrl, actionInput); + return jsonContent({ + invocation, + placement: recordValue(invocation).placement ?? { + state: 'ready', + ...spawnReceipt(recordValue(invocation)), + }, + }); + } catch (error) { + if (error instanceof VerifiedSpawnError || error instanceof FleetSpawnError) + return verifiedSpawnErrorResult(error); + throw error; + } + }, + spawnReplayScope(as) + ) ); server.registerTool( @@ -1488,7 +1592,7 @@ function registerAgentRelayTools( title: 'Remove Agent', description: 'Release a worker agent from active duty, optionally deleting it outright. ' + - 'Returns an `invocation` record acknowledging the request, which is processed asynchronously. Releasing keeps the agent registered and re-spawnable; passing `delete_agent` removes the identity permanently.', + 'Returns an `invocation` record acknowledging the request, which is processed asynchronously. Releasing keeps the agent registered; passing `delete_agent` requests permanent deletion. Use `wait` with `delete_agent` to verify the name is reusable.', inputSchema: { name: z.string().describe('Agent name'), reason: z @@ -1496,12 +1600,18 @@ function registerAgentRelayTools( .optional() .describe('Removal reason; defaults to an attributable Agent Relay MCP reason'), delete_agent: z.boolean().optional().describe('Permanently delete the agent'), + wait: z.boolean().optional().describe('With delete_agent, wait for registration clearance'), + wait_timeout_ms: z + .number() + .positive() + .optional() + .describe('Clearance timeout in milliseconds (default 30000)'), ...identityOverrideInputShape, }, outputSchema: jsonResult, annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, }, - async ({ name, reason, delete_agent, as }) => { + async ({ name, reason, delete_agent, wait, wait_timeout_ms, as }) => { const session = getSession(); const selected = as ? session.agents.get(as) : undefined; if (as && !selected) { @@ -1531,6 +1641,39 @@ function registerAgentRelayTools( throw error; } } + if (delete_agent && wait) { + // Clearance reads need workspace authority. Without it every read + // would fail, so keep the accepted removal and say why at once. + let workspace: AgentRelay | undefined; + let unavailable: string | undefined; + try { + requireWorkspaceKey(session); + workspace = new AgentRelay({ workspaceKey: session.workspaceKey!, baseUrl }); + } catch (error) { + unavailable = error instanceof Error ? error.message : String(error); + } + const removal = workspace + ? await waitForAgentRemoval({ + name, + getAgent: (name) => workspace.agents.get(name), + listAgents: () => workspace.agents.list(), + timeoutMs: wait_timeout_ms, + }) + : { cleared: false, waitedMs: 0, observedPresent: false, readError: unavailable }; + return { + ...jsonContent({ + invocation, + removal, + ...(!removal.cleared + ? { + message: + 'Removal was initiated asynchronously; verify registration clearance before respawning.', + } + : {}), + }), + ...(!removal.cleared && removal.observedPresent ? { isError: true as const } : {}), + }; + } return jsonContent({ invocation }); } ); @@ -1670,7 +1813,7 @@ export function createAgentRelayMcpServer(options: AgentRelayMcpServerOptions): const getAgentClient = (asIdentity?: string): AgentClientLike => { const agentToken = resolveAgentToken(asIdentity); - return createAgentClient({ agentToken, baseUrl: options.baseUrl }); + return withReplayScope(createAgentClient({ agentToken, baseUrl: options.baseUrl }), agentToken); }; enableInboxPiggyback( diff --git a/packages/cli/src/cli/commands/agent.test.ts b/packages/cli/src/cli/commands/agent.test.ts index 775d576da8..84b205e0eb 100644 --- a/packages/cli/src/cli/commands/agent.test.ts +++ b/packages/cli/src/cli/commands/agent.test.ts @@ -13,6 +13,7 @@ function createHarness() { const workspaceRelay = { agents: { get: vi.fn(async (name: string) => ({ id: 'agent_existing', name, status: 'online' })), + list: vi.fn(async () => [{ name: 'worker' }]), }, workspace: { register: vi.fn(async ({ name }: { name: string }) => ({ @@ -287,6 +288,53 @@ describe('agent identity lifecycle commands', () => { expect(rendered).not.toContain('Removed agent'); }); + it('waits for not-found even when the release acknowledgement says completed', async () => { + const { program, workspaceRelay, log } = createHarness(); + workspaceRelay.agents.get.mockRejectedValueOnce({ status: 404 }); + await program.parseAsync(['agent', 'remove', 'worker', '--wait'], { from: 'user' }); + expect(workspaceRelay.agents.get).toHaveBeenCalledWith('worker'); + expect(log).toHaveBeenCalledWith('Removed agent worker.'); + }); + + it('exits 8 while a released registration still owns the name', async () => { + const { program, workspaceRelay, log } = createHarness(); + workspaceRelay.agents.get.mockResolvedValue({ id: 'old', name: 'worker', status: 'released' }); + await expect( + program.parseAsync(['agent', 'remove', 'worker', '--wait', '--wait-timeout', '1'], { from: 'user' }) + ).rejects.toThrow('exit:8'); + expect(log.mock.calls.flat().join('')).not.toContain('Removed agent'); + }); + + it('keeps the asynchronous acknowledgement when every verification read is unavailable', async () => { + const { program, workspaceRelay, log, error } = createHarness(); + workspaceRelay.agents.get.mockRejectedValue(new Error('403 forbidden')); + workspaceRelay.agents.list.mockRejectedValue(new Error('403 roster forbidden')); + await program.parseAsync(['agent', 'remove', 'worker', '--wait', '--wait-timeout', '1'], { + from: 'user', + }); + expect(workspaceRelay.agents.list).toHaveBeenCalled(); + expect(log.mock.calls.flat().join('')).toContain('initiated'); + // --wait already ran; suggesting it again would misdescribe the outcome. + expect(log.mock.calls.flat().join('')).not.toContain('Use --wait'); + expect(error.mock.calls.flat().join('')).toContain('Could not verify removal: 403 roster forbidden'); + }); + + it('rejects a non-numeric --wait-timeout before removing anything', async () => { + const { program, workspaceRelay, error } = createHarness(); + await expect( + program.parseAsync(['agent', 'remove', 'worker', '--wait', '--wait-timeout', 'abc'], { from: 'user' }) + ).rejects.toThrow('exit:1'); + expect(error.mock.calls.flat().join('')).toContain('--wait-timeout'); + expect(workspaceRelay.workspace.release).not.toHaveBeenCalled(); + }); + + it('does not claim clearance or poll with --no-wait', async () => { + const { program, workspaceRelay, log } = createHarness(); + await program.parseAsync(['agent', 'remove', 'worker', '--no-wait'], { from: 'user' }); + expect(workspaceRelay.agents.get).not.toHaveBeenCalled(); + expect(log.mock.calls.flat().join('')).toContain('initiated'); + }); + it('redacts SQL and bound parameters when removal fails', async () => { const { program, workspaceRelay, error } = createHarness(); workspaceRelay.workspace.release.mockRejectedValueOnce( diff --git a/packages/cli/src/cli/commands/agent.ts b/packages/cli/src/cli/commands/agent.ts index bd7aff33fb..bd08567b03 100644 --- a/packages/cli/src/cli/commands/agent.ts +++ b/packages/cli/src/cli/commands/agent.ts @@ -8,20 +8,16 @@ import { withSdkDefaults, type SdkCommandDeps, } from '../lib/sdk-command.js'; -import { RelayError } from '@agent-relay/sdk'; +import { + AgentRemovalPendingError, + isNotFoundError, + parseRemovalWaitTimeout, + waitForAgentRemoval, +} from '../lib/agent-removal.js'; import { withAgentRegistrationDeadline, withDeadline } from '../lib/agent-registration.js'; import { attributableReleaseReason } from '../lib/release-reason.js'; -function isNotFoundError(error: unknown): boolean { - if (error instanceof RelayError) return error.code === 'not_found' || error.statusCode === 404; - const statusCode = - error && typeof error === 'object' - ? ((error as { statusCode?: unknown }).statusCode ?? (error as { status?: unknown }).status) - : undefined; - return Number(statusCode) === 404; -} - export type AgentCommandDependencies = SdkCommandDeps; function withAgentDefaults(overrides: Partial = {}): AgentCommandDependencies { @@ -152,6 +148,9 @@ export function registerAgentCommands( .description('Remove an agent while preserving attributed message history') .argument('', 'Agent name') .option('--reason ', 'Removal reason') + .option('--wait', 'Wait until the registration is cleared', false) + .option('--no-wait', 'Return after the removal is accepted') + .option('--wait-timeout ', 'Registration clearance timeout', '30000') ).action(async (name: string, opts: Record) => { await runSdk(deps, async () => { const relay = deps.createWorkspaceRelay(sdkOptionsFromOpts(opts)); @@ -160,18 +159,30 @@ export function registerAgentCommands( process.env.RELAY_AGENT_NAME ?? 'agent-relay CLI', 'agent removed' ); + const waitTimeoutMs = opts.wait === true ? parseRemovalWaitTimeout(opts.waitTimeout) : undefined; const result = await relay.workspace.release({ name, reason, deleteAgent: true }); - // The release endpoint acknowledges an async action invocation — a - // resolved promise means the request was accepted, not that the - // deletion has finished. Only claim "Removed" once the invocation - // itself reports completion; otherwise say what actually happened. - if (result.status === 'completed') { - deps.log(`Removed agent ${name}.`); - } else { + if (waitTimeoutMs !== undefined) { + const removal = await waitForAgentRemoval({ + name, + getAgent: (name) => relay.agents.get(name), + listAgents: () => relay.agents.list(), + timeoutMs: waitTimeoutMs, + }); + if (removal.cleared) { + deps.log(`Removed agent ${name}.`); + return; + } + if (removal.observedPresent) throw new AgentRemovalPendingError(name); + // Unverifiable clearance keeps the asynchronous acknowledgement. + deps.error(`Could not verify removal: ${removal.readError ?? 'registration unavailable'}`); deps.log( - `Removal of agent ${name} was initiated (status: ${result.status ?? 'pending'}) and is processed asynchronously.` + `Removal of agent ${name} was initiated (status: ${result.status ?? 'pending'}) and is processed asynchronously; registration clearance could not be verified.` ); + return; } + deps.log( + `Removal of agent ${name} was initiated (status: ${result.status ?? 'pending'}) and is processed asynchronously. Use --wait to verify the name is reusable.` + ); }); }); } diff --git a/packages/cli/src/cli/commands/fleet-agent.test.ts b/packages/cli/src/cli/commands/fleet-agent.test.ts index 30bf8f9518..abe2b0d605 100644 --- a/packages/cli/src/cli/commands/fleet-agent.test.ts +++ b/packages/cli/src/cli/commands/fleet-agent.test.ts @@ -59,6 +59,7 @@ describe('readRemoteLiveAgents', () => { expect(result).toEqual({ supported: true, + complete: true, agents: [{ name: 'å-worker' }, { name: 'worker-z' }], }); }); @@ -66,6 +67,7 @@ describe('readRemoteLiveAgents', () => { it('treats an absent marker as an unsupported old broker', () => { expect(readRemoteLiveAgents(node({ name: 'sf-mini', capabilities: [] }))).toEqual({ supported: false, + complete: false, agents: [], }); }); @@ -84,7 +86,25 @@ describe('readRemoteLiveAgents', () => { ], }) ) - ).toMatchObject({ supported: true, agents: [], warning: expect.stringContaining('1 malformed') }); + ).toMatchObject({ + supported: true, + complete: false, + agents: [], + warning: expect.stringContaining('1 malformed'), + }); + }); + + it('keeps a roster with only duplicate names complete', () => { + expect( + readRemoteLiveAgents( + node({ + name: 'sf-mini', + capabilities: [ + { name: LIVE_AGENT_CAPABILITY_NAME, kind: 'capacity', metadata: { names: ['w', 'w'] } }, + ], + }) + ) + ).toMatchObject({ supported: true, complete: true, agents: [{ name: 'w' }] }); }); }); diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index e12fc0df12..493f9e5fdd 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -49,6 +49,7 @@ vi.mock('@agent-relay/harness-driver', async (importOriginal) => ({ import { registerFleetCommands } from './fleet.js'; import { writeProjectWorkspaceKey } from '../lib/project-workspace-key.js'; import { spawnPlacementReceipt } from '../lib/spawn-lifecycle.js'; +import { MAX_TASK_BODY_BYTES } from '../lib/task-input.js'; const REPLAY_SANDBOX_ID = 'sbx_123e4567-e89b-42d3-a456-426614174000'; const REPLAY_SANDBOX_NAME = 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000'; @@ -1093,11 +1094,115 @@ describe('fleet command support', () => { const rendered = errors.join('\n'); expect(rendered).toContain('do not retry blindly'); expect(rendered).toContain('"code":"spawn_unconfirmed"'); - expect(rendered).toContain('"state":"unconfirmed_may_be_running"'); + expect(rendered).toContain('"state":"pending"'); expect(rendered).toContain('"dispatchState":"dispatched"'); expect(rendered).toContain(`"invocationId":"${invocationId}"`); }); + it.each(['live', 'terminal', 'collision', 'missing_ready'] as const)( + 'resolves timed-out targeted spawn using %s evidence', + async (evidence) => { + const invocationId = 'inv_late'; + const pending = new RelayPlacementError('spawn_unconfirmed', 'timeout', { + capability: 'spawn:codex', + attempts: 1, + node: 'target', + invocationId, + state: 'unconfirmed_may_be_running', + dispatchState: 'dispatched', + }); + const getInvocation = vi.fn(async () => + evidence === 'live' + ? { status: 'running' } + : evidence === 'terminal' || evidence === 'missing_ready' + ? { status: 'completed', output: { spawned: true, ready: evidence === 'terminal' } } + : { + status: 'failed', + error: + "failed to pre-register worker 'worker': agent 'worker' already exists and registration is create-only; use a unique name", + } + ); + const nodes = { + get: vi.fn(async () => ({ + name: 'target', + status: 'online', + lastHeartbeatAt: new Date().toISOString(), + capabilities: [{ name: 'relay:live-agents:v1', metadata: { names: ['worker'] } }], + })), + list: vi.fn(), + }; + const logs: string[] = []; + const errors: string[] = []; + const exit = vi.fn(); + const warn = vi.fn(); + const program = new Command(); + registerFleetCommands(program, { + resolveSandboxRepository: () => undefined, + sdk: { + createAgentRelay: vi.fn(() => ({ + messaging: { + placement: { + spawn: vi.fn(async () => { + throw pending; + }), + }, + commands: { getInvocation }, + }, + })) as never, + createWorkspaceRelay: vi.fn(() => ({ nodes, agents: {} })) as never, + createWorkspace: vi.fn() as never, + log: (message) => logs.push(String(message)), + error: (message) => errors.push(String(message)), + exit: exit as never, + }, + warn, + log: vi.fn(), + error: vi.fn(), + }); + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--name', + 'worker', + '--task', + 'work', + '--node', + 'target', + '--token', + 'at_test', + ], + { from: 'user' } + ); + expect(getInvocation).toHaveBeenCalledWith('spawn', invocationId); + if (evidence === 'collision' || evidence === 'missing_ready') { + expect(exit).toHaveBeenCalledWith(1); + expect(errors.join('')).toContain( + evidence === 'collision' ? '"code":"spawn_name_taken"' : '"code":"spawn_failed"' + ); + } else if (evidence === 'live') { + // Heartbeats carry worker names only: an existing worker with this + // name is indistinguishable from this invocation's worker, so the + // observation must not convert the timed-out spawn into success. + expect(exit).toHaveBeenCalledWith(8); + expect(logs).toEqual([]); + const rendered = errors.join(''); + expect(rendered).toContain('"code":"spawn_unconfirmed"'); + expect(rendered).toContain(`"invocationId":"${invocationId}"`); + expect(rendered).toContain('"evidence":"live"'); + expect(rendered).toContain('may be an earlier worker'); + } else { + expect(exit).not.toHaveBeenCalled(); + expect(JSON.parse(logs[0]!).invocation.placement).toMatchObject({ + state: 'ready', + confirmed: true, + dispatchState: 'dispatched', + }); + } + } + ); + it('fleet spawn --sandbox-provider agent37 provisions the isolated canary and uses a temporary launcher', async () => { vi.stubEnv('RELAY_AGENT_TOKEN', undefined); const placement = { @@ -1253,6 +1358,116 @@ describe('fleet command support', () => { }); }); + // relay#1930 review (cubic): the sandbox context is appended after + // readTaskInput accepted the task, so the composed task is what must fit. + it('rejects a sandbox task whose appended context pushes it past the PTY body limit, and cleans up the sandbox', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', undefined); + const revision = '0123456789abcdef0123456789abcdef01234567'; + const repositorySelection = { + repository: 'AgentWorkforce/cloud', + repositoryName: 'cloud', + revision, + projectRoot: '/local/cloud', + repositoryRelativeCwd: 'packages/web', + workerCwd: '/srv/agent-workforce/cloud/packages/web', + }; + const events: string[] = []; + const warnings: string[] = []; + const materializeCloudRelayfileRepository = vi.fn(async () => { + events.push('materialize'); + return { + cloudWorkspaceId: 'cloud-workspace', + repository: 'AgentWorkforce/cloud', + revision, + filesWritten: 4312, + sourceProfile: 'complete-v1' as const, + contentRoot: '/github/repos/AgentWorkforce/cloud/contents', + sentinelPath: '/github/repos/AgentWorkforce/cloud/.relayfile/clone.json', + }; + }); + const ensureCloudFleetSandbox = vi.fn(async () => { + events.push('ensure'); + return { + outcome: 'provisioned' as const, + providerId: 'agent37' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-live', + nodeName: 'live-node', + sandboxId: 'sandbox-live', + providerSandboxId: 'provider-live', + relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, + relayfileMounted: true, + relayfileMountPath: '/workspace', + }; + }); + const placement = { + spawn: vi.fn(async () => { + events.push('spawn'); + return { invocationId: 'inv_live', node: { name: 'live-node' } }; + }), + }; + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { + info: vi.fn(async () => ({ id: 'rw_abc' })), + register: vi.fn(async () => ({ token: 'at_live_launcher' })), + release: vi.fn(async () => ({ released: true, deleted: true })), + }, + })); + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const sdkError = vi.fn(); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + resolveSandboxRepository: vi.fn(() => repositorySelection), + materializeCloudRelayfileRepository, + ensureCloudFleetSandbox, + sdk: { + createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, + createWorkspaceRelay: createWorkspaceRelay as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: sdkError, + exit: vi.fn() as never, + }, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/local/cloud/.agentworkforce/relay/workspace-key.json', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: (message) => warnings.push(message), + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-provider', + 'agent37', + '--name', + 'cloud-live', + '--task', + 'x'.repeat(MAX_TASK_BODY_BYTES), + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + expect(sdkError.mock.calls.flat().join(' ')).toContain('UTF-8 bytes'); + expect(placement.spawn).not.toHaveBeenCalled(); + expect(deleteCloudFleetSandbox).toHaveBeenCalledWith( + expect.objectContaining({ sandboxId: 'sandbox-live' }) + ); + }); + it('plain --sandbox materializes the inferred repository through Relayfile and starts in its live relative cwd', async () => { vi.stubEnv('RELAY_AGENT_TOKEN', undefined); const revision = '0123456789abcdef0123456789abcdef01234567'; @@ -3304,6 +3519,7 @@ describe('fleet command support', () => { ).rejects.toThrow('__exit__'); expect(errors.join('\n')).toContain('dispatch may still be running'); + expect(errors.join('\n')).toContain('"code":"spawn_unconfirmed"'); expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( expect.objectContaining({ workloadProfile: 'long-running-agent', @@ -4602,6 +4818,7 @@ describe('fleet command support', () => { { from: 'user' } ); return { + program, cwd, projectRoot, local, @@ -4616,6 +4833,39 @@ describe('fleet command support', () => { }; } + it('reads --task-file before spawning through the existing task transport', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-fleet-brief-')); + try { + const brief = path.join(dir, 'brief.md'); + fs.writeFileSync(brief, ' Full brief\r\nsecond line\n'); + const fixture = setup(); + await fixture.program.parseAsync( + ['fleet', 'spawn', 'claude', '--name', 'file-worker', '--task-file', brief], + { from: 'user' } + ); + expect(fixture.errors).toEqual([]); + expect(fixture.local.spawnPty).toHaveBeenCalledWith( + expect.objectContaining({ task: ' Full brief\r\nsecond line\n' }) + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it.each([[], ['--task', 'brief', '--task-file', 'brief.md']])( + 'rejects missing or conflicting task sources: %j', + async (...args) => { + const fixture = setup(); + await expect( + fixture.program.parseAsync(['fleet', 'spawn', 'claude', '--name', 'file-worker', ...args], { + from: 'user', + }) + ).rejects.toThrow('__exit__'); + expect(fixture.errors.join('\n')).toContain('exactly one'); + expect(fixture.local.spawnPty).not.toHaveBeenCalled(); + } + ); + it('uses the caller nested directory and local broker despite ambient hosted credentials', async () => { vi.stubEnv('RELAY_WORKSPACE_KEY', 'rk_live_ambient_test'); vi.stubEnv('RELAY_AGENT_TOKEN', 'at_live_ambient_test'); @@ -5173,6 +5423,95 @@ describe('fleet command support', () => { ); }); + it('fleet release --delete-agent --wait adds clearance evidence on the selected transport', async () => { + const createWorkspaceRelay = vi.fn(() => ({ + agents: { + get: vi.fn(async () => { + throw { status: 404 }; + }), + list: vi.fn(), + }, + })); + const log = vi.fn(); + const program = new Command(); + registerFleetCommands(program, { + sdk: { + createWorkspaceRelay: createWorkspaceRelay as never, + createAgentRelay: vi.fn() as never, + createWorkspace: vi.fn() as never, + log, + error: vi.fn(), + exit: vi.fn() as never, + }, + createFleetWorkspaceClient: vi.fn(() => ({ + agents: { release: vi.fn(async () => ({ status: 'dispatched' })) }, + })) as never, + retireOwnedBindings: vi.fn(async () => undefined), + warn: vi.fn(), + }); + await program.parseAsync( + [ + 'fleet', + 'release', + 'worker', + '--delete-agent', + '--wait', + '--workspace-key', + 'rk_test', + '--base-url', + 'https://isolated.example', + ], + { from: 'user' } + ); + expect(createWorkspaceRelay).toHaveBeenCalledWith( + expect.objectContaining({ workspaceKey: 'rk_test', baseUrl: 'https://isolated.example' }) + ); + expect(JSON.parse(log.mock.calls[0]![0])).toMatchObject({ + status: 'dispatched', + removal: { cleared: true }, + }); + }); + + // relay#1930 review (cubic): Number('abc') is NaN, which the wait helper + // silently normalized to its default instead of reporting invalid input. + it.each(['abc', '0', '-5'])( + 'fleet release --wait-timeout %s is rejected before anything is released', + async (timeout) => { + const release = vi.fn(async () => ({ status: 'dispatched' })); + const sdkError = vi.fn(); + const program = new Command(); + registerFleetCommands(program, { + sdk: { + createWorkspaceRelay: vi.fn() as never, + createAgentRelay: vi.fn() as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: sdkError, + exit: vi.fn() as never, + }, + createFleetWorkspaceClient: vi.fn(() => ({ agents: { release } })) as never, + retireOwnedBindings: vi.fn(async () => undefined), + warn: vi.fn(), + }); + await program.parseAsync( + [ + 'fleet', + 'release', + 'worker', + '--delete-agent', + '--wait', + '--wait-timeout', + timeout, + '--workspace-key', + 'rk_test', + ], + { from: 'user' } + ); + expect(sdkError.mock.calls.flat().join(' ')).toContain('--wait-timeout'); + expect(release).not.toHaveBeenCalled(); + } + ); + it('fleet release --delete-agent aborts without deleting when binding retirement fails', async () => { const release = vi.fn(async () => ({ name: 'api-worker', diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index fd1ff04647..a0b6078e35 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -1,3 +1,4 @@ +import { readTaskInput, validateTaskSize } from '../lib/task-input.js'; import { randomUUID } from 'node:crypto'; import path from 'node:path'; @@ -38,7 +39,20 @@ import { readBrokerConnection } from '../lib/broker-lifecycle.js'; import { spawnAgentWithClient } from '../lib/client-factory.js'; import { formatRelativeTime, sanitizeForTerminalLine } from '../lib/formatting.js'; import { connectProjectBrokerClient } from '../lib/project-broker-client.js'; -import { isAvailableFleetNode } from '../lib/fleet-live-agents.js'; +import { + AgentRemovalPendingError, + parseRemovalWaitTimeout, + waitForAgentRemoval, +} from '../lib/agent-removal.js'; +import { + classifySpawnFailure, + mayStillBeRunning, + nameTakenSpawnError, + pendingSpawnError, + probeSpawnLiveness, + terminalSpawnOutcome, +} from '../lib/spawn-liveness.js'; +import { heartbeatAgeMs, MAX_LIVE_HEARTBEAT_AGE_MS, isAvailableFleetNode } from '../lib/fleet-live-agents.js'; import { declaredWorkforceMetadata } from '../lib/registration-metadata.js'; import { redactSecrets } from '../lib/redact.js'; import { attributableReleaseReason } from '../lib/release-reason.js'; @@ -210,13 +224,13 @@ function spawnInvocationWithMergedPlacement(invocation: Record) ...invocation, placement: { ...(sdkPlacement ?? {}), - dispatchState: receipt.dispatchState, + dispatchState: sdkPlacement?.dispatchState ?? receipt.dispatchState, ...(invocationId ? { invocationId } : {}), }, }; } -function throwForTerminalSpawnFailure(invocation: Record): void { +function throwForTerminalSpawnFailure(invocation: Record, name: string): void { const placement = spawnPlacementReceipt(invocation); if (placement.state !== 'failed') return; const invocationId = typeof placement.invocationId === 'string' ? placement.invocationId : undefined; @@ -224,7 +238,7 @@ function throwForTerminalSpawnFailure(invocation: Record): void typeof invocation.error === 'string' && invocation.error.trim() ? invocation.error : 'Fleet spawn invocation reported a terminal failure.'; - throw new RelayPlacementError('spawn_failed', message, { + const error = new RelayPlacementError('spawn_failed', message, { capability: typeof invocation.capability === 'string' ? invocation.capability : 'spawn', attempts: 1, ...(invocationId ? { invocationId } : {}), @@ -232,6 +246,7 @@ function throwForTerminalSpawnFailure(invocation: Record): void dispatchState: placement.dispatchState as 'dispatched' | 'not_dispatched' | 'unknown', receipt: invocation, }); + throw classifySpawnFailure(message) ? nameTakenSpawnError(name, error) : error; } export interface FleetCommandDependencies { @@ -442,7 +457,8 @@ export function registerFleetCommands( .description('Spawn locally by default, or select a fleet node or Cloud sandbox explicitly') .argument('', 'AI CLI to launch', parseFleetCli) .requiredOption('--name ', 'Worker agent name') - .requiredOption('--task ', 'Initial task instructions') + .option('--task ', 'Initial task instructions') + .option('--task-file ', 'Read the initial task from a local UTF-8 file') .option('--auto-place', 'Request automatic eligible-node placement in the Relay workspace') .option('--node ', 'Target a specific fleet node') .option('--target-node ', 'Alias for --node') @@ -495,7 +511,7 @@ export function registerFleetCommands( await runSdk(deps.sdk, async () => { const clientOptions = sdkOptionsFromOpts(options); const name = requiredText(options.name, 'Worker name'); - const task = requiredText(options.task, 'Task'); + const task = (await readTaskInput(options.task, options.taskFile, true, { cli }))!; let targetNode = optionalText(options.targetNode, 'Target node') ?? optionalText(options.node, 'Node'); const useSandbox = options.sandbox === true; // Explicit hosted credentials/transport and personas retain their legacy @@ -1116,32 +1132,71 @@ export function registerFleetCommands( 'The Relayfile daemon synchronizes this tree; it intentionally has no .git directory.', ].join(' ') : undefined; - const invocation = await relay.messaging.placement.spawn({ - capability: `spawn:${cli}`, - node: targetNode, - failFast: true, - confirm, - ...(confirm ? { confirmTimeoutMs: confirmTimeoutMs } : {}), - input: { + const spawnTask = + sandbox && + checkoutRepository && + sandboxRepository && + mountSandboxRelayfile && + sandbox.outcome === 'provisioned' + ? `${task}\n\nAgent Relay sandbox context: Relayfile records are available at ${sandbox.relayfileMountPath ?? '/workspace'}. The source checkout is separate; use ${workerCwd ?? 'the worker checkout'} for repository files and the mount for Relayfile records.` + : liveSandboxContext + ? `${task}\n\n${liveSandboxContext}` + : task; + // The sandbox context is appended after readTaskInput accepted the + // task, so the composed task is what the worker must receive. + validateTaskSize(spawnTask, { cli }); + let invocation; + try { + invocation = await relay.messaging.placement.spawn({ + capability: `spawn:${cli}`, + node: targetNode, + failFast: true, + confirm, + ...(confirm ? { confirmTimeoutMs: confirmTimeoutMs } : {}), + input: { + name, + cli, + task: spawnTask, + ...(channel ? { channels: [channel] } : {}), + ...(model ? { model } : {}), + ...(workerCwd ? { worker_cwd: workerCwd } : {}), + ...registrationMetadata, + ...(sessionRef ? { session_ref: sessionRef } : {}), + }, + }); + } catch (error) { + if (!(error instanceof RelayPlacementError)) throw error; + if (error.code === 'spawn_failed' && classifySpawnFailure(error.message)) + throw nameTakenSpawnError(name, error); + if (error.code !== 'spawn_unconfirmed') throw error; + const liveness = await probeSpawnLiveness({ name, - cli, - task: - sandbox && - checkoutRepository && - sandboxRepository && - mountSandboxRelayfile && - sandbox.outcome === 'provisioned' - ? `${task}\n\nAgent Relay sandbox context: Relayfile records are available at ${sandbox.relayfileMountPath ?? '/workspace'}. The source checkout is separate; use ${workerCwd ?? 'the worker checkout'} for repository files and the mount for Relayfile records.` - : liveSandboxContext - ? `${task}\n\n${liveSandboxContext}` - : task, - ...(channel ? { channels: [channel] } : {}), - ...(model ? { model } : {}), - ...(workerCwd ? { worker_cwd: workerCwd } : {}), - ...registrationMetadata, - ...(sessionRef ? { session_ref: sessionRef } : {}), - }, - }); + targetNode, + ...(error.invocationId + ? { + getInvocation: () => relay.messaging.commands.getInvocation('spawn', error.invocationId!), + } + : {}), + createClient: () => { + const workspace = deps.sdk.createWorkspaceRelay(relaycastClientOptions); + return { nodes: workspace.nodes, agents: workspace.agents }; + }, + }); + if (liveness.evidence === 'invocation_terminal') { + try { + invocation = terminalSpawnOutcome(liveness.invocation!, error, confirm); + } catch (terminalError) { + if ( + terminalError instanceof RelayPlacementError && + classifySpawnFailure(terminalError.message) + ) + throw nameTakenSpawnError(name, terminalError); + throw terminalError; + } + } else { + throw pendingSpawnError(name, error, liveness, error.message); + } + } const printableSandbox = (sandbox?.outcome === 'provisioned' || sandbox?.outcome === 'reused') && sandbox.relaycastTarget ? { @@ -1163,11 +1218,7 @@ export function registerFleetCommands( invocation: spawnInvocationWithMergedPlacement(invocation as unknown as Record), }); } catch (error) { - if ( - shouldCleanupSandbox && - sandbox?.outcome === 'provisioned' && - !(error instanceof RelayPlacementError && error.state === 'unconfirmed_may_be_running') - ) { + if (shouldCleanupSandbox && sandbox?.outcome === 'provisioned' && !mayStillBeRunning(error)) { await deps .deleteCloudFleetSandbox({ cloudWorkspaceId: sandbox.cloudWorkspaceId, @@ -1248,7 +1299,7 @@ export function registerFleetCommands( } : {}), }); - throwForTerminalSpawnFailure(invocation); + throwForTerminalSpawnFailure(invocation, name); printJson(deps.sdk, { invocation: spawnInvocationWithPlacement(invocation) }); }); }); @@ -1260,6 +1311,9 @@ export function registerFleetCommands( .argument('', 'Worker agent name') .option('--reason ', 'Release reason') .option('--delete-agent', 'Permanently delete the agent after release') + .option('--wait', 'With --delete-agent, wait for registration clearance', false) + .option('--no-wait', 'Return after release is accepted') + .option('--wait-timeout ', 'Registration clearance timeout', '30000') .option( '--unsubscribe-bindings', 'Retire provider bindings owned by this identity (implied by --delete-agent)' @@ -1269,6 +1323,8 @@ export function registerFleetCommands( warnIfInferredFromProjectSession(options, deps.warn); const workerName = requiredText(name, 'Worker name'); const deleteAgent = options.deleteAgent === true; + const waitTimeoutMs = + deleteAgent && options.wait === true ? parseRemovalWaitTimeout(options.waitTimeout) : undefined; const sdkOpts = sdkOptionsFromOpts(options); // One transport for both cleanup and release. Cleanup otherwise // substitutes the local broker session whenever --workspace-key is @@ -1317,6 +1373,22 @@ export function registerFleetCommands( reason, deleteAgent: true, }); + if (waitTimeoutMs !== undefined) { + // Construct/read on the same selected transport; unavailable credentials are uncertainty. + const removal = await waitForAgentRemoval({ + name: workerName, + getAgent: (name) => deps.sdk.createWorkspaceRelay(transport).agents.get(name), + listAgents: () => deps.sdk.createWorkspaceRelay(transport).agents.list(), + timeoutMs: waitTimeoutMs, + }); + printJson(deps.sdk, { ...deleted, removal }); + if (!removal.cleared && removal.observedPresent) throw new AgentRemovalPendingError(workerName); + if (!removal.cleared) + deps.warn( + `Removal was initiated asynchronously; could not verify clearance: ${removal.readError}` + ); + return; + } printJson(deps.sdk, deleted); return; } @@ -1649,12 +1721,27 @@ async function runFleetAgentList( const now = new Date(); const output = buildRows({ contributions, roster }, now); + const heartbeatNodes = contributions.map(({ node }) => ({ + name: node.name, + lastHeartbeatAt: node.lastHeartbeatAt ?? null, + heartbeatAgeMs: heartbeatAgeMs(node, now.getTime()), + })); + if ( + heartbeatNodes.some( + (node) => node.heartbeatAgeMs !== null && node.heartbeatAgeMs > MAX_LIVE_HEARTBEAT_AGE_MS + ) + ) { + deps.warn( + 'Some node heartbeats are older than 36 seconds. Listings are heartbeat snapshots and can differ within one heartbeat window.' + ); + } if (options.pretty === true) { deps.log(formatPretty(output)); return; } printJson(deps.sdk, { generatedAt: now.toISOString(), + nodes: heartbeatNodes, localNode: localNodeName ?? null, perNode: output.perNode, unplacedRoster: output.unplacedRoster, diff --git a/packages/cli/src/cli/commands/integration-subscribe.test.ts b/packages/cli/src/cli/commands/integration-subscribe.test.ts index 5d8bf16fe7..2eaf0423bc 100644 --- a/packages/cli/src/cli/commands/integration-subscribe.test.ts +++ b/packages/cli/src/cli/commands/integration-subscribe.test.ts @@ -30,6 +30,7 @@ interface InboundWebhook { function createRelayMock(opts: { inboundWebhooks?: InboundWebhook[] } = {}) { let counter = 0; + const subscriptions = new Map>(); return { agents: { register: vi.fn(async (i: { name: string }) => ({ id: 'a1', token: 't1', name: i.name })), @@ -37,7 +38,13 @@ function createRelayMock(opts: { inboundWebhooks?: InboundWebhook[] } = {}) { }, integrations: { subscriptions: { - create: vi.fn(async (i: unknown) => ({ id: `sub_${++counter}`, ...(i as object) })), + create: vi.fn(async (i: unknown) => { + const row = { id: `sub_${++counter}`, ...(i as object) }; + subscriptions.set(row.id, row); + return row; + }), + get: vi.fn(async (id: string) => subscriptions.get(id) ?? { id }), + list: vi.fn(async () => [...subscriptions.values()]), delete: vi.fn(async () => undefined), }, }, @@ -2099,3 +2106,214 @@ describe('confirmed agent subscription setup', () => { expect(h.relay.webhooks.createInbound).not.toHaveBeenCalled(); }); }); + +describe('subscription writeback verification', () => { + it.each([{ event: 'message.created' }, { events: ['message.created'] }, {}])( + 'rolls back incomplete persisted event coverage: %j', + async (echo) => { + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockImplementation(async (id) => ({ id, ...echo })); + const { program, relayfile, error, journal } = harness({ relay }); + await program.parseAsync(ARGS(), { from: 'user' }); + expect(error.mock.calls.flat().join(' ')).toContain('did not confirm requested events'); + expect(relayfile.bind).not.toHaveBeenCalled(); + expect(relay.integrations.subscriptions.delete).toHaveBeenCalled(); + expect(relay.webhooks.delete).toHaveBeenCalled(); + expect(relayfile.deleteWebhookSubscription).toHaveBeenCalled(); + expect(await journal.list()).toEqual([]); + } + ); + + it('journals a failed rollback delete after read-back fails', async () => { + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockRejectedValue(new Error('read unavailable')); + relay.integrations.subscriptions.delete.mockRejectedValue(new Error('delete unavailable')); + const { program, journal, relayfile } = harness({ relay }); + await program.parseAsync(ARGS(), { from: 'user' }); + expect(relayfile.bind).not.toHaveBeenCalled(); + expect(await journal.list()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'relay-subscription', id: expect.stringMatching(/^sub_/) }), + ]) + ); + }); + + it.each(['events', 'event_types', 'eventTypes'])( + 'accepts %s coverage and de-duplicates requested events', + async (field) => { + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockImplementation(async (id) => ({ + id, + [field]: ['thread.reply', 'message.created'], + })); + const { program, relayfile } = harness({ relay }); + await program.parseAsync(ARGS(['--events', 'thread.reply,message.created,thread.reply']), { + from: 'user', + }); + expect(relayfile.bind).toHaveBeenCalledOnce(); + expect(relay.integrations.subscriptions.create).toHaveBeenCalledWith( + expect.objectContaining({ + event: 'thread.reply', + events: ['thread.reply', 'message.created'], + }) + ); + } + ); + + it('rejects misspelled events before provisioning', async () => { + const { program, relay, relayfile, error } = harness(); + await program.parseAsync(ARGS(['--events', 'thread.replies']), { from: 'user' }); + expect(error.mock.calls.flat().join(' ')).toContain('Unsupported subscription events'); + expect(relay.webhooks.createInbound).not.toHaveBeenCalled(); + expect(relayfile.createWebhookSubscription).not.toHaveBeenCalled(); + }); + + it.each(['action.completed', 'message.updated'])( + 'provisions engine-supported %s subscriptions', + async (event) => { + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockImplementation(async (id) => ({ id, events: [event] })); + const { program, relayfile, error } = harness({ relay }); + await program.parseAsync(ARGS(['--events', event]), { from: 'user' }); + expect(error.mock.calls.flat().join(' ')).not.toContain('Unsupported subscription events'); + expect(relay.integrations.subscriptions.create).toHaveBeenCalledWith( + expect.objectContaining({ event, events: [event] }) + ); + expect(relayfile.bind).toHaveBeenCalledOnce(); + } + ); + + it('accepts singular thread.reply coverage', async () => { + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockImplementation(async (id) => ({ id, event: 'thread.reply' })); + const { program, relayfile } = harness({ relay }); + await program.parseAsync(ARGS(['--events', 'thread.reply']), { from: 'user' }); + expect(relayfile.bind).toHaveBeenCalledOnce(); + }); + + it.each([true, false, undefined])( + 'reports GitHub identity authorization %s honestly', + async (authorized) => { + const relayfile = createRelayfileMock([], { + listWebhookSubscriptions: vi.fn(async () => ({ + workspaceId: 'rw_test', + subscriptions: [ + { + subscriptionId: 'whsub_1', + url: 'https://cast.test/inbound', + pathGlobs: ['/github/repos/AgentWorkforce/relaycast/pulls/448/**'], + githubPrIdentityAuthorized: authorized, + }, + ], + })), + }); + const { program, log } = harness({ relayfile }); + await program.parseAsync( + [ + 'integration', + 'subscribe', + 'github', + '--resource', + '/github/repos/AgentWorkforce/relaycast/pulls/448/**', + '--to', + '#general', + ], + { from: 'user' } + ); + const output = log.mock.calls.flat().join(' '); + expect(output).not.toContain('Replies will post back'); + expect(output).toContain(authorized === true ? 'writeback is configured' : 'writeback is unverified'); + expect(relayfile.bind).toHaveBeenCalledWith( + expect.objectContaining({ resource: '/github/repos/AgentWorkforce/relaycast/pulls/448/**' }) + ); + } + ); +}); + +it('reports outbound configuration separately from inferred inbound activity', async () => { + const relay = createRelayMock(); + relay.integrations.subscriptions.list.mockResolvedValue([ + { id: 'sub_existing', events: ['thread.reply'], isActive: false }, + ] as never); + relay.messages.list.mockResolvedValue([{ createdAt: '2026-10-05T00:00:00Z' }] as never); + const relayfile = createRelayfileMock([ + { + provider: 'github', + resource: '/github/repos/AgentWorkforce/relaycast/pulls/448/**', + channel: 'general', + subscriptionId: 'sub_existing', + webhookId: 'in_existing', + }, + ]); + const { program, log } = harness({ relay, relayfile }); + await program.parseAsync(['integration', 'subscribe', '--list'], { from: 'user' }); + expect(JSON.parse(log.mock.calls[0][0]).bindings[0]).toMatchObject({ + lastDeliverySource: 'channel-inferred', + writebackSubscription: { + id: 'sub_existing', + events: ['thread.reply'], + active: false, + deliveryStatus: null, + }, + }); + relay.integrations.subscriptions.list.mockRejectedValue(new Error('unavailable')); + log.mockClear(); + await program.parseAsync(['integration', 'subscribe', '--list'], { from: 'user' }); + expect(JSON.parse(log.mock.calls[0][0]).bindings[0].writebackSubscription).toBeNull(); +}); + +// One collection read serves every bound row, so a large workspace cannot +// turn --list into one subscription request per binding. +it('reads writeback subscriptions once for all bound rows', async () => { + const relay = createRelayMock(); + relay.integrations.subscriptions.list.mockResolvedValue([ + { id: 'sub_a', events: ['thread.reply'], isActive: true }, + { id: 'sub_b', events: ['message.created'], isActive: true }, + ] as never); + const relayfile = createRelayfileMock( + ['sub_a', 'sub_b', 'sub_missing'].map((subscriptionId, index) => ({ + provider: 'github', + resource: `/github/repos/AgentWorkforce/relaycast/pulls/${index}/**`, + channel: 'general', + subscriptionId, + webhookId: `in_${index}`, + })) + ); + const { program, log } = harness({ relay, relayfile }); + await program.parseAsync(['integration', 'subscribe', '--list'], { from: 'user' }); + const rows = JSON.parse(log.mock.calls[0][0]).bindings; + expect( + rows.map((row: { writebackSubscription: { id: string } | null }) => row.writebackSubscription?.id ?? null) + ).toEqual(['sub_a', 'sub_b', null]); + expect(relay.integrations.subscriptions.list).toHaveBeenCalledTimes(1); + expect(relay.integrations.subscriptions.get).not.toHaveBeenCalled(); +}); + +it.each([ + { id: 'wrong', events: ['message.created', 'thread.reply'] }, + { isActive: false, events: ['message.created', 'thread.reply'] }, + { is_active: false, events: ['message.created', 'thread.reply'] }, +])('preserves the prior binding when read-back is invalid: %j', async (echo) => { + const prior = { + provider: 'slack', + resource: RESOURCE, + channel: 'general', + webhookId: 'in_old', + subscriptionId: 'sub_old', + }; + const relay = createRelayMock(); + relay.integrations.subscriptions.get.mockImplementation(async (id) => ({ id, ...echo })); + const relayfile = createRelayfileMock([prior]); + const { program, error } = harness({ relay, relayfile }); + await program.parseAsync(ARGS(), { from: 'user' }); + expect(error.mock.calls.flat().join(' ')).toContain('different or inactive subscription'); + expect(await relayfile.listBindings()).toEqual([prior]); + expect(relay.integrations.subscriptions.delete).not.toHaveBeenCalledWith('sub_old'); + expect(relay.webhooks.delete).not.toHaveBeenCalledWith('in_old'); +}); + +it('does not advertise thread writeback when only message.created is requested', async () => { + const { program, log } = harness(); + await program.parseAsync(ARGS(['--events', 'message.created']), { from: 'user' }); + expect(log.mock.calls.flat().join(' ')).toContain('thread.reply was not requested'); +}); diff --git a/packages/cli/src/cli/commands/integration.ts b/packages/cli/src/cli/commands/integration.ts index 881ca248ef..65d446e1c9 100644 --- a/packages/cli/src/cli/commands/integration.ts +++ b/packages/cli/src/cli/commands/integration.ts @@ -7,7 +7,11 @@ import { import { createHash, pbkdf2Sync, randomBytes } from 'node:crypto'; import { hostname } from 'node:os'; import { getProjectPaths } from '@agent-relay/config'; -import type { AgentRelayAgent } from '@agent-relay/sdk'; +import { + SUBSCRIBABLE_EVENT_TYPES, + normalizeWebhookSubscription, + type AgentRelayAgent, +} from '@agent-relay/sdk'; import { cleanupEntryKey, @@ -638,6 +642,13 @@ type ListedBinding = RelayfileBinding & { lastSuccessAt: string | null; lastError: string | null; lastChannelMessageAt: string | null; + lastDeliverySource: 'control-plane' | 'channel-inferred' | null; + writebackSubscription: { + id: string; + events: string[] | null; + active: boolean | null; + deliveryStatus: null; + } | null; githubPrIdentityAuthorized: boolean | null; }; @@ -660,6 +671,15 @@ async function enrichBindingsForList( ).catch(() => [] as Array<{ name: string; metadata?: Record }>); const channelByName = new Map(channels.map((channel) => [channel.name, channel])); + // One collection read for every bound row; per-row reads would burst one + // request per binding. Missing or unavailable engine status stays unknown. + const subscriptionById = bindings.some((binding) => binding.subscriptionId) + ? await relay.integrations.subscriptions + .list() + .then((rows) => new Map(rows.map((raw) => [String(raw.id), raw]))) + .catch(() => new Map>>()) + : new Map>>(); + return Promise.all( bindings.map(async (binding) => { const channelMeta = channelByName.get(binding.channel); @@ -688,6 +708,23 @@ async function enrichBindingsForList( ? cloudById.get(binding.webhookSubscriptionId) : undefined; const health = cloudRow?.health; + let writebackSubscription: ListedBinding['writebackSubscription'] = null; + const raw = binding.subscriptionId ? subscriptionById.get(binding.subscriptionId) : undefined; + if (raw) { + try { + const subscription = normalizeWebhookSubscription(raw); + const active = raw.isActive ?? raw.is_active; + writebackSubscription = { + id: subscription.id, + events: subscription.events ?? null, + active: typeof active === 'boolean' ? active : null, + // The pinned engine contract exposes configuration, not delivery attempts. + deliveryStatus: null, + }; + } catch { + /* A malformed engine row remains unknown. */ + } + } return { ...binding, to: agent ? `@${agent.name}` : null, @@ -698,6 +735,12 @@ async function enrichBindingsForList( lastSuccessAt: health?.lastSuccessAt ?? null, lastError: health?.lastError ?? null, lastChannelMessageAt, + lastDeliverySource: health?.lastDeliveryAt + ? 'control-plane' + : lastChannelMessageAt + ? 'channel-inferred' + : null, + writebackSubscription, githubPrIdentityAuthorized: cloudRow?.githubPrIdentityAuthorized ?? null, }; }) @@ -1578,6 +1621,14 @@ async function runSubscribeSetup( return; } + const requestedEvents = commaList(opts.events); + const events = [...new Set(requestedEvents.length ? requestedEvents : ['message.created', 'thread.reply'])]; + const supported: readonly string[] = SUBSCRIBABLE_EVENT_TYPES; + const unsupported = events.filter((event) => !supported.includes(event)); + if (unsupported.length) + throw new Error( + `Unsupported subscription events: ${unsupported.join(', ')}. Supported events: ${supported.join(', ')}.` + ); const { provider, resource, to } = await promptSubscribeOptions(deps, providerArg, opts); const local = await deps.resolveLocalRelayOptions(); await ensureProviderConnected(deps, provider, opts); @@ -1619,7 +1670,6 @@ async function runSubscribeSetup( baseUrl: resolveInboundTargetTransport(effectiveRelayOptions, relayOptions).baseUrl, }) : targetChannel(to); - const events = commaList(opts.events); const writeback = await resolveWriteback(deps, opts, channel); const inboundTarget = await createRelayfileInboundTarget(opts, local, { channel, @@ -1756,13 +1806,26 @@ async function runSubscribeSetup( }); lease.assertHealthy(`${provider} ${pathGlob}`); subscription = await relay.integrations.subscriptions.create({ - event: events.length === 1 ? events[0]! : 'message.created', - events: events.length ? events : ['message.created', 'thread.reply'], + event: events[0]!, + events, filter: { channel }, url: subscriptionTargetUrl, secret: writeback.secret, }); await upgradeAttempt({ subscriptionId: subscription.id }); + const persistedRaw = await relay.integrations.subscriptions.get(subscription.id); + const persisted = normalizeWebhookSubscription(persistedRaw); + if (persisted.id !== subscription.id || (persistedRaw.isActive ?? persistedRaw.is_active) === false) { + throw new Error( + 'Writeback subscription read-back returned a different or inactive subscription. Aborting and rolling back.' + ); + } + const missingEvents = events.filter((event) => !persisted.events?.includes(event)); + if (missingEvents.length) { + throw new Error( + `Writeback subscription did not confirm requested events: ${missingEvents.join(', ')}. Aborting and rolling back.` + ); + } lease.assertHealthy(`${provider} ${pathGlob}`); bindingInput = { provider, @@ -1875,7 +1938,17 @@ async function runSubscribeSetup( const boundLabel = pathGlob === resource ? resource : `${resource} (${pathGlob})`; deps.log(`✓ ${provider} ${boundLabel} bound -> ${to}`); deps.log('✓ Server-side inbound webhook subscription created.'); - deps.log('✓ Listening. Replies will post back in-thread.'); + const cloudRows = await listWebhookSubscriptionsForBindings(deps.relayfile, [bindingInput]); + const authorized = cloudRows.get(bindingInput.webhookSubscriptionId!)?.githubPrIdentityAuthorized; + if (!events.includes('thread.reply')) { + deps.log('✓ Listening. In-thread writeback is disabled: thread.reply was not requested.'); + } else if (provider === 'github' && authorized !== true) { + deps.log( + `✓ Listening. GitHub in-thread writeback is unverified: GitHub PR identity authorization is ${authorized === false ? 'not granted' : 'not reported'}.` + ); + } else { + deps.log('✓ Listening. In-thread writeback is configured; provider delivery is not yet confirmed.'); + } } async function runUnsubscribeOwnedBy( diff --git a/packages/cli/src/cli/commands/local-agent.test.ts b/packages/cli/src/cli/commands/local-agent.test.ts index 5faf20bbc5..4ce0a48d72 100644 --- a/packages/cli/src/cli/commands/local-agent.test.ts +++ b/packages/cli/src/cli/commands/local-agent.test.ts @@ -20,6 +20,7 @@ import { withDeliveryStatus, type LocalAgentDependencies, } from './local-agent.js'; +import { MAX_TASK_BODY_BYTES } from '../lib/task-input.js'; function harness(overrides: Partial = {}) { const client = { @@ -996,6 +997,33 @@ describe('local agent subtree', () => { } ); + // relay#1930 review (cubic): auto-routing wraps the task in a Director + // prompt, so the routed task is what must fit the PTY body limit. + it('rejects an auto-routed task whose Director prompt exceeds the PTY body limit', async () => { + const { program, client, error, exit } = harness(); + await program.parseAsync( + ['local', 'agent', 'spawn', 'claude', '--model', 'auto', '--task', 'x'.repeat(MAX_TASK_BODY_BYTES)], + { from: 'user' } + ); + expect(error.mock.calls.flat().join(' ')).toContain('UTF-8 bytes'); + expect(exit).toHaveBeenCalledWith(1); + expect(client.spawnPty).not.toHaveBeenCalled(); + }); + + it.each(['spawn', 'new'])( + '%s reports an unreadable --task-file through the CLI error path', + async (command) => { + const { program, client, error, exit } = harness(); + await program.parseAsync( + ['local', 'agent', command, 'codex', '--task-file', '/nonexistent/relay-brief.md'], + { from: 'user' } + ); + expect(error.mock.calls.flat().join(' ')).toContain('ENOENT'); + expect(exit).toHaveBeenCalledWith(1); + expect(client.spawnPty).not.toHaveBeenCalled(); + } + ); + it('release calls client.release', async () => { const { program, client } = harness(); await program.parseAsync(['local', 'agent', 'release', 'lead'], { from: 'user' }); diff --git a/packages/cli/src/cli/commands/local-agent.ts b/packages/cli/src/cli/commands/local-agent.ts index eeb23cbd79..97952a290b 100644 --- a/packages/cli/src/cli/commands/local-agent.ts +++ b/packages/cli/src/cli/commands/local-agent.ts @@ -1,3 +1,4 @@ +import { readTaskInput, validateTaskSize, type TaskTarget } from '../lib/task-input.js'; import type { Command } from 'commander'; import { AGENT37_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; @@ -52,7 +53,8 @@ function resolveAutoSpawn( provider: string, name: string, task: string | undefined, - model: string | undefined + model: string | undefined, + runtime?: TaskTarget['runtime'] ): { name: string; task: string | undefined; model: string | undefined } { if (model !== 'auto' || provider !== 'claude' || !task) { return { name, task, model }; @@ -60,6 +62,8 @@ function resolveAutoSpawn( const assessment = classifyTask(task); const team = composeTeam(assessment, task); const directorPrompt = buildDirectorPrompt(task, team); + // The Director prompt repeats the task, so it is what must fit. + validateTaskSize(directorPrompt, { cli: provider, runtime }); return { name: name === provider ? 'Director' : name, task: directorPrompt, @@ -849,6 +853,7 @@ export function registerLocalAgentCommands( .option('--name ', 'Agent name (defaults to the provider)') .option('--channels ', 'Channels to join', ['general']) .option('--task ', 'Initial task prompt') + .option('--task-file ', 'Read the initial task from a local UTF-8 file') .option('--model ', 'Model override') .option('--runtime ', 'Harness runtime: auto | native | pty', 'auto') .option('--cwd ', 'Working directory for the spawned agent') @@ -873,8 +878,12 @@ export function registerLocalAgentCommands( const resolved = resolveAutoSpawn( provider, baseName, - opts.task as string | undefined, - opts.model as string | undefined + await readTaskInput(opts.task, opts.taskFile, false, { + cli: provider, + runtime: runtime.selected, + }), + opts.model as string | undefined, + runtime.selected ); await spawnAgentWithClient(client, { name: resolved.name, @@ -904,6 +913,7 @@ export function registerLocalAgentCommands( .option('--mode ', 'Attach mode: drive | view | passthrough', 'drive') .option('--channels ', 'Channels to join', ['general']) .option('--task ', 'Initial task prompt') + .option('--task-file ', 'Read the initial task from a local UTF-8 file') .option('--model ', 'Model override') .option('--runtime ', 'Harness runtime: auto | native | pty', 'auto') .option('--cwd ', 'Working directory for the spawned agent') @@ -930,12 +940,23 @@ export function registerLocalAgentCommands( return; const brokerOptions = brokerOptionsFromOpts(options); const baseName = (options.name as string | undefined) ?? provider; - const resolved = resolveAutoSpawn( - provider, - baseName, - options.task as string | undefined, - options.model as string | undefined - ); + let resolved: ReturnType; + try { + resolved = resolveAutoSpawn( + provider, + baseName, + await readTaskInput(options.task, options.taskFile, false, { + cli: provider, + runtime: runtime.selected, + }), + options.model as string | undefined, + runtime.selected + ); + } catch (err) { + deps.error(err instanceof Error ? err.message : String(err)); + deps.exit(1); + return; + } await run( deps, async (client) => { diff --git a/packages/cli/src/cli/commands/relaycast-groups.test.ts b/packages/cli/src/cli/commands/relaycast-groups.test.ts index 64793b5fcb..f42596d3ef 100644 --- a/packages/cli/src/cli/commands/relaycast-groups.test.ts +++ b/packages/cli/src/cli/commands/relaycast-groups.test.ts @@ -84,7 +84,7 @@ function createRelayMock() { subscriptions: { create: vi.fn(async (i: unknown) => ({ id: 'sub1', ...(i as object) })), list: vi.fn(async () => []), - get: vi.fn(async (id: string) => ({ id })), + get: vi.fn(async (id: string) => ({ id, events: ['message.created', 'thread.reply'] })), delete: vi.fn(async () => undefined), }, }, @@ -170,6 +170,23 @@ describe('SDK-backed CLI groups', () => { expect(relay.channels.update).toHaveBeenCalledWith('ops', { topic: 'New topic' }); }); + // The 16 KiB ceiling is a PTY injection limit enforced where the broker + // types into a terminal. Relaycast messages may target native agents or be + // stored for history only, so the generic message commands must not apply it. + it('message post, reply and dm send publish bodies larger than the PTY injection limit', async () => { + const text = 'x'.repeat(20 * 1024); + const { program, relay, error } = harness(registerMessageCommands); + const reply = vi.fn(async (i: unknown) => ({ id: 'r1', ...(i as object) })); + (relay.messages as Record).reply = reply; + await program.parseAsync(['message', 'post', 'ops', text], { from: 'user' }); + await program.parseAsync(['message', 'reply', 'm1', text], { from: 'user' }); + await program.parseAsync(['message', 'dm', 'send', 'lead', text], { from: 'user' }); + expect(error).not.toHaveBeenCalled(); + expect(relay.messages.send).toHaveBeenCalledWith({ channel: 'ops', text }); + expect(reply).toHaveBeenCalledWith({ messageId: 'm1', text }); + expect(relay.messages.direct).toHaveBeenCalledWith({ to: 'lead', text }); + }); + it('message post routes to messages.send with the channel', async () => { const { program, relay } = harness(registerMessageCommands); await program.parseAsync(['message', 'post', 'ops', 'hello'], { from: 'user' }); @@ -690,6 +707,9 @@ describe('SDK-backed CLI groups', () => { relayfile, cleanupJournal: memoryJournal(), } satisfies Partial); + relay.integrations.subscriptions.list.mockResolvedValueOnce([ + { id: 'sub1', events: ['message.created', 'thread.reply'] } as never, + ]); await program.parseAsync(['integration', 'subscribe', '--list'], { from: 'user' }); @@ -712,6 +732,13 @@ describe('SDK-backed CLI groups', () => { lastSuccessAt: null, lastError: null, lastChannelMessageAt: null, + lastDeliverySource: null, + writebackSubscription: { + id: 'sub1', + events: ['message.created', 'thread.reply'], + active: null, + deliveryStatus: null, + }, githubPrIdentityAuthorized: null, }, ], diff --git a/packages/cli/src/cli/lib/agent-registration.ts b/packages/cli/src/cli/lib/agent-registration.ts index 132aa68a98..f8883b9d5c 100644 --- a/packages/cli/src/cli/lib/agent-registration.ts +++ b/packages/cli/src/cli/lib/agent-registration.ts @@ -6,7 +6,7 @@ export const DEFAULT_AGENT_REGISTRATION_TIMEOUT_MS = 15_000; // "bounded registration" into "registration always times out immediately". const MAX_SETTIMEOUT_DELAY_MS = 2_147_483_647; -function normalizeTimeoutMs(timeoutMs: number): number { +export function normalizeTimeoutMs(timeoutMs: number): number { if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { return DEFAULT_AGENT_REGISTRATION_TIMEOUT_MS; } @@ -17,7 +17,7 @@ function normalizeTimeoutMs(timeoutMs: number): number { // close the quote, emit an escaped literal quote, reopen. Without this, a // name containing `$(...)` or backticks would execute command substitution // if a user copies the recovery command as-is into a shell. -function shellQuote(value: string): string { +export function shellQuote(value: string): string { return `'${value.replace(/'/g, `'\\''`)}'`; } diff --git a/packages/cli/src/cli/lib/agent-removal.test.ts b/packages/cli/src/cli/lib/agent-removal.test.ts new file mode 100644 index 0000000000..4de25aebe1 --- /dev/null +++ b/packages/cli/src/cli/lib/agent-removal.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it, vi } from 'vitest'; +import { parseRemovalWaitTimeout, waitForAgentRemoval } from './agent-removal.js'; + +function harness() { + let time = 0; + return { + name: 'worker', + timeoutMs: 1000, + pollIntervalMs: 500, + now: () => time, + sleep: vi.fn(async (ms: number) => { + time += ms; + }), + getAgent: vi.fn(async (): Promise => ({ name: 'worker', status: 'released' })), + listAgents: vi.fn(async () => [{ name: 'worker' }]), + }; +} +describe('registration removal wait', () => { + it('polls released tombstones until confirmed not-found', async () => { + const h = harness(); + h.getAgent.mockResolvedValueOnce({ status: 'released' }).mockRejectedValueOnce({ statusCode: 404 }); + expect(await waitForAgentRemoval(h)).toMatchObject({ cleared: true, waitedMs: 500 }); + expect(h.listAgents).not.toHaveBeenCalled(); + }); + it('uses one deadline membership check to clear persistent tombstones', async () => { + const h = harness(); + h.listAgents.mockResolvedValueOnce([]); + expect(await waitForAgentRemoval(h)).toMatchObject({ cleared: true, waitedMs: 1000 }); + expect(h.listAgents).toHaveBeenCalledTimes(1); + }); + it('never clears on released status alone', async () => { + expect(await waitForAgentRemoval(harness())).toMatchObject({ cleared: false, observedPresent: true }); + }); + it('does not confuse transport/auth errors with presence or deletion', async () => { + const h = harness(); + h.getAgent.mockRejectedValue(new Error('403')); + h.listAgents.mockRejectedValue(new Error('offline')); + expect(await waitForAgentRemoval(h)).toMatchObject({ + cleared: false, + observedPresent: false, + readError: 'offline', + }); + }); + it('does not sleep past the deadline', async () => { + const h = harness(); + h.timeoutMs = 750; + await waitForAgentRemoval(h); + expect(h.sleep.mock.calls.map(([ms]) => ms)).toEqual([500, 250]); + }); + it('bounds never-settling reads including the final list read', async () => { + vi.useFakeTimers(); + try { + const result = waitForAgentRemoval({ + name: 'worker', + timeoutMs: 1000, + getAgent: () => new Promise(() => {}), + listAgents: () => new Promise(() => {}), + }); + await vi.advanceTimersByTimeAsync(2000); + expect(await result).toMatchObject({ cleared: false, observedPresent: false, waitedMs: 2000 }); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe('parseRemovalWaitTimeout', () => { + // Timers take whole milliseconds; a fractional deadline would expire before + // the scheduled sleep, and a sub-millisecond one would wait not at all. + it('floors to whole milliseconds and clamps to the timer limit', () => { + expect(parseRemovalWaitTimeout('1500.9')).toBe(1500); + expect(parseRemovalWaitTimeout(1e12)).toBe(2_147_483_647); + for (const value of ['0.5', '0', '-1', 'abc', 'Infinity']) { + expect(() => parseRemovalWaitTimeout(value)).toThrow('--wait-timeout'); + } + }); +}); diff --git a/packages/cli/src/cli/lib/agent-removal.ts b/packages/cli/src/cli/lib/agent-removal.ts new file mode 100644 index 0000000000..4ce9c0cc1a --- /dev/null +++ b/packages/cli/src/cli/lib/agent-removal.ts @@ -0,0 +1,95 @@ +import { RelayError, safeRelayErrorMessage } from '@agent-relay/sdk'; +import { normalizeTimeoutMs, shellQuote, withDeadline } from './agent-registration.js'; + +export const DEFAULT_REMOVAL_WAIT_TIMEOUT_MS = 30_000; +export const REMOVAL_POLL_INTERVAL_MS = 500; + +export function isNotFoundError(error: unknown): boolean { + if (error instanceof RelayError) return error.code === 'not_found' || error.statusCode === 404; + const status = + error && typeof error === 'object' + ? ((error as { statusCode?: unknown }).statusCode ?? (error as { status?: unknown }).status) + : undefined; + return Number(status) === 404; +} + +/** + * Parse `--wait-timeout` as whole milliseconds within the timer limit; reject + * what the wait would otherwise silently replace with its default. + */ +export function parseRemovalWaitTimeout(value: unknown): number { + const timeoutMs = Math.floor(Number(value)); + if (!Number.isFinite(timeoutMs) || timeoutMs < 1) { + throw new Error( + `--wait-timeout must be a positive whole number of milliseconds (got ${JSON.stringify(value)}).` + ); + } + return normalizeTimeoutMs(timeoutMs); +} + +export interface RemovalResult { + cleared: boolean; + waitedMs: number; + observedPresent: boolean; + readError?: string; +} + +export class AgentRemovalPendingError extends Error { + readonly exitCode = 8; + constructor(name: string) { + super( + `Removal of agent ${JSON.stringify(name)} was accepted, but its registration is still present. Re-check \`agent-relay agent list\` or run \`agent-relay agent remove ${shellQuote(name)} --wait\` before respawning.` + ); + } +} + +export async function waitForAgentRemoval(options: { + name: string; + getAgent: (name: string) => Promise; + listAgents: () => Promise>; + timeoutMs?: number; + pollIntervalMs?: number; + now?: () => number; + sleep?: (ms: number) => Promise; +}): Promise { + const now = options.now ?? Date.now; + const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); + const started = now(); + const budget = normalizeTimeoutMs(options.timeoutMs ?? DEFAULT_REMOVAL_WAIT_TIMEOUT_MS); + const deadline = started + budget; + let observedPresent = false; + let readError: string | undefined; + const result = (cleared: boolean): RemovalResult => ({ + cleared, + waitedMs: now() - started, + observedPresent, + ...(readError ? { readError } : {}), + }); + // Reserve one bounded read for membership corroboration at the deadline. + // A released/offline status still owns the name and is never clearance. + const read = (fn: () => Promise, ms: number) => + withDeadline(fn, () => new Error('Removal verification read timed out.'), ms); + while (now() < deadline) { + try { + await read(() => options.getAgent(options.name), Math.max(1, deadline - now())); + observedPresent = true; + readError = undefined; + } catch (error) { + if (isNotFoundError(error)) return result(true); + readError = safeRelayErrorMessage(error); + } + const remaining = deadline - now(); + if (remaining > 0) + await sleep( + Math.min(normalizeTimeoutMs(options.pollIntervalMs ?? REMOVAL_POLL_INTERVAL_MS), remaining) + ); + } + try { + const agents = await read(options.listAgents, Math.min(1_000, budget)); + if (!agents.some((agent) => agent.name === options.name)) return result(true); + observedPresent = true; + } catch (error) { + readError = safeRelayErrorMessage(error); + } + return result(false); +} diff --git a/packages/cli/src/cli/lib/fleet-live-agents.ts b/packages/cli/src/cli/lib/fleet-live-agents.ts index 41e7056b06..e22da67ada 100644 --- a/packages/cli/src/cli/lib/fleet-live-agents.ts +++ b/packages/cli/src/cli/lib/fleet-live-agents.ts @@ -22,6 +22,11 @@ export const LIVE_AGENT_CAPABILITY_NAME = 'relay:live-agents:v1'; export interface RemoteLiveAgentRead { supported: boolean; + /** + * Whether `agents` is the node's whole live set: the capability is present + * and every entry decoded. Duplicates do not make a roster incomplete. + */ + complete: boolean; agents: RemoteLiveAgent[]; warning?: string; } @@ -30,6 +35,7 @@ export interface RemoteLiveAgentRead { export function readRemoteLiveAgents(node: RelayNode): RemoteLiveAgentRead { let supported = false; let malformed = 0; + let undecoded = false; const names = new Set(); for (const capability of node.capabilities) { if (capability.name !== LIVE_AGENT_CAPABILITY_NAME) continue; @@ -37,11 +43,13 @@ export function readRemoteLiveAgents(node: RelayNode): RemoteLiveAgentRead { const rawNames = capability.metadata?.names; if (!Array.isArray(rawNames)) { malformed += 1; + undecoded = true; continue; } for (const rawName of rawNames) { if (typeof rawName !== 'string' || !rawName || names.has(rawName)) { malformed += 1; + if (typeof rawName !== 'string' || !rawName) undecoded = true; continue; } names.add(rawName); @@ -49,6 +57,7 @@ export function readRemoteLiveAgents(node: RelayNode): RemoteLiveAgentRead { } return { supported, + complete: supported && !undecoded, agents: Array.from(names, (name) => ({ name })).sort((a, b) => a.name.localeCompare(b.name)), ...(malformed > 0 ? { @@ -79,3 +88,22 @@ export function isAvailableFleetNode(node: { const isLive = node.live === undefined ? node.status === 'online' : node.live === true; return isLive && node.handlersLive !== false && !isDirectPseudoNode; } + +export const MAX_LIVE_HEARTBEAT_AGE_MS = 36_000; + +/** + * Clock skew tolerated before a server-stamped heartbeat in the future stops + * counting as fresh. Within it the age clamps to zero; beyond it the local + * clock cannot be trusted to judge the heartbeat at all, and a lagging clock + * would otherwise make an old snapshot look current. + */ +export const MAX_HEARTBEAT_CLOCK_SKEW_MS = 5_000; + +/** Age of a node's last heartbeat, or null when it cannot be judged. */ +export function heartbeatAgeMs(node: { lastHeartbeatAt?: string }, now = Date.now()): number | null { + const timestamp = Date.parse(node.lastHeartbeatAt ?? ''); + if (!Number.isFinite(timestamp)) return null; + const age = now - timestamp; + if (age < -MAX_HEARTBEAT_CLOCK_SKEW_MS) return null; + return Math.max(0, age); +} diff --git a/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts b/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts index 498134efb7..ea9f0a0b8f 100644 --- a/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts +++ b/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts @@ -139,23 +139,28 @@ describe('fleet spawn confirmation is observable from the requester (#1430)', () // MUST-FIRE — a node that reports its failure honestly still surfaced as // success before this change, because nothing read the action result. The // broker's detail (startup exit status and worker log path) must survive. - it.each(['spawn_failed: provider_auth_required', 'spawn_provider_auth_required'])( - 'preserves %s from the node', - async (reason) => { - const { client } = createClient(async (name, id) => ({ - invocation_id: id, - action_name: name, - status: 'failed', - error: `${reason}: run muse on the selected node to log in`, - })); - const error = await client.placement - .spawn(spawnInput({ confirm: true, confirmTimeoutMs: 60, confirmPollIntervalMs: 10 })) - .catch((caught: unknown) => caught); - expect(error).toBeInstanceOf(RelayPlacementError); - expect((error as RelayPlacementError).code).toBe('spawn_failed'); - expect((error as Error).message).toContain(reason); - } - ); + it.each([ + ['spawn_failed: provider_auth_required', 'run muse on the selected node to log in'], + ['spawn_provider_auth_required', 'run muse on the selected node to log in'], + [ + 'spawn_directory_trust_required', + 'Devin requires directory trust; run `devin` in the explicit spawn working directory on the selected node, trust that directory, then retry', + ], + ])('preserves %s and its remediation from the node', async (reason, detail) => { + const { client } = createClient(async (name, id) => ({ + invocation_id: id, + action_name: name, + status: 'failed', + error: `${reason}: ${detail}`, + })); + const error = await client.placement + .spawn(spawnInput({ confirm: true, confirmTimeoutMs: 60, confirmPollIntervalMs: 10 })) + .catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(RelayPlacementError); + expect((error as RelayPlacementError).code).toBe('spawn_failed'); + expect((error as Error).message).toContain(reason); + expect((error as Error).message).toContain(detail); + }); it('fails with spawn_failed and preserves the node-reported detail', async () => { const { client } = createClient(async (name, invocationId) => ({ diff --git a/packages/cli/src/cli/lib/sdk-command.test.ts b/packages/cli/src/cli/lib/sdk-command.test.ts index 01c4ab88bd..847941f466 100644 --- a/packages/cli/src/cli/lib/sdk-command.test.ts +++ b/packages/cli/src/cli/lib/sdk-command.test.ts @@ -1,3 +1,4 @@ +import { pendingSpawnError } from './spawn-liveness.js'; import { describe, expect, it, vi } from 'vitest'; import { RelayPlacementError } from '@agent-relay/sdk'; @@ -87,6 +88,24 @@ describe('runSdk structured error output', () => { }); }); + it('renders pending evidence and honors only known error exit codes', async () => { + const { deps: sdkDeps, errors } = deps(); + await runSdk(sdkDeps, async () => { + throw pendingSpawnError( + 'worker', + { invocationId: 'inv', dispatchState: 'dispatched' }, + { evidence: 'unknown' } + ); + }); + expect(sdkDeps.exit).toHaveBeenCalledWith(8); + expect(errors.join('')).toContain('"code":"spawn_unconfirmed"'); + expect(errors.join('')).toContain('"liveness":{"evidence":"unknown"}'); + await runSdk(sdkDeps, async () => { + throw Object.assign(new Error('plain'), { exitCode: 0 }); + }); + expect(sdkDeps.exit).toHaveBeenLastCalledWith(1); + }); + it('does not attach a structured receipt for a plain Error', async () => { const { deps: sdkDeps, errors } = deps(); diff --git a/packages/cli/src/cli/lib/sdk-command.ts b/packages/cli/src/cli/lib/sdk-command.ts index cdf84799dd..74b2a345d6 100644 --- a/packages/cli/src/cli/lib/sdk-command.ts +++ b/packages/cli/src/cli/lib/sdk-command.ts @@ -7,6 +7,8 @@ import { type AgentRelayAgent, } from '@agent-relay/sdk'; +import { AgentRemovalPendingError } from './agent-removal.js'; +import { FleetSpawnError } from './spawn-liveness.js'; import { defaultExit } from './exit.js'; import { createAgentRelay, createWorkspaceRelay, type SdkClientOptions } from './sdk-client.js'; import { sanitizedSpawnReceipt } from './spawn-lifecycle.js'; @@ -77,7 +79,7 @@ export async function runSdk(deps: SdkCommandDeps, fn: () => Promise): Pro await fn(); } catch (err) { const message = safeRelayErrorMessage(err); - if (err instanceof RelayPlacementError) { + if (err instanceof RelayPlacementError || err instanceof FleetSpawnError) { const structured = { error: { code: err.code, @@ -86,6 +88,10 @@ export async function runSdk(deps: SdkCommandDeps, fn: () => Promise): Pro ...(err.node ? { node: err.node } : {}), ...(err.dispatchState ? { dispatchState: err.dispatchState } : {}), ...(err.receipt ? { receipt: sanitizedSpawnReceipt(err.receipt) } : {}), + ...(err instanceof FleetSpawnError && err.liveness ? { liveness: err.liveness } : {}), + ...(err instanceof FleetSpawnError && err.diagnostic + ? { diagnostic: safeRelayErrorMessage(new Error(err.diagnostic)) } + : {}), message, }, }; @@ -93,6 +99,6 @@ export async function runSdk(deps: SdkCommandDeps, fn: () => Promise): Pro } else { deps.error(message); } - deps.exit(1); + deps.exit(err instanceof FleetSpawnError || err instanceof AgentRemovalPendingError ? err.exitCode : 1); } } diff --git a/packages/cli/src/cli/lib/spawn-liveness.test.ts b/packages/cli/src/cli/lib/spawn-liveness.test.ts new file mode 100644 index 0000000000..45f7bf7787 --- /dev/null +++ b/packages/cli/src/cli/lib/spawn-liveness.test.ts @@ -0,0 +1,249 @@ +import { describe, expect, it, vi } from 'vitest'; +import { RelayPlacementError, type RelayNode } from '@agent-relay/sdk'; +import { + classifySpawnFailure, + mayStillBeRunning, + pendingSpawnError, + probeSpawnLiveness, + terminalSpawnOutcome, +} from './spawn-liveness.js'; + +const now = Date.parse('2026-10-04T12:00:00Z'); +const node = (name = 'target', age = 0): RelayNode => ({ + name, + status: 'online', + lastHeartbeatAt: new Date(now - age).toISOString(), + capabilities: [{ name: 'relay:live-agents:v1', metadata: { names: ['worker'] } }], +}); +function client(nodes: RelayNode[] = []) { + return { + nodes: { + get: vi.fn(async (name: string) => nodes.find((n) => n.name === name) ?? null), + list: vi.fn(async () => nodes), + }, + agents: { + get: vi.fn(async (): Promise => { + throw { status: 404 }; + }), + }, + }; +} + +describe('spawn liveness resolution', () => { + it.each([ + [node(), 'live'], + [node('other'), 'live_elsewhere'], + [node('target', 36_001), 'stale'], + [{ ...node(), lastHeartbeatAt: undefined }, 'stale'], + // A heartbeat stamped within the tolerated clock skew is still fresh, but + // one far in the future cannot be judged by this clock (relay#1930 review). + [node('target', -4_000), 'live'], + [node('target', -60_000), 'stale'], + [{ ...node(), status: 'offline' }, 'absent'], + ] as const)('requires fresh target-node evidence (%j)', async (record, evidence) => { + const workspace = client([record]); + expect( + await probeSpawnLiveness({ + name: 'worker', + targetNode: 'target', + createClient: () => workspace, + now: () => now, + }) + ).toMatchObject({ evidence }); + if (evidence === 'live') expect(workspace.nodes.list).not.toHaveBeenCalled(); + }); + // relay#1930 review (cubic): the clock is read when a heartbeat is judged, + // so read latency cannot make an old heartbeat look fresh. + // relay#1930 review (cubic): a failed invocation re-read must survive into + // roster-derived evidence, as it already does for registered/absent. + it('keeps a failed invocation read on live and stale evidence', async () => { + for (const [record, evidence] of [ + [node(), 'live'], + [node('target', 36_001), 'stale'], + ] as const) { + expect( + await probeSpawnLiveness({ + name: 'worker', + targetNode: 'target', + getInvocation: async () => { + throw new Error('invocation read failed'); + }, + createClient: () => client([record]), + now: () => now, + }) + ).toMatchObject({ evidence, readError: expect.stringContaining('invocation read failed') }); + } + }); + it('measures heartbeat age after the bounded node reads, not before', async () => { + let clock = now; + const workspace = client([node('target', 30_000)]); + workspace.nodes.get.mockImplementation(async (name: string) => { + clock = now + 10_000; + return name === 'target' ? node('target', 30_000) : null; + }); + expect( + await probeSpawnLiveness({ + name: 'worker', + targetNode: 'target', + createClient: () => workspace, + now: () => clock, + }) + ).toMatchObject({ evidence: 'stale', heartbeatAgeMs: 40_000 }); + }); + // relay#1930 review (cubic): a stale or unavailable target entry must not + // hide a fresh same-named worker on another node. + it.each([ + ['stale target', node('target', 36_001)], + ['unavailable target', { ...node(), status: 'offline' }], + ] as const)( + 'scans the fleet past a %s and prefers fresh live_elsewhere evidence', + async (_label, target) => { + const workspace = client([target, node('other')]); + workspace.nodes.get.mockResolvedValue(target); + workspace.nodes.list.mockResolvedValue([node('other')]); + expect( + await probeSpawnLiveness({ + name: 'worker', + targetNode: 'target', + createClient: () => workspace, + now: () => now, + }) + ).toMatchObject({ evidence: 'live_elsewhere', node: 'other' }); + expect(workspace.nodes.list).toHaveBeenCalledOnce(); + } + ); + it('points a live_elsewhere recovery check at the whole fleet, not the requested node', () => { + const pending = pendingSpawnError( + 'worker', + { invocationId: 'inv', node: 'target' }, + { evidence: 'live_elsewhere', node: 'other', heartbeatAgeMs: 0 } + ); + expect(pending.message).toContain('check `agent-relay fleet agent list` first'); + const targeted = pendingSpawnError( + 'worker', + { invocationId: 'inv', node: 'target' }, + { evidence: 'stale' } + ); + expect(targeted.message).toContain('fleet agent list --node'); + }); + it('folds construction errors into unknown without replacing accepted dispatch evidence', async () => { + const result = await probeSpawnLiveness({ + name: 'worker', + createClient: () => { + throw new Error('no key'); + }, + }); + expect(result).toEqual({ evidence: 'unknown', readError: 'no key' }); + const pending = pendingSpawnError('worker', { invocationId: 'inv', dispatchState: 'dispatched' }, result); + expect(pending.exitCode).toBe(8); + expect(pending.message).toContain('--wait'); + expect(pending.message).not.toContain('failed'); + expect(mayStillBeRunning(pending)).toBe(true); + }); + it('preserves SDK unconfirmed errors as potentially running', () => { + expect( + mayStillBeRunning( + new RelayPlacementError('spawn_unconfirmed', 'pending', { + capability: 'spawn', + attempts: 1, + state: 'unconfirmed_may_be_running', + }) + ) + ).toBe(true); + expect(mayStillBeRunning(new Error('failure'))).toBe(false); + }); + it('reads the invocation before constructing a workspace client', async () => { + const createClient = vi.fn(() => { + throw new Error('no key'); + }); + const invocation = { status: 'completed', output: { spawned: true, ready: false } }; + const result = await probeSpawnLiveness({ + name: 'worker', + getInvocation: async () => invocation, + createClient, + }); + expect(result).toEqual({ evidence: 'invocation_terminal', invocation }); + expect(createClient).not.toHaveBeenCalled(); + expect(terminalSpawnOutcome(invocation, {}, false)).toMatchObject({ placement: { confirmed: true } }); + expect(() => terminalSpawnOutcome(invocation, {}, true)).toThrow('proof'); + expect(() => terminalSpawnOutcome({ status: 'failed', error: 'launch error' }, {}, false)).toThrow( + 'launch error' + ); + }); + it('distinguishes registered, absent and unavailable reads', async () => { + const workspace = client(); + workspace.agents.get.mockResolvedValueOnce({ status: 'released' }); + expect(await probeSpawnLiveness({ name: 'worker', createClient: () => workspace })).toMatchObject({ + evidence: 'registered', + }); + expect(await probeSpawnLiveness({ name: 'worker', createClient: () => workspace })).toMatchObject({ + evidence: 'absent', + }); + workspace.agents.get.mockRejectedValueOnce(new Error('403 forbidden')); + expect(await probeSpawnLiveness({ name: 'worker', createClient: () => workspace })).toMatchObject({ + evidence: 'unknown', + }); + }); + // relay#1930 review (cubic): an available node whose heartbeat roster is + // missing or undecodable may still run the worker, so a 404 on the agent + // registration cannot prove it absent. + it.each([ + ['an unsupported', []], + ['a malformed', [{ name: 'relay:live-agents:v1', metadata: { names: 'worker' } }]], + ] as const)('does not report absent beside %s heartbeat roster', async (_label, capabilities) => { + const workspace = client([{ ...node('other'), capabilities: [...capabilities] } as RelayNode]); + expect( + await probeSpawnLiveness({ name: 'worker', createClient: () => workspace, now: () => now }) + ).toMatchObject({ evidence: 'unknown' }); + }); + it.each(['get', 'list'] as const)( + 'does not report absent when the node %s read failed', + async (failing) => { + const workspace = client([node()]); + workspace.nodes[failing].mockRejectedValue(new Error('roster unavailable')); + const result = await probeSpawnLiveness({ + name: 'worker', + ...(failing === 'get' ? { targetNode: 'target' } : {}), + createClient: () => workspace, + now: () => now, + }); + expect(result).toEqual({ evidence: 'unknown', readError: 'roster unavailable' }); + } + ); + it('tells the operator a heartbeat cannot attribute a same-named worker to this spawn', () => { + const pending = pendingSpawnError( + 'worker', + { invocationId: 'inv', node: 'target', dispatchState: 'dispatched' }, + { evidence: 'live', node: 'target', heartbeatAgeMs: 0 } + ); + expect(pending.code).toBe('spawn_unconfirmed'); + expect(pending.message).toContain('may be an earlier worker'); + expect(pending.message).toContain('Invocation: inv'); + }); + it('bounds a hung invocation read and still probes worker evidence', async () => { + vi.useFakeTimers(); + try { + const result = probeSpawnLiveness({ + name: 'worker', + getInvocation: () => new Promise(() => {}), + createClient: () => client([node()]), + now: () => now, + }); + await vi.advanceTimersByTimeAsync(2_000); + expect(await result).toMatchObject({ evidence: 'live' }); + } finally { + vi.useRealTimers(); + } + }); + it('classifies only broker pre-registration collisions', () => { + expect( + classifySpawnFailure( + "failed to pre-register worker 'worker': agent 'worker' already exists and registration is create-only; use a unique agent name" + ) + ).toBe('spawn_name_taken'); + expect(classifySpawnFailure('failed to create sandbox: directory already exists')).toBeUndefined(); + expect( + classifySpawnFailure("failed to pre-register worker 'worker': cache entry already exists") + ).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/cli/lib/spawn-liveness.ts b/packages/cli/src/cli/lib/spawn-liveness.ts new file mode 100644 index 0000000000..a506ac9980 --- /dev/null +++ b/packages/cli/src/cli/lib/spawn-liveness.ts @@ -0,0 +1,248 @@ +import { RelayPlacementError, safeRelayErrorMessage, type RelayNode } from '@agent-relay/sdk'; +import { shellQuote, withDeadline } from './agent-registration.js'; +import { isNotFoundError } from './agent-removal.js'; +import { + heartbeatAgeMs, + isAvailableFleetNode, + MAX_LIVE_HEARTBEAT_AGE_MS, + readRemoteLiveAgents, +} from './fleet-live-agents.js'; + +export interface SpawnLiveness { + evidence: 'invocation_terminal' | 'live' | 'live_elsewhere' | 'stale' | 'registered' | 'absent' | 'unknown'; + node?: string; + heartbeatAgeMs?: number | null; + readError?: string; + invocation?: Record; +} + +interface SpawnContext { + invocationId?: string; + node?: string; + dispatchState?: string; + receipt?: Record; +} + +export class FleetSpawnError extends Error { + readonly state: 'pending' | 'failed'; + readonly exitCode: number; + readonly invocationId?: string; + readonly node?: string; + readonly dispatchState?: string; + readonly receipt?: Record; + constructor( + // `spawn_unconfirmed` is the established public code for an accepted spawn + // whose outcome is unknown (relay#1563); keep it so callers keyed on it keep + // working, and carry the pending outcome in `state` and the exit code. + readonly code: 'spawn_unconfirmed' | 'spawn_name_taken', + message: string, + context: SpawnContext, + readonly liveness?: SpawnLiveness, + readonly diagnostic?: string + ) { + super(message); + this.state = code === 'spawn_unconfirmed' ? 'pending' : 'failed'; + this.exitCode = code === 'spawn_unconfirmed' ? 8 : 1; + this.invocationId = context.invocationId; + this.node = context.node; + this.dispatchState = context.dispatchState; + this.receipt = context.receipt; + } +} + +export function mayStillBeRunning(error: unknown): boolean { + return ( + (error instanceof RelayPlacementError && error.state === 'unconfirmed_may_be_running') || + (error instanceof FleetSpawnError && error.code === 'spawn_unconfirmed') + ); +} + +export function pendingSpawnError( + name: string, + context: SpawnContext, + liveness: SpawnLiveness, + diagnostic?: string +): FleetSpawnError { + // A worker found elsewhere is invisible to a node-scoped check. + const nodeOption = + context.node && liveness.evidence !== 'live_elsewhere' ? ` --node ${shellQuote(context.node)}` : ''; + // Heartbeats carry worker names, not invocation IDs, so presence cannot be + // attributed to this spawn: it may be a worker that was already running. + const attribution = + liveness.evidence === 'live' || liveness.evidence === 'live_elsewhere' || liveness.evidence === 'stale' + ? `A worker with this name is listed by a node heartbeat, but heartbeats do not identify the invocation that started it, so it may be an earlier worker rather than this spawn. ` + : ''; + return new FleetSpawnError( + 'spawn_unconfirmed', + `Spawn of ${JSON.stringify(name)} was accepted; its outcome is pending. ` + + `Evidence: ${liveness.evidence}${liveness.node ? ` on ${JSON.stringify(liveness.node)}` : ''}. ` + + attribution + + `Invocation: ${context.invocationId ?? 'unavailable'}; dispatch: ${context.dispatchState ?? 'unknown'}. ` + + `Do not retry blindly: check \`agent-relay fleet agent list${nodeOption}\` first. ` + + `Only if no worker is running and you intend to reclaim the name, run \`agent-relay agent remove ${shellQuote(name)} --wait\` and wait for clearance before respawning.`, + context, + liveness, + diagnostic + ); +} + +export function classifySpawnFailure(message: string): 'spawn_name_taken' | undefined { + // Anchored to the broker formatter; unrelated "already exists" failures are not registration collisions. + return /(?:^|:\s)failed to pre-register worker '[^\n]+': agent '[^\n]+' already exists and registration is create-only(?:;|$)/i.test( + message + ) + ? 'spawn_name_taken' + : undefined; +} + +export function nameTakenSpawnError(name: string, error: Error & SpawnContext): FleetSpawnError { + return new FleetSpawnError( + 'spawn_name_taken', + `${safeRelayErrorMessage(error)} A removal or cleanup from a previous unsuccessful spawn may still be in flight. Re-check \`agent-relay agent list\`; if the name is present with no worker running, run \`agent-relay agent remove ${shellQuote(name)} --wait\`, wait for clearance, then retry.`, + error + ); +} + +/** Reads are bounded and client construction is fallible (agent-token-only callers are supported). */ +export async function probeSpawnLiveness(options: { + name: string; + targetNode?: string; + getInvocation?: () => Promise; + createClient: () => { + nodes: { get: (name: string) => Promise; list: () => Promise }; + agents: { get: (name: string) => Promise }; + }; + now?: () => number; +}): Promise { + let readError: string | undefined; + let rosterUnread = false; + let rosterIncomplete = false; + const read = (fn: () => Promise) => + withDeadline(fn, () => new Error('Spawn evidence read timed out.'), 2_000); + if (options.getInvocation) { + try { + const value = await read(options.getInvocation); + if (value && typeof value === 'object') { + const invocation = value as Record; + if ( + /^(completed|succeeded|success|failed|error|cancelled|canceled|denied)$/i.test( + String(invocation.status) + ) + ) { + return { evidence: 'invocation_terminal', invocation }; + } + } + } catch (error) { + readError = safeRelayErrorMessage(error); + } + } + try { + const client = options.createClient(); + try { + // Read the clock when a heartbeat is judged, after the reads: sampling it + // first would let read latency make an old heartbeat look fresh. + const clock = () => options.now?.() ?? Date.now(); + const target = options.targetNode ? await read(() => client.nodes.get(options.targetNode!)) : undefined; + const nodes = options.targetNode && target ? [target] : await read(() => client.nodes.list()); + // Prefer the targeted read; consult the fleet unless the target is an + // available node whose fresh heartbeat claims this name. + const targetAge = target ? heartbeatAgeMs(target, clock()) : null; + if ( + options.targetNode && + target && + !( + isAvailableFleetNode(target) && + targetAge !== null && + targetAge <= MAX_LIVE_HEARTBEAT_AGE_MS && + readRemoteLiveAgents(target).agents.some((a) => a.name === options.name) + ) + ) { + nodes.push(...(await read(() => client.nodes.list()))); + } + let weaker: SpawnLiveness | undefined; + const now = clock(); + for (const node of nodes) { + if (!isAvailableFleetNode(node)) continue; + const live = readRemoteLiveAgents(node); + // A node whose heartbeat roster is missing or undecodable may still be + // running the worker, so a later missing registration cannot prove absence. + if (!live.complete) rosterIncomplete = true; + if (!live.agents.some((agent) => agent.name === options.name)) continue; + const age = heartbeatAgeMs(node, now); + const evidence = + age === null || age > MAX_LIVE_HEARTBEAT_AGE_MS + ? 'stale' + : options.targetNode && node.name !== options.targetNode + ? 'live_elsewhere' + : 'live'; + // Keep a failed invocation read visible next to roster evidence. + const result: SpawnLiveness = { + evidence, + node: node.name, + heartbeatAgeMs: age, + ...(readError ? { readError } : {}), + }; + if (evidence === 'live') return result; + // Fresh evidence elsewhere outranks a stale entry. + if (!weaker || (weaker.evidence === 'stale' && evidence === 'live_elsewhere')) weaker = result; + } + if (weaker) return weaker; + } catch (error) { + readError = safeRelayErrorMessage(error); + rosterUnread = true; + } + try { + await read(() => client.agents.get(options.name)); + return { evidence: 'registered', ...(readError ? { readError } : {}) }; + } catch (error) { + // `absent` claims no worker was observed. A missing registration only + // proves that when the node roster was actually read. + if (isNotFoundError(error) && !rosterUnread && !rosterIncomplete) + return { evidence: 'absent', ...(readError ? { readError } : {}) }; + if (!isNotFoundError(error)) readError = safeRelayErrorMessage(error); + } + } catch (error) { + readError = safeRelayErrorMessage(error); + } + return { evidence: 'unknown', readError }; +} + +/** Preserve the original request's launch/readiness contract on the final invocation read. */ +export function terminalSpawnOutcome( + invocation: Record, + context: SpawnContext, + requireReady: boolean +): Record { + const output = invocation.output as Record | undefined; + if ( + !/^(completed|succeeded|success)$/i.test(String(invocation.status)) || + output?.spawned !== true || + typeof output.ready !== 'boolean' || + (requireReady && output.ready !== true) + ) { + throw new RelayPlacementError( + 'spawn_failed', + typeof invocation.error === 'string' + ? invocation.error + : 'Spawn completed without the requested launch/readiness proof.', + { + capability: 'spawn', + attempts: 1, + state: 'failed', + invocationId: context.invocationId, + node: context.node, + dispatchState: context.dispatchState as 'dispatched' | 'not_dispatched' | 'unknown', + receipt: invocation, + } + ); + } + return { + ...invocation, + placement: { + state: output.ready ? 'ready' : 'accepted', + confirmed: true, + invocationId: context.invocationId, + dispatchState: context.dispatchState, + }, + }; +} diff --git a/packages/cli/src/cli/lib/task-input.test.ts b/packages/cli/src/cli/lib/task-input.test.ts new file mode 100644 index 0000000000..1aa858edac --- /dev/null +++ b/packages/cli/src/cli/lib/task-input.test.ts @@ -0,0 +1,105 @@ +import { mkdtemp, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it, expect } from 'vitest'; +import { + readTaskInput, + MAX_INJECTION_BODY_BYTES, + MAX_TASK_BODY_BYTES, + MAX_ARGV_TASK_BYTES, + MAX_NATIVE_TASK_BYTES, +} from './task-input.js'; +describe('task input', () => { + it('requires exactly one source for fleet, allows neither locally', async () => { + await expect(readTaskInput(undefined, undefined, true)).rejects.toThrow('exactly one'); + await expect(readTaskInput('hi', 'brief.md')).rejects.toThrow('exactly one'); + expect(await readTaskInput(undefined, undefined)).toBeUndefined(); + }); + it('reads requester files without stripping whitespace or normalizing CRLF', async () => { + const dir = await mkdtemp(join(tmpdir(), 'relay-task-')); + try { + const file = join(dir, 'brief.md'); + await writeFile(file, ' héllo\r\nworld\n'); + expect(await readTaskInput(undefined, file, true)).toBe(' héllo\r\nworld\n'); + // A leading BOM is file text like any other, as it would be via --task. + await writeFile(file, 'brief\n'); + expect(await readTaskInput(undefined, file, true)).toBe('brief\n'); + await writeFile(file, ' '); + await expect(readTaskInput(undefined, file)).rejects.toThrow('empty'); + await expect(readTaskInput(undefined, join(dir, 'missing'))).rejects.toThrow(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + // Only one byte past the limit is ever read, so an oversized or endless + // --task-file fails cheaply instead of being materialized in memory. + it.skipIf(process.platform === 'win32')( + 'reads at most one byte past the limit from a task file', + async () => { + await expect(readTaskInput(undefined, '/dev/zero')).rejects.toThrow('UTF-8 bytes'); + await expect(readTaskInput(undefined, '/dev/zero', false, { cli: 'muse' })).rejects.toThrow('argv'); + }, + 2000 + ); + it('counts UTF-8 bytes', async () => { + expect(await readTaskInput('a'.repeat(MAX_TASK_BODY_BYTES), undefined)).toHaveLength(MAX_TASK_BODY_BYTES); + await expect(readTaskInput('é'.repeat(MAX_TASK_BODY_BYTES), undefined)).rejects.toThrow('UTF-8 bytes'); + }); + // The broker caps the formatted envelope at 16 KiB, so a body at that size + // would launch a worker and only then fail; leave room for the envelope. + it('rejects a body that only fits the PTY limit before its envelope is added', async () => { + expect(MAX_TASK_BODY_BYTES).toBeLessThan(MAX_INJECTION_BODY_BYTES); + await expect(readTaskInput('a'.repeat(MAX_INJECTION_BODY_BYTES), undefined)).rejects.toThrow('envelope'); + await expect(readTaskInput('a'.repeat(MAX_TASK_BODY_BYTES + 1), undefined)).rejects.toThrow( + 'UTF-8 bytes' + ); + }); + // Muse takes its startup task as one argv entry, never through the PTY + // envelope, so the broker's 16 KiB argv limit applies instead. + it('applies the Muse argv limit, not the PTY body limit, to Muse tasks', async () => { + const between = 'a'.repeat(MAX_TASK_BODY_BYTES + 1); + for (const cli of ['muse', 'MUSE.exe', '/usr/local/bin/muse', 'C:\\Tools\\Muse.CMD']) { + expect(await readTaskInput(between, undefined, false, { cli })).toBe(between); + await expect( + readTaskInput('a'.repeat(MAX_ARGV_TASK_BYTES + 1), undefined, false, { cli }) + ).rejects.toThrow('argv'); + } + await expect(readTaskInput(between, undefined, false, { cli: 'musey' })).rejects.toThrow('envelope'); + await expect(readTaskInput(between, undefined, false, { cli: 'claude' })).rejects.toThrow('envelope'); + }); + // A native runtime receives its task as a delivery frame: neither the PTY + // envelope nor the argv ceiling applies, only a bound on what is read. + it('does not apply PTY or argv limits to native-runtime tasks', async () => { + const overArgv = 'a'.repeat(MAX_ARGV_TASK_BYTES + 1); + const native = { cli: 'claude', runtime: 'native' } as const; + expect(await readTaskInput(overArgv, undefined, false, native)).toBe(overArgv); + await expect( + readTaskInput('a'.repeat(MAX_NATIVE_TASK_BYTES + 1), undefined, false, native) + ).rejects.toThrow(`${MAX_NATIVE_TASK_BYTES} UTF-8 bytes`); + const dir = await mkdtemp(join(tmpdir(), 'relay-task-')); + try { + const file = join(dir, 'brief.md'); + await writeFile(file, overArgv); + expect(await readTaskInput(undefined, file, false, native)).toBe(overArgv); + await expect(readTaskInput(undefined, file, false, { cli: 'claude', runtime: 'pty' })).rejects.toThrow( + 'envelope' + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + it('rejects a task file that is not valid UTF-8 instead of altering its text', async () => { + const dir = await mkdtemp(join(tmpdir(), 'relay-task-')); + try { + const file = join(dir, 'brief.md'); + await writeFile(file, Buffer.from([0x68, 0x69, 0xff, 0x0a])); + await expect(readTaskInput(undefined, file)).rejects.toThrow('not valid UTF-8'); + // An oversized file whose cut lands inside a character is reported by + // size, not as invalid UTF-8. + await writeFile(file, 'é'.repeat(MAX_TASK_BODY_BYTES)); + await expect(readTaskInput(undefined, file)).rejects.toThrow('UTF-8 bytes'); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/src/cli/lib/task-input.ts b/packages/cli/src/cli/lib/task-input.ts new file mode 100644 index 0000000000..4618d58a26 --- /dev/null +++ b/packages/cli/src/cli/lib/task-input.ts @@ -0,0 +1,123 @@ +import { open } from 'node:fs/promises'; + +/** Broker-wide PTY ceiling for the formatted envelope; a harness may reject a lower effective wire limit. */ +export const MAX_INJECTION_BODY_BYTES = 16 * 1024; +/** Room the broker keeps for attribution and its MCP reminder around a body. */ +export const ENVELOPE_RESERVE_BYTES = 2 * 1024; +/** Largest body that still fits {@link MAX_INJECTION_BODY_BYTES} once formatted. */ +export const MAX_TASK_BODY_BYTES = MAX_INJECTION_BODY_BYTES - ENVELOPE_RESERVE_BYTES; +/** + * The broker's portable ceiling for a prompt passed as one process argument + * (Muse's startup task, a headless provider's message). + */ +export const MAX_ARGV_TASK_BYTES = 16 * 1024; + +/** + * A native runtime receives its task as a delivery frame, so no wire limit + * applies; this only bounds how much of a `--task-file` the CLI will read. + */ +export const MAX_NATIVE_TASK_BYTES = 1024 * 1024; + +/** Who receives a task: the CLI, and the runtime once it is resolved. */ +export interface TaskTarget { + cli: string; + /** `native` delivers the task as a frame, without the PTY envelope; omitted means PTY. */ + runtime?: 'native' | 'pty'; +} + +/** Mirrors the broker's `is_muse_executable`: the basename, case-insensitive, minus a Windows launcher suffix. */ +function isMuseExecutable(cli: string): boolean { + const basename = cli.split(/[\\/]/).pop() || cli; + return basename.toLowerCase().replace(/\.(exe|cmd|bat)$/, '') === 'muse'; +} + +type TaskLimit = 'pty' | 'argv' | 'native'; + +/** Muse takes its task as one argument; a native runtime as a frame; everything else through the PTY envelope. */ +function taskLimitKind(target: TaskTarget | undefined): TaskLimit { + if (target?.runtime === 'native') return 'native'; + if (target && isMuseExecutable(target.cli)) return 'argv'; + return 'pty'; +} + +export async function readTaskInput( + task: unknown, + taskFile: unknown, + required = false, + target?: TaskTarget +): Promise { + if (task !== undefined && taskFile !== undefined) + throw new Error('Specify exactly one of --task or --task-file'); + if (task === undefined && taskFile === undefined) { + if (required) throw new Error('Specify exactly one of --task or --task-file'); + return undefined; + } + const text = + taskFile === undefined ? task : await readTaskFile(String(taskFile), taskByteLimit(target), target); + if (typeof text !== 'string' || !text.trim()) throw new Error('Task must not be empty'); + validateTaskSize(text, target); + return text; +} + +function taskByteLimit(target: TaskTarget | undefined): number { + const kind = taskLimitKind(target); + return kind === 'pty' ? MAX_TASK_BODY_BYTES : kind === 'argv' ? MAX_ARGV_TASK_BYTES : MAX_NATIVE_TASK_BYTES; +} + +/** Validate a task as its target receives it; recheck after anything is appended to it. */ +export function validateTaskSize(text: string, target?: TaskTarget): void { + validateTaskBytes(Buffer.byteLength(text, 'utf8'), target); +} + +function validateTaskBytes(bytes: number, target: TaskTarget | undefined): void { + const kind = taskLimitKind(target); + if (kind === 'pty' && bytes > MAX_TASK_BODY_BYTES) throwInjectionTooLarge(); + if (kind === 'argv') validateArgvTaskSize(bytes); + if (kind === 'native' && bytes > MAX_NATIVE_TASK_BYTES) { + throw new Error( + `Task exceeds ${MAX_NATIVE_TASK_BYTES} UTF-8 bytes; use a brief file on the node and send a short pointer` + ); + } +} + +/** + * Read a UTF-8 task file, never more than one byte past `limit`: that is enough + * to reject an oversized file without materializing all of it. Size is checked + * before decoding, since the cut can split a character, and malformed UTF-8 is + * rejected rather than replaced, so the agent never gets different text. + */ +async function readTaskFile(path: string, limit: number, target: TaskTarget | undefined): Promise { + const handle = await open(path, 'r'); + try { + const maxBytes = limit + 1; + const buffer = Buffer.alloc(maxBytes); + let length = 0; + while (length < maxBytes) { + const { bytesRead } = await handle.read(buffer, length, maxBytes - length, null); + if (bytesRead === 0) break; + length += bytesRead; + } + if (length > limit) validateTaskBytes(length, target); + try { + return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(buffer.subarray(0, length)); + } catch { + throw new Error(`Task file ${path} is not valid UTF-8`); + } + } finally { + await handle.close(); + } +} + +function validateArgvTaskSize(bytes: number): void { + if (bytes > MAX_ARGV_TASK_BYTES) { + throw new Error( + `Task exceeds the ${MAX_ARGV_TASK_BYTES}-byte portable argv limit for a single process argument; use a brief file on the node and send a short pointer` + ); + } +} + +function throwInjectionTooLarge(): never { + throw new Error( + `Task exceeds ${MAX_TASK_BODY_BYTES} UTF-8 bytes (the ${MAX_INJECTION_BODY_BYTES}-byte PTY limit less room for the message envelope); use a brief file on the node and send a short pointer` + ); +} diff --git a/packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts b/packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts index 56c7b075d7..93df18ccef 100644 --- a/packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts +++ b/packages/cli/src/cli/mcp/messaging-tools.protocol.test.ts @@ -313,3 +313,107 @@ describe('messaging delivery receipts over MCP', () => { } }); }); + +describe('channel and thread writes over MCP', () => { + function gatedWrite() { + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let n = 0; + const write = vi.fn(async () => { + await gate; + n += 1; + return { id: `message-${n}` }; + }); + return { write, release }; + } + + async function connect(getAgentClient: Parameters[1]) { + const server = new McpServer({ name: 'write-test', version: '1' }); + registerMessagingTools(server, getAgentClient); + const client = new Client({ name: 'write-client', version: '1' }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + return { + client, + close: async () => { + await client.close(); + await server.close(); + }, + }; + } + + it.each([ + ['post_message', 'send', { channel: 'events', text: 'ACK' }], + ['reply_to_thread', 'reply', { message_id: 'provider-parent', text: 'ACK' }], + ] as const)( + 'keeps independent parallel %s calls with identical text separate', + async (tool, method, args) => { + const { write, release } = gatedWrite(); + const { client, close } = await connect(() => ({ [method]: write }) as never); + try { + const first = client.callTool({ name: tool, arguments: args }); + const second = client.callTool({ name: tool, arguments: args }); + await vi.waitFor(() => expect(write).toHaveBeenCalledTimes(2)); + release(); + const results = await Promise.all([first, second]); + expect(results.map((result) => result.isError)).toEqual([undefined, undefined]); + expect(results[0].structuredContent).not.toEqual(results[1].structuredContent); + } finally { + release(); + await close(); + } + } + ); + + it.each([ + ['post_message', 'send', { channel: 'events', text: 'ACK' }], + ['reply_to_thread', 'reply', { message_id: 'provider-parent', text: 'ACK' }], + ] as const)('joins %s calls that share an idempotency key and forwards it', async (tool, method, args) => { + const { write, release } = gatedWrite(); + const { client, close } = await connect(() => ({ [method]: write }) as never); + try { + const keyed = { ...args, idempotency_key: ' retry-1 ' }; + const first = client.callTool({ name: tool, arguments: keyed }); + const retry = client.callTool({ name: tool, arguments: keyed }); + await vi.waitFor(() => expect(write).toHaveBeenCalledTimes(1)); + release(); + const results = await Promise.all([first, retry]); + expect(results[0].structuredContent).toEqual(results[1].structuredContent); + expect(write).toHaveBeenCalledTimes(1); + expect(write.mock.calls[0]).toContainEqual(expect.objectContaining({ idempotencyKey: 'retry-1' })); + } finally { + release(); + await close(); + } + }); + + it('keeps an in-flight write on the identity that was active when it was called', async () => { + const alice = gatedWrite(); + const bob = gatedWrite(); + let current = 'alice'; + const { client, close } = await connect( + () => ({ reply: current === 'alice' ? alice.write : bob.write }) as never + ); + try { + const pending = client.callTool({ + name: 'reply_to_thread', + arguments: { message_id: 'parent', text: 'ACK' }, + }); + await vi.waitFor(() => expect(alice.write).toHaveBeenCalledTimes(1)); + current = 'bob'; + alice.release(); + bob.release(); + await pending; + await client.callTool({ name: 'reply_to_thread', arguments: { message_id: 'parent', text: 'ACK' } }); + expect(alice.write).toHaveBeenCalledTimes(1); + expect(bob.write).toHaveBeenCalledTimes(1); + } finally { + alice.release(); + bob.release(); + await close(); + } + }); +}); diff --git a/packages/cli/src/cli/mcp/messaging-tools.ts b/packages/cli/src/cli/mcp/messaging-tools.ts index 02773fcc02..54d6fc51a4 100644 --- a/packages/cli/src/cli/mcp/messaging-tools.ts +++ b/packages/cli/src/cli/mcp/messaging-tools.ts @@ -23,7 +23,7 @@ import { } from '../lib/attachments.js'; import { jsonContent, jsonResult, textContent } from './tool-results.js'; import { identityOverrideInputShape, messageResult } from './tool-shapes.js'; -import { McpRequestReplay } from './request-replay.js'; +import { McpRequestReplay, replayScopeOf } from './request-replay.js'; import type { AgentClientLike } from './types.js'; const directMessageResult = z.looseObject({ @@ -74,6 +74,19 @@ function resolveEmoji(input: string): string { return aliases[normalized] ?? input; } +const idempotencyKeyInput = z + .string() + // Relaycast trims this key upstream, so trim before both the local replay + // cache and the forwarded call to keep one logical send on one key; a + // whitespace-only key would otherwise become an unkeyed send. + .trim() + .min(1) + .max(255) + .optional() + .describe( + 'Stable key for retrying this same message after a lost response; use a new key for a new message. Surrounding whitespace is trimmed and a whitespace-only key is rejected.' + ); + function requireFiles(client: AgentClientLike): RelayFilesApiLike { if (!client.files) { throw new Error('This Relay client cannot store files.'); @@ -256,6 +269,7 @@ export function registerMessagingTools( .describe( 'wait (default): queue delivery until each recipient reaches a safe idle boundary; steer: request immediate injection, which may interrupt active work.' ), + idempotency_key: idempotencyKeyInput, ...identityOverrideInputShape, }, outputSchema: jsonResult, @@ -266,14 +280,26 @@ export function registerMessagingTools( openWorldHint: true, }, }, - async ({ channel, text, attachments, mode, as }) => - jsonContent( - await getAgentClient(as).send(channel, text, { - attachments, - data: replayMessageMetadata(), - mode, - }) - ) + async ({ channel, text, attachments, mode, idempotency_key, as }, extra) => { + // Bind the acting identity now: a register_agent that moves the session + // default must not reroute a write that is already in flight. + const client = getAgentClient(as); + return replay.run( + 'post_message', + extra, + idempotency_key, + async () => + jsonContent( + await client.send(channel, text, { + attachments, + data: replayMessageMetadata(), + mode, + idempotencyKey: idempotency_key, + }) + ), + replayScopeOf(client) + ); + } ); server.registerTool( @@ -311,6 +337,7 @@ export function registerMessagingTools( inputSchema: { message_id: z.string().describe('Parent message ID'), text: z.string().describe('Reply text'), + idempotency_key: idempotencyKeyInput, ...identityOverrideInputShape, }, outputSchema: jsonResult, @@ -321,8 +348,22 @@ export function registerMessagingTools( openWorldHint: true, }, }, - async ({ message_id, text, as }) => - jsonContent(await getAgentClient(as).reply(message_id, text, { data: replayMessageMetadata() })) + async ({ message_id, text, idempotency_key, as }, extra) => { + const client = getAgentClient(as); + return replay.run( + 'reply_to_thread', + extra, + idempotency_key, + async () => + jsonContent( + await client.reply(message_id, text, { + data: replayMessageMetadata(), + idempotencyKey: idempotency_key, + }) + ), + replayScopeOf(client) + ); + } ); server.registerTool( @@ -367,18 +408,7 @@ export function registerMessagingTools( 'wait (default): queue until the recipient reaches a safe idle boundary; steer: request immediate injection, which may interrupt active work. Both modes return before reading is confirmed.' ), attachments: z.array(z.string()).optional().describe('File IDs from "upload_file"'), - idempotency_key: z - .string() - // Relaycast trims this key upstream, so trim before both the local - // replay cache and the forwarded call to keep one logical send on one - // key; a whitespace-only key would otherwise become an unkeyed send. - .trim() - .min(1) - .max(255) - .optional() - .describe( - 'Stable key for retrying this same message after a lost response; use a new key for a new message. Surrounding whitespace is trimmed and a whitespace-only key is rejected.' - ), + idempotency_key: idempotencyKeyInput, ...identityOverrideInputShape, }, outputSchema: directMessageResult, @@ -389,22 +419,30 @@ export function registerMessagingTools( openWorldHint: true, }, }, - async ({ to, text, mode, attachments, idempotency_key, as }, extra) => - replay.run('send_dm', extra, idempotency_key, async () => { - const agents = await listAgentsForRecipientResolution?.(); - const resolvedRecipient = agents ? resolveExactAgentName(agents, to) : undefined; - const message = await getAgentClient(as).dm(to, text, { - idempotencyKey: idempotency_key, - mode, - attachments, - data: replayMessageMetadata(), - }); - const receipt = compactDirectMessageReceipt( - directMessageReceipt(message, to, mode, resolvedRecipient) - ); - const result = jsonContent(receipt); - return directMessageDeliveryFailure(receipt) ? { ...result, isError: true as const } : result; - }) + async ({ to, text, mode, attachments, idempotency_key, as }, extra) => { + const client = getAgentClient(as); + return replay.run( + 'send_dm', + extra, + idempotency_key, + async () => { + const agents = await listAgentsForRecipientResolution?.(); + const resolvedRecipient = agents ? resolveExactAgentName(agents, to) : undefined; + const message = await client.dm(to, text, { + idempotencyKey: idempotency_key, + mode, + attachments, + data: replayMessageMetadata(), + }); + const receipt = compactDirectMessageReceipt( + directMessageReceipt(message, to, mode, resolvedRecipient) + ); + const result = jsonContent(receipt); + return directMessageDeliveryFailure(receipt) ? { ...result, isError: true as const } : result; + }, + replayScopeOf(client) + ); + } ); server.registerTool( diff --git a/packages/cli/src/cli/mcp/request-replay.test.ts b/packages/cli/src/cli/mcp/request-replay.test.ts new file mode 100644 index 0000000000..063e65ef58 --- /dev/null +++ b/packages/cli/src/cli/mcp/request-replay.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it, vi } from 'vitest'; +import { McpRequestReplay, replayScopeOf, withReplayScope } from './request-replay.js'; + +function deferred() { + let resolve!: (value: string) => void; + let reject!: (error: Error) => void; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + return { promise, resolve, reject }; +} + +describe('MCP request replay', () => { + it('runs distinct request IDs separately even while an identical write is pending', async () => { + const replay = new McpRequestReplay(); + const pending = deferred(); + const operation = vi.fn(() => pending.promise); + const first = replay.run('reply_to_thread', { requestId: 1, sessionId: 's' }, undefined, operation); + const second = replay.run('reply_to_thread', { requestId: 2, sessionId: 's' }, undefined, operation); + expect(operation).toHaveBeenCalledTimes(2); + pending.resolve('reply'); + await Promise.all([first, second]); + }); + + it('joins a transport replay of one request ID within a session and tool', async () => { + const replay = new McpRequestReplay(); + const pending = deferred(); + const operation = vi.fn(() => pending.promise); + const first = replay.run('post_message', { requestId: 7, sessionId: 'a' }, undefined, operation); + expect(replay.run('post_message', { requestId: 7, sessionId: 'a' }, undefined, operation)).toBe(first); + void replay.run('post_message', { requestId: 7, sessionId: 'b' }, undefined, operation); + void replay.run('reply_to_thread', { requestId: 7, sessionId: 'a' }, undefined, operation); + expect(operation).toHaveBeenCalledTimes(3); + pending.resolve('ok'); + await first; + }); + + // A session can switch its acting identity (`as`, or register_agent moving + // the default) and reuse a key; that must send, not replay another + // identity's receipt. + it('scopes a retained idempotency key to the acting identity', async () => { + const replay = new McpRequestReplay(); + const operation = vi.fn(async () => 'receipt'); + const extra = { requestId: 1, sessionId: 's' }; + await replay.run('post_message', extra, 'k', operation, 'agent-a'); + await replay.run('post_message', { ...extra, requestId: 2 }, 'k', operation, 'agent-a'); + expect(operation).toHaveBeenCalledTimes(1); + await replay.run('post_message', { ...extra, requestId: 3 }, 'k', operation, 'agent-b'); + expect(operation).toHaveBeenCalledTimes(2); + }); + + it('derives a stable token-free scope from a tagged client', () => { + const a = withReplayScope({}, 'token-a'); + expect(replayScopeOf(a)).toBe(replayScopeOf(withReplayScope({}, 'token-a'))); + expect(replayScopeOf(a)).not.toBe(replayScopeOf(withReplayScope({}, 'token-b'))); + expect(replayScopeOf(a)).not.toContain('token-a'); + expect(JSON.stringify(a)).toBe('{}'); + expect(replayScopeOf({})).toBe(''); + }); + + // Retention is checked lazily: a settling write must not leave a timer + // behind (it would land on whatever clock a later caller installed). + it('retains a completed idempotency key for five minutes without scheduling a timer', async () => { + vi.useFakeTimers(); + try { + const replay = new McpRequestReplay(); + const operation = vi.fn(async () => 'receipt'); + await replay.run('post_message', { requestId: 1 }, 'k', operation); + await Promise.resolve(); + expect(vi.getTimerCount()).toBe(0); + vi.advanceTimersByTime(5 * 60 * 1000 - 1); + await replay.run('post_message', { requestId: 2 }, 'k', operation); + expect(operation).toHaveBeenCalledTimes(1); + vi.advanceTimersByTime(1); + await replay.run('post_message', { requestId: 3 }, 'k', operation); + expect(operation).toHaveBeenCalledTimes(2); + // Retention follows the monotonic clock: stepping the wall clock back + // cannot keep a key alive past its window. + vi.setSystemTime(Date.now() - 60 * 60 * 1000); + vi.advanceTimersByTime(5 * 60 * 1000); + await replay.run('post_message', { requestId: 4 }, 'k', operation); + expect(operation).toHaveBeenCalledTimes(3); + } finally { + vi.useRealTimers(); + } + }); + + it('clears a rejected request ID for a safe retry', async () => { + const replay = new McpRequestReplay(); + const pending = deferred(); + const operation = vi.fn(() => pending.promise); + const send = () => replay.run('reply_to_thread', { requestId: 1 }, undefined, operation); + const first = send(); + expect(send()).toBe(first); + const assertion = expect(first).rejects.toThrow('unavailable'); + pending.reject(new Error('unavailable')); + await assertion; + operation.mockResolvedValue('retry'); + expect(await send()).toBe('retry'); + expect(operation).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/cli/src/cli/mcp/request-replay.ts b/packages/cli/src/cli/mcp/request-replay.ts index 3d6f085903..03b1bd2c9a 100644 --- a/packages/cli/src/cli/mcp/request-replay.ts +++ b/packages/cli/src/cli/mcp/request-replay.ts @@ -1,20 +1,63 @@ +import { createHash } from 'node:crypto'; + +const REPLAY_SCOPE = Symbol('agentRelay.replayScope'); + +/** + * Tag a Relay client with the replay scope of the agent token it acts with. + * The token decides both the acting identity and the workspace a write reaches, + * so a key reused after either changes cannot replay another context's + * receipt. Only a SHA-256 digest is kept, in a non-enumerable property. + */ +export function withReplayScope(client: T, agentToken: string): T { + Object.defineProperty(client, REPLAY_SCOPE, { + value: replayScopeForCredentials(agentToken), + enumerable: false, + }); + return client; +} + +/** + * A replay scope for whatever credentials decide who a write acts as and where + * it lands. Only a SHA-256 digest is kept, never the credentials themselves. + */ +export function replayScopeForCredentials(...credentials: string[]): string { + const material = credentials.length === 1 ? credentials[0] : JSON.stringify(credentials); + return createHash('sha256').update(material).digest('hex'); +} + +/** The replay scope a client was tagged with, or '' for an untagged client. */ +export function replayScopeOf(client: object): string { + const scope = (client as { [REPLAY_SCOPE]?: unknown })[REPLAY_SCOPE]; + return typeof scope === 'string' ? scope : ''; +} + /** * Coalesce a transport replay of one MCP request before it can repeat a * state-changing Relay call. A typed JSON-RPC request ID, scoped by MCP - * session and tool, identifies only an in-flight logical request because + * session, acting identity and tool, identifies only an in-flight logical request because * JSON-RPC permits IDs to be reused after a response. Clients that need to * retry after a lost response provide an explicit idempotency key; that key * retains the completed result briefly. Arguments deliberately do not * participate, so intentional identical sends remain separate requests. */ +const RETAIN_COMPLETED_MS = 5 * 60 * 1000; + export class McpRequestReplay { private readonly requests = new Map>(); + /** + * Retained keys in settlement order. Every key keeps the same retention on a + * monotonic clock, so this is also expiry order: pruning pops expired keys off the front, with + * no timer and no scan of live entries. + */ + private readonly retained: Array<{ key: string; pending: Promise; expiresAt: number }> = []; + private retainedHead = 0; run( tool: string, extra: unknown, idempotencyKey: string | undefined, - operation: () => Promise + operation: () => Promise, + scope = '' ): Promise { const request = extra as { requestId?: unknown; sessionId?: unknown } | undefined; const requestId = request?.requestId; @@ -26,10 +69,12 @@ export class McpRequestReplay { const key = JSON.stringify([ sessionId, + scope, tool, hasIdempotencyKey ? 'idempotency' : typeof requestId, hasIdempotencyKey ? idempotencyKey : requestId, ]); + this.pruneExpired(); const existing = this.requests.get(key) as Promise | undefined; if (existing) return existing; @@ -38,19 +83,26 @@ export class McpRequestReplay { // JSON-RPC permits an ID to be reused after its response. Only a client // supplied idempotency key can safely keep a completed result for a retry. void pending.then( - () => this.clearAfterSettlement(key, hasIdempotencyKey), + () => { + if (hasIdempotencyKey) { + this.retained.push({ key, pending, expiresAt: performance.now() + RETAIN_COMPLETED_MS }); + } else this.requests.delete(key); + }, () => this.requests.delete(key) ); return pending; } - private clearAfterSettlement(key: string, retainCompletedResult: boolean): void { - if (!retainCompletedResult) { - this.requests.delete(key); - return; + private pruneExpired(): void { + const now = performance.now(); + while (this.retainedHead < this.retained.length && this.retained[this.retainedHead].expiresAt <= now) { + const { key, pending } = this.retained[this.retainedHead++]; + if (this.requests.get(key) === pending) this.requests.delete(key); + } + // Compact once the consumed prefix dominates, keeping pruning amortized O(1). + if (this.retainedHead > 64 && this.retainedHead * 2 > this.retained.length) { + this.retained.splice(0, this.retainedHead); + this.retainedHead = 0; } - - const cleanup = setTimeout(() => this.requests.delete(key), 5 * 60 * 1000); - cleanup.unref(); } } diff --git a/packages/harness-driver/src/protocol.ts b/packages/harness-driver/src/protocol.ts index fdc7392c5f..ef5a2ec5db 100644 --- a/packages/harness-driver/src/protocol.ts +++ b/packages/harness-driver/src/protocol.ts @@ -861,6 +861,7 @@ export type WorkerToBroker = } | { type: 'delivery_verified'; + /** See the `delivery_verified` broker event for `verification` values. */ payload: { delivery_id: string; event_id: string; @@ -891,6 +892,13 @@ export type WorkerToBroker = } | { type: 'delivery_failed'; + /** + * Terminal and never replayed. PTY injection adds 'prompt_unproven' (no + * composer was recognized in time), 'body remained parked after bounded + * submit-key recovery', 'harness acceptance could not be proven' + * (neither parked nor accepted, so not evidence of loss) and + * 'injection_too_large: …' (rejected before any byte was written). + */ payload: { delivery_id: string; event_id: string; reason: string }; } | { diff --git a/packages/harnesses/src/ai-sdk/harness-host.test.ts b/packages/harnesses/src/ai-sdk/harness-host.test.ts index cfde4d64e7..58b57b25e9 100644 --- a/packages/harnesses/src/ai-sdk/harness-host.test.ts +++ b/packages/harnesses/src/ai-sdk/harness-host.test.ts @@ -162,7 +162,7 @@ describe('HarnessHost', () => { await vi.waitFor(() => expect(turns[0].submitToolResult).toHaveBeenCalledOnce()); expect(execute).toHaveBeenCalledWith( { to: 'Worker', text: 'hello' }, - expect.objectContaining({ abortSignal: expect.any(AbortSignal) }) + expect.objectContaining({ abortSignal: expect.any(AbortSignal), toolCallId: 'call-1' }) ); expect(turns[0].submitToolResult).toHaveBeenCalledWith({ toolCallId: 'call-1', diff --git a/packages/harnesses/src/ai-sdk/harness-host.ts b/packages/harnesses/src/ai-sdk/harness-host.ts index 6ee8c617c5..ef58fd5fdc 100644 --- a/packages/harnesses/src/ai-sdk/harness-host.ts +++ b/packages/harnesses/src/ai-sdk/harness-host.ts @@ -66,7 +66,14 @@ export interface HarnessHostOptions { export interface HarnessHostTool { spec: HarnessV1ToolSpec; - execute(input: unknown, options: { abortSignal?: AbortSignal }): Promise | unknown; + /** + * `toolCallId` is the model's ID for this invocation. A replay of one call + * carries the same ID; independent calls never share one. + */ + execute( + input: unknown, + options: { abortSignal?: AbortSignal; toolCallId?: string } + ): Promise | unknown; } export interface HarnessTurnHandle { @@ -348,7 +355,10 @@ export class HarnessHost { const tool = this.#tools.get(call.toolName); if (!tool) return; try { - const output = await tool.execute(parseToolInput(call.input), { abortSignal }); + const output = await tool.execute(parseToolInput(call.input), { + abortSignal, + toolCallId: call.toolCallId, + }); const control = await controlReady; await control.submitToolResult({ toolCallId: call.toolCallId, output }); } catch (error) { diff --git a/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts b/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts index bbdcb42cb6..aa9c8efc5c 100644 --- a/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts +++ b/packages/harnesses/src/ai-sdk/native-relay-tools.test.ts @@ -57,3 +57,65 @@ describe('native Relay host tools', () => { expect(NATIVE_RELAY_INSTRUCTIONS).toContain('Do not ask the user how to use Relay'); }); }); + +it.each([ + ['send_dm', 'dm', { to: 'nativeClaude', text: 'ACK' }], + ['post_message', 'send', { channel: 'events', text: 'ACK' }], + ['reply_to_thread', 'reply', { message_id: 'parent', text: 'ACK' }], +] as const)('joins only a replay of the same native %s tool call', async (tool, method, args) => { + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const write = vi.fn(async () => { + await gate; + return { id: `reply-${write.mock.calls.length}` }; + }); + const options = { env: { RELAY_AGENT_TOKEN: 'test' }, agentClient: { [method]: write } as never }; + const execute = createNativeRelayTools(options).find((item) => item.spec.name === tool)!.execute; + const first = execute(args, { toolCallId: 'call-1' }); + const replay = execute(args, { toolCallId: 'call-1' }); + const independent = execute(args, { toolCallId: 'call-2' }); + const unidentified = [execute(args, {}), execute(args, {})]; + await Promise.resolve(); + await Promise.resolve(); + expect(write).toHaveBeenCalledTimes(4); + release(); + expect(await first).toBe(await replay); + await Promise.all([independent, ...unidentified]); + await execute(args, { toolCallId: 'call-1' }); + expect(write).toHaveBeenCalledTimes(5); +}); + +// A replay that arrives after the first write settled (for example while its +// tool result is still being submitted) reaches Relay again, so the write +// carries a key derived from the tool call for Relay to deduplicate. +it.each([ + ['send_dm', 'dm', { to: 'nativeClaude', text: 'ACK' }, 'nativeClaude'], + ['post_message', 'send', { channel: 'events', text: 'ACK' }, 'events'], + ['reply_to_thread', 'reply', { message_id: 'parent', text: 'ACK' }, 'parent'], +] as const)('sends native %s with a tool-call idempotency key', async (tool, method, args, target) => { + const write = vi.fn(async () => ({ id: 'm1' })); + const execute = createNativeRelayTools({ + env: { RELAY_AGENT_TOKEN: 'test' }, + agentClient: { [method]: write } as never, + }).find((item) => item.spec.name === tool)!.execute; + await execute(args, { toolCallId: 'call-1' }); + await execute(args, { toolCallId: 'call-1' }); + await execute(args, {}); + expect(write).toHaveBeenNthCalledWith(1, target, 'ACK', { idempotencyKey: `native:${tool}:call-1` }); + expect(write).toHaveBeenNthCalledWith(2, target, 'ACK', { idempotencyKey: `native:${tool}:call-1` }); + expect(write).toHaveBeenNthCalledWith(3, target, 'ACK'); +}); + +it('clears a rejected native reply so the same tool call can be retried', async () => { + const reply = vi.fn().mockRejectedValueOnce(new Error('offline')).mockResolvedValue({ id: 'retry' }); + const execute = createNativeRelayTools({ + env: { RELAY_AGENT_TOKEN: 'test' }, + agentClient: { reply } as never, + }).find((item) => item.spec.name === 'reply_to_thread')!.execute; + const args = { message_id: 'parent', text: 'ACK' }; + await expect(execute(args, { toolCallId: 'call-1' })).rejects.toThrow('offline'); + await expect(execute(args, { toolCallId: 'call-1' })).resolves.toEqual({ id: 'retry' }); + expect(reply).toHaveBeenCalledTimes(2); +}); diff --git a/packages/harnesses/src/ai-sdk/native-relay-tools.ts b/packages/harnesses/src/ai-sdk/native-relay-tools.ts index faf9bb06b6..a1c2ab432b 100644 --- a/packages/harnesses/src/ai-sdk/native-relay-tools.ts +++ b/packages/harnesses/src/ai-sdk/native-relay-tools.ts @@ -53,6 +53,32 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha const workspace = options.workspaceClient ?? (workspaceKey ? createWorkspaceClient({ workspaceKey, baseUrl }) : undefined); + // One map per native session, keyed by the model's tool call ID so only a + // replay of the same invocation joins a pending write. Text never forms the + // key: independent identical replies are separate messages. A completed + // write is never retained locally; a replay after it settled reaches Relay + // with the same tool-call idempotency key, which Relay deduplicates. + const pendingWrites = new Map>(); + function write( + tool: string, + toolCallId: string | undefined, + operation: (options?: { idempotencyKey: string }) => Promise + ) { + if (!toolCallId) return Promise.resolve().then(() => operation()); + const key = JSON.stringify([tool, toolCallId]); + const existing = pendingWrites.get(key); + if (existing) return existing; + const pending = Promise.resolve().then(() => + operation({ idempotencyKey: `native:${tool}:${toolCallId}` }) + ); + pendingWrites.set(key, pending); + void pending.then( + () => pendingWrites.delete(key), + () => pendingWrites.delete(key) + ); + return pending; + } + const tools: HarnessHostTool[] = [ { spec: { @@ -68,9 +94,13 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha additionalProperties: false, }, }, - execute: (value) => { + execute: (value, { toolCallId } = {}) => { const input = objectInput(value); - return agent.dm(requiredString(input, 'to'), requiredString(input, 'text')); + const to = requiredString(input, 'to'); + const text = requiredString(input, 'text'); + return write('send_dm', toolCallId, (options) => + options ? agent.dm(to, text, options) : agent.dm(to, text) + ); }, }, { @@ -87,9 +117,13 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha additionalProperties: false, }, }, - execute: (value) => { + execute: (value, { toolCallId } = {}) => { const input = objectInput(value); - return agent.send(requiredString(input, 'channel'), requiredString(input, 'text')); + const target = requiredString(input, 'channel'); + const text = requiredString(input, 'text'); + return write('post_message', toolCallId, (options) => + options ? agent.send(target, text, options) : agent.send(target, text) + ); }, }, { @@ -148,9 +182,13 @@ export function createNativeRelayTools(options: NativeRelayToolOptions = {}): Ha additionalProperties: false, }, }, - execute: (value) => { + execute: (value, { toolCallId } = {}) => { const input = objectInput(value); - return agent.reply(requiredString(input, 'message_id'), requiredString(input, 'text')); + const target = requiredString(input, 'message_id'); + const text = requiredString(input, 'text'); + return write('reply_to_thread', toolCallId, (options) => + options ? agent.reply(target, text, options) : agent.reply(target, text) + ); }, }, { diff --git a/packages/integration-prompts/src/builders.ts b/packages/integration-prompts/src/builders.ts index a59cf99095..a595c9c93a 100644 --- a/packages/integration-prompts/src/builders.ts +++ b/packages/integration-prompts/src/builders.ts @@ -55,6 +55,9 @@ export function fullInjectInstructions(descriptors: IntegrationDescriptor[]): st lines.push('No integration event subscriptions are active for this project.'); } else { lines.push('Active integration event subscriptions for this project:'); + lines.push( + 'Reply once in the Relay thread to respond to a subscription event; when writeback is configured and authorized, that reply posts back to the provider. Use resource file writes to initiate new integration actions.' + ); for (const subscription of subscriptions) { lines.push(subscriptionLine(subscription)); } diff --git a/packages/sdk/src/__tests__/integrations.test.ts b/packages/sdk/src/__tests__/integrations.test.ts index 2979c0ddfe..b4cfc1da2d 100644 --- a/packages/sdk/src/__tests__/integrations.test.ts +++ b/packages/sdk/src/__tests__/integrations.test.ts @@ -54,6 +54,28 @@ describe('SDK integrations / capabilities / workspace passthrough', () => { expect(subs).toHaveLength(1); }); + it('normalizes subscription event coverage without losing engine configuration', async () => { + const relaycast = createRelaycastMock(); + relaycast.subscriptions.get.mockResolvedValue({ + id: 'sub1', + event_types: ['thread.reply'], + is_active: false, + } as never); + const client = new RelaycastMessagingClient({ relaycast: relaycast as never }); + expect(await client.integrations.subscriptions.get('sub1')).toMatchObject({ + id: 'sub1', + events: ['thread.reply'], + is_active: false, + }); + expect(await client.integrations.subscriptions.create({ event: 'thread.reply' })).toMatchObject({ + id: 'sub1', + events: ['thread.reply'], + }); + expect(await client.integrations.subscriptions.list()).toEqual([ + { id: 'sub1', event: 'message.created', events: ['message.created'] }, + ]); + }); + it('delegates capability (command) operations to relaycast', async () => { const relaycast = createRelaycastMock(); const client = new RelaycastMessagingClient({ relaycast: relaycast as never }); diff --git a/packages/sdk/src/__tests__/relaycast-translate.test.ts b/packages/sdk/src/__tests__/relaycast-translate.test.ts index 6e3d5ce052..f6d888ce0f 100644 --- a/packages/sdk/src/__tests__/relaycast-translate.test.ts +++ b/packages/sdk/src/__tests__/relaycast-translate.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { toRelayNode } from '../messaging/relaycast-translate.js'; +import { normalizeWebhookSubscription, toRelayNode } from '../messaging/relaycast-translate.js'; describe('toRelayNode fleet liveness', () => { it('omits stale active-agent load for an offline node', () => { @@ -18,3 +18,22 @@ describe('toRelayNode fleet liveness', () => { expect(node.activeAgents).toBeUndefined(); }); }); + +describe('subscription event normalization', () => { + it.each([ + { events: ['message.created', 'thread.reply'] }, + { event_types: ['message.created', 'thread.reply'] }, + { eventTypes: ['message.created', 'thread.reply'] }, + ])('normalizes plural events: %j', (raw) => { + expect(normalizeWebhookSubscription({ id: 'sub', ...raw }).events).toEqual([ + 'message.created', + 'thread.reply', + ]); + }); + + it('falls back to a singular event only when plural events are absent', () => { + expect(normalizeWebhookSubscription({ event: 'thread.reply' }).events).toEqual(['thread.reply']); + expect(normalizeWebhookSubscription({ events: [], event: 'thread.reply' }).events).toEqual([]); + expect(normalizeWebhookSubscription({}).events).toBeUndefined(); + }); +}); diff --git a/packages/sdk/src/messaging/index.ts b/packages/sdk/src/messaging/index.ts index 334b3fdfa5..39e38d45df 100644 --- a/packages/sdk/src/messaging/index.ts +++ b/packages/sdk/src/messaging/index.ts @@ -16,3 +16,6 @@ export { type RelaySpawnPlacementState, } from './relaycast.js'; export * from './thin-client.js'; + +export { normalizeWebhookSubscription } from './relaycast-translate.js'; +export { SUBSCRIBABLE_EVENT_TYPES } from './subscribable-events.js'; diff --git a/packages/sdk/src/messaging/relaycast-translate.ts b/packages/sdk/src/messaging/relaycast-translate.ts index ecab4a16d2..a925573c09 100644 --- a/packages/sdk/src/messaging/relaycast-translate.ts +++ b/packages/sdk/src/messaging/relaycast-translate.ts @@ -267,9 +267,12 @@ export function normalizeInboundWebhook(raw: unknown): RelayInboundWebhook { /** Normalize a relaycast event subscription into `RelayWebhookSubscription`. */ export function normalizeWebhookSubscription(raw: unknown): RelayWebhookSubscription { const record = asRecord(raw); - const events = Array.isArray(record.events) - ? record.events.filter((event): event is string => typeof event === 'string') - : undefined; + const rawEvents = record.events ?? record.event_types ?? record.eventTypes; + const events = Array.isArray(rawEvents) + ? rawEvents.filter((event): event is string => typeof event === 'string') + : typeof record.event === 'string' + ? [record.event] + : undefined; return { id: readStr(record, 'id') ?? '', ...(readStr(record, 'url') ? { url: readStr(record, 'url') } : {}), diff --git a/packages/sdk/src/messaging/relaycast.ts b/packages/sdk/src/messaging/relaycast.ts index 0af7d12e35..fe5929175f 100644 --- a/packages/sdk/src/messaging/relaycast.ts +++ b/packages/sdk/src/messaging/relaycast.ts @@ -660,12 +660,19 @@ export class RelaycastMessagingClient implements RelayMessagingClient { this.requireWebhooks().trigger(id, payload ?? {}), }, subscriptions: { - create: async (input: RelayCreateSubscriptionInput): Promise => - (await this.requireSubscriptions().create(input)) as RelayEventSubscription, + create: async (input: RelayCreateSubscriptionInput): Promise => { + const raw = (await this.requireSubscriptions().create(input)) as RelayEventSubscription; + return { ...raw, ...normalizeWebhookSubscription(raw) }; + }, list: async (): Promise => - (await this.requireSubscriptions().list()) as RelayEventSubscription[], - get: async (id: string): Promise => - (await this.requireSubscriptions().get(id)) as RelayEventSubscription, + ((await this.requireSubscriptions().list()) as RelayEventSubscription[]).map((raw) => ({ + ...raw, + ...normalizeWebhookSubscription(raw), + })), + get: async (id: string): Promise => { + const raw = (await this.requireSubscriptions().get(id)) as RelayEventSubscription; + return { ...raw, ...normalizeWebhookSubscription(raw) }; + }, delete: async (id: string): Promise => { await this.requireSubscriptions().delete(id); }, diff --git a/packages/sdk/src/messaging/subscribable-events.ts b/packages/sdk/src/messaging/subscribable-events.ts new file mode 100644 index 0000000000..2b2495dcf0 --- /dev/null +++ b/packages/sdk/src/messaging/subscribable-events.ts @@ -0,0 +1,7 @@ +import { SubscribableEventTypeSchema, type SubscribableEventType } from '@relaycast/types'; + +/** + * Every event a Relaycast subscription accepts, taken from the engine's own + * schema so a CLI allowlist cannot drift narrower than the server. + */ +export const SUBSCRIBABLE_EVENT_TYPES: readonly SubscribableEventType[] = SubscribableEventTypeSchema.options; diff --git a/packages/sdk/src/messaging/thin-client.ts b/packages/sdk/src/messaging/thin-client.ts index 3ab59467ca..c36f1ca889 100644 --- a/packages/sdk/src/messaging/thin-client.ts +++ b/packages/sdk/src/messaging/thin-client.ts @@ -104,6 +104,8 @@ export interface RelayAgentThinClient { channel: string, text: string, options?: { + /** Stable key for retries of one logical send, including across processes. */ + idempotencyKey?: string; attachments?: string[]; data?: Record | null; mode?: RelayMessageMode; @@ -113,7 +115,11 @@ export interface RelayAgentThinClient { reply( messageId: string, text: string, - options?: { data?: Record | null } + options?: { + /** Stable key for retries of one logical reply, including across processes. */ + idempotencyKey?: string; + data?: Record | null; + } ): Promise; thread(messageId: string, options?: { limit?: number }): Promise; dm( @@ -135,6 +141,8 @@ export interface RelayAgentThinClient { conversationId: string, text: string, options?: { + /** Stable key for retries of one logical send, including across processes. */ + idempotencyKey?: string; attachments?: string[]; data?: Record | null; mode?: RelayMessageMode; diff --git a/packages/sdk/src/messaging/types.ts b/packages/sdk/src/messaging/types.ts index 13174ed4b8..dedd37ca29 100644 --- a/packages/sdk/src/messaging/types.ts +++ b/packages/sdk/src/messaging/types.ts @@ -512,6 +512,7 @@ export interface RelayCreateWebhookInput { export interface RelayEventSubscription { id: string; + events?: string[]; event?: string; createdAt?: string; [key: string]: unknown; diff --git a/tests/e2e/fleet/fleet-e2e.test.ts b/tests/e2e/fleet/fleet-e2e.test.ts index acc5e449f1..04acf9574d 100644 --- a/tests/e2e/fleet/fleet-e2e.test.ts +++ b/tests/e2e/fleet/fleet-e2e.test.ts @@ -556,7 +556,10 @@ describe.skipIf(!pre.ok)('two-node fleet scenario matrix', () => { const inv = await getInvocation(engine, driverToken, 'spawn', first.invocationId!); return inv.status === 'completed' || inv.status === 'failed' ? inv : null; }, - { label: 'resumable spawn settled', timeoutMs: 30_000, intervalMs: 300 } + // A verified spawn carrying a task now completes only once that task's + // receipt is verified after readiness, so allow for readiness plus + // delivery verification rather than readiness alone. + { label: 'resumable spawn settled', timeoutMs: 60_000, intervalMs: 300 } ); expect(firstDone.status).toBe('completed'); // resumable spawn carried session_ref through token authority expect(firstDone.output).toMatchObject({ spawned: true, ready: true, name }); @@ -620,7 +623,7 @@ describe.skipIf(!pre.ok)('two-node fleet scenario matrix', () => { const inv = await getInvocation(engine, driverToken, 'spawn', resume.invocationId!); return ['completed', 'failed'].includes(inv.status) ? inv : null; }, - { label: 'resumed worker confirmed ready', timeoutMs: 30_000 } + { label: 'resumed worker confirmed ready', timeoutMs: 60_000 } ); expect(resumed.status).toBe('completed'); expect(resumed.output).toMatchObject({ spawned: true, ready: true, name }); @@ -630,7 +633,9 @@ describe.skipIf(!pre.ok)('two-node fleet scenario matrix', () => { args: expect.arrayContaining(['resume', sessionRef]), }); }, - 180_000 + // Two 60 s spawn settles, two 15 s brief observations and a 30 s release + // wait alone reach 180 s, so leave room for the actions between them. + 240_000 ); it('propagates unsupported generic session resume as a terminal failure with no retained identity', async () => { diff --git a/tests/e2e/fleet/nodes/stub-agent.cjs b/tests/e2e/fleet/nodes/stub-agent.cjs index f753fc14d2..f8631dd37f 100644 --- a/tests/e2e/fleet/nodes/stub-agent.cjs +++ b/tests/e2e/fleet/nodes/stub-agent.cjs @@ -35,10 +35,30 @@ function recordBriefNonce(nonce) { ); } +// Invoked as `codex`, model Codex's composer closely enough for the broker's +// chunked initial-task path: raw input (no kernel echo, so discarded +// pre-ready keystrokes never appear), typed text rendered after the `›` +// prompt, and Enter submitting the turn and redrawing an empty composer. A +// composer that keeps the submitted text visible looks like a parked task, so +// the broker would retry the submit and report the delivery incomplete. +const codexComposer = path.basename(process.argv[1]) === 'codex'; +if (codexComposer && process.stdin.isTTY) process.stdin.setRawMode(true); + +function renderCodexComposer(text) { + for (const ch of text) { + if (ch === '\r') process.stdout.write('\r\n› '); + else if (ch === '\n') process.stdout.write('\r\n '); + else process.stdout.write(ch); + } +} + try { + // Decode as a stream: a UTF-8 character can straddle two PTY reads. + process.stdin.setEncoding('utf8'); process.stdin.resume(); process.stdin.on('data', (chunk) => { if (!ready) return; + if (codexComposer) renderCodexComposer(chunk.toString()); input += chunk.toString(); // Require a delimiter after the nonce. PTY chunks can split anywhere, so // treating the current buffer end as a complete token could record a diff --git a/tests/relayflows/cases/1615-api-send-recipient-reachability/run.mjs b/tests/relayflows/cases/1615-api-send-recipient-reachability/run.mjs index 97366bb490..6739f14655 100644 --- a/tests/relayflows/cases/1615-api-send-recipient-reachability/run.mjs +++ b/tests/relayflows/cases/1615-api-send-recipient-reachability/run.mjs @@ -51,7 +51,13 @@ const fakeClaudeSource = String.raw`#!/usr/bin/env node process.stdin.setRawMode?.(true); process.stdin.resume(); process.stdout.write('RELAYFLOW_RECIPIENT_READY\r\n❯'); -process.stdin.on('data', (chunk) => process.stdout.write(chunk)); +// Like the real composer, redraw the prompt after Enter submits a line, so +// the broker's pre-write composer check can prove readiness for the next +// message instead of seeing only the previous message's echo at the cursor. +process.stdin.on('data', (chunk) => { + process.stdout.write(chunk); + if (chunk.includes(13)) process.stdout.write('\r\n❯'); +}); `; let broker;