Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@ All notable changes to Agent Relay will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
## [Unreleased - Patch]

### Fixed

- Broker PTY delivery to Claude no longer stalls forever behind a failed draft: the `❯` + no-break-space prompt is recognised, blank line-break residue left in the composer is erased, and deliveries re-sent to a `blocked_on_send` worker now count against the retry budget and dead-letter instead of being retried indefinitely.

## [14.0.0] - 2026-10-13

Expand Down
107 changes: 106 additions & 1 deletion crates/broker/src/broker/delivery_verification.rs
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,10 @@ fn composer_candidates(snapshot: &Snapshot, cli: &str) -> Vec<String> {
.collect();
}
let is_prompt = |line: &str| {
let trimmed = line.trim_start();
// Claude paints its prompt as `❯` + U+00A0 (no-break space); treat it
// as an ordinary space so the live composer is recognised.
let normalized = line.trim_start().replace('\u{a0}', " ");
let trimmed = normalized.as_str();
if lower.contains("codex") {
trimmed == "›"
|| trimmed.starts_with("› ")
Expand Down Expand Up @@ -546,6 +549,62 @@ pub(crate) fn failed_draft_released(cli: &str, expected_echo: &str, snapshot: &S
|| (composer_is_idle(snapshot, cli) && !composer_holds_tail(snapshot, cli, &tail))
}

/// Most blank rows below a bare prompt that may be cleared as residue.
pub(crate) const MAX_BLANK_COMPOSER_RESIDUE_ROWS: usize = 4;

/// Rows of line-break residue left in an otherwise empty Claude composer.
///
/// A failed injection can leave only its line breaks behind: the composer
/// renders as a bare `❯` row followed by blank rows, with the cursor on the
/// last blank row and the composer frame's rule below it. That is not idle (the
/// cursor is not on the prompt row), so the failed-draft latch would wait
/// forever. Returns the number of blank rows between the prompt and the cursor
/// when — and only when — no text exists anywhere in that composer, so erasing
/// them can never touch a human draft. `None` for every other state.
pub(crate) fn blank_composer_residue_rows(cli: &str, snapshot: &Snapshot) -> Option<usize> {
if !cli.to_ascii_lowercase().contains("claude")
|| snapshot.has_visible_text_at_or_after_cursor()
{
return None;
}
let screen = snapshot.to_plain();
let lines: Vec<&str> = screen.lines().collect();
let cursor = (snapshot.cursor.0 as usize).checked_sub(1)?;
if cursor >= lines.len() {
return None;
}
let mut row = cursor;
while lines[row].trim().is_empty() {
if cursor - row >= MAX_BLANK_COMPOSER_RESIDUE_ROWS {
return None;
}
row = row.checked_sub(1)?;
}
if !matches!(lines[row].trim(), "❯" | ">") {
return None;
}
let blank_rows = cursor - row;
if blank_rows == 0 {
return None;
}
// The composer must close with its frame rule below the cursor; any text
// there could be the rest of a draft the cursor was moved up into.
let below = lines[cursor + 1..]
.iter()
.find(|line| !line.trim().is_empty())?;
let rule = below.trim();
if !rule.chars().all(|c| matches!(c, '─' | '━' | '╌' | '-')) {
return None;
}
Some(blank_rows)
}

/// Bytes that erase [`blank_composer_residue_rows`] line breaks: one
/// backspace per blank row joins each back onto the prompt row.
pub(crate) fn blank_composer_residue_cleanup(rows: usize) -> Vec<u8> {
vec![0x7f; rows]
}

/// Distinguish terminal echo from actual harness acceptance.
///
/// Activity is definitive acceptance. A body is considered parked only when
Expand Down Expand Up @@ -1886,4 +1945,50 @@ mod tests {
throttle.record(DeliveryOutcome::Success);
assert_eq!(throttle.delay(), Duration::from_millis(100));
}

/// Last 24 rows of desktop-upgrade-sf's PTY (relay broker snapshot, 2026-10-19):
/// Claude idle after a turn, but its composer holds one line break left by
/// a failed injection, so the cursor sits at [22,3] under a bare `❯ `.
const LIVE_CLAUDE_NEWLINE_RESIDUE: &str = " codex-relay-desktop-249119, this time about dogpatch-mini, which needed no\r\n upgrade.\r\n\r\n It prompted a quick read-only check of sf-mini:\r\n\r\n - Installed: Relay Desktop 2026.10.47, build 84, running. I installed\r\n 2026.10.45 (build 81) on Oct 16, so someone else has upgraded it since.\r\n - Latest release: v2026.10.48, published Oct 18 04:26 UTC. sf-mini is one\r\n release behind.\r\n - Build number: dogpatch-mini reports 2026.10.48 as build 84, the same build\r\n number sf-mini shows for 2026.10.47. That could be a typo in their report or\r\n a release build number that wasn't bumped. It doesn't matter for this\r\n machine.\r\n\r\n If you want sf-mini on 2026.10.48, say so and I'll do the same upgrade as\r\n before.\r\n\r\n✻ Worked for 31s · done 11:43 PM\r\n\r\n────────────────────────────────────────────────────────────────────────────────\r\n❯ \r\n\r\n────────────────────────────────────────────────────────────────────────────────\r\n ⏵⏵ bypass permissions on (shift+tab to cycle)";

#[cfg(unix)]
#[tokio::test]
async fn blank_composer_residue_holds_the_failed_draft_latch_until_cleared() {
let expected = "Nudge: please ack the relay-desktop#577 read-only validation ask";
let (_pty, residue) =
codex_snapshot(&format!("{LIVE_CLAUDE_NEWLINE_RESIDUE}\x1b[22;3H")).await;
assert_eq!(residue.cursor, (22, 3));
// The bug: an empty-but-two-row composer never reads as idle, so the
// latch never released and every later delivery waited forever.
assert!(!composer_is_idle(&residue, "claude"));
assert!(!failed_draft_released("claude", expected, &residue));
assert_eq!(blank_composer_residue_rows("claude", &residue), Some(1));
assert_eq!(blank_composer_residue_cleanup(1), vec![0x7f]);
assert_eq!(blank_composer_residue_rows("codex", &residue), None);

// Once the line break is erased the cursor is back on the prompt row.
let (_pty, cleared) =
codex_snapshot(&format!("{LIVE_CLAUDE_NEWLINE_RESIDUE}\x1b[21;3H")).await;
// The live prompt row is `❯` + U+00A0; it must still anchor the composer.
assert_eq!(cleared.to_plain().lines().nth(20), Some("❯\u{a0}"));
assert!(composer_is_idle(&cleared, "claude"));
assert!(failed_draft_released("claude", expected, &cleared));
assert_eq!(blank_composer_residue_rows("claude", &cleared), None);
}

#[cfg(unix)]
#[tokio::test]
async fn blank_composer_residue_never_matches_a_composer_holding_text() {
let text_on_cursor_row = " codex-relay-desktop-249119, this time about dogpatch-mini, which needed no\r\n upgrade.\r\n\r\n It prompted a quick read-only check of sf-mini:\r\n\r\n - Installed: Relay Desktop 2026.10.47, build 84, running. I installed\r\n 2026.10.45 (build 81) on Oct 16, so someone else has upgraded it since.\r\n - Latest release: v2026.10.48, published Oct 18 04:26 UTC. sf-mini is one\r\n release behind.\r\n - Build number: dogpatch-mini reports 2026.10.48 as build 84, the same build\r\n number sf-mini shows for 2026.10.47. That could be a typo in their report or\r\n a release build number that wasn't bumped. It doesn't matter for this\r\n machine.\r\n\r\n If you want sf-mini on 2026.10.48, say so and I'll do the same upgrade as\r\n before.\r\n\r\n✻ Worked for 31s · done 11:43 PM\r\n\r\n────────────────────────────────────────────────────────────────────────────────\r\n❯ \r\ndraft continues\r\n────────────────────────────────────────────────────────────────────────────────\r\n ⏵⏵ bypass permissions on (shift+tab to cycle)";
let (_pty, snap) = codex_snapshot(&format!("{text_on_cursor_row}\x1b[22;3H")).await;
assert_eq!(blank_composer_residue_rows("claude", &snap), None);

let text_on_prompt_row = " codex-relay-desktop-249119, this time about dogpatch-mini, which needed no\r\n upgrade.\r\n\r\n It prompted a quick read-only check of sf-mini:\r\n\r\n - Installed: Relay Desktop 2026.10.47, build 84, running. I installed\r\n 2026.10.45 (build 81) on Oct 16, so someone else has upgraded it since.\r\n - Latest release: v2026.10.48, published Oct 18 04:26 UTC. sf-mini is one\r\n release behind.\r\n - Build number: dogpatch-mini reports 2026.10.48 as build 84, the same build\r\n number sf-mini shows for 2026.10.47. That could be a typo in their report or\r\n a release build number that wasn't bumped. It doesn't matter for this\r\n machine.\r\n\r\n If you want sf-mini on 2026.10.48, say so and I'll do the same upgrade as\r\n before.\r\n\r\n✻ Worked for 31s · done 11:43 PM\r\n\r\n────────────────────────────────────────────────────────────────────────────────\r\n❯ my own draft\r\n\r\n────────────────────────────────────────────────────────────────────────────────\r\n ⏵⏵ bypass permissions on (shift+tab to cycle)";
let (_pty, snap) = codex_snapshot(&format!("{text_on_prompt_row}\x1b[22;3H")).await;
assert_eq!(blank_composer_residue_rows("claude", &snap), None);

let text_after_cursor = " codex-relay-desktop-249119, this time about dogpatch-mini, which needed no\r\n upgrade.\r\n\r\n It prompted a quick read-only check of sf-mini:\r\n\r\n - Installed: Relay Desktop 2026.10.47, build 84, running. I installed\r\n 2026.10.45 (build 81) on Oct 16, so someone else has upgraded it since.\r\n - Latest release: v2026.10.48, published Oct 18 04:26 UTC. sf-mini is one\r\n release behind.\r\n - Build number: dogpatch-mini reports 2026.10.48 as build 84, the same build\r\n number sf-mini shows for 2026.10.47. That could be a typo in their report or\r\n a release build number that wasn't bumped. It doesn't matter for this\r\n machine.\r\n\r\n If you want sf-mini on 2026.10.48, say so and I'll do the same upgrade as\r\n before.\r\n\r\n✻ Worked for 31s · done 11:43 PM\r\n\r\n────────────────────────────────────────────────────────────────────────────────\r\n❯ \r\n later text\r\n────────────────────────────────────────────────────────────────────────────────\r\n ⏵⏵ bypass permissions on (shift+tab to cycle)";
let (_pty, snap) = codex_snapshot(&format!("{text_after_cursor}\x1b[22;1H")).await;
assert_eq!(blank_composer_residue_rows("claude", &snap), None);
}
}
98 changes: 82 additions & 16 deletions crates/broker/src/pty_worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,11 @@ use tokio::{
use crate::broker::{
continuity::parse_continuity_command,
delivery_verification::{
assess_harness_acceptance, current_timestamp_ms, delivery_injected_event_payload,
delivery_queued_event_payload, draft_visible_at_cursor, failed_draft_released,
DeliveryOutcome, HarnessAcceptance, PendingActivity, PendingVerification, ThrottleState,
VerificationOutput, ACTIVITY_BUFFER_KEEP_BYTES, ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW,
assess_harness_acceptance, blank_composer_residue_cleanup, blank_composer_residue_rows,
current_timestamp_ms, delivery_injected_event_payload, delivery_queued_event_payload,
draft_visible_at_cursor, failed_draft_released, DeliveryOutcome, HarnessAcceptance,
PendingActivity, PendingVerification, ThrottleState, VerificationOutput,
ACTIVITY_BUFFER_KEEP_BYTES, ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW,
MAX_VERIFICATION_ATTEMPTS, VERIFICATION_WINDOW,
},
injection_format::{format_injection_for_worker_with_workspace, McpReminderThrottle},
Expand Down Expand Up @@ -115,6 +116,24 @@ struct FailedComposerLatch {
latched_at: Instant,
/// The terminal stuck signal has been sent for this latch.
escalated: bool,
/// Blank-row residue cleanups already sent for this latch.
residue_cleanups: u8,
/// When the last residue cleanup was written.
last_residue_cleanup: Option<Instant>,
}

/// Most blank-row residue cleanups one latch may send before it gives up.
const MAX_RESIDUE_CLEANUPS: u8 = 3;

/// Minimum spacing between residue cleanups so the harness can repaint.
const RESIDUE_CLEANUP_SPACING: Duration = Duration::from_secs(2);

/// Whether the latch may erase blank-row composer residue now.
fn residue_cleanup_due(latch: &FailedComposerLatch, now: Instant) -> bool {
latch.residue_cleanups < MAX_RESIDUE_CLEANUPS
&& latch
.last_residue_cleanup
.is_none_or(|at| now.duration_since(at) >= RESIDUE_CLEANUP_SPACING)
}

/// How long a visibly stuck draft may hold injection before the worker sends
Expand Down Expand Up @@ -161,6 +180,8 @@ fn latch_failed_written_delivery(
expected_echo: expected_echo.to_string(),
latched_at: Instant::now(),
escalated: false,
residue_cleanups: 0,
last_residue_cleanup: None,
});
}

Expand Down Expand Up @@ -3110,22 +3131,41 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> {
// Release the failed-draft latch once the composer no longer
// holds that body (the harness took it late, or a human
// submitted or cleared it). Until then queued deliveries wait.
if let Some(latch) = failed_composer_latch.as_ref() {
// A composer left holding only blank rows (line breaks from the
// failed body) is not idle and would hold the latch forever;
// erase exactly those rows, never any text.
if let Some(latch) = failed_composer_latch.as_mut() {
if pending_verifications.is_empty()
&& pending_recovery_writes.is_empty()
&& active_injection.is_none()
&& failed_draft_released(
&resolved_cli,
&latch.expected_echo,
&Snapshot::capture(&pty),
)
{
tracing::info!(
delivery_id = %latch.delivery_id,
event_id = %latch.event_id,
"failed delivery left the composer; resuming injection"
);
failed_composer_latch = None;
let snapshot = Snapshot::capture(&pty);
if failed_draft_released(&resolved_cli, &latch.expected_echo, &snapshot) {
tracing::info!(
delivery_id = %latch.delivery_id,
event_id = %latch.event_id,
"failed delivery left the composer; resuming injection"
);
failed_composer_latch = None;
} else if let Some(rows) =
blank_composer_residue_rows(&resolved_cli, &snapshot)
{
let now = Instant::now();
if residue_cleanup_due(latch, now) {
latch.residue_cleanups = latch.residue_cleanups.saturating_add(1);
latch.last_residue_cleanup = Some(now);
tracing::warn!(
delivery_id = %latch.delivery_id,
rows,
attempt = latch.residue_cleanups,
"failed draft left only blank rows in the composer; erasing them"
);
warn_on_auto_response_write(
pty.submit_write(blank_composer_residue_cleanup(rows)),
"failed_draft_residue_cleanup",
);
}
}
}
}
// A draft still stuck after the bounded timeout gets one
Expand Down Expand Up @@ -4879,6 +4919,30 @@ mod tests {
);
}

#[test]
fn residue_cleanup_is_bounded_and_spaced() {
let mut latch = FailedComposerLatch {
delivery_id: DeliveryId::from("del_failed"),
event_id: EventId::from("evt_failed"),
expected_echo: "Relay message from Lead [evt]: failed".to_string(),
latched_at: Instant::now(),
escalated: false,
residue_cleanups: 0,
last_residue_cleanup: None,
};
let start = Instant::now();
assert!(residue_cleanup_due(&latch, start));
latch.residue_cleanups = 1;
latch.last_residue_cleanup = Some(start);
assert!(!residue_cleanup_due(&latch, start), "spaced");
assert!(residue_cleanup_due(&latch, start + RESIDUE_CLEANUP_SPACING));
latch.residue_cleanups = MAX_RESIDUE_CLEANUPS;
assert!(
!residue_cleanup_due(&latch, start + RESIDUE_CLEANUP_SPACING * 10),
"bounded"
);
}

#[test]
fn latched_worker_reports_blocked_only_when_the_latch_is_the_sole_blocker() {
let latch = FailedComposerLatch {
Expand All @@ -4887,6 +4951,8 @@ mod tests {
expected_echo: "Relay message from Lead [evt]: failed".to_string(),
latched_at: Instant::now(),
escalated: false,
residue_cleanups: 0,
last_residue_cleanup: None,
};
assert!(failed_draft_latch_is_sole_blocker(
Some(&latch),
Expand Down
Loading
Loading