From 16a92605891730d9b196ac5e40cc196d1140a44d Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Mon, 19 Oct 2026 19:34:11 -0700 Subject: [PATCH] fix(broker): unstick PTY deliveries held behind a failed-draft latch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live on sf-mini/finn-mini (2026-10-19) desktop-upgrade-sf/finn held 3 and 9 deliveries for 42-56h (~675 resends, failed_attempts always 0, surfaced_and_acked=0) behind a failed_composer_latch that never released. - Recognise Claude's prompt row `❯` + U+00A0 as a composer anchor; the ASCII-only match left composer_is_idle false on an empty composer. - Erase blank-row residue (line breaks from the failed body) under a bare prompt with one backspace per row, bounded to 3 spaced tries and only when no text exists anywhere in the composer. - retry_pending_delivery: a handoff to a worker in BlockedOnSend no longer resets failed_attempts, so MAX_DELIVERY_RETRIES dead-letters it visibly. Working workers keep the old behaviour. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +- .../src/broker/delivery_verification.rs | 107 +++++++++++++++++- crates/broker/src/pty_worker.rs | 98 +++++++++++++--- crates/broker/src/runtime/delivery.rs | 30 ++++- crates/broker/src/runtime/tests.rs | 76 +++++++++++++ 5 files changed, 296 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd3e446790..9de4db565f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Patch] + +### Fixed + +- Broker PTY delivery to Claude no longer stalls forever behind a failed draft: the `❯` + no-break-space prompt is recognised, blank line-break residue left in the composer is erased, and deliveries re-sent to a `blocked_on_send` worker now count against the retry budget and dead-letter instead of being retried indefinitely. ## [14.0.0] - 2026-10-13 diff --git a/crates/broker/src/broker/delivery_verification.rs b/crates/broker/src/broker/delivery_verification.rs index 0ceb1dd080..12a31b58ab 100644 --- a/crates/broker/src/broker/delivery_verification.rs +++ b/crates/broker/src/broker/delivery_verification.rs @@ -362,7 +362,10 @@ fn composer_candidates(snapshot: &Snapshot, cli: &str) -> Vec { .collect(); } let is_prompt = |line: &str| { - let trimmed = line.trim_start(); + // Claude paints its prompt as `❯` + U+00A0 (no-break space); treat it + // as an ordinary space so the live composer is recognised. + let normalized = line.trim_start().replace('\u{a0}', " "); + let trimmed = normalized.as_str(); if lower.contains("codex") { trimmed == "›" || trimmed.starts_with("› ") @@ -546,6 +549,62 @@ pub(crate) fn failed_draft_released(cli: &str, expected_echo: &str, snapshot: &S || (composer_is_idle(snapshot, cli) && !composer_holds_tail(snapshot, cli, &tail)) } +/// Most blank rows below a bare prompt that may be cleared as residue. +pub(crate) const MAX_BLANK_COMPOSER_RESIDUE_ROWS: usize = 4; + +/// Rows of line-break residue left in an otherwise empty Claude composer. +/// +/// A failed injection can leave only its line breaks behind: the composer +/// renders as a bare `❯` row followed by blank rows, with the cursor on the +/// last blank row and the composer frame's rule below it. That is not idle (the +/// cursor is not on the prompt row), so the failed-draft latch would wait +/// forever. Returns the number of blank rows between the prompt and the cursor +/// when — and only when — no text exists anywhere in that composer, so erasing +/// them can never touch a human draft. `None` for every other state. +pub(crate) fn blank_composer_residue_rows(cli: &str, snapshot: &Snapshot) -> Option { + if !cli.to_ascii_lowercase().contains("claude") + || snapshot.has_visible_text_at_or_after_cursor() + { + return None; + } + let screen = snapshot.to_plain(); + let lines: Vec<&str> = screen.lines().collect(); + let cursor = (snapshot.cursor.0 as usize).checked_sub(1)?; + if cursor >= lines.len() { + return None; + } + let mut row = cursor; + while lines[row].trim().is_empty() { + if cursor - row >= MAX_BLANK_COMPOSER_RESIDUE_ROWS { + return None; + } + row = row.checked_sub(1)?; + } + if !matches!(lines[row].trim(), "❯" | ">") { + return None; + } + let blank_rows = cursor - row; + if blank_rows == 0 { + return None; + } + // The composer must close with its frame rule below the cursor; any text + // there could be the rest of a draft the cursor was moved up into. + let below = lines[cursor + 1..] + .iter() + .find(|line| !line.trim().is_empty())?; + let rule = below.trim(); + if !rule.chars().all(|c| matches!(c, '─' | '━' | '╌' | '-')) { + return None; + } + Some(blank_rows) +} + +/// Bytes that erase [`blank_composer_residue_rows`] line breaks: one +/// backspace per blank row joins each back onto the prompt row. +pub(crate) fn blank_composer_residue_cleanup(rows: usize) -> Vec { + vec![0x7f; rows] +} + /// Distinguish terminal echo from actual harness acceptance. /// /// Activity is definitive acceptance. A body is considered parked only when @@ -1886,4 +1945,50 @@ mod tests { throttle.record(DeliveryOutcome::Success); assert_eq!(throttle.delay(), Duration::from_millis(100)); } + + /// Last 24 rows of desktop-upgrade-sf's PTY (relay broker snapshot, 2026-10-19): + /// Claude idle after a turn, but its composer holds one line break left by + /// a failed injection, so the cursor sits at [22,3] under a bare `❯ `. + const LIVE_CLAUDE_NEWLINE_RESIDUE: &str = " codex-relay-desktop-249119, this time about dogpatch-mini, which needed no\r\n upgrade.\r\n\r\n It prompted a quick read-only check of sf-mini:\r\n\r\n - Installed: Relay Desktop 2026.10.47, build 84, running. I installed\r\n 2026.10.45 (build 81) on Oct 16, so someone else has upgraded it since.\r\n - Latest release: v2026.10.48, published Oct 18 04:26 UTC. sf-mini is one\r\n release behind.\r\n - Build number: dogpatch-mini reports 2026.10.48 as build 84, the same build\r\n number sf-mini shows for 2026.10.47. That could be a typo in their report or\r\n a release build number that wasn't bumped. It doesn't matter for this\r\n machine.\r\n\r\n If you want sf-mini on 2026.10.48, say so and I'll do the same upgrade as\r\n before.\r\n\r\n✻ Worked for 31s · done 11:43 PM\r\n\r\n────────────────────────────────────────────────────────────────────────────────\r\n❯ \r\n\r\n────────────────────────────────────────────────────────────────────────────────\r\n ⏵⏵ bypass permissions on (shift+tab to cycle)"; + + #[cfg(unix)] + #[tokio::test] + async fn blank_composer_residue_holds_the_failed_draft_latch_until_cleared() { + let expected = "Nudge: please ack the relay-desktop#577 read-only validation ask"; + let (_pty, residue) = + codex_snapshot(&format!("{LIVE_CLAUDE_NEWLINE_RESIDUE}\x1b[22;3H")).await; + assert_eq!(residue.cursor, (22, 3)); + // The bug: an empty-but-two-row composer never reads as idle, so the + // latch never released and every later delivery waited forever. + assert!(!composer_is_idle(&residue, "claude")); + assert!(!failed_draft_released("claude", expected, &residue)); + assert_eq!(blank_composer_residue_rows("claude", &residue), Some(1)); + assert_eq!(blank_composer_residue_cleanup(1), vec![0x7f]); + assert_eq!(blank_composer_residue_rows("codex", &residue), None); + + // Once the line break is erased the cursor is back on the prompt row. + let (_pty, cleared) = + codex_snapshot(&format!("{LIVE_CLAUDE_NEWLINE_RESIDUE}\x1b[21;3H")).await; + // The live prompt row is `❯` + U+00A0; it must still anchor the composer. + assert_eq!(cleared.to_plain().lines().nth(20), Some("❯\u{a0}")); + assert!(composer_is_idle(&cleared, "claude")); + assert!(failed_draft_released("claude", expected, &cleared)); + assert_eq!(blank_composer_residue_rows("claude", &cleared), None); + } + + #[cfg(unix)] + #[tokio::test] + async fn blank_composer_residue_never_matches_a_composer_holding_text() { + let text_on_cursor_row = " codex-relay-desktop-249119, this time about dogpatch-mini, which needed no\r\n upgrade.\r\n\r\n It prompted a quick read-only check of sf-mini:\r\n\r\n - Installed: Relay Desktop 2026.10.47, build 84, running. I installed\r\n 2026.10.45 (build 81) on Oct 16, so someone else has upgraded it since.\r\n - Latest release: v2026.10.48, published Oct 18 04:26 UTC. sf-mini is one\r\n release behind.\r\n - Build number: dogpatch-mini reports 2026.10.48 as build 84, the same build\r\n number sf-mini shows for 2026.10.47. That could be a typo in their report or\r\n a release build number that wasn't bumped. It doesn't matter for this\r\n machine.\r\n\r\n If you want sf-mini on 2026.10.48, say so and I'll do the same upgrade as\r\n before.\r\n\r\n✻ Worked for 31s · done 11:43 PM\r\n\r\n────────────────────────────────────────────────────────────────────────────────\r\n❯ \r\ndraft continues\r\n────────────────────────────────────────────────────────────────────────────────\r\n ⏵⏵ bypass permissions on (shift+tab to cycle)"; + let (_pty, snap) = codex_snapshot(&format!("{text_on_cursor_row}\x1b[22;3H")).await; + assert_eq!(blank_composer_residue_rows("claude", &snap), None); + + let text_on_prompt_row = " codex-relay-desktop-249119, this time about dogpatch-mini, which needed no\r\n upgrade.\r\n\r\n It prompted a quick read-only check of sf-mini:\r\n\r\n - Installed: Relay Desktop 2026.10.47, build 84, running. I installed\r\n 2026.10.45 (build 81) on Oct 16, so someone else has upgraded it since.\r\n - Latest release: v2026.10.48, published Oct 18 04:26 UTC. sf-mini is one\r\n release behind.\r\n - Build number: dogpatch-mini reports 2026.10.48 as build 84, the same build\r\n number sf-mini shows for 2026.10.47. That could be a typo in their report or\r\n a release build number that wasn't bumped. It doesn't matter for this\r\n machine.\r\n\r\n If you want sf-mini on 2026.10.48, say so and I'll do the same upgrade as\r\n before.\r\n\r\n✻ Worked for 31s · done 11:43 PM\r\n\r\n────────────────────────────────────────────────────────────────────────────────\r\n❯ my own draft\r\n\r\n────────────────────────────────────────────────────────────────────────────────\r\n ⏵⏵ bypass permissions on (shift+tab to cycle)"; + let (_pty, snap) = codex_snapshot(&format!("{text_on_prompt_row}\x1b[22;3H")).await; + assert_eq!(blank_composer_residue_rows("claude", &snap), None); + + let text_after_cursor = " codex-relay-desktop-249119, this time about dogpatch-mini, which needed no\r\n upgrade.\r\n\r\n It prompted a quick read-only check of sf-mini:\r\n\r\n - Installed: Relay Desktop 2026.10.47, build 84, running. I installed\r\n 2026.10.45 (build 81) on Oct 16, so someone else has upgraded it since.\r\n - Latest release: v2026.10.48, published Oct 18 04:26 UTC. sf-mini is one\r\n release behind.\r\n - Build number: dogpatch-mini reports 2026.10.48 as build 84, the same build\r\n number sf-mini shows for 2026.10.47. That could be a typo in their report or\r\n a release build number that wasn't bumped. It doesn't matter for this\r\n machine.\r\n\r\n If you want sf-mini on 2026.10.48, say so and I'll do the same upgrade as\r\n before.\r\n\r\n✻ Worked for 31s · done 11:43 PM\r\n\r\n────────────────────────────────────────────────────────────────────────────────\r\n❯ \r\n later text\r\n────────────────────────────────────────────────────────────────────────────────\r\n ⏵⏵ bypass permissions on (shift+tab to cycle)"; + let (_pty, snap) = codex_snapshot(&format!("{text_after_cursor}\x1b[22;1H")).await; + assert_eq!(blank_composer_residue_rows("claude", &snap), None); + } } diff --git a/crates/broker/src/pty_worker.rs b/crates/broker/src/pty_worker.rs index 21f2f9ce7c..bf94b35526 100644 --- a/crates/broker/src/pty_worker.rs +++ b/crates/broker/src/pty_worker.rs @@ -28,10 +28,11 @@ use tokio::{ use crate::broker::{ continuity::parse_continuity_command, delivery_verification::{ - assess_harness_acceptance, current_timestamp_ms, delivery_injected_event_payload, - delivery_queued_event_payload, draft_visible_at_cursor, failed_draft_released, - DeliveryOutcome, HarnessAcceptance, PendingActivity, PendingVerification, ThrottleState, - VerificationOutput, ACTIVITY_BUFFER_KEEP_BYTES, ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW, + assess_harness_acceptance, blank_composer_residue_cleanup, blank_composer_residue_rows, + current_timestamp_ms, delivery_injected_event_payload, delivery_queued_event_payload, + draft_visible_at_cursor, failed_draft_released, DeliveryOutcome, HarnessAcceptance, + PendingActivity, PendingVerification, ThrottleState, VerificationOutput, + ACTIVITY_BUFFER_KEEP_BYTES, ACTIVITY_BUFFER_MAX_BYTES, ACTIVITY_WINDOW, MAX_VERIFICATION_ATTEMPTS, VERIFICATION_WINDOW, }, injection_format::{format_injection_for_worker_with_workspace, McpReminderThrottle}, @@ -115,6 +116,24 @@ struct FailedComposerLatch { latched_at: Instant, /// The terminal stuck signal has been sent for this latch. escalated: bool, + /// Blank-row residue cleanups already sent for this latch. + residue_cleanups: u8, + /// When the last residue cleanup was written. + last_residue_cleanup: Option, +} + +/// Most blank-row residue cleanups one latch may send before it gives up. +const MAX_RESIDUE_CLEANUPS: u8 = 3; + +/// Minimum spacing between residue cleanups so the harness can repaint. +const RESIDUE_CLEANUP_SPACING: Duration = Duration::from_secs(2); + +/// Whether the latch may erase blank-row composer residue now. +fn residue_cleanup_due(latch: &FailedComposerLatch, now: Instant) -> bool { + latch.residue_cleanups < MAX_RESIDUE_CLEANUPS + && latch + .last_residue_cleanup + .is_none_or(|at| now.duration_since(at) >= RESIDUE_CLEANUP_SPACING) } /// How long a visibly stuck draft may hold injection before the worker sends @@ -161,6 +180,8 @@ fn latch_failed_written_delivery( expected_echo: expected_echo.to_string(), latched_at: Instant::now(), escalated: false, + residue_cleanups: 0, + last_residue_cleanup: None, }); } @@ -3110,22 +3131,41 @@ pub(crate) async fn run_pty_worker(cmd: PtyCommand) -> Result<()> { // Release the failed-draft latch once the composer no longer // holds that body (the harness took it late, or a human // submitted or cleared it). Until then queued deliveries wait. - if let Some(latch) = failed_composer_latch.as_ref() { + // A composer left holding only blank rows (line breaks from the + // failed body) is not idle and would hold the latch forever; + // erase exactly those rows, never any text. + if let Some(latch) = failed_composer_latch.as_mut() { if pending_verifications.is_empty() && pending_recovery_writes.is_empty() && active_injection.is_none() - && failed_draft_released( - &resolved_cli, - &latch.expected_echo, - &Snapshot::capture(&pty), - ) { - tracing::info!( - delivery_id = %latch.delivery_id, - event_id = %latch.event_id, - "failed delivery left the composer; resuming injection" - ); - failed_composer_latch = None; + let snapshot = Snapshot::capture(&pty); + if failed_draft_released(&resolved_cli, &latch.expected_echo, &snapshot) { + tracing::info!( + delivery_id = %latch.delivery_id, + event_id = %latch.event_id, + "failed delivery left the composer; resuming injection" + ); + failed_composer_latch = None; + } else if let Some(rows) = + blank_composer_residue_rows(&resolved_cli, &snapshot) + { + let now = Instant::now(); + if residue_cleanup_due(latch, now) { + latch.residue_cleanups = latch.residue_cleanups.saturating_add(1); + latch.last_residue_cleanup = Some(now); + tracing::warn!( + delivery_id = %latch.delivery_id, + rows, + attempt = latch.residue_cleanups, + "failed draft left only blank rows in the composer; erasing them" + ); + warn_on_auto_response_write( + pty.submit_write(blank_composer_residue_cleanup(rows)), + "failed_draft_residue_cleanup", + ); + } + } } } // A draft still stuck after the bounded timeout gets one @@ -4879,6 +4919,30 @@ mod tests { ); } + #[test] + fn residue_cleanup_is_bounded_and_spaced() { + let mut latch = FailedComposerLatch { + delivery_id: DeliveryId::from("del_failed"), + event_id: EventId::from("evt_failed"), + expected_echo: "Relay message from Lead [evt]: failed".to_string(), + latched_at: Instant::now(), + escalated: false, + residue_cleanups: 0, + last_residue_cleanup: None, + }; + let start = Instant::now(); + assert!(residue_cleanup_due(&latch, start)); + latch.residue_cleanups = 1; + latch.last_residue_cleanup = Some(start); + assert!(!residue_cleanup_due(&latch, start), "spaced"); + assert!(residue_cleanup_due(&latch, start + RESIDUE_CLEANUP_SPACING)); + latch.residue_cleanups = MAX_RESIDUE_CLEANUPS; + assert!( + !residue_cleanup_due(&latch, start + RESIDUE_CLEANUP_SPACING * 10), + "bounded" + ); + } + #[test] fn latched_worker_reports_blocked_only_when_the_latch_is_the_sole_blocker() { let latch = FailedComposerLatch { @@ -4887,6 +4951,8 @@ mod tests { expected_echo: "Relay message from Lead [evt]: failed".to_string(), latched_at: Instant::now(), escalated: false, + residue_cleanups: 0, + last_residue_cleanup: None, }; assert!(failed_draft_latch_is_sole_blocker( Some(&latch), diff --git a/crates/broker/src/runtime/delivery.rs b/crates/broker/src/runtime/delivery.rs index 31b005df5f..f98ef7f434 100644 --- a/crates/broker/src/runtime/delivery.rs +++ b/crates/broker/src/runtime/delivery.rs @@ -7,7 +7,9 @@ pub(crate) struct PendingDelivery { pub(super) attempts: u32, /// Consecutive broker-to-worker handoff failures. Successful writes reset /// this count because waiting for the PTY to acknowledge an already queued - /// delivery is not a failed delivery attempt. + /// delivery is not a failed delivery attempt. The exception is a worker + /// that reported `blocked_on_send`: it drops the re-send as a duplicate, + /// so the handoff counts against the budget and the delivery dead-letters. pub(super) failed_attempts: u32, pub(super) next_retry_at: Instant, pub(super) queued_at_ms: u64, @@ -973,6 +975,11 @@ pub(crate) async fn insert_and_attempt_delivery( Ok(delivery_id) } +/// `last_error` for a delivery handed to a worker that reported +/// `blocked_on_send`: the handoff succeeded but nothing was injected. +pub(crate) const BLOCKED_ON_SEND_NOT_INJECTED: &str = + "worker blocked_on_send; delivery not injected"; + pub(crate) async fn retry_pending_delivery( delivery_id: &DeliveryId, workers: &mut WorkerRegistry, @@ -1027,6 +1034,18 @@ pub(crate) async fn retry_pending_delivery( return Ok(DeliveryAttemptOutcome::Noop); } + // A worker that reported `blocked_on_send` keeps queued deliveries behind + // a gate it cannot clear by itself, and drops a re-sent delivery it + // already holds as a duplicate. A successful handoff to it is therefore + // not progress: count it against the retry budget so the delivery + // dead-letters visibly instead of being re-sent forever. A worker that is + // merely busy (Working) keeps the old behaviour so a long turn never + // dead-letters wait-mode messages. + let worker_blocked_on_send = workers + .workers + .get(&pending.worker_name) + .is_some_and(|handle| handle.state == crate::worker::AgentWorkState::BlockedOnSend); + match workers .deliver(&pending.worker_name, pending.delivery.clone()) .await @@ -1034,10 +1053,15 @@ pub(crate) async fn retry_pending_delivery( Ok(()) => { if let Some(current) = pending_deliveries.get_mut(delivery_id) { current.attempts = current.attempts.saturating_add(1); - current.failed_attempts = 0; current.next_retry_at = Instant::now() + delivery_ack_timeout(¤t.delivery.injection_mode, retry_interval); - current.last_error = None; + if worker_blocked_on_send { + current.failed_attempts = current.failed_attempts.saturating_add(1); + current.last_error = Some(BLOCKED_ON_SEND_NOT_INJECTED.to_string()); + } else { + current.failed_attempts = 0; + current.last_error = None; + } return Ok(DeliveryAttemptOutcome::Attempted { worker_name: current.worker_name.clone(), attempts: current.attempts, diff --git a/crates/broker/src/runtime/tests.rs b/crates/broker/src/runtime/tests.rs index 1606cedac4..85249c53bc 100644 --- a/crates/broker/src/runtime/tests.rs +++ b/crates/broker/src/runtime/tests.rs @@ -11291,3 +11291,79 @@ async fn already_acked_attachment_replay_bypasses_download_staging() { Ok(FleetControlCommand::Send(BrokerToRelaycast::DeliveryAck(_))) )); } + +/// A worker that reported `blocked_on_send` drops a re-sent delivery it already +/// holds as a duplicate, so a successful handoff to it is not progress. Before +/// this fix every 5-minute re-send reset `failed_attempts` to 0 and the +/// delivery stayed pending for days (desktop-upgrade-sf/finn, 2026-10-19). +#[tokio::test] +async fn resend_to_blocked_on_send_worker_counts_against_retry_budget_and_dead_letters() { + let mut workers = make_worker_registry_with_worker("blocked-worker").await; + workers + .workers + .get_mut("blocked-worker") + .expect("fixture worker exists") + .state = AgentWorkState::BlockedOnSend; + let id = DeliveryId::new("del_blocked"); + let mut pending_deliveries = HashMap::from([( + id.clone(), + make_pending_delivery("del_blocked", "blocked-worker"), + )]); + + for expected in 1..=MAX_DELIVERY_RETRIES { + let outcome = retry_pending_delivery( + &id, + &mut workers, + &mut pending_deliveries, + Duration::from_millis(1), + ) + .await + .expect("handoff to a live worker succeeds"); + assert!(matches!(outcome, DeliveryAttemptOutcome::Attempted { .. })); + let current = pending_deliveries.get(&id).expect("still pending"); + assert_eq!(current.failed_attempts, expected, "no reset while blocked"); + assert_eq!( + current.last_error.as_deref(), + Some(super::delivery::BLOCKED_ON_SEND_NOT_INJECTED) + ); + } + + let outcome = retry_pending_delivery( + &id, + &mut workers, + &mut pending_deliveries, + Duration::from_millis(1), + ) + .await + .expect("exhausted budget classifies as terminal"); + match outcome { + DeliveryAttemptOutcome::Failed { last_error, .. } => { + assert_eq!(last_error, super::delivery::BLOCKED_ON_SEND_NOT_INJECTED) + } + other => panic!("expected dead-letter outcome, got {other:?}"), + } + assert!(pending_deliveries.is_empty()); +} + +/// A busy (Working) worker keeps the old behaviour: a long turn must never +/// dead-letter wait-mode deliveries queued behind it. +#[tokio::test] +async fn resend_to_working_worker_still_resets_failed_attempts() { + let mut workers = make_worker_registry_with_worker("busy-worker").await; + let id = DeliveryId::new("del_busy"); + let mut pending = make_pending_delivery("del_busy", "busy-worker"); + pending.failed_attempts = MAX_DELIVERY_RETRIES - 1; + let mut pending_deliveries = HashMap::from([(id.clone(), pending)]); + + retry_pending_delivery( + &id, + &mut workers, + &mut pending_deliveries, + Duration::from_millis(1), + ) + .await + .expect("handoff succeeds"); + let current = pending_deliveries.get(&id).expect("still pending"); + assert_eq!(current.failed_attempts, 0); + assert_eq!(current.last_error, None); +}