diff --git a/CHANGELOG.md b/CHANGELOG.md index ba93110501..584d3d38b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,23 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Minor] + +### Added + +- `agent-relay fleet spawn --sandbox` (unpinned or `--sandbox-provider agent37`) now uses Cloud's durable `async-v1` preparation and prints each preparation phase, so a slow Relayfile initial sync no longer ends as an unknown outcome. Older Cloud deployments keep working through the synchronous response. +- `@agent-relay/cloud` `ensureCloudFleetSandbox` accepts `preparationMode: 'async-v1'` and an `onPreparationProgress` callback; `CloudFleetSandboxProvisionError` exposes `sandboxAbsent` and a typed `preparationFailure` (`code`, `phase`, `causeStage`). + +### Fixed + +- When Cloud ends a sandbox preparation (for example `relayfile_mount_failed` / `initial_sync_deadline`), `fleet spawn --sandbox` names the cause and confirms no sandbox was left running, without a redundant cleanup call or unknown-outcome guidance. +- A lost Cloud response during `fleet spawn --sandbox` preparation no longer causes the spawn or a preparation step to be resubmitted. +- `fleet spawn --sandbox` prints the generated sandbox identity before provisioning; re-running an interrupted spawn with `--sandbox-id ` resumes that sandbox instead of creating a second one. +- Against a Cloud that does not support async preparation, a typed failure such as `relayfile_mount_failed` or a capacity rejection is reported at once with its cause instead of after a status-polling wait. +- Workspace-scoped commands such as `agent-relay fleet release` continue using the workspace's Relaycast server when `--base-url` is omitted; explicit conflicting `--base-url` values are still rejected. +- Fleet spawns now recover from transient Relaycast `d1_pressure`; persistent registration pressure returns a named `d1_pressure` error within 25 seconds, and any success arriving after retry-count or wall-clock exhaustion is explicitly deregistered. +- Acknowledged `agent.deregister`, cumulative `delivery.ack`, and non-task action results now recover from transient Relaycast `d1_pressure`; exhausted non-task action results are replayed after reconnect. +- `agent-relay fleet spawn` reports registration `d1_pressure` as retryable only when the registration is proven to have failed before any change was made. ## [13.1.5] - 2026-10-08 diff --git a/crates/broker/src/node_control.rs b/crates/broker/src/node_control.rs index b6ae4ce16d..314cb32941 100644 --- a/crates/broker/src/node_control.rs +++ b/crates/broker/src/node_control.rs @@ -58,6 +58,12 @@ const INITIAL_RECONNECT_DELAY: Duration = Duration::from_secs(1); const MAX_RECONNECT_DELAY: Duration = Duration::from_secs(30); const D1_PRESSURE_RETRY_INITIAL_DELAY: Duration = Duration::from_secs(1); const D1_PRESSURE_RETRY_MAX_DELAY: Duration = Duration::from_secs(8); +// Five retries consume 1 + 2 + 4 + 8 + 8 = 23 seconds. Keep the wall-clock +// budget below the 30-second fleet registration waiter so persistent pressure +// reaches the spawn caller as a named d1_pressure failure, not a generic +// agent_register_timeout. +const CONTROL_D1_PRESSURE_MAX_RETRIES: u32 = 5; +const CONTROL_D1_PRESSURE_RETRY_BUDGET: Duration = Duration::from_secs(25); const REGISTER_AGENT_PENDING_TTL: Duration = Duration::from_secs(300); const RELAYCAST_DEFAULT_BASE_URL: &str = "https://cast.agentrelay.com"; const CREATE_NODE_RETRY_BACKOFFS_MS: [u64; 3] = [200, 400, 800]; @@ -70,6 +76,28 @@ fn d1_pressure_retry_delay(retry_attempts: u32) -> Duration { (D1_PRESSURE_RETRY_INITIAL_DELAY * multiplier).min(D1_PRESSURE_RETRY_MAX_DELAY) } +#[derive(Clone, Copy)] +struct ControlRetryPolicy { + initial_delay: Duration, + max_delay: Duration, + max_retries: u32, + budget: Duration, +} + +impl ControlRetryPolicy { + const DEFAULT: Self = Self { + initial_delay: D1_PRESSURE_RETRY_INITIAL_DELAY, + max_delay: D1_PRESSURE_RETRY_MAX_DELAY, + max_retries: CONTROL_D1_PRESSURE_MAX_RETRIES, + budget: CONTROL_D1_PRESSURE_RETRY_BUDGET, + }; + + fn retry_delay(self, retry_attempts: u32) -> Duration { + let multiplier = 1u32 << retry_attempts.saturating_sub(1).min(3); + (self.initial_delay * multiplier).min(self.max_delay) + } +} + /// Whether a fresh re-mint should still be attempted at this consecutive-401 /// count. The count is incremented for the current 401 *before* this is called, /// so the very first 401 (count 1) is still within budget. Crucially this is @@ -724,8 +752,12 @@ pub(crate) struct AgentRegistrationToken { struct PendingAgentRegistration { isolates_channels: bool, name: String, - reply: oneshot::Sender>, + request: AgentRegister, + reply: Option>>, created_at: Instant, + d1_pressure_retry_attempts: u32, + d1_pressure_retry_at: Option, + last_d1_pressure: Option, } #[derive(Debug)] @@ -2321,6 +2353,7 @@ pub(crate) async fn run_node_control_client_with_completions( // token once, but never follow an unbounded chain of cache changes. let mut proof_conflict_cache_retry_used = false; let mut retained_completion: Option = None; + let mut retained_pressure_frames: VecDeque = VecDeque::new(); loop { // Retained completion frames stay in their unbounded lane until a @@ -2488,11 +2521,13 @@ pub(crate) async fn run_node_control_client_with_completions( &mut command_rx, &mut completion_rx, &mut retained_completion, + &mut retained_pressure_frames, &event_tx, &mut registration, &mut inventory, &mut load, INVENTORY_REFRESH_INTERVAL, + ControlRetryPolicy::DEFAULT, ) .await; if matches!(result, ControlRunResult::Shutdown) { @@ -2669,23 +2704,204 @@ struct PendingInventorySync { superseded: bool, } +struct PendingRetryableControlFrame { + frame: BrokerToRelaycast, + created_at: Instant, + retry_attempts: u32, + retry_at: Option, + last_d1_pressure: Option, + replayed_after_pressure: bool, +} + +#[derive(Debug, PartialEq, Eq)] +enum ControlPressureRetry { + Unmatched, + Scheduled(Duration), + Exhausted, +} + struct ApplicationLiveness { deadline: Duration, + control_retry_policy: ControlRetryPolicy, last_acknowledged: Instant, pending_inventory_syncs: VecDeque, + pending_retryable_control_frames: HashMap, + exhausted_retryable_control_frames: HashMap, ready: bool, } impl ApplicationLiveness { + #[cfg(test)] fn new(deadline: Duration) -> Self { + Self::new_with_retry_policy(deadline, ControlRetryPolicy::DEFAULT) + } + + fn new_with_retry_policy(deadline: Duration, control_retry_policy: ControlRetryPolicy) -> Self { Self { deadline, + control_retry_policy, last_acknowledged: Instant::now(), pending_inventory_syncs: VecDeque::new(), + pending_retryable_control_frames: HashMap::new(), + exhausted_retryable_control_frames: HashMap::new(), ready: false, } } + fn track_retryable_control_frame(&mut self, id: String, frame: BrokerToRelaycast) { + self.exhausted_retryable_control_frames.remove(&id); + self.pending_retryable_control_frames.insert( + id, + PendingRetryableControlFrame { + frame, + created_at: Instant::now(), + retry_attempts: 0, + retry_at: None, + last_d1_pressure: None, + replayed_after_pressure: false, + }, + ); + } + + fn mark_retryable_control_frame_replayed(&mut self, id: &str) { + if let Some(pending) = self.pending_retryable_control_frames.get_mut(id) { + pending.replayed_after_pressure = true; + } + } + + fn complete_retryable_control_frame(&mut self, id: &str) { + self.pending_retryable_control_frames.remove(id); + } + + fn consume_exhausted_retryable_control_frame(&mut self, id: &str) -> bool { + self.exhausted_retryable_control_frames.remove(id).is_some() + } + + fn schedule_control_d1_pressure_retry( + &mut self, + id: &str, + reason: &str, + now: Instant, + ) -> ControlPressureRetry { + let Some(pending) = self.pending_retryable_control_frames.get_mut(id) else { + return ControlPressureRetry::Unmatched; + }; + pending.last_d1_pressure = Some(reason.to_string()); + let deadline = pending.created_at + self.control_retry_policy.budget; + if pending.retry_attempts >= self.control_retry_policy.max_retries || now >= deadline { + return ControlPressureRetry::Exhausted; + } + let next_attempt = pending.retry_attempts.saturating_add(1); + let delay = self.control_retry_policy.retry_delay(next_attempt); + if now + .checked_add(delay) + .is_none_or(|retry_at| retry_at > deadline) + { + return ControlPressureRetry::Exhausted; + } + pending.retry_attempts = next_attempt; + pending.retry_at = Some(now + delay); + ControlPressureRetry::Scheduled(delay) + } + + fn next_control_retry_event_delay(&self, now: Instant) -> Option { + // Successful delivery acknowledgements and non-task action results are + // intentionally one-way: the engine sends no positive reply. Their + // entries therefore live until this bounded budget elapses. Memory is + // bounded to roughly one budget window of control traffic, and this + // O(n) deadline scan covers only that window rather than session age. + self.pending_retryable_control_frames + .values() + .flat_map(|pending| { + [ + pending.retry_at, + Some(pending.created_at + self.control_retry_policy.budget), + ] + }) + .flatten() + .chain( + self.exhausted_retryable_control_frames + .values() + .map(|exhausted_at| *exhausted_at + self.control_retry_policy.budget), + ) + .map(|event_at| event_at.saturating_duration_since(now)) + .min() + } + + fn expire_retryable_control_frames( + &mut self, + now: Instant, + ) -> Vec<(String, String, BrokerToRelaycast, bool)> { + self.exhausted_retryable_control_frames + .retain(|_, exhausted_at| { + now.saturating_duration_since(*exhausted_at) < self.control_retry_policy.budget + }); + let expired: Vec = self + .pending_retryable_control_frames + .iter() + .filter(|(_, pending)| { + now.saturating_duration_since(pending.created_at) + >= self.control_retry_policy.budget + }) + .map(|(id, _)| id.clone()) + .collect(); + expired + .into_iter() + .filter_map(|id| { + let pending = self.pending_retryable_control_frames.remove(&id)?; + pending.last_d1_pressure.map(|reason| { + // At most one response can still be in flight because a retry is + // scheduled only after the previous pressure response arrives. + // Keep a one-shot tombstone so that late terminal response cannot + // produce a second task outcome after the timeout path below. + self.exhausted_retryable_control_frames + .insert(id.clone(), now); + ( + id, + format!("d1_pressure: {reason} (retry budget exhausted)"), + pending.frame, + pending.replayed_after_pressure, + ) + }) + }) + .collect() + } + + fn take_retryable_control_frame(&mut self, id: &str) -> Option { + self.pending_retryable_control_frames.remove(id) + } + + fn retain_non_task_action_results( + &mut self, + retained_pressure_frames: &mut VecDeque, + ) { + let retained_ids: Vec = self + .pending_retryable_control_frames + .iter() + .filter(|(id, pending)| { + !id.starts_with(crate::runtime::task_request_prefix()) + && matches!(&pending.frame, BrokerToRelaycast::ActionResult(_)) + && pending.last_d1_pressure.is_some() + }) + .map(|(id, _)| id.clone()) + .collect(); + for id in retained_ids { + if let Some(pending) = self.pending_retryable_control_frames.remove(&id) { + retained_pressure_frames.push_back(pending.frame); + } + } + } + + fn take_due_control_retry(&mut self, now: Instant) -> Option { + let pending = self + .pending_retryable_control_frames + .values_mut() + .filter(|pending| pending.retry_at.is_some_and(|retry_at| retry_at <= now)) + .min_by_key(|pending| pending.retry_at)?; + pending.retry_at = None; + Some(pending.frame.clone()) + } + fn track_inventory_sync(&mut self, frame: InventorySync) { // A newer authoritative snapshot supersedes every older retry payload. // Keep the probes for correlation/liveness, but never let an old frame @@ -2954,6 +3170,7 @@ async fn handle_connected_command( pending_deregistrations: &mut HashMap>>, application_liveness: &mut ApplicationLiveness, retained_completion: &mut Option, + retained_pressure_frames: &mut VecDeque, ) -> std::ops::ControlFlow where S: Sink + Unpin, @@ -3021,13 +3238,16 @@ where } Continue(()) } - Some(FleetControlCommand::Send(message)) => { + Some(FleetControlCommand::Send(mut message)) => { // A `delivery_ack` is the engine's only evidence that a // frame was consumed. The runtime records the *decision* // to ack before handing it here and cannot wait for the // wire, so the probe learns the outcome at the one place // that knows it. See `NodeDeliveryProbe::record_ack_sent`. let is_ack = matches!(message, BrokerToRelaycast::DeliveryAck(_)); + if let Some((request_id, _)) = prepare_retryable_control_frame(&mut message) { + application_liveness.track_retryable_control_frame(request_id, message.clone()); + } let sent = send_wire(sink, &message).await; if let (true, Some(probe)) = (is_ack, config.probe.as_ref()) { if sent.is_ok() { @@ -3039,7 +3259,18 @@ where if sent.is_err() { if let BrokerToRelaycast::ActionResult(result) = &message { tracing::warn!(invocation_id = %result.invocation_id, frame_kind = "action.result", - "dropping fleet frame after wire send failure"); + "retaining fleet result for replay after reconnect"); + if result.task.is_none() + && !result + .id + .as_deref() + .is_some_and(|id| id.starts_with(crate::runtime::task_request_prefix())) + { + if let Some(id) = result.id.as_deref() { + application_liveness.complete_retryable_control_frame(id); + } + retained_pressure_frames.push_back(message); + } } return Break(ControlRunResult::Disconnected { application_ready: application_liveness.ready, @@ -3047,8 +3278,17 @@ where } Continue(()) } - Some(FleetControlCommand::RetainedSend(completion)) => { + Some(FleetControlCommand::RetainedSend(mut completion)) => { + let retry_id = prepare_retryable_control_frame(&mut completion.message) + .map(|(request_id, _)| request_id); + if let Some(request_id) = retry_id.as_ref() { + application_liveness + .track_retryable_control_frame(request_id.clone(), completion.message.clone()); + } if send_wire(sink, &completion.message).await.is_err() { + if let Some(request_id) = retry_id { + application_liveness.complete_retryable_control_frame(&request_id); + } *retained_completion = Some(completion); return Break(ControlRunResult::Disconnected { application_ready: application_liveness.ready, @@ -3061,11 +3301,10 @@ where let request_id = format!("agent_deregister_{}", Uuid::new_v4().simple()); request.id = Some(request_id.clone()); pending_deregistrations.retain(|_, pending| !pending.is_closed()); - pending_deregistrations.insert(request_id, reply); - if send_wire(sink, &BrokerToRelaycast::AgentDeregister(request)) - .await - .is_err() - { + pending_deregistrations.insert(request_id.clone(), reply); + let message = BrokerToRelaycast::AgentDeregister(request); + application_liveness.track_retryable_control_frame(request_id, message.clone()); + if send_wire(sink, &message).await.is_err() { return Break(ControlRunResult::Disconnected { application_ready: application_liveness.ready, }); @@ -3078,16 +3317,21 @@ where .clone() .unwrap_or_else(|| format!("agent_register_{}", Uuid::new_v4().simple())); request.id = Some(request_id.clone()); + let wire_request = request.clone(); pending_agent_registrations.insert( request_id, PendingAgentRegistration { isolates_channels: request.auto_join_general == Some(false), name: request.name.clone(), - reply, + request, + reply: Some(reply), created_at: Instant::now(), + d1_pressure_retry_attempts: 0, + d1_pressure_retry_at: None, + last_d1_pressure: None, }, ); - if send_wire(sink, &BrokerToRelaycast::AgentRegister(request)) + if send_wire(sink, &BrokerToRelaycast::AgentRegister(wire_request)) .await .is_err() { @@ -3114,6 +3358,38 @@ async fn run_connected_once( inventory: &mut Vec, load: &mut FleetLoadSnapshot, inventory_refresh_interval: Duration, +) -> ControlRunResult { + let (_completion_tx, mut completion_rx) = mpsc::unbounded_channel(); + let mut retained_completion = None; + let mut retained_pressure_frames = VecDeque::new(); + run_connected_once_with_completions( + config, + command_rx, + &mut completion_rx, + &mut retained_completion, + &mut retained_pressure_frames, + event_tx, + registration, + inventory, + load, + inventory_refresh_interval, + ControlRetryPolicy::DEFAULT, + ) + .await +} + +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +async fn run_connected_once_with_retry_policy( + config: &FleetControlConfig, + command_rx: &mut mpsc::Receiver, + event_tx: &mpsc::Sender, + registration: &mut Option, + inventory: &mut Vec, + load: &mut FleetLoadSnapshot, + inventory_refresh_interval: Duration, + retry_policy: ControlRetryPolicy, + retained_pressure_frames: &mut VecDeque, ) -> ControlRunResult { let (_completion_tx, mut completion_rx) = mpsc::unbounded_channel(); let mut retained_completion = None; @@ -3122,11 +3398,13 @@ async fn run_connected_once( command_rx, &mut completion_rx, &mut retained_completion, + retained_pressure_frames, event_tx, registration, inventory, load, inventory_refresh_interval, + retry_policy, ) .await } @@ -3137,11 +3415,13 @@ async fn run_connected_once_with_completions( command_rx: &mut mpsc::Receiver, completion_rx: &mut mpsc::UnboundedReceiver, retained_completion: &mut Option, + retained_pressure_frames: &mut VecDeque, event_tx: &mpsc::Sender, registration: &mut Option, inventory: &mut Vec, load: &mut FleetLoadSnapshot, inventory_refresh_interval: Duration, + control_retry_policy: ControlRetryPolicy, ) -> ControlRunResult { let Some(mut node_register) = registration.clone() else { return ControlRunResult::ConnectFailed; @@ -3237,7 +3517,10 @@ async fn run_connected_once_with_completions( .checked_mul(2) .unwrap_or(Duration::MAX) .max(read_idle_timeout); - let mut application_liveness = ApplicationLiveness::new(application_liveness_timeout); + let mut application_liveness = ApplicationLiveness::new_with_retry_policy( + application_liveness_timeout, + control_retry_policy, + ); let deferred_commands = match register_node_session( &mut sink, @@ -3285,6 +3568,20 @@ async fn run_connected_once_with_completions( application_ready: false, }; } + while let Some(mut frame) = retained_pressure_frames.pop_front() { + let request_id = prepare_retryable_control_frame(&mut frame) + .map(|(id, _)| id) + .expect("pressure-retained control frames must remain retryable"); + application_liveness.track_retryable_control_frame(request_id.clone(), frame.clone()); + application_liveness.mark_retryable_control_frame_replayed(&request_id); + if send_wire(&mut sink, &frame).await.is_err() { + application_liveness.complete_retryable_control_frame(&request_id); + retained_pressure_frames.push_front(frame); + return ControlRunResult::Disconnected { + application_ready: false, + }; + } + } // The idle check runs on the heartbeat tick, so the tick must be shorter // than the window it polices; an overridden (test) window keeps that ratio. @@ -3321,6 +3618,7 @@ async fn run_connected_once_with_completions( &mut pending_deregistrations, &mut application_liveness, retained_completion, + retained_pressure_frames, ) .await { @@ -3335,6 +3633,9 @@ async fn run_connected_once_with_completions( "node_control_disconnected" }; fail_deferred_commands(deferred_commands.collect(), reason, inventory, load); + if matches!(result, ControlRunResult::Disconnected { .. }) { + application_liveness.retain_non_task_action_results(retained_pressure_frames); + } return result; } } @@ -3344,12 +3645,104 @@ async fn run_connected_once_with_completions( let inventory_retry = tokio::time::sleep(inventory_retry_delay.unwrap_or(application_liveness.deadline)); tokio::pin!(inventory_retry); + let agent_registration_retry_delay = next_agent_registration_retry_event_delay( + &pending_agent_registrations, + Instant::now(), + application_liveness.control_retry_policy, + ); + let agent_registration_retry = tokio::time::sleep( + agent_registration_retry_delay.unwrap_or(application_liveness.deadline), + ); + tokio::pin!(agent_registration_retry); + let control_retry_delay = + application_liveness.next_control_retry_event_delay(Instant::now()); + let control_retry = + tokio::time::sleep(control_retry_delay.unwrap_or(application_liveness.deadline)); + tokio::pin!(control_retry); tokio::select! { + _ = &mut control_retry, if control_retry_delay.is_some() => { + let now = Instant::now(); + let mut reconnect_for_retained_result = false; + for (id, reason, frame, replayed_after_pressure) in application_liveness.expire_retryable_control_frames(now) { + let mut surfaced_to_owner = false; + if let Some(pending) = pending_deregistrations.remove(&id) { + let _ = pending.send(Err(reason.clone())); + surfaced_to_owner = true; + } + if id.starts_with(crate::runtime::task_request_prefix()) { + let _ = event_tx.send(FleetControlEvent::Message(RelaycastToBroker::Error( + crate::fleet_wire::Error { + v: FLEET_WIRE_VERSION, + id: id.clone(), + ok: false, + code: "d1_pressure".to_string(), + message: reason.clone(), + }, + ))).await; + surfaced_to_owner = true; + } + if !id.starts_with(crate::runtime::task_request_prefix()) + && matches!(&frame, BrokerToRelaycast::ActionResult(_)) + { + retained_pressure_frames.push_back(frame); + reconnect_for_retained_result |= !replayed_after_pressure; + surfaced_to_owner = true; + tracing::warn!( + target = "relay_broker::fleet", + node_id = %config.node_id, + id, + reason, + replayed_after_pressure, + "retaining action.result after d1_pressure exhaustion" + ); + } + if !surfaced_to_owner { + tracing::error!( + target = "relay_broker::fleet", + node_id = %config.node_id, + id, + reason, + "control frame d1_pressure retry budget exhausted" + ); + } + } + if reconnect_for_retained_result { + application_liveness.retain_non_task_action_results(retained_pressure_frames); + drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; + } + if let Some(frame) = application_liveness.take_due_control_retry(now) { + if send_wire(&mut sink, &frame).await.is_err() { + application_liveness.retain_non_task_action_results(retained_pressure_frames); + drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; + } + } + } + _ = &mut agent_registration_retry, if agent_registration_retry_delay.is_some() => { + let now = Instant::now(); + expire_d1_pressure_agent_registrations( + &mut pending_agent_registrations, + now, + application_liveness.control_retry_policy, + ); + if let Some(frame) = take_due_agent_registration_retry( + &mut pending_agent_registrations, + now, + ) { + if send_wire(&mut sink, &BrokerToRelaycast::AgentRegister(frame)).await.is_err() { + application_liveness.retain_non_task_action_results(retained_pressure_frames); + drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; + } + } + } _ = &mut inventory_retry, if inventory_retry_delay.is_some() => { let Some(frame) = application_liveness.take_due_retry(Instant::now()) else { continue; }; if send_wire(&mut sink, &BrokerToRelaycast::InventorySync(frame)).await.is_err() { + application_liveness.retain_non_task_action_results(retained_pressure_frames); drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } @@ -3368,9 +3761,13 @@ async fn run_connected_once_with_completions( &mut pending_deregistrations, &mut application_liveness, retained_completion, + retained_pressure_frames, ) .await { + if matches!(result, ControlRunResult::Disconnected { .. }) { + application_liveness.retain_non_task_action_results(retained_pressure_frames); + } return result; } } @@ -3392,6 +3789,7 @@ async fn run_connected_once_with_completions( idle_secs = idle.as_secs(), "no inbound node-control frame within the read-idle window; reconnecting" ); + application_liveness.retain_non_task_action_results(retained_pressure_frames); drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } @@ -3406,16 +3804,19 @@ async fn run_connected_once_with_completions( pending_inventory_syncs = application_liveness.pending_inventory_syncs.len(), "node-control application acknowledgement deadline exceeded while transport remained active; reconnecting" ); + application_liveness.retain_non_task_action_results(retained_pressure_frames); drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } if send_wire(&mut sink, &BrokerToRelaycast::NodeHeartbeat(load.heartbeat(&node_register))).await.is_err() { + application_liveness.retain_non_task_action_results(retained_pressure_frames); drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } // Guarantees the peer owes us a frame every interval, so an idle // engine is distinguishable from a dead connection. if send_ws_frame(&mut sink, Message::Ping(Vec::new())).await.is_err() { + application_liveness.retain_non_task_action_results(retained_pressure_frames); drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } @@ -3429,11 +3830,13 @@ async fn run_connected_once_with_completions( ) .await { + application_liveness.retain_non_task_action_results(retained_pressure_frames); return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } } message = stream.next() => { let Some(message) = message else { + application_liveness.retain_non_task_action_results(retained_pressure_frames); drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; }; @@ -3441,6 +3844,7 @@ async fn run_connected_once_with_completions( Ok(message) => message, Err(error) => { tracing::warn!(target = "relay_broker::fleet", error = %error, "fleet node ws read failed"); + application_liveness.retain_non_task_action_results(retained_pressure_frames); drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } @@ -3455,6 +3859,7 @@ async fn run_connected_once_with_completions( &mut pending_agent_registrations, &mut pending_deregistrations, &mut application_liveness, + retained_pressure_frames, &config.node_id, &mut sink, config.probe.as_ref(), @@ -3462,6 +3867,7 @@ async fn run_connected_once_with_completions( ) .await { + application_liveness.retain_non_task_action_results(retained_pressure_frames); drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } @@ -3505,6 +3911,7 @@ async fn handle_server_message( pending_agent_registrations: &mut HashMap, pending_deregistrations: &mut HashMap>>, application_liveness: &mut ApplicationLiveness, + retained_pressure_frames: &mut VecDeque, node_id: &str, sink: &mut S, probe: Option<&std::sync::Arc>, @@ -3547,6 +3954,18 @@ where } match frame { RelaycastToBroker::Reply(reply) => { + if application_liveness + .consume_exhausted_retryable_control_frame(&reply.id) + { + tracing::debug!( + target = "relay_broker::fleet", + node_id, + id = %reply.id, + "consumed late reply for an exhausted control-frame retry" + ); + return true; + } + application_liveness.complete_retryable_control_frame(&reply.id); if reply.id.starts_with(crate::runtime::task_request_prefix()) { return event_tx .send(FleetControlEvent::Message(RelaycastToBroker::Reply( @@ -3590,13 +4009,105 @@ where complete_agent_registration( reply, pending_agent_registrations, + application_liveness, sink, ) .await } } } - RelaycastToBroker::Error(error) => { + RelaycastToBroker::Error(mut error) => { + if error.code == "d1_pressure" { + match application_liveness.schedule_control_d1_pressure_retry( + &error.id, + &error.message, + Instant::now(), + ) { + ControlPressureRetry::Scheduled(delay) => { + tracing::warn!( + target = "relay_broker::fleet", + node_id, + id = %error.id, + retry_delay_ms = delay.as_millis(), + "retryable d1_pressure on a control frame; retry scheduled on the live control socket" + ); + return true; + } + ControlPressureRetry::Exhausted => { + let exhausted = application_liveness + .take_retryable_control_frame(&error.id); + let reason = format!( + "d1_pressure: {} (retry budget exhausted)", + error.message, + ); + if !error + .id + .starts_with(crate::runtime::task_request_prefix()) + && exhausted.as_ref().is_some_and(|pending| { + matches!( + &pending.frame, + BrokerToRelaycast::ActionResult(_) + ) + }) + { + let exhausted = + exhausted.expect("checked action.result"); + let replayed_after_pressure = + exhausted.replayed_after_pressure; + retained_pressure_frames.push_back(exhausted.frame); + tracing::warn!( + target = "relay_broker::fleet", + node_id, + id = %error.id, + reason, + replayed_after_pressure, + "retaining action.result after d1_pressure exhaustion" + ); + return replayed_after_pressure; + } + if let Some(pending) = + pending_deregistrations.remove(&error.id) + { + let _ = pending.send(Err(reason)); + return true; + } + if error + .id + .starts_with(crate::runtime::task_request_prefix()) + { + error.code = "d1_pressure".to_string(); + error.message = reason; + return event_tx + .send(FleetControlEvent::Message( + RelaycastToBroker::Error(error), + )) + .await + .is_ok(); + } + tracing::error!( + target = "relay_broker::fleet", + node_id, + id = %error.id, + reason, + "control frame d1_pressure retry budget exhausted" + ); + return true; + } + ControlPressureRetry::Unmatched => {} + } + } + if application_liveness + .consume_exhausted_retryable_control_frame(&error.id) + { + tracing::debug!( + target = "relay_broker::fleet", + node_id, + id = %error.id, + "consumed late error for an exhausted control-frame retry" + ); + return true; + } + application_liveness.complete_retryable_control_frame(&error.id); if error.id.starts_with(crate::runtime::task_request_prefix()) { return event_tx .send(FleetControlEvent::Message(RelaycastToBroker::Error( @@ -3616,23 +4127,40 @@ where pending_agent_registrations, ); } - // Surface every engine rejection at error level. A node.register or - // heartbeat rejection (e.g. node_name_conflict) matches no pending - // agent registration below, so without this it vanishes silently — - // leaving the node half-registered with dead heartbeats and no signal. - tracing::error!( - target = "relay_broker::fleet", - code = %error.code, - message = %error.message, - id = %error.id, - "engine rejected a node control frame" - ); - fail_agent_registration( - &error.id, - format!("{}: {}", error.code, error.message), - pending_agent_registrations, - ); if error.code == "d1_pressure" { + let reason = error.message.clone(); + match schedule_agent_registration_d1_pressure_retry( + &error.id, + &reason, + pending_agent_registrations, + Instant::now(), + application_liveness.control_retry_policy, + ) { + AgentRegistrationPressureRetry::Scheduled(delay) => { + tracing::warn!( + target = "relay_broker::fleet", + node_id, + id = %error.id, + retry_delay_ms = delay.as_millis(), + "retryable d1_pressure on agent registration; retry scheduled on the live control socket" + ); + return true; + } + AgentRegistrationPressureRetry::Exhausted => { + tracing::error!( + target = "relay_broker::fleet", + node_id, + id = %error.id, + "agent registration d1_pressure retry budget exhausted" + ); + fail_agent_registration_d1_pressure_exhausted( + &error.id, + pending_agent_registrations, + ); + return true; + } + AgentRegistrationPressureRetry::Unmatched => {} + } if application_liveness.discard_superseded(&error.id) { tracing::debug!( target = "relay_broker::fleet", @@ -3655,6 +4183,22 @@ where return true; } } + // Surface every unhandled engine rejection at error level. A + // node.register or heartbeat rejection (e.g. node_name_conflict) + // matches no pending operation below, so without this it vanishes + // silently. Expected, retryable pressure returned above at WARN. + tracing::error!( + target = "relay_broker::fleet", + code = %error.code, + message = %error.message, + id = %error.id, + "engine rejected a node control frame" + ); + fail_agent_registration( + &error.id, + format!("{}: {}", error.code, error.message), + pending_agent_registrations, + ); let rejected_liveness_probe = application_liveness.reject(&error.id); // A reply proves the application is responsive, but rejecting // the authoritative inventory probe means the control session @@ -3688,6 +4232,7 @@ where async fn complete_agent_registration( reply: crate::fleet_wire::Reply, pending_agent_registrations: &mut HashMap, + application_liveness: &mut ApplicationLiveness, sink: &mut S, ) -> bool where @@ -3712,7 +4257,12 @@ where let name = data.as_ref()?.name.as_deref()?; let key = pending_agent_registrations .iter() - .find(|(_, pending)| pending.name == name) + .find(|(_, pending)| pending.reply.is_some() && pending.name == name) + .or_else(|| { + pending_agent_registrations + .iter() + .find(|(_, pending)| pending.name == name) + }) .map(|(key, _)| key.clone())?; pending_agent_registrations .remove(&key) @@ -3729,9 +4279,9 @@ where let data = match data { Some(data) => data, None => { - let _ = pending - .reply - .send(Err("invalid_agent_register_reply_data".to_string())); + if let Some(reply) = pending.reply { + let _ = reply.send(Err("invalid_agent_register_reply_data".to_string())); + } return true; } }; @@ -3741,30 +4291,29 @@ where token: data.token, delivery_ack_seq: data.delivery_ack_seq, }; - match pending.reply.send(Ok(token.clone())) { - Ok(()) => true, - Err(Ok(token)) => { - tracing::warn!( - target = "relay_broker::fleet", - id = %request_id, - name = %pending.name, - agent_id = %token.agent_id, - "late agent.register success after caller stopped waiting; sending compensating agent.deregister" - ); - send_wire( - sink, - &BrokerToRelaycast::AgentDeregister(AgentDeregister { - v: FLEET_WIRE_VERSION, - id: Some(request_id), - agent_id: token.agent_id, - name: Some(pending.name), - }), - ) - .await - .is_ok() - } - Err(Err(_)) => true, - } + let late_token = match pending.reply { + Some(reply) => match reply.send(Ok(token.clone())) { + Ok(()) => return true, + Err(Ok(token)) => token, + Err(Err(_)) => return true, + }, + None => token, + }; + tracing::warn!( + target = "relay_broker::fleet", + id = %request_id, + name = %pending.name, + agent_id = %late_token.agent_id, + "late agent.register success after caller stopped waiting; sending compensating agent.deregister" + ); + let cleanup = BrokerToRelaycast::AgentDeregister(AgentDeregister { + v: FLEET_WIRE_VERSION, + id: Some(request_id.clone()), + agent_id: late_token.agent_id, + name: Some(pending.name), + }); + application_liveness.track_retryable_control_frame(request_id, cleanup.clone()); + send_wire(sink, &cleanup).await.is_ok() } fn fail_agent_registration( @@ -3773,7 +4322,9 @@ fn fail_agent_registration( pending_agent_registrations: &mut HashMap, ) { if let Some(pending) = pending_agent_registrations.remove(id) { - let _ = pending.reply.send(Err(reason)); + if let Some(reply) = pending.reply { + let _ = reply.send(Err(reason)); + } } } @@ -3812,18 +4363,126 @@ fn fail_unsupported_channel_isolation( } } -fn expire_agent_registrations( +#[derive(Debug, PartialEq, Eq)] +enum AgentRegistrationPressureRetry { + Unmatched, + Scheduled(Duration), + Exhausted, +} + +fn schedule_agent_registration_d1_pressure_retry( + id: &str, + reason: &str, + pending_agent_registrations: &mut HashMap, + now: Instant, + retry_policy: ControlRetryPolicy, +) -> AgentRegistrationPressureRetry { + let Some(pending) = pending_agent_registrations.get_mut(id) else { + return AgentRegistrationPressureRetry::Unmatched; + }; + pending.last_d1_pressure = Some(reason.to_string()); + let deadline = pending.created_at + retry_policy.budget; + if pending.d1_pressure_retry_attempts >= retry_policy.max_retries || now >= deadline { + return AgentRegistrationPressureRetry::Exhausted; + } + + let next_attempt = pending.d1_pressure_retry_attempts.saturating_add(1); + let delay = retry_policy.retry_delay(next_attempt); + if now + .checked_add(delay) + .is_none_or(|retry_at| retry_at > deadline) + { + return AgentRegistrationPressureRetry::Exhausted; + } + pending.d1_pressure_retry_attempts = next_attempt; + pending.d1_pressure_retry_at = Some(now + delay); + AgentRegistrationPressureRetry::Scheduled(delay) +} + +fn next_agent_registration_retry_event_delay( + pending_agent_registrations: &HashMap, + now: Instant, + retry_policy: ControlRetryPolicy, +) -> Option { + pending_agent_registrations + .values() + .filter(|pending| pending.last_d1_pressure.is_some()) + .flat_map(|pending| { + [ + pending.d1_pressure_retry_at, + Some(pending.created_at + retry_policy.budget), + ] + }) + .flatten() + .map(|event_at| event_at.saturating_duration_since(now)) + .min() +} + +fn expire_d1_pressure_agent_registrations( pending_agent_registrations: &mut HashMap, now: Instant, + retry_policy: ControlRetryPolicy, ) { let expired: Vec = pending_agent_registrations .iter() - .filter_map(|(id, pending)| { - (now.saturating_duration_since(pending.created_at) >= REGISTER_AGENT_PENDING_TTL) - .then_some(id.clone()) + .filter(|(_, pending)| { + pending.last_d1_pressure.is_some() + && now.saturating_duration_since(pending.created_at) >= retry_policy.budget }) + .map(|(id, _)| id.clone()) .collect(); - + for id in expired { + fail_agent_registration_d1_pressure_exhausted(&id, pending_agent_registrations); + } +} + +fn fail_agent_registration_d1_pressure_exhausted( + id: &str, + pending_agent_registrations: &mut HashMap, +) { + let Some(pending) = pending_agent_registrations.get_mut(id) else { + return; + }; + let reason = pending + .last_d1_pressure + .take() + .unwrap_or_else(|| "Node liveness retry pending".to_string()); + pending.d1_pressure_retry_at = None; + if let Some(reply) = pending.reply.take() { + let _ = reply.send(Err(format!( + "d1_pressure: {reason} (retry budget exhausted)" + ))); + } +} + +fn take_due_agent_registration_retry( + pending_agent_registrations: &mut HashMap, + now: Instant, +) -> Option { + let pending = pending_agent_registrations + .values_mut() + .filter(|pending| { + pending + .d1_pressure_retry_at + .is_some_and(|retry_at| retry_at <= now) + }) + .min_by_key(|pending| pending.d1_pressure_retry_at)?; + pending.d1_pressure_retry_at = None; + Some(pending.request.clone()) +} + +fn expire_agent_registrations( + pending_agent_registrations: &mut HashMap, + now: Instant, +) { + let expired: Vec = pending_agent_registrations + .iter() + .filter_map(|(id, pending)| { + (now.saturating_duration_since(pending.created_at) >= REGISTER_AGENT_PENDING_TTL) + .then_some(id.clone()) + }) + .collect(); + for id in expired { if let Some(pending) = pending_agent_registrations.remove(&id) { tracing::warn!( @@ -3832,9 +4491,9 @@ fn expire_agent_registrations( name = %pending.name, "agent.register pending reply expired without engine response" ); - let _ = pending - .reply - .send(Err("agent_register_pending_expired".to_string())); + if let Some(reply) = pending.reply { + let _ = reply.send(Err("agent_register_pending_expired".to_string())); + } } } } @@ -3844,7 +4503,9 @@ fn drain_agent_registrations( reason: &str, ) { for (_, pending) in pending_agent_registrations.drain() { - let _ = pending.reply.send(Err(reason.to_string())); + if let Some(reply) = pending.reply { + let _ = reply.send(Err(reason.to_string())); + } } } @@ -3884,6 +4545,58 @@ fn fail_deferred_commands( } } +/// Tracks safely replayable steady-state control frames which also have a +/// concrete recovery owner. The engine's `d1_pressure` contract guarantees +/// that the rejected operation was not applied, so exact replay is safe for: +/// +/// - acknowledged `agent.deregister`, +/// - cumulative `delivery.ack` (a missed terminal ack is recovered by durable +/// redelivery), and +/// - non-task `action.result`, which gets a stable id and is retained for +/// redelivery after reconnect if the bounded pressure budget is exhausted. +/// +/// `node.register`, `inventory.sync`, and `agent.register` have dedicated +/// pending state because their successful replies open liveness or return a +/// token. Heartbeats intentionally are not retried: a newer periodic snapshot +/// supersedes them. No separate channel-join frame exists; the only join sent +/// by this broker is part of the exact `agent.register` frame. One-way +/// deregistrations remain untracked because they have no retained owner. +fn prepare_retryable_control_frame( + message: &mut BrokerToRelaycast, +) -> Option<(String, &'static str)> { + let (id, kind) = match message { + BrokerToRelaycast::AgentDeregister(request) => (request.id.as_mut()?, "agent.deregister"), + BrokerToRelaycast::DeliveryAck(request) => ( + request + .id + .get_or_insert_with(|| format!("delivery_ack_{}", Uuid::new_v4().simple())), + "delivery.ack", + ), + BrokerToRelaycast::ActionResult(request) + if request.task.is_some() + || request + .id + .as_deref() + .is_some_and(|id| id.starts_with(crate::runtime::task_request_prefix())) => + { + return None; + } + BrokerToRelaycast::ActionResult(request) => ( + request + .id + .get_or_insert_with(|| format!("action_result_{}", Uuid::new_v4().simple())), + "action.result", + ), + // Task owners have a shorter acknowledgement deadline than this + // socket's pressure budget. Surface their correlated d1_pressure + // immediately so the owner, rather than a stale socket timer, decides + // whether to retry the task. + BrokerToRelaycast::ActionAccept(_) => return None, + _ => return None, + }; + Some((id.clone(), kind)) +} + async fn send_wire(sink: &mut S, message: &BrokerToRelaycast) -> Result<()> where S: Sink + Unpin, @@ -4064,7 +4777,7 @@ mod tests { let mut application_liveness = ApplicationLiveness::new(Duration::from_secs(1)); let (delivered, mut delivery_ack) = oneshot::channel(); let mut retained_completion = None; - + let mut retained_pressure_frames = VecDeque::new(); let result = handle_connected_command( Some(FleetControlCommand::RetainedSend(RetainedFleetCompletion { message: BrokerToRelaycast::ActionResult(ActionResult { @@ -4089,6 +4802,7 @@ mod tests { &mut pending_deregistrations, &mut application_liveness, &mut retained_completion, + &mut retained_pressure_frames, ) .await; @@ -4140,6 +4854,33 @@ mod tests { } } + fn test_pending_agent_registration( + name: &str, + isolates_channels: bool, + id: &str, + reply: oneshot::Sender>, + created_at: Instant, + ) -> PendingAgentRegistration { + PendingAgentRegistration { + isolates_channels, + name: name.to_string(), + request: AgentRegister { + auto_join_general: isolates_channels.then_some(false), + v: FLEET_WIRE_VERSION, + id: Some(id.to_string()), + name: name.to_string(), + invocation_id: None, + session_ref: None, + resumable: None, + }, + reply: Some(reply), + created_at, + d1_pressure_retry_attempts: 0, + d1_pressure_retry_at: None, + last_d1_pressure: None, + } + } + async fn read_http_request(socket: &mut TcpStream) -> String { let mut buffer = [0_u8; 8192]; let read = socket.read(&mut buffer).await.unwrap(); @@ -5254,21 +5995,23 @@ mod tests { let mut pending = HashMap::from([ ( "isolated".to_string(), - PendingAgentRegistration { - isolates_channels: true, - name: "isolated".to_string(), - reply: isolated_tx, - created_at: Instant::now(), - }, + test_pending_agent_registration( + "isolated", + true, + "isolated", + isolated_tx, + Instant::now(), + ), ), ( "legacy".to_string(), - PendingAgentRegistration { - isolates_channels: false, - name: "legacy".to_string(), - reply: legacy_tx, - created_at: Instant::now(), - }, + test_pending_agent_registration( + "legacy", + false, + "legacy", + legacy_tx, + Instant::now(), + ), ), ]); let (events, _) = mpsc::channel(1); @@ -5283,6 +6026,7 @@ mod tests { &mut pending, &mut deregistrations, &mut ApplicationLiveness::new(Duration::from_secs(1)), + &mut VecDeque::new(), "node-test", &mut sink, None, @@ -5334,6 +6078,7 @@ mod tests { &mut HashMap::new(), &mut HashMap::new(), &mut liveness, + &mut VecDeque::new(), "node-test", &mut futures_util::sink::drain(), None, @@ -5374,6 +6119,7 @@ mod tests { &mut HashMap::new(), &mut HashMap::new(), &mut ApplicationLiveness::new(Duration::from_secs(1)), + &mut VecDeque::new(), "node-test", &mut futures_util::sink::drain(), None, @@ -5449,6 +6195,94 @@ mod tests { assert!(!liveness.ready); } + #[test] + fn d1_pressure_safe_control_frame_audit_assigns_ids_and_replays_exact_frames() { + let frames = vec![ + BrokerToRelaycast::AgentDeregister(AgentDeregister { + v: FLEET_WIRE_VERSION, + id: Some("agent_deregister_1".to_string()), + agent_id: "agt-1".to_string(), + name: Some("agent-a".to_string()), + }), + delivery_ack("agent-a", 7), + ]; + + for mut frame in frames { + let (id, _) = prepare_retryable_control_frame(&mut frame) + .expect("audited frame must be retryable"); + let mut state = ApplicationLiveness::new(Duration::from_secs(30)); + state.track_retryable_control_frame(id.clone(), frame.clone()); + let now = Instant::now(); + assert_eq!( + state.schedule_control_d1_pressure_retry(&id, "Node liveness retry pending", now,), + ControlPressureRetry::Scheduled(Duration::from_secs(1)) + ); + assert_eq!( + state.take_due_control_retry(now + Duration::from_secs(1)), + Some(frame), + "the retry must preserve every field and correlation id" + ); + } + + let mut heartbeat = BrokerToRelaycast::NodeHeartbeat( + FleetLoadSnapshot::default().heartbeat(&build_node_register( + &test_manifest(), + "node-test", + "host-test", + "broker/test", + None, + )), + ); + assert_eq!(prepare_retryable_control_frame(&mut heartbeat), None); + + let mut task_result = BrokerToRelaycast::ActionResult(ActionResult { + v: FLEET_WIRE_VERSION, + id: Some(format!("{}result_1", crate::runtime::task_request_prefix())), + invocation_id: "inv-1".to_string(), + result: ActionResultPayload::Output(ActionResultOutput { + output: json!({"ok": true}), + }), + task: None, + }); + assert_eq!(prepare_retryable_control_frame(&mut task_result), None); + let mut task_accept = BrokerToRelaycast::ActionAccept(crate::fleet_wire::ActionAccept { + v: FLEET_WIRE_VERSION, + id: "task_request_1".to_string(), + invocation_id: "inv-task".to_string(), + execution_id: "exec-1".to_string(), + worker_generation: "generation-1".to_string(), + }); + assert_eq!(prepare_retryable_control_frame(&mut task_accept), None); + + let mut one_way_result = BrokerToRelaycast::ActionResult(ActionResult { + v: FLEET_WIRE_VERSION, + id: None, + invocation_id: "inv-spawn".to_string(), + result: ActionResultPayload::Output(ActionResultOutput { + output: json!({"ok": true}), + }), + task: None, + }); + let (result_id, kind) = prepare_retryable_control_frame(&mut one_way_result) + .expect("non-task action.result must gain a retained retry owner"); + assert_eq!(kind, "action.result"); + assert!(result_id.starts_with("action_result_")); + assert!(matches!( + one_way_result, + BrokerToRelaycast::ActionResult(ActionResult { id: Some(id), .. }) if id == result_id + )); + let mut one_way_deregister = BrokerToRelaycast::AgentDeregister(AgentDeregister { + v: FLEET_WIRE_VERSION, + id: None, + agent_id: "agt-one-way".to_string(), + name: Some("agent-one-way".to_string()), + }); + assert_eq!( + prepare_retryable_control_frame(&mut one_way_deregister), + None + ); + } + #[test] fn newer_inventory_snapshot_cancels_an_older_retry_payload() { let mut liveness = ApplicationLiveness::new(Duration::from_secs(120)); @@ -5498,6 +6332,7 @@ mod tests { &mut HashMap::new(), &mut HashMap::new(), &mut liveness, + &mut VecDeque::new(), "node-test", &mut futures_util::sink::drain(), None, @@ -5524,18 +6359,208 @@ mod tests { ); } + #[tokio::test] + async fn late_d1_pressure_after_control_retry_timeout_is_consumed_once() { + let mut liveness = ApplicationLiveness::new(Duration::from_secs(120)); + let id = format!("{}late-pressure", crate::runtime::task_request_prefix()); + let frame = BrokerToRelaycast::ActionAccept(crate::fleet_wire::ActionAccept { + v: FLEET_WIRE_VERSION, + id: id.clone(), + invocation_id: "inv-late-pressure".to_string(), + execution_id: "exec-late-pressure".to_string(), + worker_generation: "generation-late-pressure".to_string(), + }); + liveness.track_retryable_control_frame(id.clone(), frame); + let now = Instant::now(); + assert!(matches!( + liveness.schedule_control_d1_pressure_retry(&id, "Node liveness retry pending", now,), + ControlPressureRetry::Scheduled(_) + )); + assert_eq!( + liveness + .expire_retryable_control_frames(now + CONTROL_D1_PRESSURE_RETRY_BUDGET) + .len(), + 1 + ); + + let (events, mut receiver) = mpsc::channel(1); + let healthy = handle_server_message( + Message::Text( + json!({ + "v": 1, + "id": id.clone(), + "type": "error", + "ok": false, + "code": "d1_pressure", + "message": "Node liveness retry pending" + }) + .to_string(), + ), + &events, + &mut HashMap::new(), + &mut HashMap::new(), + &mut liveness, + &mut VecDeque::new(), + "node-test", + &mut futures_util::sink::drain(), + None, + None, + ) + .await; + + assert!(healthy); + assert!(matches!( + receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + assert!(!liveness.consume_exhausted_retryable_control_frame(&id)); + } + + #[test] + fn exhausted_control_retry_tombstone_has_its_own_cleanup_deadline() { + let policy = ControlRetryPolicy { + initial_delay: Duration::from_millis(5), + max_delay: Duration::from_millis(5), + max_retries: 1, + budget: Duration::from_millis(50), + }; + let mut liveness = + ApplicationLiveness::new_with_retry_policy(Duration::from_secs(1), policy); + let id = format!("{}tombstone", crate::runtime::task_request_prefix()); + liveness.track_retryable_control_frame( + id.clone(), + BrokerToRelaycast::ActionAccept(crate::fleet_wire::ActionAccept { + v: FLEET_WIRE_VERSION, + id: id.clone(), + invocation_id: "inv-tombstone".to_string(), + execution_id: "exec-tombstone".to_string(), + worker_generation: "generation-tombstone".to_string(), + }), + ); + let now = Instant::now(); + assert!(matches!( + liveness.schedule_control_d1_pressure_retry(&id, "Node liveness retry pending", now,), + ControlPressureRetry::Scheduled(_) + )); + assert_eq!( + liveness + .expire_retryable_control_frames(now + policy.budget) + .len(), + 1 + ); + assert_eq!( + liveness.next_control_retry_event_delay(now + policy.budget), + Some(policy.budget) + ); + assert!(liveness + .expire_retryable_control_frames(now + policy.budget + policy.budget) + .is_empty()); + assert!(liveness.exhausted_retryable_control_frames.is_empty()); + } + + #[test] + fn disconnect_retains_only_action_results_that_observed_pressure() { + let mut liveness = ApplicationLiveness::new(Duration::from_secs(1)); + let id = "action_result_disconnect".to_string(); + let frame = BrokerToRelaycast::ActionResult(ActionResult { + v: FLEET_WIRE_VERSION, + id: Some(id.clone()), + invocation_id: "inv-disconnect".to_string(), + result: ActionResultPayload::Output(ActionResultOutput { + output: json!({"ok": true}), + }), + task: None, + }); + liveness.track_retryable_control_frame(id.clone(), frame.clone()); + let mut retained = VecDeque::new(); + liveness.retain_non_task_action_results(&mut retained); + assert!( + retained.is_empty(), + "a successful write has no replay proof" + ); + + assert!(matches!( + liveness.schedule_control_d1_pressure_retry( + &id, + "Node liveness retry pending", + Instant::now(), + ), + ControlPressureRetry::Scheduled(_) + )); + liveness.retain_non_task_action_results(&mut retained); + assert_eq!(retained, VecDeque::from([frame])); + } + + #[tokio::test] + async fn replayed_action_result_exhaustion_waits_for_an_organic_reconnect() { + let policy = ControlRetryPolicy { + initial_delay: Duration::from_millis(1), + max_delay: Duration::from_millis(1), + max_retries: 0, + budget: Duration::from_millis(10), + }; + let mut liveness = + ApplicationLiveness::new_with_retry_policy(Duration::from_secs(1), policy); + let id = "action_result_replayed".to_string(); + liveness.track_retryable_control_frame( + id.clone(), + BrokerToRelaycast::ActionResult(ActionResult { + v: FLEET_WIRE_VERSION, + id: Some(id.clone()), + invocation_id: "inv-replayed".to_string(), + result: ActionResultPayload::Output(ActionResultOutput { + output: json!({"ok": true}), + }), + task: None, + }), + ); + liveness.mark_retryable_control_frame_replayed(&id); + let (events, _receiver) = mpsc::channel(1); + let mut retained = VecDeque::new(); + let stays_connected = handle_server_message( + Message::Text( + json!({ + "v": 1, + "id": id, + "type": "error", + "ok": false, + "code": "d1_pressure", + "message": "Node liveness retry pending" + }) + .to_string(), + ), + &events, + &mut HashMap::new(), + &mut HashMap::new(), + &mut liveness, + &mut retained, + "node-test", + &mut futures_util::sink::drain(), + None, + None, + ) + .await; + assert!(stays_connected, "a replay must not start a reconnect loop"); + assert_eq!( + retained.len(), + 1, + "the result remains queued for a later reconnect" + ); + } + #[test] fn expire_agent_registrations_bounds_pending_map() { let created_at = Instant::now(); let (reply_tx, mut reply_rx) = oneshot::channel(); let mut pending = HashMap::from([( "agent_register_1".to_string(), - PendingAgentRegistration { - isolates_channels: false, - name: "agent-a".to_string(), - reply: reply_tx, + test_pending_agent_registration( + "agent-a", + false, + "agent_register_1", + reply_tx, created_at, - }, + ), )]); expire_agent_registrations( @@ -5561,12 +6586,13 @@ mod tests { let (reply_tx, reply_rx) = oneshot::channel(); let mut pending = HashMap::from([( "agent_register_req".to_string(), - PendingAgentRegistration { - isolates_channels: false, - name: "agent-a".to_string(), - reply: reply_tx, - created_at: Instant::now(), - }, + test_pending_agent_registration( + "agent-a", + false, + "agent_register_req", + reply_tx, + Instant::now(), + ), )]); let mut sink = futures_util::sink::drain(); let reply = crate::fleet_wire::Reply { @@ -5580,7 +6606,8 @@ mod tests { }), }; - assert!(complete_agent_registration(reply, &mut pending, &mut sink).await); + let mut liveness = ApplicationLiveness::new(Duration::from_secs(30)); + assert!(complete_agent_registration(reply, &mut pending, &mut liveness, &mut sink).await); assert!( pending.is_empty(), "the pending registration must be consumed by the name-based fallback" @@ -5596,6 +6623,101 @@ mod tests { ); } + #[tokio::test] + async fn complete_agent_registration_name_fallback_skips_exhausted_same_name_request() { + let (stale_tx, _stale_rx) = oneshot::channel(); + let mut stale = test_pending_agent_registration( + "agent-a", + false, + "agent_register_stale", + stale_tx, + Instant::now(), + ); + stale.reply = None; + let (active_tx, active_rx) = oneshot::channel(); + let active = test_pending_agent_registration( + "agent-a", + false, + "agent_register_active", + active_tx, + Instant::now(), + ); + let mut pending = HashMap::from([ + ("agent_register_stale".to_string(), stale), + ("agent_register_active".to_string(), active), + ]); + let reply = crate::fleet_wire::Reply { + v: FLEET_WIRE_VERSION, + id: "196331520553345024".to_string(), + ok: true, + data: json!({ + "name": "agent-a", + "agent_id": "agt-active", + "token": "at_active" + }), + }; + let mut liveness = ApplicationLiveness::new(Duration::from_secs(30)); + + assert!( + complete_agent_registration( + reply, + &mut pending, + &mut liveness, + &mut futures_util::sink::drain(), + ) + .await + ); + assert!(pending.contains_key("agent_register_stale")); + assert!(!pending.contains_key("agent_register_active")); + assert_eq!(active_rx.await.unwrap().unwrap().agent_id, "agt-active"); + } + + #[tokio::test] + async fn late_agent_registration_cleanup_is_tracked_for_pressure_retry() { + let (reply_tx, _reply_rx) = oneshot::channel(); + let mut exhausted = test_pending_agent_registration( + "agent-a", + false, + "agent_register_late", + reply_tx, + Instant::now(), + ); + exhausted.reply = None; + let mut pending = HashMap::from([("agent_register_late".to_string(), exhausted)]); + let reply = crate::fleet_wire::Reply { + v: FLEET_WIRE_VERSION, + id: "196331520553345024".to_string(), + ok: true, + data: json!({ + "name": "agent-a", + "agent_id": "agt-late", + "token": "at_late" + }), + }; + let mut liveness = ApplicationLiveness::new(Duration::from_secs(30)); + + assert!( + complete_agent_registration( + reply, + &mut pending, + &mut liveness, + &mut futures_util::sink::drain(), + ) + .await + ); + let tracked = liveness + .pending_retryable_control_frames + .get("agent_register_late") + .expect("compensating deregistration must retain a retry owner"); + assert!(matches!( + &tracked.frame, + BrokerToRelaycast::AgentDeregister(request) + if request.id.as_deref() == Some("agent_register_late") + && request.agent_id == "agt-late" + && request.name.as_deref() == Some("agent-a") + )); + } + #[tokio::test] async fn complete_agent_registration_ignores_non_agent_reply() { // A `node.register`/`inventory.sync` reply carries a fresh engine @@ -5605,12 +6727,13 @@ mod tests { let (reply_tx, mut reply_rx) = oneshot::channel(); let mut pending = HashMap::from([( "agent_register_req".to_string(), - PendingAgentRegistration { - isolates_channels: false, - name: "agent-a".to_string(), - reply: reply_tx, - created_at: Instant::now(), - }, + test_pending_agent_registration( + "agent-a", + false, + "agent_register_req", + reply_tx, + Instant::now(), + ), )]); let mut sink = futures_util::sink::drain(); let reply = crate::fleet_wire::Reply { @@ -5620,7 +6743,8 @@ mod tests { data: json!({ "node_id": "node-test", "online": true }), }; - assert!(complete_agent_registration(reply, &mut pending, &mut sink).await); + let mut liveness = ApplicationLiveness::new(Duration::from_secs(30)); + assert!(complete_agent_registration(reply, &mut pending, &mut liveness, &mut sink).await); assert_eq!( pending.len(), 1, @@ -5770,8 +6894,17 @@ mod tests { )) .await .unwrap(); - let ack = next_non_heartbeat_node_to_server(&mut ws).await; - assert_eq!(ack, delivery_ack("agent-a", 1)); + match next_non_heartbeat_node_to_server(&mut ws).await { + BrokerToRelaycast::DeliveryAck(ack) => { + assert_eq!(ack.agent, "agent-a"); + assert_eq!(ack.up_to_seq, 1); + assert!(ack + .id + .as_deref() + .is_some_and(|id| id.starts_with("delivery_ack_"))); + } + other => panic!("expected delivery.ack, got {other:?}"), + } ws.send(Message::Text( serde_json::to_string(&RelaycastToBroker::ActionInvoke(ActionInvoke { @@ -6006,18 +7139,411 @@ mod tests { server.abort(); } - #[tokio::test] - async fn registration_d1_pressure_backoff_remains_shutdown_interruptible() { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum SafeControlRetryCase { + DeliveryAck, + ActionResult, + AgentDeregister, + } + + async fn assert_safe_control_frame_socket_retry( + case: SafeControlRetryCase, + persistent: bool, + disconnect_during_backoff: bool, + ) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let ws_url = format!("ws://{}/v1/node/ws", listener.local_addr().unwrap()); - let (command_tx, mut command_rx) = mpsc::channel(1); - let (event_tx, _event_rx) = mpsc::channel(1); - let mut registration = Some(build_node_register( - &test_manifest(), - "node-test", - "host-test", - "broker/test", - None, + let (command_tx, mut command_rx) = mpsc::channel(8); + let (event_tx, mut event_rx) = mpsc::channel(8); + let (server_done_tx, server_done_rx) = oneshot::channel(); + let retry_policy = ControlRetryPolicy { + initial_delay: Duration::from_millis(100), + max_delay: Duration::from_millis(100), + max_retries: 2, + budget: Duration::from_millis(350), + }; + let server_max_retries = retry_policy.max_retries; + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut ws = accept_async(stream).await.unwrap(); + assert!(matches!( + next_node_to_server(&mut ws).await, + BrokerToRelaycast::NodeRegister(_) + )); + assert!(matches!( + next_node_to_server(&mut ws).await, + BrokerToRelaycast::InventorySync(_) + )); + assert!(matches!( + next_node_to_server(&mut ws).await, + BrokerToRelaycast::NodeHeartbeat(_) + )); + + let initial = next_non_heartbeat_node_to_server(&mut ws).await; + match (&case, &initial) { + (SafeControlRetryCase::DeliveryAck, BrokerToRelaycast::DeliveryAck(_)) + | (SafeControlRetryCase::ActionResult, BrokerToRelaycast::ActionResult(_)) + | (SafeControlRetryCase::AgentDeregister, BrokerToRelaycast::AgentDeregister(_)) => { + } + _ => panic!("unexpected frame for {case:?}: {initial:?}"), + } + let id = match &initial { + BrokerToRelaycast::DeliveryAck(frame) => frame.id.clone(), + BrokerToRelaycast::ActionResult(frame) => frame.id.clone(), + BrokerToRelaycast::AgentDeregister(frame) => frame.id.clone(), + _ => unreachable!(), + } + .expect("retryable control frame must carry a stable id"); + + let pressure_responses = if persistent { + server_max_retries + 1 + } else { + 1 + }; + for response_index in 0..pressure_responses { + ws.send(Message::Text( + json!({ + "v": 1, + "id": id.clone(), + "type": "error", + "ok": false, + "code": "d1_pressure", + "message": "Node liveness retry pending" + }) + .to_string(), + )) + .await + .unwrap(); + if !disconnect_during_backoff + && (response_index < pressure_responses - 1 || !persistent) + { + if response_index == 0 { + assert!( + tokio::time::timeout( + Duration::from_millis(20), + next_non_heartbeat_node_to_server(&mut ws), + ) + .await + .is_err(), + "{case:?} retried without backoff" + ); + } + let retry = tokio::time::timeout( + Duration::from_millis(500), + next_non_heartbeat_node_to_server(&mut ws), + ) + .await + .expect("retry should arrive inside the short test budget"); + assert_eq!(retry, initial, "{case:?} retry changed the frame"); + } + } + + if disconnect_during_backoff { + ws.close(None).await.unwrap(); + } else if persistent { + if case == SafeControlRetryCase::DeliveryAck { + assert!( + tokio::time::timeout( + Duration::from_millis(150), + next_non_heartbeat_node_to_server(&mut ws), + ) + .await + .is_err(), + "delivery.ack exceeded its retry bound" + ); + } + } else { + ws.send(Message::Text( + json!({"v": 1, "id": id.clone(), "type": "reply", "ok": true, "data": {}}) + .to_string(), + )) + .await + .unwrap(); + } + server_done_tx.send(()).unwrap(); + if !disconnect_during_backoff { + while ws.next().await.is_some() {} + } + }); + + let config = FleetControlConfig { + ws_url, + node_token: Some("nt_test".to_string()), + node_id: "node-test".to_string(), + node_name: "host-test".to_string(), + broker_version: "broker/test".to_string(), + token_minter: None, + session_token: None, + read_idle_timeout: None, + probe: None, + terminal_reconnect_tx: None, + }; + let mut registration = Some(build_node_register( + &test_manifest(), + "node-test", + "host-test", + "broker/test", + None, + )); + let mut inventory = Vec::new(); + let mut load = FleetLoadSnapshot::default(); + let mut retained_pressure_frames = VecDeque::new(); + let session = run_connected_once_with_retry_policy( + &config, + &mut command_rx, + &event_tx, + &mut registration, + &mut inventory, + &mut load, + Duration::from_secs(3_600), + retry_policy, + &mut retained_pressure_frames, + ); + // Keep the command lane open while a pressure-triggered reconnect is + // expected; otherwise dropping the driver's sender can race the close + // frame and make the session report an intentional shutdown instead. + let _command_lane_guard = command_tx.clone(); + let driver = async move { + assert_eq!(event_rx.recv().await.unwrap(), FleetControlEvent::Connected); + let deregistration_reply = match case { + SafeControlRetryCase::DeliveryAck => { + command_tx + .send(FleetControlCommand::Send(delivery_ack("agent-a", 7))) + .await + .unwrap(); + None + } + SafeControlRetryCase::ActionResult => { + let message = BrokerToRelaycast::ActionResult(ActionResult { + v: FLEET_WIRE_VERSION, + id: None, + invocation_id: "inv-pressure".to_string(), + result: ActionResultPayload::Output(ActionResultOutput { + output: json!({"spawned": true}), + }), + task: None, + }); + if persistent { + command_tx + .send(FleetControlCommand::Send(message)) + .await + .unwrap(); + } else { + let (delivered, _delivery_rx) = oneshot::channel(); + command_tx + .send(FleetControlCommand::RetainedSend(RetainedFleetCompletion { + message, + delivered, + })) + .await + .unwrap(); + } + None + } + SafeControlRetryCase::AgentDeregister => { + let (reply, receiver) = oneshot::channel(); + command_tx + .send(FleetControlCommand::DeregisterAgent { + request: AgentDeregister { + v: FLEET_WIRE_VERSION, + id: None, + agent_id: "agt-pressure".to_string(), + name: Some("agent-pressure".to_string()), + }, + reply, + }) + .await + .unwrap(); + Some(receiver) + } + }; + server_done_rx.await.unwrap(); + let deregistration_result = match deregistration_reply { + Some(receiver) => Some(receiver.await.unwrap()), + None => None, + }; + if !((persistent || disconnect_during_backoff) + && case == SafeControlRetryCase::ActionResult) + { + command_tx + .send(FleetControlCommand::Shutdown) + .await + .unwrap(); + } + deregistration_result + }; + + let ((result, deregistration_result), ()) = + tokio::time::timeout(Duration::from_secs(5), async { + let (result, deregistration_result) = tokio::join!(session, driver); + ((result, deregistration_result), server.await.unwrap()) + }) + .await + .expect("safe control retry case must finish inside its short test budget"); + + if (persistent || disconnect_during_backoff) && case == SafeControlRetryCase::ActionResult { + assert!(matches!(result, ControlRunResult::Disconnected { .. })); + assert_eq!(retained_pressure_frames.len(), 1); + assert!(matches!( + retained_pressure_frames.front(), + Some(BrokerToRelaycast::ActionResult(result)) + if result.id.as_deref().is_some_and(|id| id.starts_with("action_result_")) + && result.invocation_id == "inv-pressure" + )); + + let expected = retained_pressure_frames + .front() + .cloned() + .expect("retained action.result"); + let reconnect_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let reconnect_ws_url = format!( + "ws://{}/v1/node/ws", + reconnect_listener.local_addr().unwrap() + ); + let (reconnect_done_tx, reconnect_done_rx) = oneshot::channel(); + let reconnect_server = tokio::spawn(async move { + let (stream, _) = reconnect_listener.accept().await.unwrap(); + let mut ws = accept_async(stream).await.unwrap(); + assert!(matches!( + next_node_to_server(&mut ws).await, + BrokerToRelaycast::NodeRegister(_) + )); + assert!(matches!( + next_node_to_server(&mut ws).await, + BrokerToRelaycast::InventorySync(_) + )); + assert!(matches!( + next_node_to_server(&mut ws).await, + BrokerToRelaycast::NodeHeartbeat(_) + )); + let replayed = next_non_heartbeat_node_to_server(&mut ws).await; + assert_eq!(replayed, expected, "reconnect must replay the exact result"); + let id = match replayed { + BrokerToRelaycast::ActionResult(result) => result.id.unwrap(), + _ => unreachable!(), + }; + ws.send(Message::Text( + json!({"v": 1, "id": id, "type": "reply", "ok": true, "data": {}}).to_string(), + )) + .await + .unwrap(); + reconnect_done_tx.send(()).unwrap(); + while ws.next().await.is_some() {} + }); + let reconnect_config = FleetControlConfig { + ws_url: reconnect_ws_url, + node_token: Some("nt_test".to_string()), + node_id: "node-test".to_string(), + node_name: "host-test".to_string(), + broker_version: "broker/test".to_string(), + token_minter: None, + session_token: None, + read_idle_timeout: None, + probe: None, + terminal_reconnect_tx: None, + }; + let (reconnect_command_tx, mut reconnect_command_rx) = mpsc::channel(2); + let (reconnect_event_tx, mut reconnect_event_rx) = mpsc::channel(2); + let reconnect_session = run_connected_once_with_retry_policy( + &reconnect_config, + &mut reconnect_command_rx, + &reconnect_event_tx, + &mut registration, + &mut inventory, + &mut load, + Duration::from_secs(3_600), + retry_policy, + &mut retained_pressure_frames, + ); + let reconnect_driver = async move { + assert_eq!( + reconnect_event_rx.recv().await.unwrap(), + FleetControlEvent::Connected + ); + reconnect_done_rx.await.unwrap(); + reconnect_command_tx + .send(FleetControlCommand::Shutdown) + .await + .unwrap(); + }; + let (reconnect_result, ()) = tokio::time::timeout(Duration::from_secs(5), async { + let (result, ()) = tokio::join!(reconnect_session, reconnect_driver); + (result, reconnect_server.await.unwrap()) + }) + .await + .expect("retained result must replay after reconnect"); + assert_eq!(reconnect_result, ControlRunResult::Shutdown); + assert!(retained_pressure_frames.is_empty()); + } else { + assert_eq!(result, ControlRunResult::Shutdown); + } + match (case, persistent, deregistration_result) { + (SafeControlRetryCase::AgentDeregister, false, Some(Ok(()))) => {} + (SafeControlRetryCase::AgentDeregister, true, Some(Err(reason))) => { + assert!(reason.starts_with("d1_pressure:"), "got {reason}"); + } + (SafeControlRetryCase::AgentDeregister, _, other) => { + panic!("unexpected deregistration result: {other:?}") + } + (_, _, None) => {} + (_, _, other) => panic!("unexpected reply owner: {other:?}"), + } + } + + #[tokio::test] + async fn delivery_ack_retries_transient_pressure_on_same_socket() { + assert_safe_control_frame_socket_retry(SafeControlRetryCase::DeliveryAck, false, false) + .await; + } + + #[tokio::test] + async fn delivery_ack_stops_after_persistent_pressure_budget() { + assert_safe_control_frame_socket_retry(SafeControlRetryCase::DeliveryAck, true, false) + .await; + } + + #[tokio::test] + async fn action_result_retries_transient_pressure_on_same_socket() { + assert_safe_control_frame_socket_retry(SafeControlRetryCase::ActionResult, false, false) + .await; + } + + #[tokio::test] + async fn action_result_is_retained_after_persistent_pressure_budget() { + assert_safe_control_frame_socket_retry(SafeControlRetryCase::ActionResult, true, false) + .await; + } + + #[tokio::test] + async fn action_result_is_retained_when_socket_drops_during_pressure_backoff() { + assert_safe_control_frame_socket_retry(SafeControlRetryCase::ActionResult, false, true) + .await; + } + + #[tokio::test] + async fn agent_deregister_retries_transient_pressure_on_same_socket() { + assert_safe_control_frame_socket_retry(SafeControlRetryCase::AgentDeregister, false, false) + .await; + } + + #[tokio::test] + async fn agent_deregister_fails_named_after_persistent_pressure_budget() { + assert_safe_control_frame_socket_retry(SafeControlRetryCase::AgentDeregister, true, false) + .await; + } + + #[tokio::test] + async fn registration_d1_pressure_backoff_remains_shutdown_interruptible() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let ws_url = format!("ws://{}/v1/node/ws", listener.local_addr().unwrap()); + let (command_tx, mut command_rx) = mpsc::channel(1); + let (event_tx, _event_rx) = mpsc::channel(1); + let mut registration = Some(build_node_register( + &test_manifest(), + "node-test", + "host-test", + "broker/test", + None, )); let mut inventory = Vec::new(); let mut load = FleetLoadSnapshot { @@ -6230,6 +7756,372 @@ mod tests { let _ = command_tx.send(FleetControlCommand::Shutdown).await; } + #[tokio::test] + async fn node_control_agent_register_retries_d1_pressure_then_succeeds() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let ws_url = format!("ws://{}/v1/node/ws", listener.local_addr().unwrap()); + let (command_tx, command_rx) = mpsc::channel(32); + let (event_tx, mut event_rx) = mpsc::channel(32); + + tokio::spawn(run_node_control_client( + FleetControlConfig { + ws_url, + node_token: Some("nt_test".to_string()), + node_id: "node-test".to_string(), + node_name: "host-test".to_string(), + broker_version: "broker/test".to_string(), + token_minter: None, + session_token: None, + read_idle_timeout: None, + probe: None, + terminal_reconnect_tx: None, + }, + command_rx, + event_tx, + )); + + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut ws = accept_async(stream).await.unwrap(); + + assert!(matches!( + next_node_to_server(&mut ws).await, + BrokerToRelaycast::NodeRegister(_) + )); + assert!(matches!( + next_node_to_server(&mut ws).await, + BrokerToRelaycast::InventorySync(_) + )); + assert!(matches!( + next_node_to_server(&mut ws).await, + BrokerToRelaycast::NodeHeartbeat(_) + )); + + let initial = match next_non_heartbeat_node_to_server(&mut ws).await { + BrokerToRelaycast::AgentRegister(request) => request, + other => panic!("expected agent.register, got {other:?}"), + }; + let request_id = initial.id.clone().expect("agent.register id"); + + for (attempt, no_retry_before, retry_within) in [ + (1, Duration::from_millis(500), Duration::from_secs(2)), + (2, Duration::from_millis(1_500), Duration::from_secs(3)), + ] { + ws.send(Message::Text( + json!({ + "v": 1, + "id": request_id, + "type": "error", + "ok": false, + "code": "d1_pressure", + "message": "Node liveness retry pending" + }) + .to_string(), + )) + .await + .unwrap(); + assert!( + tokio::time::timeout( + no_retry_before, + next_non_heartbeat_node_to_server(&mut ws), + ) + .await + .is_err(), + "agent.register retry {attempt} must honor exponential backoff" + ); + let retried = + tokio::time::timeout(retry_within, next_non_heartbeat_node_to_server(&mut ws)) + .await + .expect( + "retryable agent.register d1_pressure should retry on the live socket", + ); + assert_eq!(retried, BrokerToRelaycast::AgentRegister(initial.clone())); + } + + ws.send(Message::Text( + serde_json::to_string(&RelaycastToBroker::Reply(crate::fleet_wire::Reply { + v: FLEET_WIRE_VERSION, + id: request_id, + ok: true, + data: json!({ + "name": "agent-a", + "agent_id": "agt-1", + "token": "at_test" + }), + })) + .unwrap(), + )) + .await + .unwrap(); + }); + + command_tx + .send(FleetControlCommand::RegisterNode { + manifest: test_manifest(), + resume_cursor: None, + }) + .await + .unwrap(); + assert_eq!(event_rx.recv().await.unwrap(), FleetControlEvent::Connected); + + let (reply_tx, reply_rx) = oneshot::channel(); + command_tx + .send(FleetControlCommand::RegisterAgent { + request: AgentRegister { + auto_join_general: Some(false), + v: FLEET_WIRE_VERSION, + id: None, + name: "agent-a".to_string(), + invocation_id: Some("inv-1".to_string()), + session_ref: Some("session-1".to_string()), + resumable: Some(true), + }, + reply: reply_tx, + }) + .await + .unwrap(); + + let token = tokio::time::timeout(Duration::from_secs(8), reply_rx) + .await + .expect("agent.register retry budget elapsed") + .unwrap() + .expect("transient d1_pressure must not fail the spawn"); + assert_eq!(token.agent_id, "agt-1"); + assert_eq!(token.token, "at_test"); + + tokio::time::timeout(Duration::from_secs(8), server) + .await + .unwrap() + .unwrap(); + let _ = command_tx.send(FleetControlCommand::Shutdown).await; + } + + async fn assert_agent_register_pressure_exhaustion_fails_closed_and_cleans_late_success( + retry_policy: ControlRetryPolicy, + expected_attempts: Option, + ) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let ws_url = format!("ws://{}/v1/node/ws", listener.local_addr().unwrap()); + let (command_tx, mut command_rx) = mpsc::channel(32); + let (event_tx, mut event_rx) = mpsc::channel(32); + let (late_success_tx, mut late_success_rx) = oneshot::channel(); + let (cleanup_seen_tx, cleanup_seen_rx) = oneshot::channel(); + let server_max_retries = retry_policy.max_retries; + + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut ws = accept_async(stream).await.unwrap(); + assert!(matches!( + next_node_to_server(&mut ws).await, + BrokerToRelaycast::NodeRegister(_) + )); + assert!(matches!( + next_node_to_server(&mut ws).await, + BrokerToRelaycast::InventorySync(_) + )); + assert!(matches!( + next_node_to_server(&mut ws).await, + BrokerToRelaycast::NodeHeartbeat(_) + )); + + let mut initial: Option = None; + let mut attempts = 0u32; + loop { + enum ServerEvent { + LateSuccess, + Frame(BrokerToRelaycast), + } + let event = tokio::select! { + result = &mut late_success_rx => { + result.unwrap(); + ServerEvent::LateSuccess + } + frame = next_non_heartbeat_node_to_server(&mut ws) => { + ServerEvent::Frame(frame) + } + }; + match event { + ServerEvent::LateSuccess => { + assert!(attempts > 0, "pressure must be observed before exhaustion"); + if let Some(expected) = expected_attempts { + assert_eq!(attempts, expected, "retry-count exhaustion attempt count"); + } else { + assert!( + attempts < server_max_retries + 1, + "budget exhaustion must preempt retry-count exhaustion" + ); + } + break; + } + ServerEvent::Frame(BrokerToRelaycast::AgentRegister(request)) => { + attempts = attempts.saturating_add(1); + if let Some(initial) = initial.as_ref() { + assert_eq!( + &request, initial, + "every retry must preserve the exact frame" + ); + } else { + initial = Some(request.clone()); + } + ws.send(Message::Text( + json!({ + "v": 1, + "id": request.id, + "type": "error", + "ok": false, + "code": "d1_pressure", + "message": "Node liveness retry pending" + }) + .to_string(), + )) + .await + .unwrap(); + } + ServerEvent::Frame(other) => { + panic!("expected agent.register, got {other:?}") + } + } + } + let initial = initial.expect("initial agent.register frame"); + let request_id = initial.id.expect("agent.register request id"); + ws.send(Message::Text( + serde_json::to_string(&RelaycastToBroker::Reply(crate::fleet_wire::Reply { + v: FLEET_WIRE_VERSION, + id: request_id.clone(), + ok: true, + data: json!({ + "name": "agent-a", + "agent_id": "agt-late", + "token": "at-late" + }), + })) + .unwrap(), + )) + .await + .unwrap(); + match next_non_heartbeat_node_to_server(&mut ws).await { + BrokerToRelaycast::AgentDeregister(deregister) => { + assert_eq!(deregister.id.as_deref(), Some(request_id.as_str())); + assert_eq!(deregister.agent_id, "agt-late"); + assert_eq!(deregister.name.as_deref(), Some("agent-a")); + } + other => panic!("expected compensating agent.deregister, got {other:?}"), + } + cleanup_seen_tx.send(()).unwrap(); + while ws.next().await.is_some() {} + }); + + let config = FleetControlConfig { + ws_url, + node_token: Some("nt_test".to_string()), + node_id: "node-test".to_string(), + node_name: "host-test".to_string(), + broker_version: "broker/test".to_string(), + token_minter: None, + session_token: None, + read_idle_timeout: None, + probe: None, + terminal_reconnect_tx: None, + }; + let mut registration = Some(build_node_register( + &test_manifest(), + "node-test", + "host-test", + "broker/test", + None, + )); + let mut inventory = Vec::new(); + let mut load = FleetLoadSnapshot::default(); + let mut retained_pressure_frames = VecDeque::new(); + let session = run_connected_once_with_retry_policy( + &config, + &mut command_rx, + &event_tx, + &mut registration, + &mut inventory, + &mut load, + Duration::from_secs(3_600), + retry_policy, + &mut retained_pressure_frames, + ); + let driver = async move { + assert_eq!(event_rx.recv().await.unwrap(), FleetControlEvent::Connected); + let (reply_tx, reply_rx) = oneshot::channel(); + command_tx + .send(FleetControlCommand::RegisterAgent { + request: AgentRegister { + auto_join_general: Some(false), + v: FLEET_WIRE_VERSION, + id: None, + name: "agent-a".to_string(), + invocation_id: Some("inv-1".to_string()), + session_ref: Some("session-1".to_string()), + resumable: Some(true), + }, + reply: reply_tx, + }) + .await + .unwrap(); + let error = reply_rx + .await + .unwrap() + .expect_err("persistent d1_pressure must fail closed"); + late_success_tx.send(()).unwrap(); + cleanup_seen_rx.await.unwrap(); + command_tx + .send(FleetControlCommand::Shutdown) + .await + .unwrap(); + error + }; + let ((result, error), ()) = tokio::time::timeout(Duration::from_secs(5), async { + let (result, error) = tokio::join!(session, driver); + ((result, error), server.await.unwrap()) + }) + .await + .expect("short persistent-pressure policy must finish reliably"); + assert_eq!(result, ControlRunResult::Shutdown); + assert!( + error.starts_with("d1_pressure:"), + "spawn caller must see the retryable root cause, got {error}" + ); + } + + #[tokio::test] + async fn node_control_agent_register_persistent_d1_pressure_fails_closed_and_cleans_late_success( + ) { + let retry_policy = ControlRetryPolicy { + initial_delay: Duration::from_millis(10), + max_delay: Duration::from_millis(20), + max_retries: 2, + // This case proves retry-count exhaustion. Keep the independent + // wall-clock guard nonbinding even on a heavily loaded runner. + budget: Duration::from_secs(30), + }; + assert_agent_register_pressure_exhaustion_fails_closed_and_cleans_late_success( + retry_policy, + Some(retry_policy.max_retries + 1), + ) + .await; + } + + #[tokio::test] + async fn node_control_agent_register_budget_exhaustion_fails_closed_and_cleans_late_success() { + let retry_policy = ControlRetryPolicy { + initial_delay: Duration::from_millis(10), + max_delay: Duration::from_millis(20), + max_retries: 100, + // Shorter than the first retry delay, so the budget path wins by + // construction rather than depending on runner scheduling. + budget: Duration::from_millis(1), + }; + assert_agent_register_pressure_exhaustion_fails_closed_and_cleans_late_success( + retry_policy, + None, + ) + .await; + } + #[tokio::test] async fn node_control_client_mints_initial_token_when_none_supplied() { // Broker startup no longer mints the node token on the API-readiness @@ -9006,6 +10898,7 @@ mod tests { &mut registrations, &mut deregistrations, &mut liveness, + &mut VecDeque::new(), "node-test", &mut sink, None, diff --git a/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts b/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts index 5840fbf539..a7e4ffe660 100644 --- a/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts +++ b/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts @@ -41,6 +41,9 @@ describe('fleet CLI lifecycle routing', () => { relayHome = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-fleet-lifecycle-home-')); vi.stubEnv('AGENT_RELAY_PROJECT', projectRoot); vi.stubEnv('AGENT_RELAY_HOME', relayHome); + vi.stubEnv('RELAY_WORKSPACE_KEY', 'rk_live_workspace'); + vi.stubEnv('RELAY_API_KEY', ''); + vi.stubEnv('RELAY_BASE_URL', 'https://cast.agentrelay.com'); const dataDir = path.join(projectRoot, '.agentworkforce', 'relay'); writeProjectWorkspaceKey(dataDir, 'rk_live_workspace', { workspaceId: TARGET.workspaceId }); diff --git a/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts b/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts index e668abb889..b6f8f1ae4d 100644 --- a/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts +++ b/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts @@ -174,7 +174,10 @@ describe('fleet sandbox command regressions', () => { expect(resolveRepository).toHaveBeenCalledWith(outsideGit, undefined, undefined, 'strict'); expect(materialize).not.toHaveBeenCalled(); - expect(ensure).toHaveBeenCalledWith(expect.objectContaining({ mountRelayfile: true })); + expect(ensure).toHaveBeenCalledWith( + expect.objectContaining({ mountRelayfile: true }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) + ); expect((ensure.mock.calls[0]?.[0] as Record).relayfilePaths).toBeUndefined(); expect(placement).toHaveBeenCalledWith( expect.objectContaining({ input: expect.objectContaining({ worker_cwd: '/workspace' }) }) @@ -257,7 +260,8 @@ describe('fleet sandbox command regressions', () => { sandboxId: REPLAY_SANDBOX_ID, name: REPLAY_SANDBOX_NAME, mountRelayfile: true, - }) + }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); const ensureInput = ensure.mock.calls[0]?.[0] as Record; expect(ensureInput.repos).toBeUndefined(); @@ -324,7 +328,8 @@ describe('fleet sandbox command regressions', () => { expect.objectContaining({ repos: [selection.repository], repoRevisions: { [selection.repository]: REVISION }, - }) + }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); }); }); diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 93cf4a155d..e12fc0df12 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1191,6 +1191,7 @@ describe('fleet command support', () => { expect(ensureInput).toEqual({ workspaceId: 'rw_abc', requiredCapability: 'spawn:codex', + preparationMode: 'async-v1', maxAgents: 1, mountRelayfile: true, relayfilePaths: ['/live-review/run-123/**'], @@ -1264,6 +1265,7 @@ describe('fleet command support', () => { workerCwd: '/srv/agent-workforce/cloud/packages/web', }; const events: string[] = []; + const warnings: string[] = []; const materializeCloudRelayfileRepository = vi.fn(async () => { events.push('materialize'); return { @@ -1329,7 +1331,7 @@ describe('fleet command support', () => { deleteCloudFleetSandbox: vi.fn(async () => undefined), createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, - warn: () => undefined, + warn: (message) => warnings.push(message), error: () => undefined, }); @@ -1365,9 +1367,24 @@ describe('fleet command support', () => { '/github/repos/AgentWorkforce/cloud/.relayfile/**', '/.skills/**', ], - }) + }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0] as Record; + const ensureOptions = ensureCloudFleetSandbox.mock.calls[0]?.[1]; + ensureOptions?.onPreparationProgress?.({ + sandboxId: REPLAY_SANDBOX_ID, + state: 'pending', + phase: 'relayfile_mount_bootstrap', + generation: 2, + }); + // The generated identity is printed before Cloud work so an interrupted + // run can be resumed with --sandbox-id instead of creating a second sandbox. + expect(warnings).toEqual([ + `Cloud sandbox identity: ${String(ensureInput.sandboxId)}. If this command is interrupted, re-run it with --sandbox-id ${String(ensureInput.sandboxId)} to resume the same sandbox instead of creating another.`, + 'Cloud sandbox preparation: relayfile_mount_bootstrap (pending, generation 2).', + ]); + expect(ensureInput.sandboxId).toMatch(/^sbx_[0-9a-f-]{36}$/); expect(ensureInput.repos).toBeUndefined(); expect(ensureInput.repoRevisions).toBeUndefined(); expect(placement.spawn).toHaveBeenCalledWith( @@ -1482,7 +1499,8 @@ describe('fleet command support', () => { expect.objectContaining({ mountRelayfile: true, relayfilePaths: ['/github/repos/AgentWorkforce/agent-assistant/contents/**', '/memory/**'], - }) + }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0] as Record; expect(ensureInput.repos).toBeUndefined(); @@ -1602,7 +1620,8 @@ describe('fleet command support', () => { expect.objectContaining({ mountRelayfile: true, relayfilePaths: ['/github/repos/AgentWorkforce/cloud/contents/**', '/memory/**'], - }) + }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); const spawnInput = placement.spawn.mock.calls[0]?.[0]?.input as { task?: string }; expect(placement.spawn).toHaveBeenCalledWith( @@ -1970,7 +1989,8 @@ describe('fleet command support', () => { { from: 'user' } ); expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( - expect.objectContaining({ relayfilePaths: paths16 }) + expect.objectContaining({ relayfilePaths: paths16 }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); const tooMany = new Command(); @@ -2195,7 +2215,12 @@ describe('fleet command support', () => { expect(findProjectRoot).toHaveBeenCalledWith(path.resolve(process.cwd(), 'packages/web')); expect(selectionOptions[0]).toMatchObject({ projectRoot: "/local/cloud/packages/web team's" }); const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0] as Record; + // The exact checkout revision is part of the durable async request, so a + // resumed preparation clones the same commit it was accepted for. expect(ensureInput).toMatchObject({ + preparationMode: 'async-v1', + forceProvision: true, + providerId: 'agent37', repos: ['AgentWorkforce/cloud'], repoRevisions: { 'AgentWorkforce/cloud': revision }, }); @@ -2412,7 +2437,12 @@ describe('fleet command support', () => { expect(resolveWorkspaceByKey).toHaveBeenCalledWith('rk_live_pinned'); expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( - expect.objectContaining({ workspaceId: 'cloud-pinned', workloadProfile: 'long-running-agent' }) + expect.objectContaining({ + workspaceId: 'cloud-pinned', + workloadProfile: 'long-running-agent', + preparationMode: 'async-v1', + }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); expect(createWorkspaceRelay).toHaveBeenCalledWith( expect.objectContaining({ @@ -2672,6 +2702,10 @@ describe('fleet command support', () => { { from: 'user' } ); + expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( + expect.objectContaining({ preparationMode: 'async-v1', providerId: 'agent37', mountRelayfile: false }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) + ); expect(createWorkspaceRelay).toHaveBeenCalledWith({ projectRoot: process.cwd(), token: undefined, @@ -3273,7 +3307,9 @@ describe('fleet command support', () => { expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( expect.objectContaining({ workloadProfile: 'long-running-agent', - }) + preparationMode: 'async-v1', + }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); }); @@ -3482,7 +3518,8 @@ describe('fleet command support', () => { { from: 'user' } ); expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( - expect.objectContaining({ workspaceId: 'rw_captured' }) + expect.objectContaining({ workspaceId: 'rw_captured', preparationMode: 'async-v1' }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); }); @@ -3548,7 +3585,8 @@ describe('fleet command support', () => { ); expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( - expect.objectContaining({ workspaceId: cloudWorkspaceId }) + expect.objectContaining({ workspaceId: cloudWorkspaceId, preparationMode: 'async-v1' }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); expect(createWorkspaceRelay).toHaveBeenCalledWith({ projectRoot: process.cwd(), @@ -3715,6 +3753,10 @@ describe('fleet command support', () => { /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ ); expect(ensureInput?.name).toBe(`fleet-sandbox-${ensureInput?.sandboxId?.slice('sbx_'.length)}`); + expect(ensureInput?.preparationMode).toBe('async-v1'); + expect(ensureCloudFleetSandbox.mock.calls[0]?.[1]).toEqual({ + onPreparationProgress: expect.any(Function), + }); }); it('rejects a replay sandbox name that does not match its identity', async () => { @@ -3890,6 +3932,7 @@ describe('fleet command support', () => { expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); expect(warnings.join('\n')).toContain(`check Cloud Fleet for node '${REPLAY_SANDBOX_NAME}'`); + expect(warnings.join('\n')).not.toContain('no sandbox was left running'); expect(warnings.join('\n')).toContain(`--sandbox-id '${REPLAY_SANDBOX_ID}'`); }); @@ -3964,6 +4007,81 @@ describe('fleet command support', () => { expect(warnings.join('\n')).not.toContain('check Cloud Fleet'); }); + it('names a terminal relayfile_mount_failed cause and reports no sandbox left running without cleanup', async () => { + const warnings: string[] = []; + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const terminalMessage = + 'Cloud fleet sandbox preparation failed during relayfile_mount (relayfile_mount_failed, initial_sync_deadline): Relayfile mount failed during relayfile_mount. Retry provisioning the sandbox.'; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + resolveSandboxRepository: () => undefined, + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => { + throw new CloudFleetSandboxProvisionError(terminalMessage, { + cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + sandboxId: REPLAY_SANDBOX_ID, + nodeName: REPLAY_SANDBOX_NAME, + providerId: 'agent37', + sandboxAbsent: true, + preparationFailure: { + code: 'relayfile_mount_failed', + phase: 'relayfile_mount', + causeStage: 'initial_sync_deadline', + }, + }); + }), + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: (...args: unknown[]) => warnings.push(args.join(' ')), + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-id', + REPLAY_SANDBOX_ID, + '--sandbox-name', + REPLAY_SANDBOX_NAME, + '--workspace-id', + 'rw_abc', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); + expect(warnings).toEqual([ + `${terminalMessage} Cloud confirmed sandbox '${REPLAY_SANDBOX_ID}' is not running; no sandbox was left running.`, + ]); + expect(warnings.join('\n')).not.toContain('outcome is unknown'); + }); + it('preserves the caller Daytona ID after a matched malformed provisioned response', async () => { const warnings: string[] = []; const deleteCloudFleetSandbox = vi.fn(async () => undefined); diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index c896490750..fd1ff04647 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -13,6 +13,7 @@ import { resolveWorkspaceByKey, type CloudFleetSandboxProviderId, type CloudRelayfileRepositoryMaterialization, + type EnsureCloudFleetSandboxInput, type EnsureCloudFleetSandboxResult, } from '@agent-relay/cloud'; import { HarnessDriverClient } from '@agent-relay/harness-driver'; @@ -457,7 +458,10 @@ export function registerFleetCommands( '--sandbox-name ', 'Explicit sandbox node name (custom unless --sandbox-id requires matching fleet-sandbox-)' ) - .option('--sandbox-id ', 'Reuse a caller-declared sbx_ identity for an exact replay') + .option( + '--sandbox-id ', + 'Reuse a caller-declared sbx_ identity for an exact replay; re-run an interrupted --sandbox spawn with the identity it printed to resume it' + ) .option('--workspace-id ', 'Explicit Relay workspace identity required for sandbox provisioning') .option('--sandbox-provider ', 'Sandbox provider: daytona, e2b, or agent37') .option( @@ -790,10 +794,20 @@ export function registerFleetCommands( sandboxProvider === undefined || sandboxProvider === 'agent37' ? 'long-running-agent' : 'standard-long-running-agent'; + const useAsyncPreparation = + sandboxId !== undefined && (sandboxProvider === undefined || sandboxProvider === 'agent37'); + if (useAsyncPreparation && sandboxIdOption === undefined) { + // The generated identity is not persisted. Print it before Cloud work + // starts so an interrupted run can be resumed instead of duplicated. + deps.warn( + `Cloud sandbox identity: ${sandboxId}. If this command is interrupted, re-run it with --sandbox-id ${sandboxId} to resume the same sandbox instead of creating another.` + ); + } try { - sandbox = await deps.ensureCloudFleetSandbox({ + const ensureInput: EnsureCloudFleetSandboxInput = { workspaceId: relayWorkspaceId, requiredCapability: `spawn:${cli}`, + ...(useAsyncPreparation ? { preparationMode: 'async-v1' as const } : {}), maxAgents: 1, mountRelayfile: mountSandboxRelayfile, ...(sandboxMountPaths === undefined ? {} : { relayfilePaths: sandboxMountPaths }), @@ -807,10 +821,27 @@ export function registerFleetCommands( ...(checkoutRepository && sandboxRepository ? { repoRevisions: { [sandboxRepository.repository]: sandboxRepository.revision } } : {}), - }); + }; + sandbox = useAsyncPreparation + ? await deps.ensureCloudFleetSandbox(ensureInput, { + onPreparationProgress: (progress) => { + deps.warn( + `Cloud sandbox preparation: ${progress.phase} (${progress.state}, generation ${progress.generation}).` + ); + }, + }) + : await deps.ensureCloudFleetSandbox(ensureInput); assertSandboxRepositoryRevision(sandbox, checkoutRepository ? sandboxRepository : undefined); } catch (error) { - if ( + if (error instanceof CloudFleetSandboxProvisionError && error.sandboxAbsent) { + // Cloud's terminal record for this exact identity already proves the + // provider sandbox is gone; a delete here would only race its reaper. + deps.warn( + `${error.message} Cloud confirmed sandbox '${ + error.sandboxId ?? sandboxId ?? 'the requested sandbox' + }' is not running; no sandbox was left running.` + ); + } else if ( shouldCleanupSandbox && error instanceof CloudFleetSandboxProvisionError && error.confirmedProvisioned && diff --git a/packages/cli/src/cli/commands/integration-subscribe.test.ts b/packages/cli/src/cli/commands/integration-subscribe.test.ts index b28d090a53..5d8bf16fe7 100644 --- a/packages/cli/src/cli/commands/integration-subscribe.test.ts +++ b/packages/cli/src/cli/commands/integration-subscribe.test.ts @@ -13,6 +13,7 @@ import { type RelayfileBinding, } from './integration.js'; import type { PendingCleanupEntry } from './integration-cleanup-journal.js'; +import { writeProjectWorkspaceKey } from '../lib/project-workspace-key.js'; afterEach(() => { vi.unstubAllEnvs(); @@ -345,6 +346,40 @@ describe('integration subscribe', () => { ); }); + it('prefers a persisted route over a conflicting broker-session fallback', async () => { + const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-integration-persisted-route-')); + const relayHome = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-integration-persisted-home-')); + try { + vi.stubEnv('AGENT_RELAY_PROJECT', projectRoot); + vi.stubEnv('AGENT_RELAY_HOME', relayHome); + writeProjectWorkspaceKey(path.join(projectRoot, '.agentworkforce/relay'), 'rk_live_local', { + workspaceId: 'rw_test', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + const { program, error } = harness({ + resolveLocalRelayOptions: async () => ({ + workspaceKey: 'rk_live_local', + baseUrl: 'https://cast.agentrelay.com', + }), + }); + + await program.parseAsync(ARGS(), { from: 'user' }); + + expect(error).not.toHaveBeenCalled(); + expect(fetch).toHaveBeenCalledWith( + new URL('/v1/integrations/relayfile/inbound-target', 'https://agent37-cast.agentrelay.com'), + expect.objectContaining({ + headers: expect.objectContaining({ Authorization: 'Bearer rk_live_agent37' }), + }) + ); + } finally { + fs.rmSync(projectRoot, { recursive: true, force: true }); + fs.rmSync(relayHome, { recursive: true, force: true }); + } + }); + it('allows an explicit Relaycast base URL for inbound-target provisioning', async () => { const { program } = harness(); diff --git a/packages/cli/src/cli/commands/integration.ts b/packages/cli/src/cli/commands/integration.ts index 28a5c485fa..881ca248ef 100644 --- a/packages/cli/src/cli/commands/integration.ts +++ b/packages/cli/src/cli/commands/integration.ts @@ -415,11 +415,12 @@ function shouldRetryWithLocalWorkspaceKey(error: unknown): boolean { ); } +/** Add a local session as a non-authoritative retry fallback. */ function localRetryOptions(options: SdkClientOptions, local: LocalRelayOptions): SdkClientOptions { return { ...options, workspaceKey: local.workspaceKey, - baseUrl: options.baseUrl ?? local.baseUrl, + ...(options.baseUrl === undefined && local.baseUrl ? { fallbackBaseUrl: local.baseUrl } : {}), }; } @@ -544,15 +545,16 @@ function parseRelayfileInboundTargetResponse(body: unknown): { url: string; secr return { url: parsedUrl.toString(), secret }; } +/** Resolve the HTTPS Relaycast transport used to provision an inbound target. */ function resolveInboundTargetTransport(options: SdkClientOptions, callerOptions: SdkClientOptions) { const selected = { ...options }; // A legacy broker session can advertise its loopback HTTP API. It is not - // the Relaycast gateway. Preserve genuine HTTPS session origins, including - // custom self-hosts, and let persisted workspace routing validate the pair. - if (!callerOptions.baseUrl && selected.baseUrl) { - const url = new URL(selected.baseUrl); + // the Relaycast gateway. Preserve genuine HTTPS session fallbacks, including + // custom self-hosts, while keeping them subordinate to persisted routing. + if (!callerOptions.baseUrl && selected.fallbackBaseUrl) { + const url = new URL(selected.fallbackBaseUrl); if (url.protocol === 'http:' && ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)) { - selected.baseUrl = undefined; + selected.fallbackBaseUrl = undefined; } } const transport = resolveWorkspaceTransport(selected); diff --git a/packages/cli/src/cli/commands/relaycast-groups.test.ts b/packages/cli/src/cli/commands/relaycast-groups.test.ts index 633bcb3c17..64793b5fcb 100644 --- a/packages/cli/src/cli/commands/relaycast-groups.test.ts +++ b/packages/cli/src/cli/commands/relaycast-groups.test.ts @@ -329,7 +329,10 @@ describe('SDK-backed CLI groups', () => { expect(resolveLocalRelayOptions).toHaveBeenCalled(); expect(createAgentRelay).toHaveBeenNthCalledWith( 2, - expect.objectContaining({ workspaceKey: 'rk_live_local', baseUrl: 'https://relay.local' }) + expect.objectContaining({ + workspaceKey: 'rk_live_local', + fallbackBaseUrl: 'https://relay.local', + }) ); expect(secondRelay.integrations.webhooks.create).toHaveBeenCalledWith({ channel: 'deploy-status', @@ -379,7 +382,10 @@ describe('SDK-backed CLI groups', () => { expect(resolveLocalRelayOptions).toHaveBeenCalled(); expect(createAgentRelay).toHaveBeenNthCalledWith( 2, - expect.objectContaining({ workspaceKey: 'rk_live_local', baseUrl: 'https://relay.local' }) + expect.objectContaining({ + workspaceKey: 'rk_live_local', + fallbackBaseUrl: 'https://relay.local', + }) ); expect(secondRelay.webhooks.createInbound).toHaveBeenCalledWith({ channel: 'general', diff --git a/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts b/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts index a32f44558a..498134efb7 100644 --- a/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts +++ b/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts @@ -176,6 +176,42 @@ describe('fleet spawn confirmation is observable from the requester (#1430)', () expect((error as Error).message).toContain('/tmp/worker-1430.log'); }); + it('maps exhausted agent registration pressure to a retryable spawn error', async () => { + const { client } = createClient(async (name, invocationId) => ({ + invocation_id: invocationId, + action_name: name, + status: 'failed', + error: + "node agent.register failed for agent 'worker-pressure': d1_pressure: Node liveness retry pending (retry budget exhausted)", + })); + + const error = await client.placement + .spawn(spawnInput({ confirm: true, confirmTimeoutMs: 1_000, confirmPollIntervalMs: 10 })) + .catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(RelayPlacementError); + expect((error as RelayPlacementError).code).toBe('spawn_retryable'); + expect((error as Error).message).toContain('d1_pressure'); + expect((error as Error).message).toContain('retry the spawn'); + }); + + it('does not call a later free-form d1_pressure detail pre-mutation', async () => { + const { client } = createClient(async (name, invocationId) => ({ + invocation_id: invocationId, + action_name: name, + status: 'failed', + error: 'spawn cleanup failed after mutation: d1_pressure remained elevated', + })); + + const error = await client.placement + .spawn(spawnInput({ confirm: true, confirmTimeoutMs: 1_000, confirmPollIntervalMs: 10 })) + .catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(RelayPlacementError); + expect((error as RelayPlacementError).code).toBe('spawn_failed'); + expect((error as Error).message).not.toContain('No registration mutation was applied'); + }); + // MUST-NOT-FIRE — a healthy node. This is the arm a repaired node represents: // confirmation must not turn a working spawn into an error. it('resolves when the node confirms the spawn completed', async () => { diff --git a/packages/cli/src/cli/lib/sdk-client.test.ts b/packages/cli/src/cli/lib/sdk-client.test.ts index 584c461a12..743d3347fa 100644 --- a/packages/cli/src/cli/lib/sdk-client.test.ts +++ b/packages/cli/src/cli/lib/sdk-client.test.ts @@ -21,6 +21,7 @@ let dir: string; let projectRoot: string; const original = process.env.AGENT_RELAY_HOME; const originalProject = process.env.AGENT_RELAY_PROJECT; +const originalBaseUrl = process.env.RELAY_BASE_URL; beforeEach(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-sdk-client-')); @@ -29,6 +30,7 @@ beforeEach(() => { // `resolveWorkspaceKey` reads the CWD broker key from) to a temp dir. projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-sdk-project-')); process.env.AGENT_RELAY_PROJECT = projectRoot; + delete process.env.RELAY_BASE_URL; }); afterEach(() => { @@ -36,6 +38,8 @@ afterEach(() => { else process.env.AGENT_RELAY_HOME = original; if (originalProject === undefined) delete process.env.AGENT_RELAY_PROJECT; else process.env.AGENT_RELAY_PROJECT = originalProject; + if (originalBaseUrl === undefined) delete process.env.RELAY_BASE_URL; + else process.env.RELAY_BASE_URL = originalBaseUrl; fs.rmSync(dir, { recursive: true, force: true }); fs.rmSync(projectRoot, { recursive: true, force: true }); }); @@ -504,6 +508,98 @@ describe('sdk client option resolution', () => { }); }); + it('prefers a persisted isolated route over an ambient canonical base URL', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + + expect( + resolveWorkspaceTransport({ + env: { + AGENT_RELAY_HOME: dir, + AGENT_RELAY_PROJECT: projectRoot, + RELAY_BASE_URL: 'https://cast.agentrelay.com', + }, + }) + ).toEqual({ + workspaceKey: 'rk_live_agent37', + baseUrl: 'https://agent37-cast.agentrelay.com', + source: 'project', + }); + }); + + it('rejects an explicit base URL that conflicts with the persisted route', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + + expect(() => + resolveWorkspaceTransport({ + baseUrl: 'https://cast.agentrelay.com', + env: { AGENT_RELAY_HOME: dir, AGENT_RELAY_PROJECT: projectRoot }, + }) + ).toThrow('The requested Relaycast base URL does not match the persisted workspace route.'); + }); + + it('keeps explicit and ambient base URL precedence when no route is persisted', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { workspaceId: 'rw_abc' }); + const baseEnv = { + AGENT_RELAY_HOME: dir, + AGENT_RELAY_PROJECT: projectRoot, + RELAY_BASE_URL: 'https://ambient.example.test', + }; + + expect(resolveWorkspaceTransport({ env: baseEnv })).toEqual({ + workspaceKey: 'rk_live_canonical', + baseUrl: 'https://ambient.example.test', + source: 'project', + }); + expect( + resolveWorkspaceTransport({ + baseUrl: 'https://explicit.example.test', + env: baseEnv, + }) + ).toEqual({ + workspaceKey: 'rk_live_canonical', + baseUrl: 'https://explicit.example.test', + source: 'project', + }); + }); + + it('constructs the client with the persisted origin and its route-scoped credential', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + + const relay = createAgentRelay({ + env: { + AGENT_RELAY_HOME: dir, + AGENT_RELAY_PROJECT: projectRoot, + RELAY_BASE_URL: 'https://cast.agentrelay.com', + }, + }) as unknown as { + workspaceKey?: string; + messagingOptions: { baseUrl?: string }; + }; + + expect({ + workspaceKey: relay.workspaceKey, + baseUrl: relay.messagingOptions.baseUrl, + }).toEqual({ + workspaceKey: 'rk_live_agent37', + baseUrl: 'https://agent37-cast.agentrelay.com', + }); + }); + it.each([undefined, 'https://agentrelay.com/cloud', 'https://dev.agentrelay.com/cloud/'])( 'rejects a dev canonical route with unmatched persisted Cloud API %s', (relaycastCloudApiUrl) => { diff --git a/packages/cli/src/cli/lib/sdk-client.ts b/packages/cli/src/cli/lib/sdk-client.ts index 673c250a19..235e3b13ae 100644 --- a/packages/cli/src/cli/lib/sdk-client.ts +++ b/packages/cli/src/cli/lib/sdk-client.ts @@ -18,6 +18,8 @@ import { export interface SdkClientOptions { workspaceKey?: string; token?: string; + /** Non-authoritative session URL used only when no persisted route or explicit URL exists. */ + fallbackBaseUrl?: string; baseUrl?: string; env?: NodeJS.ProcessEnv; /** Explicit project root for nested invocations such as packages/web. */ @@ -93,22 +95,28 @@ function selectionForTransport(options: SdkClientOptions): WorkspaceSelection | return canonicalSelection; } +/** Resolve an origin while keeping a persisted route paired with its credential. */ function resolveBaseUrlForSelection( selection: WorkspaceSelection | undefined, options: SdkClientOptions ): string | undefined { const persisted = validatePersistedRelaycastBaseUrl(selection); - const requested = trimOrUndefined(options.baseUrl) ?? trimOrUndefined(env(options).RELAY_BASE_URL); - if (persisted && requested) { + const explicit = trimOrUndefined(options.baseUrl); + const fallback = trimOrUndefined(options.fallbackBaseUrl) ?? trimOrUndefined(env(options).RELAY_BASE_URL); + const requested = explicit ?? fallback; + // Precedence is explicit --base-url (validated against a persisted route), + // then the persisted route, fallbackBaseUrl, and finally RELAY_BASE_URL. + // This keeps a server-selected route paired with its credential. + if (persisted && explicit) { let parsed: URL; try { - parsed = new URL(requested); + parsed = new URL(explicit); } catch { throw new Error('The requested Relaycast base URL is invalid.'); } - const authority = /^https:\/\/([^/?#]+)/i.exec(requested)?.[1] ?? ''; + const authority = /^https:\/\/([^/?#]+)/i.exec(explicit)?.[1] ?? ''; if ( - !/^https:\/\/[^/?#]+\/?$/i.test(requested) || + !/^https:\/\/[^/?#]+\/?$/i.test(explicit) || parsed.protocol !== 'https:' || parsed.username || parsed.password || diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index dd4a24cb7f..c32e710ce6 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -21,6 +21,7 @@ import { materializeCloudRelayfileRepository, normalizeRelaycastTarget, } from './fleet-sandbox.js'; +import { CloudAuthError } from './types.js'; const auth = { accessToken: 'access', @@ -57,6 +58,23 @@ const PROVIDER_OUTCOME_MATRIX = NON_AGENT_PROVIDER_IDS.flatMap((providerId) => (['provisioned', 'reused'] as const).map((outcome) => ({ providerId, outcome })) ); +function preparationEnvelope( + state: 'pending' | 'ready' | 'terminal' | 'cleanup_pending', + phase: string, + generation: number, + extra: Record = {} +) { + return { + version: 1, + mode: 'async-v1', + sandboxId: SANDBOX_ID, + state, + phase, + generation, + ...extra, + }; +} + describe('Cloud fleet sandbox client', () => { beforeEach(() => { vi.resetAllMocks(); @@ -431,6 +449,58 @@ describe('Cloud fleet sandbox client', () => { ).resolves.toMatchObject({ relaycastTarget: DEV_RELAYCAST_TARGET }); }); + it('validates an async-prepared Relaycast target against the auth that read the ready record', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'agent_relay_cli_bootstrap', 0), { + status: 202, + }), + // Only the read of the ready record carries DEV auth, so this proves + // normalization uses the auth that read the record, not the ensure's. + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 1, { + result: { + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-async-dev', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-async-dev', + relayWorkspaceId: 'rw_abc', + relaycastTarget: DEV_RELAYCAST_TARGET, + relayfileMounted: true, + }, + }) + ), + auth: devAuth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ + nodeId: 'node-async-dev', + relaycastTarget: DEV_RELAYCAST_TARGET, + relaycastCloudApiUrl: DEV_CLOUD_API_URL, + }); + }); + it('resolves the unified workspace and provisions a ready mounted sandbox', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -505,6 +575,1385 @@ describe('Cloud fleet sandbox client', () => { }); }); + it('advances async preparation one durable phase per request and reports deduplicated progress', async () => { + const progress = vi.fn(); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth: refreshedAuth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'agent_relay_cli_bootstrap', 0), { + status: 202, + }), + auth: refreshedAuth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'agent_relay_cli_bootstrap', 0), { + status: 202, + }), + auth: refreshedAuth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'relayfile_mount_bootstrap', 1), { + status: 202, + }), + auth: refreshedAuth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 2, { + result: { + outcome: 'provisioned', + nodeId: 'node-async', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-async', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + relayfileMountPath: '/workspace', + providerId: 'agent37', + }, + }) + ), + auth: refreshedAuth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1, onPreparationProgress: progress } + ) + ).resolves.toMatchObject({ + outcome: 'provisioned', + sandboxId: SANDBOX_ID, + nodeId: 'node-async', + providerId: 'agent37', + }); + + const requests = mocks.authorizedApiFetch.mock.calls.slice(1); + expect(requests.map((call) => [call[1], call[2]?.method])).toEqual([ + ['/api/v1/fleet/nodes/sandbox/ensure', 'POST'], + [`/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + [`/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`, 'POST'], + [`/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`, 'POST'], + ]); + const ensureBody = JSON.parse(String(requests[0]?.[2]?.body)); + expect(ensureBody).toMatchObject({ + preparationMode: 'async-v1', + sandboxId: SANDBOX_ID, + }); + // Cloud routes unpinned async-v1 to Agent37 itself; naming the provider + // would switch Cloud to the strict explicit-provider path (relay#1656). + expect(ensureBody).not.toHaveProperty('providerId'); + expect(progress.mock.calls.map(([value]) => value)).toEqual([ + expect.objectContaining({ state: 'pending', phase: 'agent_relay_cli_bootstrap', generation: 0 }), + expect.objectContaining({ state: 'pending', phase: 'relayfile_mount_bootstrap', generation: 1 }), + expect.objectContaining({ state: 'ready', phase: 'broker_visible', generation: 2 }), + ]); + }); + + it('accepts an older Cloud capability-routing an unpinned async request to another provider', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeId: 'node-routed', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: DAYTONA_PROVIDER_SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + providerId: 'daytona', + }, + { status: 201 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }) + ).resolves.toMatchObject({ + outcome: 'provisioned', + sandboxId: SANDBOX_ID, + providerId: 'daytona', + }); + const ensureBody = JSON.parse(String(mocks.authorizedApiFetch.mock.calls[1]?.[2]?.body)); + expect(ensureBody).toMatchObject({ preparationMode: 'async-v1', sandboxId: SANDBOX_ID }); + expect(ensureBody).not.toHaveProperty('providerId'); + }); + + it('accepts a synchronous 201 result when an older Cloud ignores async preparation mode', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeId: 'node-from-old-cloud', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-from-old-cloud', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + relayfileMountPath: '/workspace', + providerId: 'agent37', + }, + { status: 201 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }) + ).resolves.toMatchObject({ + outcome: 'provisioned', + nodeId: 'node-from-old-cloud', + sandboxId: SANDBOX_ID, + }); + + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + expect(JSON.parse(String(mocks.authorizedApiFetch.mock.calls[1]?.[2]?.body))).toMatchObject({ + preparationMode: 'async-v1', + sandboxId: SANDBOX_ID, + }); + }); + + it('retries synchronously when an older Cloud rejects the async preparation field', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'Unrecognized key: preparationMode' }, { status: 400 }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeId: 'node-after-compatibility-retry', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-compatibility-retry', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + providerId: 'agent37', + }, + { status: 201 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }) + ).resolves.toMatchObject({ + outcome: 'provisioned', + nodeId: 'node-after-compatibility-retry', + sandboxId: SANDBOX_ID, + }); + + const ensureRequests = mocks.authorizedApiFetch.mock.calls.slice(1); + expect(ensureRequests).toHaveLength(2); + expect(JSON.parse(String(ensureRequests[0]?.[2]?.body))).toHaveProperty('preparationMode', 'async-v1'); + expect(JSON.parse(String(ensureRequests[1]?.[2]?.body))).not.toHaveProperty('preparationMode'); + }); + + it('does not replay ensure for an unrelated 400 response', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'Repository revision is invalid' }, { status: 400 }), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }) + ).rejects.toThrow('Repository revision is invalid'); + + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + }); + + it('retries an accepted async preparation after rate limiting through durable status', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'rate limited' }, { status: 429, headers: { 'retry-after': '0' } }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'agent_relay_cli_bootstrap', 1), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 2, { + result: { + outcome: 'provisioned', + nodeId: 'node-after-rate-limit', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-rate-limit', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: false, + providerId: 'agent37', + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-after-rate-limit', sandboxId: SANDBOX_ID }); + + expect(mocks.authorizedApiFetch.mock.calls.slice(2).map((call) => call[2]?.method)).toEqual([ + 'GET', + 'GET', + 'POST', + ]); + }); + + it('recovers a dropped initial async ensure response by reading durable status without replaying ensure', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockRejectedValueOnce(new TypeError('connection reset')) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 1, { + result: { + outcome: 'provisioned', + nodeId: 'node-recovered', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-recovered', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: false, + providerId: 'agent37', + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-recovered', sandboxId: SANDBOX_ID }); + + expect( + mocks.authorizedApiFetch.mock.calls.filter((call) => call[1] === '/api/v1/fleet/nodes/sandbox/ensure') + ).toHaveLength(1); + expect(mocks.authorizedApiFetch.mock.calls[2]?.[2]?.method).toBe('GET'); + }); + + it('recovers an initial async 5xx by reading durable status without replaying ensure', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'gateway response lost' }, { status: 504 }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 1, { + result: { + outcome: 'provisioned', + nodeId: 'node-after-gateway', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-gateway', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: false, + providerId: 'agent37', + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-after-gateway', sandboxId: SANDBOX_ID }); + + expect( + mocks.authorizedApiFetch.mock.calls.filter((call) => call[1] === '/api/v1/fleet/nodes/sandbox/ensure') + ).toHaveLength(1); + expect(mocks.authorizedApiFetch.mock.calls[2]?.[2]?.method).toBe('GET'); + }); + + it('recovers a successful async ensure with a truncated envelope through durable status', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: new Response(null, { status: 202 }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 1, { + result: { + outcome: 'provisioned', + nodeId: 'node-after-truncated-ensure', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-truncated-ensure', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: false, + providerId: 'agent37', + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-after-truncated-ensure', sandboxId: SANDBOX_ID }); + + expect( + mocks.authorizedApiFetch.mock.calls.filter((call) => call[1] === '/api/v1/fleet/nodes/sandbox/ensure') + ).toHaveLength(1); + expect(mocks.authorizedApiFetch.mock.calls[2]?.[2]?.method).toBe('GET'); + }); + + it('preserves an initial async sandbox identity mismatch instead of polling the wrong record', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ + ...preparationEnvelope('pending', 'provider_allocation', 0), + sandboxId: 'sbx_00000000-0000-4000-8000-000000000099', + }), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).rejects.toThrow(/returned sandboxId .* instead of requested sandboxId/u); + + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + }); + + it('bounds status polling when the initial async request was never accepted', async () => { + vi.spyOn(Date, 'now').mockReturnValueOnce(0).mockReturnValue(110_001); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockRejectedValueOnce(new TypeError('connection refused')) + .mockResolvedValueOnce({ + response: Response.json({ error: 'not found' }, { status: 404 }), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + sandboxId: SANDBOX_ID, + providerId: undefined, + outcomeUnknown: true, + confirmedProvisioned: false, + }); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + }); + + it('wraps the internal async deadline with accepted cleanup identity', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { timeoutMs: 5, preparationPollIntervalMs: 20 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + sandboxId: SANDBOX_ID, + providerId: 'agent37', + confirmedProvisioned: true, + outcomeUnknown: false, + }); + }); + + it('attributes Agent37 cleanup identity once status confirms a lost unpinned async ensure', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockRejectedValueOnce(new TypeError('fetch failed')) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0)), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { timeoutMs: 40, preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + sandboxId: SANDBOX_ID, + providerId: 'agent37', + confirmedProvisioned: true, + outcomeUnknown: false, + }); + }); + + it('rejects a confirmed unpinned async ready result attributed to another provider', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 1, { + result: { + outcome: 'provisioned', + nodeId: 'node-wrong-provider', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: DAYTONA_PROVIDER_SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + providerId: 'daytona', + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }) + ).rejects.toThrow('Cloud returned provider daytona instead of requested provider agent37.'); + }); + + it('preserves caller cancellation after async preparation was accepted', async () => { + const controller = new AbortController(); + const cancelled = new Error('caller cancelled'); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockImplementationOnce(async () => { + controller.abort(cancelled); + return { + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }; + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { signal: controller.signal, preparationPollIntervalMs: 1 } + ) + ).rejects.toBe(cancelled); + }); + + it('reads durable status after a dropped async advance before issuing another advance', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockRejectedValueOnce(new TypeError('advance response dropped')) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'agent_relay_cli_bootstrap', 1), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 2, { + result: { + outcome: 'provisioned', + nodeId: 'node-after-drop', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-drop', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: false, + providerId: 'agent37', + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-after-drop' }); + + expect(mocks.authorizedApiFetch.mock.calls.slice(2).map((call) => call[2]?.method)).toEqual([ + 'GET', + 'POST', + 'GET', + 'POST', + ]); + }); + + it('surfaces a durable terminal async failure without retrying it', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('terminal', 'cleanup', 1, { + failure: { + code: 'preparation_failed', + error: 'Sandbox preparation failed.', + phase: 'relayfile_mount_bootstrap', + }, + }), + { status: 502 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + confirmedProvisioned: false, + outcomeUnknown: false, + sandboxAbsent: true, + sandboxId: SANDBOX_ID, + preparationFailure: { code: 'preparation_failed', phase: 'relayfile_mount_bootstrap' }, + }); + expect((error as Error).message).toContain('relayfile_mount_bootstrap'); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(3); + }); + + it('reports the production initial_sync_deadline mount failure as a typed proven-absent failure, not an unknown outcome', async () => { + const preparationPath = `/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`; + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'relayfile_mount', 3), { status: 202 }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'relayfile_mount', 3), { status: 202 }), + auth, + }) + // Cloud build 6b8f1de8, smoke B: the mount's initial sync hit the 250 s + // deadline. Under async-v1 that tick moves the record to cleanup. + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('cleanup_pending', 'cleanup', 4, { + failure: { + code: 'relayfile_mount_failed', + error: 'Relayfile mount failed during relayfile_mount. Retry provisioning the sandbox.', + phase: 'relayfile_mount', + causeStage: 'initial_sync_deadline', + }, + }), + { status: 202 } + ), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('terminal', 'cleanup', 5, { + failure: { + code: 'relayfile_mount_failed', + error: 'Relayfile mount failed during relayfile_mount. Retry provisioning the sandbox.', + phase: 'relayfile_mount', + causeStage: 'initial_sync_deadline', + }, + }), + { status: 502 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + sandboxId: SANDBOX_ID, + nodeName: SANDBOX_NAME, + sandboxAbsent: true, + confirmedProvisioned: false, + outcomeUnknown: false, + preparationFailure: { + code: 'relayfile_mount_failed', + phase: 'relayfile_mount', + causeStage: 'initial_sync_deadline', + }, + }); + expect((error as Error).message).toContain('relayfile_mount_failed, initial_sync_deadline'); + // One ensure, never replayed; Cloud-owned cleanup is observed, never advanced. + expect(mocks.authorizedApiFetch.mock.calls.slice(1).map((call) => [call[1], call[2]?.method])).toEqual([ + ['/api/v1/fleet/nodes/sandbox/ensure', 'POST'], + [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + [preparationPath, 'POST'], + [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + ]); + }); + + it('paces preparation by the Retry-After that Cloud sends with pending progress', async () => { + const pending = () => + new Response(JSON.stringify(preparationEnvelope('pending', 'relayfile_mount', 3)), { + status: 202, + headers: { 'content-type': 'application/json', 'retry-after': '0' }, + }); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ response: pending(), auth }) + .mockResolvedValueOnce({ response: pending(), auth }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 4, { + result: { + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-paced', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-paced', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + }, + }) + ), + auth, + }); + + // A 60 s default interval would time this test out if Retry-After were ignored. + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 60_000 } + ) + ).resolves.toMatchObject({ nodeId: 'node-paced' }); + }); + + it('surfaces an older Cloud typed relayfile_mount_failed 503 immediately instead of polling a missing status route', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + // An older Cloud ignores preparationMode and answers synchronously with + // the production smoke B failure. + .mockResolvedValueOnce({ + response: Response.json( + { + error: + 'Relayfile mount failed during relayfile_mount: initial sync deadline exceeded. Retry provisioning the sandbox.', + code: 'relayfile_mount_failed', + phase: 'relayfile_mount', + causeStage: 'initial_sync_deadline', + sandboxId: SANDBOX_ID, + }, + { status: 503 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ sandboxId: SANDBOX_ID, sandboxAbsent: false }); + expect((error as Error).message).toContain('(relayfile_mount_failed, initial_sync_deadline)'); + // No status reads: the typed body is a complete synchronous answer. + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + }); + + it('surfaces an older Cloud typed capacity 503 as a definitive pre-allocation rejection without polling', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + error: 'Sandbox capacity is exhausted before allocation; no sandbox was created', + code: 'sandbox_capacity_exhausted', + capacity: [{ provider: 'agent37', current: 14, limit: 10 }], + retryable: true, + no_sandbox_created: true, + }, + { status: 503 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toMatchObject({ + code: 'sandbox_capacity_exhausted', + noSandboxCreated: true, + retryable: true, + capacity: [{ provider: 'agent37', current: 14, limit: 10 }], + }); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + }); + + it('reconciles a lost final advance response by exact identity before reporting terminal', async () => { + const preparationPath = `/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`; + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'relayfile_mount', 3), { status: 202 }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'relayfile_mount', 3), { status: 202 }), + auth, + }) + // The advance that ran the failing mount tick loses its response. + .mockRejectedValueOnce(new TypeError('fetch failed')) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('terminal', 'cleanup', 5, { + failure: { + code: 'relayfile_mount_failed', + error: 'Relayfile mount failed during relayfile_mount. Retry provisioning the sandbox.', + phase: 'relayfile_mount', + causeStage: 'initial_sync_deadline', + }, + }), + { status: 502 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toMatchObject({ + sandboxId: SANDBOX_ID, + sandboxAbsent: true, + outcomeUnknown: false, + preparationFailure: { code: 'relayfile_mount_failed', causeStage: 'initial_sync_deadline' }, + }); + expect(mocks.authorizedApiFetch.mock.calls.slice(1).map((call) => [call[1], call[2]?.method])).toEqual([ + ['/api/v1/fleet/nodes/sandbox/ensure', 'POST'], + [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + [preparationPath, 'POST'], + [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + ]); + }); + + it('observes cleanup_pending without advancing until Cloud reports terminal', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('cleanup_pending', 'cleanup', 1), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('cleanup_pending', 'cleanup', 1), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('terminal', 'cleanup', 2, { + failure: { + code: 'preparation_deadline_exceeded', + error: 'Sandbox preparation deadline exceeded.', + phase: 'cleanup', + }, + }), + { status: 502 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ confirmedProvisioned: false, sandboxAbsent: true, sandboxId: SANDBOX_ID }); + expect(mocks.authorizedApiFetch.mock.calls.slice(2).map((call) => call[2]?.method)).toEqual([ + 'GET', + 'GET', + ]); + }); + + it('surfaces an initial async authentication failure without polling', async () => { + const authError = new CloudAuthError('AUTH_REFRESH_EXPIRED', 'Sign in again.'); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockRejectedValueOnce(authError); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + providerId: 'agent37', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).rejects.toBe(authError); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + }); + + it('reconciles an envelope-shaped but invalid 5xx ensure body through durable status', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { version: 1, mode: 'async-v1', sandboxId: SANDBOX_ID, state: 'exploded', generation: -1 }, + { status: 502 } + ), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 2, { + result: { + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-after-invalid-5xx', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-invalid-5xx', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-after-invalid-5xx', sandboxId: SANDBOX_ID }); + expect(mocks.authorizedApiFetch.mock.calls[2]?.[2]?.method).toBe('GET'); + }); + + it('reconciles a truncated successful advance through durable status instead of failing the sandbox', async () => { + const preparationPath = `/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`; + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'relayfile_mount', 3), { status: 202 }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'relayfile_mount', 3), { status: 202 }), + auth, + }) + // The advance committed its tick, but the body was cut off in transit. + .mockResolvedValueOnce({ response: new Response('{"version":1,"mo', { status: 200 }), auth }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 4, { + result: { + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-after-truncated-advance', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-truncated-advance', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-after-truncated-advance', sandboxId: SANDBOX_ID }); + expect(mocks.authorizedApiFetch.mock.calls.slice(1).map((call) => [call[1], call[2]?.method])).toEqual([ + ['/api/v1/fleet/nodes/sandbox/ensure', 'POST'], + [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + [preparationPath, 'POST'], + [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + ]); + }); + + it('surfaces a compatibility-retry authentication failure unchanged', async () => { + const authError = new CloudAuthError('AUTH_REFRESH_EXPIRED', 'Sign in again.'); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'Unrecognized key: preparationMode' }, { status: 400 }), + auth, + }) + .mockRejectedValueOnce(authError); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }) + ).rejects.toBe(authError); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(3); + }); + + it('preserves caller cancellation during the compatibility retry', async () => { + const controller = new AbortController(); + const cancelled = new Error('caller cancelled'); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'Unrecognized key: preparationMode' }, { status: 400 }), + auth, + }) + .mockImplementationOnce(async () => { + controller.abort(cancelled); + throw new DOMException('This operation was aborted', 'AbortError'); + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { signal: controller.signal } + ) + ).rejects.toBe(cancelled); + }); + + it('surfaces an async polling authentication failure without retrying it', async () => { + const authError = new CloudAuthError('AUTH_REFRESH_EXPIRED', 'Sign in again.'); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockRejectedValueOnce(authError); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + providerId: 'agent37', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).rejects.toBe(authError); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(3); + }); + + it('marks an invalid ready result safe only after exact async cleanup identity is proven', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 1, { + result: { + outcome: 'provisioned', + nodeId: 'node-invalid-ready', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + providerId: 'agent37', + }, + }) + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + repos: ['AgentWorkforce/relay'], + repoRevisions: { 'AgentWorkforce/relay': '0123456789abcdef0123456789abcdef01234567' }, + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + sandboxId: SANDBOX_ID, + nodeName: SANDBOX_NAME, + providerId: 'agent37', + confirmedProvisioned: true, + outcomeUnknown: false, + }); + }); + it('rejects an invalid sandbox identity before contacting Cloud', async () => { await expect( ensureCloudFleetSandbox({ diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 612825a5d7..b71bb4810b 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -1,6 +1,6 @@ import { authorizedApiFetch, ensureCloudSession } from './auth.js'; import { redactCredentialValues } from './redact.js'; -import { defaultApiUrl } from './types.js'; +import { CloudAuthError, defaultApiUrl } from './types.js'; type JsonRecord = Record; @@ -25,6 +25,9 @@ const DEFAULT_RESOLUTION_TIMEOUT_MS = 120_000; // not abandon a successful server-side request without receiving its sandbox // identity (which prevents the CLI from cleaning it up safely). const DEFAULT_ENSURE_TIMEOUT_MS = 480_000; +const DEFAULT_ASYNC_PREPARATION_TIMEOUT_MS = 35 * 60_000; +const ASYNC_PREPARATION_REQUEST_TIMEOUT_MS = 110_000; +const DEFAULT_ASYNC_PREPARATION_POLL_INTERVAL_MS = 1_000; const DEFAULT_DELETE_TIMEOUT_MS = 30_000; const DEFAULT_RELAYFILE_REPOSITORY_MATERIALIZE_TIMEOUT_MS = 20 * 60_000; const DEFAULT_RELAYFILE_REPOSITORY_POLL_INTERVAL_MS = 2_000; @@ -35,6 +38,16 @@ export type CloudFleetSandboxRequestOptions = { apiUrl?: string; signal?: AbortSignal; timeoutMs?: number; + /** Receives safe durable phase changes while an async-v1 sandbox is prepared. */ + onPreparationProgress?: (progress: CloudFleetSandboxPreparationProgress) => void; + preparationPollIntervalMs?: number; +}; + +export type CloudFleetSandboxPreparationProgress = { + sandboxId: string; + state: 'pending' | 'ready' | 'terminal' | 'cleanup_pending'; + phase: string; + generation: number; }; export type MaterializeCloudRelayfileRepositoryInput = { @@ -75,6 +88,14 @@ export type CloudFleetSandboxCapacityExhaustion = { readonly limit: number; }; +/** Credential-free typed failure from a terminal async-v1 preparation record. */ +export type CloudFleetSandboxPreparationFailure = { + readonly code: string; + readonly phase: string; + /** Mount command boundary, e.g. initial_sync_deadline, when Cloud reported one. */ + readonly causeStage?: string; +}; + /** Describes provisioning failure without granting unproven cleanup authority. */ export class CloudFleetSandboxProvisionError extends Error { readonly cloudWorkspaceId?: string; @@ -93,6 +114,15 @@ export class CloudFleetSandboxProvisionError extends Error { readonly retryable: boolean; /** Safe aggregate counts for each provider that blocked allocation. */ readonly capacity: readonly CloudFleetSandboxCapacityExhaustion[]; + /** + * Cloud's durable async-v1 record for this exact sandbox identity is + * terminal, which Cloud writes only after the provider rejected allocation, + * dispatch was abandoned, or the provider sandbox was proven absent. No + * sandbox for this identity is left running and no cleanup is owed. + */ + readonly sandboxAbsent: boolean; + /** Typed Cloud failure for a terminal preparation, e.g. relayfile_mount_failed. */ + readonly preparationFailure?: CloudFleetSandboxPreparationFailure; constructor( message: string, @@ -107,6 +137,8 @@ export class CloudFleetSandboxProvisionError extends Error { noSandboxCreated?: boolean; retryable?: boolean; capacity?: readonly CloudFleetSandboxCapacityExhaustion[]; + sandboxAbsent?: boolean; + preparationFailure?: CloudFleetSandboxPreparationFailure; cause?: unknown; } = {} ) { @@ -122,6 +154,9 @@ export class CloudFleetSandboxProvisionError extends Error { this.noSandboxCreated = !this.confirmedProvisioned && identity.noSandboxCreated === true; this.retryable = identity.retryable === true; this.capacity = identity.capacity?.map((entry) => ({ ...entry })) ?? []; + this.sandboxAbsent = !this.confirmedProvisioned && identity.sandboxAbsent === true; + this.preparationFailure = + identity.preparationFailure === undefined ? undefined : { ...identity.preparationFailure }; } } @@ -130,6 +165,8 @@ class CloudFleetSandboxIdentityMismatchError extends Error {} export type EnsureCloudFleetSandboxInput = { /** Cloud UUID or unified rw_* workspace id. */ workspaceId: string; + /** Opt in to Cloud's durable, poll-driven Agent37 preparation contract. */ + preparationMode?: 'async-v1'; /** Caller-declared one-time Cloud identity used to resume a cut-off provision. */ sandboxId?: string; name?: string; @@ -432,6 +469,31 @@ function endpointError(action: string, response: Response, payload: unknown): Er ); } +/** A Cloud error body that names its failure, as opposed to a gateway page. */ +function isTypedErrorBody(payload: unknown): boolean { + return isObject(payload) && readString(payload, 'code') !== undefined; +} + +function typedCauseSuffix(payload: unknown): string { + if (!isObject(payload)) return ''; + const code = readString(payload, 'code'); + if (!code) return ''; + const causeStage = readString(payload, 'causeStage'); + return ` (${code}${causeStage === undefined ? '' : `, ${causeStage}`})`; +} + +function retryAfterDelayMs(response: Response): number | null { + const value = response.headers.get('retry-after')?.trim(); + if (!value) return null; + const seconds = Number(value); + if (Number.isFinite(seconds) && seconds >= 0) { + return Math.min(MAX_TIMER_MS, Math.max(0, Math.ceil(seconds * 1000))); + } + const retryAt = Date.parse(value); + if (!Number.isFinite(retryAt)) return null; + return Math.min(MAX_TIMER_MS, Math.max(0, retryAt - Date.now())); +} + function requiredString(payload: JsonRecord, key: string, context: string): string { const value = readString(payload, key); if (!value) throw new Error(`${context} response is missing ${key}.`); @@ -751,6 +813,19 @@ function confirmsProvisionedSandboxIdentity( return readString(payload, 'providerId') === requestedProviderId; } +function confirmsAsyncPreparedSandboxIdentity( + payload: unknown, + expectedSandboxId: string, + expectedNodeName: string | undefined, + requestedProviderId: CloudFleetSandboxProviderId | undefined +): boolean { + if (!isObject(payload) || requestedProviderId === undefined) return false; + if (readString(payload, 'outcome') !== 'provisioned') return false; + if (readString(payload, 'sandboxId') !== expectedSandboxId) return false; + if (expectedNodeName !== undefined && readString(payload, 'nodeName') !== expectedNodeName) return false; + return readString(payload, 'providerId') === requestedProviderId; +} + function normalizeEnsureResult( payload: unknown, cloudWorkspaceId: string, @@ -868,6 +943,110 @@ function normalizeEnsureResult( throw new Error('Cloud fleet sandbox response has an unknown outcome.'); } +const ASYNC_PREPARATION_STATES = new Set(['pending', 'ready', 'terminal', 'cleanup_pending']); +// Phase is progress metadata only: state decides the outcome. Cloud's async-v1 +// record currently reports provider_allocation, agent_relay_cli_bootstrap, +// relayfile_mount_bootstrap, spawn_cli_bootstrap, repo_clone, +// cli_credential_setup, relayfile_mount, workspace_skills, fleet_enrollment, +// finalize, broker_visible and cleanup. Accept any bounded phase token so a +// newer Cloud adding a phase is not misread as an invalid (unknown) outcome. +const ASYNC_PREPARATION_PHASE_TOKEN = /^[a-z][a-z0-9_]{0,63}$/; + +type AsyncPreparationEnvelope = CloudFleetSandboxPreparationProgress & { + result?: JsonRecord; + failure?: { code: string; error: string; phase: string; causeDetail?: string; causeStage?: string }; +}; + +function readAsyncPreparationEnvelope( + payload: unknown, + expectedSandboxId: string +): AsyncPreparationEnvelope | null { + if (!isObject(payload) || payload.mode !== 'async-v1' || payload.version !== 1) return null; + const sandboxId = readString(payload, 'sandboxId'); + if (sandboxId !== expectedSandboxId) { + throw new CloudFleetSandboxIdentityMismatchError( + `Cloud returned sandboxId ${sandboxId ?? 'missing'} instead of requested sandboxId ${expectedSandboxId}.` + ); + } + const state = readString(payload, 'state'); + const phase = readString(payload, 'phase'); + const generation = readNumber(payload, 'generation'); + if ( + !state || + !ASYNC_PREPARATION_STATES.has(state) || + !phase || + !ASYNC_PREPARATION_PHASE_TOKEN.test(phase) || + generation === undefined || + !Number.isSafeInteger(generation) || + generation < 0 + ) { + throw new Error('Cloud fleet sandbox preparation response was invalid.'); + } + const envelope: AsyncPreparationEnvelope = { + sandboxId, + state: state as AsyncPreparationEnvelope['state'], + phase, + generation, + }; + if (payload.result !== undefined) { + if (!isObject(payload.result)) { + throw new Error('Cloud fleet sandbox preparation result was invalid.'); + } + envelope.result = payload.result; + } + if (payload.failure !== undefined) { + if (!isObject(payload.failure)) { + throw new Error('Cloud fleet sandbox preparation failure was invalid.'); + } + const code = readString(payload.failure, 'code'); + const error = readString(payload.failure, 'error'); + const failurePhase = readString(payload.failure, 'phase'); + const causeDetail = readString(payload.failure, 'causeDetail'); + const causeStage = readString(payload.failure, 'causeStage'); + if (!code || !error || !failurePhase) { + throw new Error('Cloud fleet sandbox preparation failure was invalid.'); + } + envelope.failure = { + code, + error, + phase: failurePhase, + ...(causeDetail === undefined ? {} : { causeDetail }), + ...(causeStage === undefined ? {} : { causeStage }), + }; + } + return envelope; +} + +function isLegacyEnsureOutcome(payload: unknown): boolean { + if (!isObject(payload)) return false; + return ['provisioned', 'reused', 'provisioning_timeout'].includes(readString(payload, 'outcome') ?? ''); +} + +function rejectsAsyncPreparationMode(response: Response, payload: unknown): boolean { + if (response.status !== 400 || !isObject(payload)) return false; + const diagnostic = [ + readString(payload, 'code'), + readString(payload, 'error'), + readString(payload, 'message'), + ] + .filter((value): value is string => value !== undefined) + .join(' ') + .toLowerCase(); + return diagnostic.includes('preparationmode') || diagnostic.includes('preparation_mode'); +} + +function asyncPreparationRequestSignal(overall: AbortSignal): AbortSignal { + return AbortSignal.any([overall, AbortSignal.timeout(ASYNC_PREPARATION_REQUEST_TIMEOUT_MS)]); +} + +function asyncPreparationInitialRequestSignal(overall: AbortSignal): AbortSignal { + // Until every Cloud deployment understands async-v1, the first request may + // be handled synchronously by an older server. Preserve the legacy client's + // 480-second response window for that one capability probe; every status or + // advance request remains capped at 110 seconds. + return AbortSignal.any([overall, AbortSignal.timeout(DEFAULT_ENSURE_TIMEOUT_MS)]); +} + /** * Materialize one exact GitHub revision into the selected workspace's live * Relayfile tree and wait until the decoded working-tree mount can consume it. @@ -1020,6 +1199,21 @@ export async function ensureCloudFleetSandbox( } validateRequestedRepos(input.repos); const repoRevisions = validateRepoRevisions(input.repos, input.repoRevisions); + const asyncPreparation = input.preparationMode === 'async-v1'; + if (asyncPreparation && sandboxIdentity.sandboxId === undefined) { + throw new Error('Async Cloud fleet preparation requires a caller-declared sandboxId.'); + } + // Cloud routes an unpinned async-v1 request to Agent37 itself, so the wire + // carries only the caller's provider: pinning the implied default would turn + // capability routing into the strict explicit-provider path. Once Cloud has + // confirmed a durable async record, attribute it to Agent37 so a malformed + // ready result can never lose the provider needed for exact cleanup. + const asyncProviderId = asyncPreparation ? (input.providerId ?? 'agent37') : input.providerId; + const preparationPollIntervalMs = normalizeTimerMs( + options.preparationPollIntervalMs ?? DEFAULT_ASYNC_PREPARATION_POLL_INTERVAL_MS, + false, + 'Cloud fleet preparation poll interval' + ); const session = await ensureCloudSession({ apiUrl: options.apiUrl || defaultApiUrl(), @@ -1027,35 +1221,288 @@ export async function ensureCloudFleetSandbox( }); const resolutionSignal = boundedSignal(options, DEFAULT_RESOLUTION_TIMEOUT_MS); const resolved = await resolveCloudWorkspaceId(workspaceId, session.auth, resolutionSignal); - const signal = boundedSignal(options, DEFAULT_ENSURE_TIMEOUT_MS); + const signal = boundedSignal( + options, + asyncPreparation ? DEFAULT_ASYNC_PREPARATION_TIMEOUT_MS : DEFAULT_ENSURE_TIMEOUT_MS + ); + let activeAuth = resolved.auth; + const ensureRequest = { + workspaceId: resolved.cloudWorkspaceId, + requiredCapability, + ...(asyncPreparation ? { preparationMode: 'async-v1' } : {}), + ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), + ...(sandboxIdentity.name === undefined ? {} : { name: sandboxIdentity.name }), + ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), + ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), + ...(input.relayfilePaths === undefined ? {} : { relayfilePaths: [...input.relayfilePaths] }), + ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), + ...(input.providerId !== undefined ? { providerId: input.providerId } : {}), + ...(input.workloadProfile !== undefined ? { workloadProfile: input.workloadProfile } : {}), + ...(input.waitTimeoutMs !== undefined ? { waitTimeoutMs: input.waitTimeoutMs } : {}), + ...(input.repos !== undefined && input.repos.length > 0 ? { repos: [...input.repos] } : {}), + ...(repoRevisions === undefined ? {} : { repoRevisions }), + }; + const ensureBody = JSON.stringify(ensureRequest); + + let sawAcceptedPreparation = false; + // Before Cloud confirms an async record, an older Cloud may have routed the + // request synchronously, so only the caller's own provider is attributable. + const attributedProvider = (): { providerId?: CloudFleetSandboxProviderId } => { + const providerId = sawAcceptedPreparation ? asyncProviderId : input.providerId; + return providerId === undefined ? {} : { providerId }; + }; + let lastProgressSignature: string | undefined; + const consumePreparation = (payload: unknown): EnsureCloudFleetSandboxResult | null => { + const envelope = readAsyncPreparationEnvelope(payload, sandboxIdentity.sandboxId!); + if (!envelope) return null; + sawAcceptedPreparation = true; + const signature = `${envelope.state}:${envelope.phase}:${envelope.generation}`; + if (signature !== lastProgressSignature) { + options.onPreparationProgress?.({ + sandboxId: envelope.sandboxId, + state: envelope.state, + phase: envelope.phase, + generation: envelope.generation, + }); + lastProgressSignature = signature; + } + if (envelope.state === 'ready') { + try { + if (!envelope.result) { + throw new Error('Cloud fleet sandbox preparation completed without a result.'); + } + return normalizeEnsureResult( + envelope.result, + resolved.cloudWorkspaceId, + sandboxIdentity.sandboxId!, + sandboxIdentity.name, + asyncProviderId, + repoRevisions, + activeAuth.apiUrl + ); + } catch (error) { + const confirmedProvisioned = confirmsAsyncPreparedSandboxIdentity( + envelope.result, + sandboxIdentity.sandboxId!, + sandboxIdentity.name, + asyncProviderId + ); + throw new CloudFleetSandboxProvisionError( + error instanceof Error ? error.message : 'Cloud fleet sandbox preparation result was invalid.', + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: sandboxIdentity.sandboxId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(asyncProviderId === undefined ? {} : { providerId: asyncProviderId }), + ...(confirmedProvisioned ? { confirmedProvisioned: true } : { outcomeUnknown: true }), + cause: error, + } + ); + } + } + if (envelope.state === 'terminal') { + // Cloud writes terminal only after it rejected allocation, abandoned + // dispatch, or proved the provider sandbox absent; it owns that cleanup. + const failure = envelope.failure; + // Cloud persists the safe mount cause stage (e.g. initial_sync_deadline) + // in the durable failure record (cloud#4137 839d71e1b). + const causeStage = failure?.causeStage; + const typed = failure + ? [failure.code, ...(causeStage === undefined ? [] : [causeStage])].join(', ') + : ''; + throw new CloudFleetSandboxProvisionError( + redactCredentialValues( + failure + ? `Cloud fleet sandbox preparation failed during ${failure.phase} (${typed}): ${failure.error}` + : 'Cloud fleet sandbox preparation failed.' + ), + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: sandboxIdentity.sandboxId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(asyncProviderId === undefined ? {} : { providerId: asyncProviderId }), + sandboxAbsent: true, + ...(failure === undefined + ? {} + : { + preparationFailure: { + code: failure.code, + phase: failure.phase, + ...(causeStage === undefined ? {} : { causeStage }), + }, + }), + } + ); + } + return null; + }; + + const pollPreparation = async ( + initialPayload?: unknown, + initialDelayMs: number | null = null + ): Promise => { + const unconfirmedDeadline = Date.now() + ASYNC_PREPARATION_REQUEST_TIMEOUT_MS; + let payload = initialPayload; + let method: 'GET' | 'POST' = initialPayload === undefined ? 'GET' : 'POST'; + let nextPollDelayMs: number | null = initialDelayMs; + for (;;) { + if (payload !== undefined) { + const ready = consumePreparation(payload); + if (ready) return ready; + } + if (!sawAcceptedPreparation && Date.now() >= unconfirmedDeadline) { + throw new Error('Cloud fleet sandbox preparation was not accepted before the status deadline.'); + } + const requestedDelayMs = nextPollDelayMs ?? preparationPollIntervalMs; + nextPollDelayMs = null; + const delayMs = sawAcceptedPreparation + ? requestedDelayMs + : Math.min(requestedDelayMs, Math.max(0, unconfirmedDeadline - Date.now())); + await waitForDelay(delayMs, signal); + // Do not start another full request after the short lost-ensure + // reconciliation window. A long configured poll interval must not turn + // the 110-second uncertainty bound into a 220-second network attempt. + if (!sawAcceptedPreparation && Date.now() >= unconfirmedDeadline) { + throw new Error('Cloud fleet sandbox preparation was not accepted before the status deadline.'); + } + const path = `/api/v1/fleet/nodes/sandbox/${encodeURIComponent( + sandboxIdentity.sandboxId! + )}/preparation${ + method === 'GET' ? `?workspaceId=${encodeURIComponent(resolved.cloudWorkspaceId)}` : '' + }`; + let pollResponse: Response; + try { + const polled = await authorizedApiFetch( + activeAuth, + path, + { + method, + signal: asyncPreparationRequestSignal(signal), + ...(method === 'POST' + ? { body: JSON.stringify({ workspaceId: resolved.cloudWorkspaceId }) } + : {}), + }, + { interactive: false } + ); + activeAuth = polled.auth; + pollResponse = polled.response; + } catch (fetchError) { + if (signal.aborted) throw signal.reason ?? fetchError; + const errorName = fetchError instanceof Error ? fetchError.name : ''; + if ( + !(fetchError instanceof TypeError) && + errorName !== 'TimeoutError' && + errorName !== 'AbortError' + ) { + throw fetchError; + } + if (!sawAcceptedPreparation && Date.now() >= unconfirmedDeadline) { + throw fetchError; + } + // A lost advance response is not replayed. Read durable status first; + // only a confirmed pending record permits a later advance. + method = 'GET'; + payload = undefined; + continue; + } + payload = await readJson(pollResponse); + let envelope: AsyncPreparationEnvelope | null; + try { + envelope = readAsyncPreparationEnvelope(payload, sandboxIdentity.sandboxId!); + } catch (error) { + // A record for another identity is a diagnostic, not a lost response. + if (error instanceof CloudFleetSandboxIdentityMismatchError) throw error; + envelope = null; + } + if (envelope) { + const ready = consumePreparation(payload); + if (ready) return ready; + // Cleanup is owned by Cloud. Keep observing it until Cloud publishes a + // terminal result; advancing here could race the reaper or restart work + // on a sandbox that is already being destroyed. + method = envelope.state === 'cleanup_pending' ? 'GET' : 'POST'; + // Each advance goes through Cloud's ensure limiter; pace by its hint. + nextPollDelayMs = retryAfterDelayMs(pollResponse); + payload = undefined; + continue; + } + if ( + method === 'GET' && + pollResponse.status === 404 && + !sawAcceptedPreparation && + Date.now() < unconfirmedDeadline + ) { + payload = undefined; + continue; + } + if (!pollResponse.ok) { + if ( + (pollResponse.status >= 500 || pollResponse.status === 429) && + (sawAcceptedPreparation || Date.now() < unconfirmedDeadline) + ) { + if (pollResponse.status === 429) { + nextPollDelayMs = retryAfterDelayMs(pollResponse) ?? preparationPollIntervalMs; + } + method = 'GET'; + payload = undefined; + continue; + } + throw endpointError('read fleet sandbox preparation status', pollResponse, payload); + } + // A truncated or malformed success may hide a committed tick. Treat it as + // a lost response: read durable status before any further advance. This + // also covers Cloud's bare `{ state }` fallback body (no `mode`/`version`), + // returned when it cannot re-read the record after a failed tick. + method = 'GET'; + payload = undefined; + } + }; + let response: Response; - let responseAuth = resolved.auth; try { - ({ response, auth: responseAuth } = await authorizedApiFetch( - resolved.auth, + const ensured = await authorizedApiFetch( + activeAuth, '/api/v1/fleet/nodes/sandbox/ensure', { method: 'POST', - signal, - body: JSON.stringify({ - workspaceId: resolved.cloudWorkspaceId, - requiredCapability, - ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), - ...(sandboxIdentity.name === undefined ? {} : { name: sandboxIdentity.name }), - ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), - ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), - ...(input.relayfilePaths === undefined ? {} : { relayfilePaths: [...input.relayfilePaths] }), - ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), - ...(input.providerId !== undefined ? { providerId: input.providerId } : {}), - ...(input.workloadProfile !== undefined ? { workloadProfile: input.workloadProfile } : {}), - ...(input.waitTimeoutMs !== undefined ? { waitTimeoutMs: input.waitTimeoutMs } : {}), - ...(input.repos !== undefined && input.repos.length > 0 ? { repos: [...input.repos] } : {}), - ...(repoRevisions === undefined ? {} : { repoRevisions }), - }), + signal: asyncPreparation ? asyncPreparationInitialRequestSignal(signal) : signal, + body: ensureBody, }, { interactive: false } - )); + ); + activeAuth = ensured.auth; + response = ensured.response; } catch (error) { + if (asyncPreparation) { + if (options.signal?.aborted) throw options.signal.reason ?? error; + if (error instanceof CloudAuthError) throw error; + const errorName = error instanceof Error ? error.name : ''; + const retryableTransport = + error instanceof TypeError || errorName === 'TimeoutError' || errorName === 'AbortError'; + if (!retryableTransport && !signal.aborted) throw error; + try { + return await pollPreparation(); + } catch (pollError) { + if (pollError instanceof CloudFleetSandboxProvisionError) throw pollError; + if (pollError instanceof CloudAuthError) throw pollError; + if (options.signal?.aborted) throw options.signal.reason ?? pollError; + throw new CloudFleetSandboxProvisionError( + redactCredentialValues( + `Cloud fleet sandbox preparation status could not be confirmed: ${ + pollError instanceof Error ? pollError.message : String(pollError) + }` + ), + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: sandboxIdentity.sandboxId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...attributedProvider(), + ...(sawAcceptedPreparation ? { confirmedProvisioned: true } : { outcomeUnknown: true }), + cause: pollError, + } + ); + } + } throw new CloudFleetSandboxProvisionError( redactCredentialValues( `Cloud fleet sandbox request ended without a complete response: ${ @@ -1072,7 +1519,115 @@ export async function ensureCloudFleetSandbox( } ); } - const payload = await readJson(response); + let payload = await readJson(response); + let legacyCompatibilityResponse = asyncPreparation && response.ok && isLegacyEnsureOutcome(payload); + if (asyncPreparation && rejectsAsyncPreparationMode(response, payload)) { + try { + const retried = await authorizedApiFetch( + activeAuth, + '/api/v1/fleet/nodes/sandbox/ensure', + { + method: 'POST', + signal: asyncPreparationInitialRequestSignal(signal), + body: JSON.stringify({ ...ensureRequest, preparationMode: undefined }), + }, + { interactive: false } + ); + activeAuth = retried.auth; + response = retried.response; + payload = await readJson(response); + legacyCompatibilityResponse = true; + } catch (error) { + // Match the initial and status paths: authentication failures and caller + // cancellation are not lost responses and must surface unchanged. + if (error instanceof CloudAuthError) throw error; + if (options.signal?.aborted) throw options.signal.reason ?? error; + throw new CloudFleetSandboxProvisionError( + redactCredentialValues( + `Cloud fleet sandbox compatibility request ended without a complete response: ${ + error instanceof Error ? error.message : String(error) + }` + ), + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: sandboxIdentity.sandboxId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), + outcomeUnknown: true, + cause: error, + } + ); + } + } + if (asyncPreparation && !legacyCompatibilityResponse) { + let envelope: AsyncPreparationEnvelope | null; + try { + envelope = readAsyncPreparationEnvelope(payload, sandboxIdentity.sandboxId!); + } catch (error) { + // A mismatched public identity is neither a truncated response nor an + // ambiguous gateway failure. Polling the caller-declared identity would + // hide the diagnostic and could point cleanup at the wrong sandbox. + if (error instanceof CloudFleetSandboxIdentityMismatchError) throw error; + envelope = null; + } + if (envelope) { + const ready = consumePreparation(payload); + if (ready) return ready; + try { + return await pollPreparation(undefined, retryAfterDelayMs(response)); + } catch (pollError) { + if (pollError instanceof CloudFleetSandboxProvisionError) throw pollError; + if (pollError instanceof CloudAuthError) throw pollError; + if (options.signal?.aborted) throw options.signal.reason ?? pollError; + throw new CloudFleetSandboxProvisionError( + redactCredentialValues( + `Cloud fleet sandbox preparation did not complete: ${ + pollError instanceof Error ? pollError.message : String(pollError) + }` + ), + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: sandboxIdentity.sandboxId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(asyncProviderId === undefined ? {} : { providerId: asyncProviderId }), + confirmedProvisioned: true, + cause: pollError, + } + ); + } + } + // A successful/accepted response can still lose or truncate its body after + // Cloud durably recorded the preparation, and a gateway 5xx can hide an + // accepted request. Reconcile those through the declared sandbox's status + // endpoint instead of replaying ensure. A typed 5xx error body (`code`) is + // a complete synchronous answer, e.g. from an older Cloud that ignored + // preparationMode: parse it below so its cause and capacity detail + // surface immediately instead of polling a status route it lacks. + if (response.ok || (response.status >= 500 && !isTypedErrorBody(payload))) { + try { + return await pollPreparation(); + } catch (pollError) { + if (pollError instanceof CloudFleetSandboxProvisionError) throw pollError; + if (pollError instanceof CloudAuthError) throw pollError; + if (options.signal?.aborted) throw options.signal.reason ?? pollError; + throw new CloudFleetSandboxProvisionError( + redactCredentialValues( + `Cloud fleet sandbox preparation status could not be confirmed: ${ + pollError instanceof Error ? pollError.message : String(pollError) + }` + ), + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: sandboxIdentity.sandboxId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...attributedProvider(), + ...(sawAcceptedPreparation ? { confirmedProvisioned: true } : { outcomeUnknown: true }), + cause: pollError, + } + ); + } + } + } if (!response.ok) { const returnedSandboxId = isObject(payload) ? readString(payload, 'sandboxId') : undefined; if ( @@ -1110,7 +1665,7 @@ export async function ensureCloudFleetSandbox( // replayable as an unknown outcome, even for legacy custom-name callers; // never copy an unverified response ID into cleanup authority. if (response.status >= 500) { - throw new CloudFleetSandboxProvisionError(error.message, { + throw new CloudFleetSandboxProvisionError(`${error.message}${typedCauseSuffix(payload)}`, { cloudWorkspaceId: resolved.cloudWorkspaceId, ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), @@ -1139,7 +1694,7 @@ export async function ensureCloudFleetSandbox( sandboxIdentity.name, input.providerId, repoRevisions, - responseAuth.apiUrl + activeAuth.apiUrl ); } catch (error) { const confirmedProvisioned = confirmsProvisionedSandboxIdentity( diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 598a5af253..2c5652db96 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -105,6 +105,8 @@ export { type CloudFleetSandboxWorkloadProfile, type DeleteCloudFleetSandboxInput, type CloudFleetSandboxRequestOptions, + type CloudFleetSandboxPreparationFailure, + type CloudFleetSandboxPreparationProgress, type CloudFleetRelaycastRoute, type CloudFleetRelaycastTarget, } from './fleet-sandbox.js'; diff --git a/packages/sdk/src/messaging/relaycast-placement.ts b/packages/sdk/src/messaging/relaycast-placement.ts index 26d76ce9fa..50a8872d7f 100644 --- a/packages/sdk/src/messaging/relaycast-placement.ts +++ b/packages/sdk/src/messaging/relaycast-placement.ts @@ -93,6 +93,8 @@ export class RelayPlacementError extends Error { | 'unmapped_repo' /** The node ran the action and reported a failure. */ | 'spawn_failed' + /** The node rejected spawn setup before mutation and a caller may retry. */ + | 'spawn_retryable' /** * The node accepted the invocation but never reported a terminal result. * A node running an obsolete broker advertises `spawn:` capacity diff --git a/packages/sdk/src/messaging/relaycast.ts b/packages/sdk/src/messaging/relaycast.ts index 74c4f115e5..3497b2385e 100644 --- a/packages/sdk/src/messaging/relaycast.ts +++ b/packages/sdk/src/messaging/relaycast.ts @@ -1070,9 +1070,15 @@ export class RelaycastMessagingClient implements RelayMessagingClient { return invocation; } if (status && CONFIRM_FAILURE_STATUSES.has(status)) { + const nodeError = invocation?.error?.trim(); + const retryablePressure = + errorContext.capability.startsWith('spawn:') && + /^node agent\.register failed for agent '[^']+': d1_pressure:/.test(nodeError ?? ''); throw new RelayPlacementError( - 'spawn_failed', - invocation?.error?.trim() || `node '${context.node}' reported ${status} for ${actionName}`, + retryablePressure ? 'spawn_retryable' : 'spawn_failed', + retryablePressure + ? `${nodeError}. No registration mutation was applied; retry the spawn.` + : nodeError || `node '${context.node}' reported ${status} for ${actionName}`, { ...errorContext, state: 'failed', diff --git a/tests/relayflows/cases/1763-zero-config-sandbox/run.mjs b/tests/relayflows/cases/1763-zero-config-sandbox/run.mjs index 505b511b3d..e35e6bf8cd 100644 --- a/tests/relayflows/cases/1763-zero-config-sandbox/run.mjs +++ b/tests/relayflows/cases/1763-zero-config-sandbox/run.mjs @@ -43,11 +43,21 @@ test('fleet sandbox CLI materializes exact source through a scoped live Relayfil if (!output) throw new Error('Missing observation path.'); const requests = []; mocks.ensureCloudSession.mockResolvedValue({ auth, client: {} }); - mocks.authorizedApiFetch.mockImplementation(async (_auth, _path, request) => { + mocks.authorizedApiFetch.mockImplementation(async (_auth, requestPath, request) => { const body = request?.body ? JSON.parse(request.body) : null; requests.push({ body }); if (requests.length === 1) return { response: Response.json({ cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99' }), auth }; - return { response: Response.json({ outcome: 'provisioned', cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', nodeId: 'node-proof', nodeName: body?.name ?? 'sandbox-proof', sandboxId: body?.sandboxId ?? 'sbx_123e4567-e89b-42d3-a456-426614174000', relayWorkspaceId: 'rw-proof', relayfileMounted: true, providerId: 'agent37', relaycastTarget: { route: 'agent37-isolated', baseUrl: 'https://agent37-cast.agentrelay.com', workspaceId: 'rw-proof', relaycastApiKey: 'rk_live_probe' }, repoRevisions: body?.repoRevisions ?? undefined }, { status: 201 }), auth }; + const ensureBody = requests[1]?.body ?? {}; + const sandboxId = ensureBody.sandboxId ?? 'sbx_123e4567-e89b-42d3-a456-426614174000'; + const result = { outcome: 'provisioned', cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', nodeId: 'node-proof', nodeName: ensureBody.name ?? 'sandbox-proof', sandboxId, providerSandboxId: 'provider-proof', relayWorkspaceId: 'rw-proof', relayfileMounted: true, providerId: 'agent37', relaycastTarget: { route: 'agent37-isolated', baseUrl: 'https://agent37-cast.agentrelay.com', workspaceId: 'rw-proof', relaycastApiKey: 'rk_live_probe' }, repoRevisions: ensureBody.repoRevisions ?? undefined }; + if (${JSON.stringify(arm)} === 'head') { + if (requestPath === '/api/v1/fleet/nodes/sandbox/ensure' && request?.method === 'POST') return { response: Response.json({ version: 1, mode: 'async-v1', sandboxId, state: 'pending', phase: 'provider_allocation', generation: 0 }, { status: 202 }), auth }; + const preparationPath = '/api/v1/fleet/nodes/sandbox/' + encodeURIComponent(sandboxId) + '/preparation'; + if (requestPath === preparationPath + '?workspaceId=50587328-441d-4acb-b8f3-dbe1b3c5de99' && request?.method === 'GET') return { response: Response.json({ version: 1, mode: 'async-v1', sandboxId, state: 'pending', phase: 'agent_relay_cli_bootstrap', generation: 1 }, { status: 202 }), auth }; + if (requestPath === preparationPath && request?.method === 'POST') return { response: Response.json({ version: 1, mode: 'async-v1', sandboxId, state: 'ready', phase: 'broker_visible', generation: 2, result }), auth }; + throw new Error('Unexpected async preparation request: ' + request?.method + ' ' + requestPath); + } + return { response: Response.json(result, { status: 201 }), auth }; }); const logs = [], warnings = [], deletes = [], releases = [], materializations = [], spawnInputs = []; const program = new Command(); program.exitOverride(); diff --git a/tests/relayflows/cases/1919-fleet-release-persisted-route/case.json b/tests/relayflows/cases/1919-fleet-release-persisted-route/case.json new file mode 100644 index 0000000000..5d1831551e --- /dev/null +++ b/tests/relayflows/cases/1919-fleet-release-persisted-route/case.json @@ -0,0 +1,21 @@ +{ + "version": 1, + "id": "1919-fleet-release-persisted-route", + "kind": "bugfix", + "title": "Keep fleet release on the persisted workspace route", + "runner": { + "command": ["node", "tests/relayflows/cases/1919-fleet-release-persisted-route/run.mjs"] + }, + "requirements": [], + "timeoutSeconds": 900, + "expected": { + "base": { + "outcome": "bug", + "signature": "fleet_release_rejected_persisted_route" + }, + "head": { + "outcome": "fixed", + "signature": "fleet_release_used_persisted_route" + } + } +} diff --git a/tests/relayflows/cases/1919-fleet-release-persisted-route/probe.test.mts b/tests/relayflows/cases/1919-fleet-release-persisted-route/probe.test.mts new file mode 100644 index 0000000000..756eb8d875 --- /dev/null +++ b/tests/relayflows/cases/1919-fleet-release-persisted-route/probe.test.mts @@ -0,0 +1,96 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { Command } from 'commander'; +import { expect, test, vi } from 'vitest'; + +// The runner copies this probe into /.relay-pr-proof/. +import { registerFleetCommands } from '../packages/cli/src/cli/commands/fleet.js'; +import { resolveWorkspaceTransport } from '../packages/cli/src/cli/lib/sdk-client.js'; +import { writeProjectWorkspaceKey } from '../packages/cli/src/cli/lib/project-workspace-key.js'; + +const OBSERVATION_PATH = process.env.RELAY_PR1919_OBSERVATION_PATH; +const ISOLATED_ORIGIN = 'https://agent37-cast.agentrelay.com'; + +test('fleet release uses the persisted route over an ambient base URL', async () => { + if (!OBSERVATION_PATH) throw new Error('Missing RELAY_PR1919_OBSERVATION_PATH.'); + + const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-proof-1919-project-')); + const relayHome = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-proof-1919-home-')); + const release = vi.fn(async () => ({ name: 'route-probe', released: true, deleted: false })); + const errors: string[] = []; + let selectedTransport: ReturnType | undefined; + + vi.stubEnv('AGENT_RELAY_PROJECT', projectRoot); + vi.stubEnv('AGENT_RELAY_HOME', relayHome); + vi.stubEnv('RELAY_WORKSPACE_KEY', 'rk_live_workspace'); + vi.stubEnv('RELAY_API_KEY', ''); + vi.stubEnv('RELAY_BASE_URL', 'https://cast.agentrelay.com'); + + try { + const dataDir = path.join(projectRoot, '.agentworkforce', 'relay'); + writeProjectWorkspaceKey(dataDir, 'rk_live_workspace', { + workspaceId: 'rw_route_probe', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: ISOLATED_ORIGIN, + relaycastApiKey: 'rk_live_agent37_route_probe', + }); + + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + resolveSandboxRepository: () => undefined, + createFleetWorkspaceClient: ((options: Parameters[0]) => { + selectedTransport = resolveWorkspaceTransport(options); + return { agents: { release } }; + }) as never, + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn() as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: (message: unknown) => errors.push(String(message)), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + log: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }); + + let commandError: string | undefined; + try { + await program.parseAsync(['fleet', 'release', 'route-probe'], { from: 'user' }); + } catch (error) { + commandError = error instanceof Error ? error.message : String(error); + } + + const observation = { + released: release.mock.calls.length === 1, + baseUrl: selectedTransport?.baseUrl ?? null, + workspaceKey: selectedTransport?.workspaceKey ?? null, + error: errors[0] ?? commandError ?? null, + }; + const baseObserved = + observation.released === false && + observation.baseUrl === null && + observation.workspaceKey === null && + observation.error?.includes( + 'The requested Relaycast base URL does not match the persisted workspace route.' + ); + const headObserved = + observation.released === true && + observation.baseUrl === ISOLATED_ORIGIN && + observation.workspaceKey === 'rk_live_agent37_route_probe' && + observation.error === null; + + expect(baseObserved || headObserved, JSON.stringify(observation)).toBe(true); + fs.writeFileSync(OBSERVATION_PATH, JSON.stringify({ ...observation, baseObserved, headObserved })); + } finally { + vi.unstubAllEnvs(); + fs.rmSync(projectRoot, { recursive: true, force: true }); + fs.rmSync(relayHome, { recursive: true, force: true }); + } +}); diff --git a/tests/relayflows/cases/1919-fleet-release-persisted-route/run.mjs b/tests/relayflows/cases/1919-fleet-release-persisted-route/run.mjs new file mode 100644 index 0000000000..2f280f6805 --- /dev/null +++ b/tests/relayflows/cases/1919-fleet-release-persisted-route/run.mjs @@ -0,0 +1,136 @@ +#!/usr/bin/env node + +import { execFileSync, spawnSync } from 'node:child_process'; +import { access, copyFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const CASE_ID = '1919-fleet-release-persisted-route'; +const arm = requiredValue('RELAY_PR_PROOF_ARM'); +const targetDir = requiredDirectory('RELAY_PR_PROOF_TARGET_DIR'); +const harnessDir = requiredDirectory('RELAY_PR_PROOF_HARNESS_DIR'); +const resultPath = requiredValue('RELAY_PR_PROOF_RESULT_PATH'); + +if (arm !== 'base' && arm !== 'head') { + throw new Error(`RELAY_PR_PROOF_ARM must be base or head, received ${JSON.stringify(arm)}.`); +} +const expectedSha = requiredValue(arm === 'base' ? 'RELAY_PR_PROOF_BASE_SHA' : 'RELAY_PR_PROOF_HEAD_SHA'); +const actualSha = execFileSync('git', ['-C', targetDir, 'rev-parse', 'HEAD'], { + encoding: 'utf8', +}).trim(); +if (actualSha !== expectedSha) { + throw new Error(`Target checkout ${actualSha} does not match exact ${arm} SHA ${expectedSha}.`); +} +const runnerPath = fileURLToPath(import.meta.url); +if (!isWithin(harnessDir, runnerPath)) { + throw new Error('The RelayFlow runner must execute from the exact-head harness checkout.'); +} + +await rm(resultPath, { force: true }); +const proofDir = path.join(targetDir, '.relay-pr-proof'); +const probePath = path.join(proofDir, '1919-fleet-release-persisted-route.test.mts'); +const observationPath = path.join(proofDir, '1919-fleet-release-persisted-route-observation.json'); +const configPath = path.join(proofDir, '1919-vitest.config.mts'); +const caseDir = path.dirname(runnerPath); + +try { + const vitestEntry = path.join(targetDir, 'node_modules', 'vitest', 'vitest.mjs'); + if (!(await pathExists(vitestEntry))) { + run('npm', ['ci', '--ignore-scripts', '--no-audit', '--no-fund'], targetDir, 'dependency installation'); + } + + await mkdir(proofDir, { recursive: true }); + await rm(probePath, { force: true }); + await rm(configPath, { force: true }); + await rm(observationPath, { force: true }); + await copyFile(path.join(caseDir, 'probe.test.mts'), probePath); + await writeFile( + configPath, + `import { defineConfig } from 'vitest/config';\nimport targetConfig from '../vitest.config.ts';\n\nexport default defineConfig({ resolve: targetConfig.resolve, test: { environment: 'node', include: ['.relay-pr-proof/1919-fleet-release-persisted-route.test.mts'] } });\n`, + { encoding: 'utf8', flag: 'wx' } + ); + run( + process.execPath, + [vitestEntry, 'run', '--config', configPath, '--reporter=verbose'], + targetDir, + 'fleet release route probe', + { RELAY_PR1919_OBSERVATION_PATH: observationPath } + ); + + const observation = JSON.parse(await readFile(observationPath, 'utf8')); + let outcome; + let signature; + let details; + if (observation.baseObserved === true && observation.headObserved === false) { + outcome = 'bug'; + signature = 'fleet_release_rejected_persisted_route'; + details = + 'The exact base CLI rejected fleet release because ambient RELAY_BASE_URL conflicted with the persisted isolated route.'; + } else if (observation.baseObserved === false && observation.headObserved === true) { + outcome = 'fixed'; + signature = 'fleet_release_used_persisted_route'; + details = + 'The exact head CLI released through the persisted isolated origin and its route-scoped credential despite ambient RELAY_BASE_URL.'; + } else { + throw new Error(`Unexpected fleet release route observation: ${JSON.stringify(observation)}.`); + } + + await mkdir(path.dirname(resultPath), { recursive: true }); + await writeFile( + resultPath, + `${JSON.stringify({ version: 1, caseId: CASE_ID, arm, outcome, signature, details })}\n`, + 'utf8' + ); +} finally { + await rm(probePath, { force: true }); + await rm(configPath, { force: true }); + await rm(observationPath, { force: true }); +} + +/** Read a required non-empty RelayFlow environment value. */ +function requiredValue(name) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`Missing required environment variable ${name}.`); + return value; +} + +/** Resolve a required RelayFlow directory to an absolute path. */ +function requiredDirectory(name) { + return path.resolve(requiredValue(name)); +} + +/** Return true when a candidate path stays inside the expected checkout. */ +function isWithin(directory, candidate) { + const relative = path.relative(directory, candidate); + return ( + relative === '' || + (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) + ); +} + +/** Return true when a filesystem path is accessible. */ +async function pathExists(candidate) { + try { + await access(candidate); + return true; + } catch { + return false; + } +} + +/** Run a proof subprocess synchronously and surface a stable labeled failure. */ +function run(command, args, cwd, label, extraEnv = {}) { + const completed = spawnSync(command, args, { + cwd, + env: { ...process.env, ...extraEnv }, + stdio: ['ignore', 'inherit', 'inherit'], + }); + if (completed.error) throw new Error(`${label} could not start: ${completed.error.message}`); + if (completed.status !== 0) { + throw new Error( + `${label} failed with ${ + completed.signal ? `signal ${completed.signal}` : `exit code ${completed.status ?? 'unknown'}` + }.` + ); + } +}