diff --git a/.github/workflows/package-validation.yml b/.github/workflows/package-validation.yml index 3b3126795a..f09aa05d77 100644 --- a/.github/workflows/package-validation.yml +++ b/.github/workflows/package-validation.yml @@ -354,5 +354,4 @@ jobs: - name: Smoke standalone lifecycle env: AGENT_RELAY_STARTUP_DEBUG: 1 - RELAY_WORKSPACE_KEY: ${{ secrets.RELAY_CI_WORKSPACE_KEY }} run: bash scripts/ci-standalone-smoke.sh "$STANDALONE_CLI" "$STANDALONE_BROKER" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 52f8d317fc..0f7ef3bc63 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1521,8 +1521,6 @@ jobs: echo "✓ Uncompressed binary verified" - name: Smoke standalone lifecycle - env: - RELAY_WORKSPACE_KEY: ${{ secrets.RELAY_CI_WORKSPACE_KEY }} run: | if [ "$(uname -m)" != "${{ matrix.expected_arch }}" ]; then echo "Skipping lifecycle smoke for ${{ matrix.expected_arch }} on $(uname -m) host" diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cbd402e81..af25ae67fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,21 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Minor] + +### Changed + +- Operators can keep using custom `--sandbox-name` values with `agent-relay fleet spawn --sandbox`; launches without a custom name generate a stable `sbx_` identity automatically, while only `--sandbox-id` replay requires the matching deterministic lowercase `fleet-sandbox-` name. + +### Fixed + +- Ambiguous Cloud sandbox responses retain their stable `--sandbox-id` for replay instead of automatically deleting an allocation whose outcome is unknown. +- Persisting an Agent37 Relaycast target keeps the canonical Cloud workspace key as the durable selector and stores the route-scoped transport credential separately, so later commands can reuse the original explicit key. +- Relaycast credentials and origins now resolve as one transport pair for attach, observer, Fleet, and Relayfile provisioning commands; stale sandbox responses cannot overwrite a project workspace that was rebound while provisioning was in flight. +- Legacy non-Agent37 Cloud sandbox responses remain usable when they omit the newer Relaycast target, while any target Cloud does return is verified and persisted for both newly provisioned and reused providers. +- Standalone package smoke workspaces remain valid for five minutes, and startup overrides are capped at four minutes so shutdown and cleanup verification always retain a full-minute lease margin. +- Startup retries now require Relaycast's typed pre-commit storage-admission codes instead of replaying every 5xx response from an unkeyed workspace or agent-registration request. +- Agent registration retries now honor Relaycast's typed cooldown (capped at one minute) instead of immediately retrying through the same write-capacity window, while a three-minute aggregate deadline prevents hung requests from stranding callers. ## [11.10.4] - 2026-09-08 diff --git a/crates/broker/src/relaycast/auth.rs b/crates/broker/src/relaycast/auth.rs index 46f7ac196c..d5a0f4408d 100644 --- a/crates/broker/src/relaycast/auth.rs +++ b/crates/broker/src/relaycast/auth.rs @@ -933,17 +933,26 @@ const RELAYCAST_HTTP_TIMEOUT: std::time::Duration = std::time::Duration::from_se /// 34099838274 lost three jobs to exactly that. const TRANSIENT_STARTUP_RETRY_BACKOFFS_MS: [u64; 2] = [200, 400]; -/// The server-side statuses worth replaying: 500, 502, 503, 504. A 501 is a -/// contract mismatch rather than a transient and is deliberately excluded, as -/// are transport errors — a timed-out `POST /v1/agents` may already have -/// created the agent, and re-sending it is the AR-448 duplicate shape. +/// Replay only server failures whose typed error code establishes that the +/// request failed at the storage-admission boundary. Retrying every 5xx by +/// status is unsafe for these unkeyed POSTs: an application-level 503 or a 500 +/// returned after commit could create the AR-448 duplicate shape when replayed. +/// +/// `database_overloaded` is Relaycast's D1 admission failure and +/// `workspace_storage_unavailable` is emitted when workspace persistence never +/// starts. Both are explicitly pre-commit contracts. Transport errors remain +/// terminal because a timed-out request may already have committed. fn is_transient_server_error(error: &RelayError) -> bool { matches!( error, RelayError::Api { + code, status: 500 | 502 | 503 | 504, .. - } + } if matches!( + code.trim(), + "database_overloaded" | "workspace_storage_unavailable" + ) ) } @@ -1535,10 +1544,10 @@ mod tests { use super::{ hash_identity_key, is_agent_token_invalid, is_agent_token_invalid_anyhow, - is_agent_token_invalid_code, reclaim_legacy_identity, relay_error_to_anyhow, - relay_request_with_timeout, resolve_relaycast_base_url, retry_transient_relay_error, - stable_node_identity_key, AuthClient, AuthHttpError, CredentialCache, - AGENT_TOKEN_INVALID_CODE, DEFAULT_RELAYCAST_BASE_URL, + is_agent_token_invalid_code, is_transient_server_error, reclaim_legacy_identity, + relay_error_to_anyhow, relay_request_with_timeout, resolve_relaycast_base_url, + retry_transient_relay_error, stable_node_identity_key, AuthClient, AuthHttpError, + CredentialCache, AGENT_TOKEN_INVALID_CODE, DEFAULT_RELAYCAST_BASE_URL, }; use relaycast::RelayError; @@ -2076,6 +2085,135 @@ mod tests { } } + #[tokio::test] + async fn unknown_application_503_is_not_retried() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let calls = AtomicUsize::new(0); + let error = retry_transient_relay_error("probing an application failure", || { + calls.fetch_add(1, Ordering::SeqCst); + async { + Err::<(), _>(RelayError::api( + "application_temporarily_unavailable", + "the application rejected the request", + 503, + )) + } + }) + .await + .expect_err("an unclassified 503 must not replay an unkeyed POST"); + + assert_eq!(calls.load(Ordering::SeqCst), 1); + match error { + RelayError::Api { + code, + status, + attempts, + .. + } => { + assert_eq!(code, "application_temporarily_unavailable"); + assert_eq!(status, 503); + assert_eq!(attempts, 1); + } + other => panic!("expected the original terminal API error, got {other}"), + } + } + + #[test] + fn transient_retry_requires_both_a_safe_code_and_server_status() { + assert!(is_transient_server_error(&RelayError::api( + "database_overloaded", + "overloaded", + 503, + ))); + assert!(is_transient_server_error(&RelayError::api( + "workspace_storage_unavailable", + "storage unavailable", + 502, + ))); + assert!(!is_transient_server_error(&RelayError::api( + "database_overloaded", + "conflict", + 409, + ))); + assert!(!is_transient_server_error(&RelayError::api( + "unknown_error", + "server failure", + 500, + ))); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn unknown_application_503_exits_registration_without_a_replay() { + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + + use axum::{ + extract::State, http::StatusCode as AxumStatusCode, routing::post, Json, Router, + }; + + async fn reject_registration( + State(attempts): State>, + ) -> (AxumStatusCode, Json) { + attempts.fetch_add(1, Ordering::SeqCst); + ( + AxumStatusCode::SERVICE_UNAVAILABLE, + Json(json!({ + "ok": false, + "error": { + "code": "application_temporarily_unavailable", + "message": "The request could not be completed." + } + })), + ) + } + + let _env_guard = clear_relay_env(); + // SAFETY: test-only, serialized by RELAY_ENV_MUTEX via clear_relay_env. + unsafe { + std::env::set_var("AGENT_RELAY_WORKSPACE_KEY", "rk_live_env"); + } + + let attempts = Arc::new(AtomicUsize::new(0)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server_state = attempts.clone(); + let server = tokio::spawn(async move { + axum::serve( + listener, + Router::new() + .route("/v1/agents", post(reject_registration)) + .with_state(server_state), + ) + .await + }); + + let error = AuthClient::new(Some(format!("http://{address}"))) + .startup_session(Some("lead")) + .await + .expect_err("an unclassified application 503 must be terminal"); + + let message = format!("{error:#}"); + assert!( + message.contains("application_temporarily_unavailable"), + "{message}" + ); + assert!(message.contains("attempts: 1"), "{message}"); + assert_eq!( + attempts.load(Ordering::SeqCst), + 1, + "an unkeyed registration must not be replayed on an unknown 503" + ); + + server.abort(); + // SAFETY: test-only, serialized by RELAY_ENV_MUTEX via clear_relay_env. + unsafe { + std::env::remove_var("AGENT_RELAY_WORKSPACE_KEY"); + } + } + /// The retry budget is bounded, and the terminal error still carries the /// registration diagnostics PR #1673 added. An operator chasing a sandbox /// worker that spawns but never registers (AgentWorkforce/cloud#3401) diff --git a/crates/broker/src/relaycast/ws.rs b/crates/broker/src/relaycast/ws.rs index 297557005b..a82ef52cc1 100644 --- a/crates/broker/src/relaycast/ws.rs +++ b/crates/broker/src/relaycast/ws.rs @@ -6,12 +6,12 @@ use std::{ use anyhow::{Context, Result}; use relaycast::{ - agent::DmOptions, format_registration_error, - retry_agent_registration as sdk_retry_agent_registration, ActionDefinition, ActionInvocation, - AgentClient, AgentIdentityRecoveryResponse, AgentRegistrationClient, AgentRegistrationError, - AgentRegistrationRetryOutcome, CompleteInvocationRequest, CreateObserverTokenRequest, - EmitSessionEventRequest, MessageListQuery, ObserverToken, RegisterActionRequest, RelayCast, - RelayCastOptions, RelayError, ReleaseAgentRequest, TakeOverAgentRequest, UpdateAgentRequest, + agent::DmOptions, format_registration_error, registration_is_retryable, ActionDefinition, + ActionInvocation, AgentClient, AgentIdentityRecoveryResponse, AgentRegistrationClient, + AgentRegistrationError, AgentRegistrationRetryOutcome, CompleteInvocationRequest, + CreateObserverTokenRequest, EmitSessionEventRequest, MessageListQuery, ObserverToken, + RegisterActionRequest, RelayCast, RelayCastOptions, RelayError, ReleaseAgentRequest, + TakeOverAgentRequest, UpdateAgentRequest, }; use serde_json::Value; @@ -49,8 +49,6 @@ pub struct RelaycastHttpClient { pub type RelaycastRegistrationError = AgentRegistrationError; pub type RegRetryOutcome = AgentRegistrationRetryOutcome; -#[cfg(test)] -pub(crate) use relaycast::registration_is_retryable; pub(crate) use relaycast::registration_retry_after_secs; /// Why the broker is asking `register_agent_token` for a token. @@ -1442,7 +1440,87 @@ pub fn format_worker_preregistration_error( format_registration_error(name, error).replace("register agent", "pre-register worker") } +const MAX_AGENT_REGISTRATION_ATTEMPTS: usize = 3; +const DEFAULT_AGENT_REGISTRATION_RETRY_DELAY: Duration = Duration::from_secs(2); +const MAX_AGENT_REGISTRATION_RETRY_DELAY: Duration = Duration::from_secs(60); +const MAX_AGENT_REGISTRATION_ELAPSED: Duration = Duration::from_secs(180); + +fn agent_registration_retry_delay(error: &RelaycastRegistrationError) -> Duration { + registration_retry_after_secs(error) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_AGENT_REGISTRATION_RETRY_DELAY) + .min(MAX_AGENT_REGISTRATION_RETRY_DELAY) +} + +async fn retry_agent_registration_with( + mut request: F, + mut sleep: S, +) -> Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, + S: FnMut(Duration) -> SleepFut, + SleepFut: std::future::Future, +{ + for attempt in 0..MAX_AGENT_REGISTRATION_ATTEMPTS { + match request().await { + Ok(token) => return Ok(token), + Err(error) + if registration_is_retryable(&error) + && attempt + 1 < MAX_AGENT_REGISTRATION_ATTEMPTS => + { + let delay = agent_registration_retry_delay(&error); + tracing::warn!( + attempt = attempt + 1, + retry_in_ms = delay.as_millis(), + error = %error, + "transient Relaycast agent registration failure; retrying" + ); + sleep(delay).await; + } + Err(error) if registration_is_retryable(&error) => { + return Err(RegRetryOutcome::RetryableExhausted(error)); + } + Err(error) => return Err(RegRetryOutcome::Fatal(error)), + } + } + unreachable!("the registration retry loop always returns on its final attempt") +} + +async fn retry_agent_registration_with_budget( + agent_name: &str, + budget: Duration, + request: F, + sleep: S, +) -> Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, + S: FnMut(Duration) -> SleepFut, + SleepFut: std::future::Future, +{ + match tokio::time::timeout(budget, retry_agent_registration_with(request, sleep)).await { + Ok(result) => result, + Err(_) => Err(RegRetryOutcome::RetryableExhausted( + RelaycastRegistrationError::Transport { + agent_name: agent_name.to_string(), + detail: format!( + "registration retry budget exhausted after {}s", + budget.as_secs() + ), + }, + )), + } +} + /// Attempt to register an agent token with up to 3 retries for transient errors. +/// +/// The Relaycast SDK's typed rate-limit errors carry its cooldown. Honor +/// that duration (under a hard one-minute cap) instead of immediately retrying +/// through the same admission window. Transport failures retain the short SDK +/// fallback because they do not carry a retry delay. The complete operation is +/// capped below Cloud's five-minute step-provisioning budget, so a hung request +/// or a second full cooldown cannot strand the caller indefinitely. pub async fn retry_agent_registration( http: &RelaycastHttpClient, name: &str, @@ -1454,7 +1532,13 @@ pub async fn retry_agent_registration( detail: "SDK relay client not initialized".to_string(), }) })?; - sdk_retry_agent_registration(registration, name, cli).await + retry_agent_registration_with_budget( + name, + MAX_AGENT_REGISTRATION_ELAPSED, + || registration.register_agent_token(name, cli), + |delay| tokio::time::sleep(delay), + ) + .await } /// The declared fields alone, trimmed, with blanks omitted. @@ -1522,9 +1606,11 @@ mod tests { use crate::{fleet_wire::AgentRegistrationMetadata, ids::ChannelName}; use super::{ - format_worker_preregistration_error, registration_is_retryable, - registration_retry_after_secs, ImpersonationAwareRegistrationError, MessageInjectionMode, - RecipientReachability, RegisterIntent, RelaycastHttpClient, + agent_registration_retry_delay, format_worker_preregistration_error, + registration_is_retryable, registration_retry_after_secs, retry_agent_registration_with, + retry_agent_registration_with_budget, ImpersonationAwareRegistrationError, + MessageInjectionMode, RecipientReachability, RegRetryOutcome, RegisterIntent, + RelaycastHttpClient, }; fn seeded_http_client(base_url: &str) -> RelaycastHttpClient { @@ -1549,6 +1635,172 @@ mod tests { assert_eq!(registration_retry_after_secs(&error), Some(60)); } + #[test] + fn registration_retry_delay_honors_typed_cooldown_with_a_hard_cap() { + let advertised = AgentRegistrationError::RateLimited { + agent_name: "worker-a".to_string(), + retry_after_secs: 17, + detail: "rate limited".to_string(), + }; + let excessive = AgentRegistrationError::Blocked { + agent_name: "worker-a".to_string(), + retry_after_secs: 600, + }; + let transport = AgentRegistrationError::Transport { + agent_name: "worker-a".to_string(), + detail: "connection reset".to_string(), + }; + + assert_eq!( + agent_registration_retry_delay(&advertised), + Duration::from_secs(17) + ); + assert_eq!( + agent_registration_retry_delay(&excessive), + Duration::from_secs(60) + ); + assert_eq!( + agent_registration_retry_delay(&transport), + Duration::from_secs(2) + ); + } + + #[tokio::test] + async fn registration_retry_waits_for_the_typed_rate_limit_delay() { + let mut outcomes = std::collections::VecDeque::from([ + Err(AgentRegistrationError::RateLimited { + agent_name: "worker-a".to_string(), + retry_after_secs: 23, + detail: "rate limited".to_string(), + }), + Ok("at_live_after_retry".to_string()), + ]); + let mut delays = Vec::new(); + + let token = retry_agent_registration_with( + || std::future::ready(outcomes.pop_front().expect("bounded test outcome")), + |delay| { + delays.push(delay); + std::future::ready(()) + }, + ) + .await + .expect("the retry after the advertised delay should succeed"); + + assert_eq!(token, "at_live_after_retry"); + assert_eq!(delays, vec![Duration::from_secs(23)]); + } + + #[tokio::test] + async fn registration_retry_preserves_terminal_classification_and_attempt_budget() { + let mut transient_outcomes = std::collections::VecDeque::from([ + Err(AgentRegistrationError::Transport { + agent_name: "worker-a".to_string(), + detail: "first".to_string(), + }), + Err(AgentRegistrationError::Transport { + agent_name: "worker-a".to_string(), + detail: "second".to_string(), + }), + Err(AgentRegistrationError::Transport { + agent_name: "worker-a".to_string(), + detail: "third".to_string(), + }), + ]); + let mut transient_delays = Vec::new(); + + let exhausted = retry_agent_registration_with( + || { + std::future::ready( + transient_outcomes + .pop_front() + .expect("three-attempt test outcome"), + ) + }, + |delay| { + transient_delays.push(delay); + std::future::ready(()) + }, + ) + .await; + + assert!(matches!( + exhausted, + Err(RegRetryOutcome::RetryableExhausted( + AgentRegistrationError::Transport { detail, .. } + )) if detail == "third" + )); + assert_eq!(transient_delays, vec![Duration::from_secs(2); 2]); + assert!(transient_outcomes.is_empty()); + + let mut fatal_delays = Vec::new(); + let fatal = retry_agent_registration_with( + || { + std::future::ready(Err(AgentRegistrationError::Api { + agent_name: "worker-a".to_string(), + status: 401, + detail: "unauthorized".to_string(), + })) + }, + |delay| { + fatal_delays.push(delay); + std::future::ready(()) + }, + ) + .await; + + assert!(matches!( + fatal, + Err(RegRetryOutcome::Fatal(AgentRegistrationError::Api { + status: 401, + .. + })) + )); + assert!(fatal_delays.is_empty()); + } + + #[tokio::test] + async fn registration_retry_budget_cancels_a_hung_request() { + let result = retry_agent_registration_with_budget( + "worker-a", + Duration::from_millis(10), + || std::future::pending::>(), + |_| std::future::ready(()), + ) + .await; + + assert!(matches!( + result, + Err(RegRetryOutcome::RetryableExhausted( + AgentRegistrationError::Transport { detail, .. } + )) if detail.contains("retry budget exhausted") + )); + } + + #[tokio::test] + async fn registration_retry_budget_cancels_a_cooldown_sleep() { + let result = retry_agent_registration_with_budget( + "worker-a", + Duration::from_millis(10), + || { + std::future::ready(Err(AgentRegistrationError::RateLimited { + agent_name: "worker-a".to_string(), + retry_after_secs: 60, + detail: "rate limited".to_string(), + })) + }, + |_| std::future::pending::<()>(), + ) + .await; + + assert!(matches!( + result, + Err(RegRetryOutcome::RetryableExhausted( + AgentRegistrationError::Transport { detail, .. } + )) if detail.contains("retry budget exhausted") + )); + } + #[test] fn format_registration_error_includes_worker_name() { let error = AgentRegistrationError::Transport { diff --git a/packages/cli/README.md b/packages/cli/README.md index 5065737f7f..01990324f6 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -179,7 +179,15 @@ sees the same synced Relayfile workspace. Use `--sandbox-provider daytona` or `--sandbox-provider e2b` to require an operator-enabled provider; omit the flag to let Cloud's sandbox router choose. Pass `--no-sandbox-relayfile` only when a deliberately bare sandbox is desired. If provisioning times out or the spawn -fails, Relay asks Cloud to delete the newly created sandbox. +fails, Relay asks Cloud to delete the newly created sandbox. Runs without a +custom name use one `sbx_` identity and the matching +`fleet-sandbox-` node name. Legacy custom `--sandbox-name` values remain supported when no +`--sandbox-id` is supplied; in that mode Cloud receives no sandbox identity. +When replaying with `--sandbox-id ` (lowercase RFC 4122 UUID), pass +its matching deterministic `--sandbox-name` or let Relay derive it. If +provisioning ends with an unknown outcome, rerun the command with the warning's +`--sandbox-id` to replay the same Cloud identity instead of adopting another +fleet node. Large workspaces should select only the live subtree an agent needs. Pass one or more explicit directory roots after `--sandbox-relayfile-path`; Cloud diff --git a/packages/cli/src/cli/ci-standalone-smoke.test.ts b/packages/cli/src/cli/ci-standalone-smoke.test.ts index ed6e4d4d7e..a4934f9663 100644 --- a/packages/cli/src/cli/ci-standalone-smoke.test.ts +++ b/packages/cli/src/cli/ci-standalone-smoke.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from 'node:child_process'; -import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; @@ -15,12 +15,94 @@ function makeExecutable(directory: string, name: string, contents: string): stri return path; } -function createFakeBinaries(): { cli: string; broker: string; invocationLog: string } { +function createFakeBinaries(): { cli: string; broker: string; invocationLog: string; toolsPath: string } { const directory = mkdtempSync(join(tmpdir(), 'relay-standalone-smoke-test-')); temporaryDirectories.push(directory); const invocationLog = join(directory, 'invocations.log'); writeFileSync(invocationLog, ''); + const toolsPath = join(directory, 'tools'); + mkdirSync(toolsPath); + makeExecutable( + toolsPath, + 'curl', + `#!/usr/bin/env bash +set -euo pipefail +output=/dev/null +headers=/dev/null +method=GET +url= +while [ "\$#" -gt 0 ]; do + case "\$1" in + --output) output="\$2"; shift 2 ;; + --dump-header) headers="\$2"; shift 2 ;; + --connect-timeout|--max-time) shift 2 ;; + --request) method="\$2"; shift 2 ;; + --write-out) shift 2 ;; + --data|--header) shift 2 ;; + *) url="\$1"; shift ;; + esac +done +if [[ "$method" = POST && "$url" = */v1/workspaces ]]; then + printf '%s' '{"data":{"api_key":"rk_live_fake_smoke_key","workspace_id":"rw_fake_smoke"}}' > "\$output" + echo 201 +elif [[ "$method" = DELETE && "$url" = */v1/workspace ]]; then + delete_status="\${FAKE_DELETE_STATUS:-204}" + if [ "$delete_status" != 200 ] && [ "$delete_status" != 204 ]; then + printf '%s' '{"error":{"code":"internal_error","message":"rk_live_delete_body_must_not_print"}}' > "\$output" + fi + echo "$delete_status" +elif [[ "$method" = GET && "$url" = */v1/workspace ]]; then + verify_status="\${FAKE_VERIFY_STATUS:-401}" + if [ -n "\${FAKE_VERIFY_STATUSES:-}" ]; then + verify_count_file="\${INVOCATION_LOG}.verify-count" + verify_count=0 + if [ -f "\$verify_count_file" ]; then + verify_count="\$(<"\$verify_count_file")" + fi + IFS=',' read -r -a verify_statuses <<< "\$FAKE_VERIFY_STATUSES" + verify_index="\$verify_count" + if [ "\$verify_index" -ge "\${#verify_statuses[@]}" ]; then + verify_index="\$((\${#verify_statuses[@]} - 1))" + fi + verify_status="\${verify_statuses[\$verify_index]}" + printf '%s\n' "\$((verify_count + 1))" > "\$verify_count_file" + fi + if [ -n "\${FAKE_VERIFY_RETRY_AFTER:-}" ]; then + retry_after_header="\${FAKE_VERIFY_RETRY_AFTER_HEADER:-Retry-After}" + printf 'HTTP/2 %s\r\n%s: %s\r\n\r\n' "\$verify_status" "\$retry_after_header" "\$FAKE_VERIFY_RETRY_AFTER" > "\$headers" + fi + echo "\$verify_status" +else + echo 500 +fi +` + ); + makeExecutable( + toolsPath, + 'sleep', + `#!/usr/bin/env bash +printf 'sleep %s\n' "\${1:-}" >> "$INVOCATION_LOG" +` + ); + makeExecutable( + toolsPath, + 'jq', + `#!/usr/bin/env bash +set -euo pipefail +if [ "\${1:-}" = "-cn" ]; then + echo '{"name":"fake","expires_in_seconds":300}' +elif [ "\${1:-}" = "-er" ] && [[ "\${2:-}" = *api_key* ]]; then + echo 'rk_live_fake_smoke_key' +elif [ "\${1:-}" = "-er" ] && [[ "\${2:-}" = *workspace_id* ]]; then + echo 'rw_fake_smoke' +elif [ "\${1:-}" = "-er" ] && [[ "\${2:-}" = *error.code* ]] && [ -n "\${FAKE_DELETE_ERROR_CODE:-}" ]; then + echo "\$FAKE_DELETE_ERROR_CODE" +else + exit 1 +fi +` + ); const broker = makeExecutable( directory, 'broker', @@ -48,8 +130,8 @@ case "\${2:-}" in echo "Cleaned up (was not running)" ;; up) - if [ -z "\${RELAY_WORKSPACE_KEY:-}" ]; then - echo "workspace key missing" >&2 + if [ "\${RELAY_BASE_URL:-}" != "https://cast.agentrelay.com" ]; then + echo "trusted Relaycast base URL missing" >&2 exit 65 fi if [ -n "\${RELAY_WORKSPACES_JSON:-}" ]; then @@ -82,7 +164,7 @@ esac ` ); - return { cli, broker, invocationLog }; + return { cli, broker, invocationLog, toolsPath }; } afterEach(() => { @@ -103,18 +185,37 @@ describe('ci-standalone-smoke workspace reuse', () => { expect(cleanupSubshellIndex).toBeGreaterThan(trapDisarmIndex); }); - it('injects the same dedicated secret at every workflow call site', () => { + it('creates an ephemeral workspace on the trusted engine and wires its base URL explicitly', () => { + const script = readFileSync(smokeScript, 'utf8'); + expect(script).toContain('TRUSTED_RELAY_BASE_URL="https://cast.agentrelay.com"'); + expect(script).toContain('WORKSPACE_LEASE_SECONDS=300'); + expect(script).toContain('CURL_CONNECT_TIMEOUT_SECONDS=10'); + expect(script).toContain('CURL_MAX_TIME_SECONDS=60'); + expect(script).toContain('--connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS"'); + expect(script).toContain('--max-time "$CURL_MAX_TIME_SECONDS"'); + expect(script).toContain('expires_in_seconds: $expires'); + expect(script).toContain('printf \'::add-mask::%s\\n\' "$WORKSPACE_KEY"'); + expect(script).toContain('--request DELETE'); + expect(script).toContain('Ephemeral workspace deletion verified'); for (const workflow of ['.github/workflows/package-validation.yml', '.github/workflows/publish.yml']) { - expect(readFileSync(resolve(workflow), 'utf8')).toContain( - 'RELAY_WORKSPACE_KEY: ${{ secrets.RELAY_CI_WORKSPACE_KEY }}' - ); + expect(readFileSync(resolve(workflow), 'utf8')).not.toContain('RELAY_CI_WORKSPACE_KEY'); } }); + it('reports the create HTTP status and sanitized error code when no key is returned', () => { + const script = readFileSync(smokeScript, 'utf8'); + expect(script).toContain('CREATE_ERROR_CODE='); + expect(script).toContain('did not contain an API key (HTTP ${CREATE_STATUS:-unknown}'); + expect(script).toContain('error code ${CREATE_ERROR_CODE}'); + expect(script).not.toContain('cat "$WORKSPACE_RESPONSE"'); + }); + it('keeps the outer startup deadline above the broker aggregate handshake budget', () => { const script = readFileSync(smokeScript, 'utf8'); const brokerSession = readFileSync(resolve('crates/broker/src/runtime/session.rs'), 'utf8'); const minimumSeconds = Number(script.match(/MIN_STARTUP_TIMEOUT_SECONDS=([0-9]+)/)?.[1]); + const maximumSeconds = Number(script.match(/MAX_STARTUP_TIMEOUT_SECONDS=([0-9]+)/)?.[1]); + const leaseSeconds = Number(script.match(/WORKSPACE_LEASE_SECONDS=([0-9]+)/)?.[1]); const smokeSeconds = Number( script.match(/AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS:-([0-9]+)}/)?.[1] ); @@ -124,6 +225,7 @@ describe('ci-standalone-smoke workspace reuse', () => { expect(minimumSeconds).toBeGreaterThanOrEqual(brokerSeconds + 10); expect(smokeSeconds).toBeGreaterThanOrEqual(minimumSeconds); + expect(leaseSeconds - maximumSeconds).toBeGreaterThanOrEqual(60); }); it('rejects unsafe startup-timeout overrides before invoking binaries', () => { @@ -132,15 +234,15 @@ describe('ci-standalone-smoke workspace reuse', () => { ['050', 'without leading zeros'], ['060', 'without leading zeros'], ['08', 'without leading zeros'], - ['99999', 'must be no more than 86400s'], - ['9223372036854775808', 'between 50s and 86400s'], + ['241', 'must be no more than 240s'], + ['99999', 'must be no more than 240s'], + ['9223372036854775808', 'between 50s and 240s'], ]) { const { cli, broker, invocationLog } = createFakeBinaries(); const result = spawnSync('bash', [smokeScript, cli, broker], { encoding: 'utf8', env: { ...process.env, - RELAY_WORKSPACE_KEY: 'rk_live_test_only', AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS: override, INVOCATION_LOG: invocationLog, }, @@ -152,47 +254,170 @@ describe('ci-standalone-smoke workspace reuse', () => { } }); - it('fails closed before invoking binaries when the dedicated key is missing or whitespace-only', () => { - const unusableKeys: Array<[label: string, value: string | undefined]> = [ - ['unset', undefined], - ['empty', ''], - ['single space', ' '], - ['tab', '\t'], - ]; + it('mints and cleans up the ephemeral workspace without printing its key', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + }, + timeout: 10_000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('Ephemeral workspace deletion verified'); + // The GitHub workflow command necessarily carries the value once so the + // runner can mask it; all ordinary output must remain free of the key. + const ordinaryOutput = result.stdout.replace(/::add-mask::[^\n]*\n/g, ''); + expect(ordinaryOutput).not.toContain('rk_live_fake_smoke_key'); + expect(result.stderr).not.toContain('rk_live_fake_smoke_key'); + }); + + it('accepts a successful delete without adding a follow-up read dependency', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + FAKE_DELETE_STATUS: '204', + FAKE_VERIFY_STATUS: '503', + }, + timeout: 10_000, + }); - for (const [label, value] of unusableKeys) { - const { cli, broker, invocationLog } = createFakeBinaries(); - const env = { ...process.env }; - if (value === undefined) { - delete env.RELAY_WORKSPACE_KEY; - } else { - env.RELAY_WORKSPACE_KEY = value; - } - env.INVOCATION_LOG = invocationLog; + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('Ephemeral workspace deletion verified'); + expect(result.stderr).not.toContain('follow-up HTTP'); + }); - const result = spawnSync('bash', [smokeScript, cli, broker], { - encoding: 'utf8', - env, - }); + it('accepts an ambiguous delete response only when the follow-up proves absence', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + FAKE_DELETE_STATUS: '500', + FAKE_DELETE_ERROR_CODE: 'internal_error', + FAKE_VERIFY_STATUS: '401', + }, + timeout: 10_000, + }); - expect(result.status, `${label}: ${result.stderr}`).toBe(2); - expect(result.stderr, label).toContain('Refusing to start'); - expect(result.stderr, label).toContain('throwaway workspace'); - expect(result.stderr, label).not.toContain('binary not found'); - expect(readFileSync(invocationLog, 'utf8'), label).toBe(''); - } + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain( + 'Ephemeral workspace deletion verified after ambiguous DELETE HTTP 500, error code internal_error' + ); + expect(result.stdout).not.toContain('rk_live_delete_body_must_not_print'); + expect(result.stderr).not.toContain('rk_live_delete_body_must_not_print'); + }); + + it('retries a transient verification read and honors bounded Retry-After', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + FAKE_DELETE_STATUS: '500', + FAKE_DELETE_ERROR_CODE: 'internal_error', + FAKE_VERIFY_STATUSES: '503,401', + FAKE_VERIFY_RETRY_AFTER: '3', + FAKE_VERIFY_RETRY_AFTER_HEADER: 'retry-after', + }, + timeout: 10_000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain( + 'Ephemeral workspace deletion verified after ambiguous DELETE HTTP 500, error code internal_error' + ); + expect(readFileSync(invocationLog, 'utf8')).toContain('sleep 3'); + }); + + it('classifies an unsafe delete error code instead of logging it', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + FAKE_DELETE_STATUS: '500', + FAKE_DELETE_ERROR_CODE: 'rk_live_untrusted_code', + FAKE_VERIFY_STATUS: '401', + }, + timeout: 10_000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('ambiguous DELETE HTTP 500, error code other'); + expect(result.stdout).not.toContain('rk_live_untrusted_code'); + expect(result.stderr).not.toContain('rk_live_untrusted_code'); + }); + + it('fails closed when an ambiguous delete leaves the workspace readable', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + FAKE_DELETE_STATUS: '500', + FAKE_DELETE_ERROR_CODE: 'internal_error', + FAKE_VERIFY_STATUS: '200', + }, + timeout: 10_000, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + 'ephemeral workspace cleanup returned HTTP 500, error code internal_error' + ); + expect(result.stderr).toContain('ephemeral workspace deletion was not proved (follow-up HTTP 200)'); + expect(result.stdout).not.toContain('rk_live_delete_body_must_not_print'); + expect(result.stderr).not.toContain('rk_live_delete_body_must_not_print'); + }); + + it('fails closed when an ambiguous delete cannot be verified', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + FAKE_DELETE_STATUS: '500', + FAKE_DELETE_ERROR_CODE: 'internal_error', + FAKE_VERIFY_STATUS: '503', + }, + timeout: 10_000, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + 'ephemeral workspace cleanup returned HTTP 500, error code internal_error' + ); + expect(result.stderr).toContain('ephemeral workspace deletion was not proved (follow-up HTTP 503)'); }); it('passes the shared key through the isolated lifecycle and joins its workspace', () => { - const { cli, broker, invocationLog } = createFakeBinaries(); + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); const result = spawnSync('bash', [smokeScript, cli, broker], { encoding: 'utf8', env: { ...process.env, - RELAY_WORKSPACE_KEY: 'rk_live_test_only', RELAY_WORKSPACES_JSON: '[{"workspace_id":"rw_wrong","api_key":"rk_wrong"}]', AGENT_RELAY_STANDALONE_BROKER_NAME: 'relay-ci-test-a', INVOCATION_LOG: invocationLog, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, }, timeout: 10_000, }); @@ -203,15 +428,15 @@ describe('ci-standalone-smoke workspace reuse', () => { }); it('rejects a lifecycle that reports a newly created workspace', () => { - const { cli, broker, invocationLog } = createFakeBinaries(); + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); const result = spawnSync('bash', [smokeScript, cli, broker], { encoding: 'utf8', env: { ...process.env, - RELAY_WORKSPACE_KEY: 'rk_live_test_only', AGENT_RELAY_STANDALONE_BROKER_NAME: 'relay-ci-test-b', FAKE_WORKSPACE_MODE: 'created', INVOCATION_LOG: invocationLog, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, }, timeout: 10_000, }); @@ -222,7 +447,7 @@ describe('ci-standalone-smoke workspace reuse', () => { }); it('rejects readiness written after the startup deadline', () => { - const { cli, broker, invocationLog } = createFakeBinaries(); + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); const bashEnv = join(dirname(invocationLog), 'accelerated-clock.sh'); writeFileSync( bashEnv, @@ -232,12 +457,12 @@ describe('ci-standalone-smoke workspace reuse', () => { encoding: 'utf8', env: { ...process.env, - RELAY_WORKSPACE_KEY: 'rk_live_test_only', AGENT_RELAY_STANDALONE_BROKER_NAME: 'relay-ci-test-c', AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS: '50', FAKE_READY_AFTER_SECOND_DOWN: '1', INVOCATION_LOG: invocationLog, BASH_ENV: bashEnv, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, }, timeout: 10_000, }); diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index a02af8a467..b272327d0c 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -46,6 +46,21 @@ vi.mock('@agent-relay/harness-driver', async (importOriginal) => ({ import { registerFleetCommands } from './fleet.js'; import { writeProjectWorkspaceKey } from '../lib/project-workspace-key.js'; +const REPLAY_SANDBOX_ID = 'sbx_123e4567-e89b-42d3-a456-426614174000'; +const REPLAY_SANDBOX_NAME = 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000'; +const AGENT37_RELAYCAST_TARGET = { + route: 'agent37-isolated' as const, + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_agent37_target', +}; +const CANONICAL_RELAYCAST_TARGET = { + route: 'canonical' as const, + baseUrl: 'https://cast.agentrelay.com', + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_canonical_target', +}; + const LIVE_AGENT_CAPABILITY_NAME = 'relay:live-agents:v1'; const liveAgentCapabilities = (...names: string[]) => [ { @@ -643,7 +658,7 @@ describe('fleet command support', () => { }); }); - it('fleet spawn --sandbox-provider e2b provisions E2B, mounts Relayfile, and uses a temporary launcher', async () => { + it('fleet spawn --sandbox-provider agent37 provisions the isolated canary and uses a temporary launcher', async () => { vi.stubEnv('RELAY_AGENT_TOKEN', undefined); const placement = { spawn: vi.fn(async () => ({ @@ -663,12 +678,19 @@ describe('fleet command support', () => { const createAgentRelay = vi.fn(() => ({ messaging: { placement } })); const ensureCloudFleetSandbox = vi.fn(async () => ({ outcome: 'provisioned' as const, - providerId: 'e2b' as const, + providerId: 'agent37' as const, cloudWorkspaceId: 'cloud-workspace', nodeId: 'node-1', nodeName: 'e2b-codex', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_agent37_target', + }, relayfileMounted: true, relayfileMountPath: '/workspace', })); @@ -686,6 +708,13 @@ describe('fleet command support', () => { exit: vi.fn() as never, }, ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/tmp/agent-relay-test/workspace-key.json', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, deleteCloudFleetSandbox, createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, @@ -700,11 +729,13 @@ describe('fleet command support', () => { 'codex', '--sandbox', '--sandbox-provider', - 'e2b', + 'agent37', + '--sandbox-id', + REPLAY_SANDBOX_ID, + '--sandbox-name', + REPLAY_SANDBOX_NAME, '--sandbox-relayfile-path', '/live-review/run-123/**', - '--sandbox-name', - 'e2b-codex', '--name', 'sandbox-worker', '--task', @@ -714,18 +745,21 @@ describe('fleet command support', () => { ], { from: 'user' } ); - expect(ensureCloudFleetSandbox).toHaveBeenCalledWith({ + const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0]; + expect(ensureInput).toEqual({ workspaceId: 'rw_abc', requiredCapability: 'spawn:codex', maxAgents: 1, mountRelayfile: true, relayfilePaths: ['/live-review/run-123/**'], + sandboxId: REPLAY_SANDBOX_ID, forceProvision: true, - providerId: 'e2b', + providerId: 'agent37', workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, - name: 'e2b-codex', + name: REPLAY_SANDBOX_NAME, }); + expect(ensureInput?.name).toBe(`fleet-sandbox-${ensureInput?.sandboxId?.slice('sbx_'.length)}`); expect(register).toHaveBeenCalledWith( expect.objectContaining({ name: expect.stringMatching(/^fleet-spawn-launcher-[a-f0-9]{8}$/), @@ -734,9 +768,9 @@ describe('fleet command support', () => { { strict: true } ); expect(createAgentRelay).toHaveBeenCalledWith({ - workspaceKey: 'rk_live_test', + workspaceKey: 'rk_live_agent37_target', token: 'at_live_launcher', - baseUrl: undefined, + baseUrl: 'https://agent37-cast.agentrelay.com', }); expect(placement.spawn).toHaveBeenCalledWith( expect.objectContaining({ @@ -756,109 +790,305 @@ describe('fleet command support', () => { }) ); expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); + expect(createWorkspaceRelay).toHaveBeenNthCalledWith(1, { + workspaceKey: 'rk_live_agent37_target', + baseUrl: 'https://agent37-cast.agentrelay.com', + }); expect(JSON.parse(logs[0]!)).toMatchObject({ - sandbox: { providerId: 'e2b', nodeName: 'e2b-codex', relayfileMountPath: '/workspace' }, + sandbox: { providerId: 'agent37', nodeName: 'e2b-codex', relayfileMountPath: '/workspace' }, invocation: { invocationId: 'inv_sandbox' }, - attachCommand: "agent-relay node agent attach 'sandbox-worker' --node 'e2b-codex' --mode drive", + attachCommand: + "agent-relay node agent attach 'sandbox-worker' --node 'e2b-codex' --mode drive --base-url 'https://agent37-cast.agentrelay.com'", }); }); - it('fleet spawn --sandbox deletes a freshly provisioned sandbox when dispatch fails', async () => { - const placement = { spawn: vi.fn(async () => Promise.reject(new Error('dispatch failed'))) }; - const deleteCloudFleetSandbox = vi.fn(async () => undefined); + it('fleet spawn --sandbox reuses an Agent37 target without a Relayfile mount', async () => { + const placement = { + spawn: vi.fn(async () => ({ invocationId: 'inv_reused', node: { name: 'agent37-codex' } })), + }; + const register = vi.fn(async () => ({ token: 'at_live_agent37' })); + const release = vi.fn(async () => ({ released: true, deleted: true })); + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })), register, release }, + })); const ensureCloudFleetSandbox = vi.fn(async () => ({ - outcome: 'provisioned' as const, + outcome: 'reused' as const, cloudWorkspaceId: 'cloud-workspace', nodeId: 'node-1', nodeName: 'agent37-codex', - sandboxId: 'sandbox-1', - relayWorkspaceId: 'rw_abc', - relayfileMounted: true, - relayfileMountPath: '/workspace', + status: 'online', + activeAgents: 0, + maxAgents: 1, providerId: 'agent37' as const, + relaycastTarget: AGENT37_RELAYCAST_TARGET, })); - const errors: string[] = []; const program = new Command(); program.exitOverride(); registerFleetCommands(program, { sdk: { createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, - createWorkspaceRelay: vi.fn(() => ({ - workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, - })) as never, + createWorkspaceRelay: createWorkspaceRelay as never, createWorkspace: vi.fn() as never, log: vi.fn(), - error: (...args: unknown[]) => errors.push(args.join(' ')), - exit: (() => { - throw new Error('__exit__'); - }) as never, + error: vi.fn(), + exit: vi.fn() as never, }, ensureCloudFleetSandbox, - deleteCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/tmp/agent-relay-test/workspace-key.json', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, + deleteCloudFleetSandbox: vi.fn(async () => undefined), createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, warn: () => undefined, error: () => undefined, }); - await expect( - program.parseAsync( - [ - 'fleet', - 'spawn', - 'codex', - '--sandbox', - '--name', - 'sandbox-worker', - '--task', - 'Work', - '--workspace-key', - 'rk_live_test', - '--token', - 'at_live_lead', - ], - { from: 'user' } - ) - ).rejects.toThrow('__exit__'); + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-provider', + 'agent37', + '--no-sandbox-relayfile', + '--workspace-id', + 'rw_abc', + '--name', + 'reused-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); - expect(errors.join('\n')).toContain('dispatch failed'); - expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( - expect.objectContaining({ - workloadProfile: 'long-running-agent', - }) + expect(createWorkspaceRelay).toHaveBeenCalledWith({ + workspaceKey: AGENT37_RELAYCAST_TARGET.relaycastApiKey, + baseUrl: AGENT37_RELAYCAST_TARGET.baseUrl, + }); + expect(register).toHaveBeenCalledWith( + expect.objectContaining({ name: expect.stringMatching(/^fleet-spawn-launcher-/) }), + { strict: true } ); - expect(deleteCloudFleetSandbox).toHaveBeenCalledWith({ - cloudWorkspaceId: 'cloud-workspace', - sandboxId: 'sandbox-1', - providerId: 'agent37', + expect(release).toHaveBeenCalled(); + }); + + it('applies and persists a returned target for a reused non-Agent37 provider', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', 'at_live_canonical_ambient'); + const placement = { + spawn: vi.fn(async () => ({ invocationId: 'inv_reused_e2b', node: { name: 'e2b-codex' } })), + }; + const createAgentRelay = vi.fn(() => ({ messaging: { placement } })); + const register = vi.fn(async () => ({ token: 'at_live_launcher' })); + const release = vi.fn(async () => ({ released: true, deleted: true })); + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })), register, release }, + })); + const persistWorkspaceRelaycastTarget = vi.fn(() => true); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: createAgentRelay as never, + createWorkspaceRelay: createWorkspaceRelay as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => ({ + outcome: 'reused' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-e2b', + nodeName: 'e2b-codex', + status: 'online', + activeAgents: 0, + maxAgents: 1, + providerId: 'e2b' as const, + relaycastTarget: CANONICAL_RELAYCAST_TARGET, + })), + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/tmp/agent-relay-test/workspace-key.json', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-provider', + 'e2b', + '--no-sandbox-relayfile', + '--workspace-id', + 'rw_abc', + '--name', + 'reused-e2b-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + + expect(createWorkspaceRelay).toHaveBeenCalledWith({ + workspaceKey: CANONICAL_RELAYCAST_TARGET.relaycastApiKey, + baseUrl: CANONICAL_RELAYCAST_TARGET.baseUrl, + }); + expect(register).toHaveBeenCalledWith( + expect.objectContaining({ name: expect.stringMatching(/^fleet-spawn-launcher-/) }), + { strict: true } + ); + expect(createAgentRelay).toHaveBeenCalledWith({ + workspaceKey: CANONICAL_RELAYCAST_TARGET.relaycastApiKey, + token: 'at_live_launcher', + baseUrl: CANONICAL_RELAYCAST_TARGET.baseUrl, + }); + expect(persistWorkspaceRelaycastTarget).toHaveBeenCalledWith( + expect.objectContaining({ key: 'rk_live_test' }), + CANONICAL_RELAYCAST_TARGET + ); + }); + + it('does not inherit a persisted Agent37 target for a legacy sandbox without a returned target', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', 'at_live_unproven_ambient'); + const placement = { + spawn: vi.fn(async () => ({ invocationId: 'inv_legacy_reused', node: { name: 'e2b-codex' } })), + }; + const createAgentRelay = vi.fn(() => ({ messaging: { placement } })); + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { + register: vi.fn(async () => ({ token: 'at_live_legacy_launcher' })), + release: vi.fn(async () => ({ released: true, deleted: true })), + }, + })); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: createAgentRelay as never, + createWorkspaceRelay: createWorkspaceRelay as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => ({ + outcome: 'reused' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-e2b', + nodeName: 'e2b-codex', + status: 'online', + activeAgents: 0, + maxAgents: 1, + providerId: 'e2b' as const, + })), + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/tmp/agent-relay-test/workspace-key.json', + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: AGENT37_RELAYCAST_TARGET.baseUrl, + relaycastApiKey: AGENT37_RELAYCAST_TARGET.relaycastApiKey, + }), + persistWorkspaceRelaycastTarget: vi.fn(() => true), + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-provider', + 'e2b', + '--no-sandbox-relayfile', + '--workspace-id', + 'rw_abc', + '--name', + 'legacy-reused-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + + expect(createWorkspaceRelay).toHaveBeenCalledWith({ + workspaceKey: 'rk_live_test', + token: undefined, + baseUrl: undefined, + ignorePersistedRelaycastTarget: true, + }); + expect(createAgentRelay).toHaveBeenCalledWith({ + workspaceKey: 'rk_live_test', + token: 'at_live_legacy_launcher', + baseUrl: undefined, + ignorePersistedRelaycastTarget: true, }); }); - it('cleans up when Cloud reports a post-provision response failure with a sandbox ID', async () => { + it('fleet spawn --sandbox cleans up when the target cannot be persisted', async () => { const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const ensureCloudFleetSandbox = vi.fn(async () => ({ + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-1', + nodeName: 'agent37-codex', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, + relayfileMounted: true, + })); + const createAgentRelay = vi.fn(); const program = new Command(); program.exitOverride(); registerFleetCommands(program, { sdk: { - createAgentRelay: vi.fn() as never, + createAgentRelay: createAgentRelay as never, createWorkspaceRelay: vi.fn(() => ({ workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, })) as never, createWorkspace: vi.fn() as never, log: vi.fn(), error: vi.fn(), - exit: (() => { + exit: vi.fn(() => { throw new Error('__exit__'); }) as never, }, - ensureCloudFleetSandbox: vi.fn(async () => { - throw new CloudFleetSandboxProvisionError('malformed response', { - cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', - sandboxId: 'sandbox-1', - nodeName: 'daytona-codex', - outcomeUnknown: true, - }); + ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/tmp/agent-relay-test/workspace-key.json', + workspaceId: 'rw_abc', }), + persistWorkspaceRelaycastTarget: () => false, deleteCloudFleetSandbox, createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, @@ -873,53 +1103,69 @@ describe('fleet command support', () => { 'spawn', 'codex', '--sandbox', - '--sandbox-name', - 'daytona-codex', + '--sandbox-provider', + 'agent37', '--name', 'sandbox-worker', '--task', 'Work', '--workspace-key', 'rk_live_test', - '--token', - 'at_live_lead', ], { from: 'user' } ) ).rejects.toThrow('__exit__'); - expect(deleteCloudFleetSandbox).toHaveBeenCalledWith({ - cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + cloudWorkspaceId: 'cloud-workspace', sandboxId: 'sandbox-1', }); + expect(createAgentRelay).not.toHaveBeenCalled(); }); - it('pins cleanup to the requested E2B provider when Cloud fails closed after provisioning', async () => { + it('fleet spawn --sandbox deletes a freshly provisioned sandbox when dispatch fails', async () => { + const placement = { spawn: vi.fn(async () => Promise.reject(new Error('dispatch failed'))) }; const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const ensureCloudFleetSandbox = vi.fn(async () => ({ + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-1', + nodeName: 'agent37-codex', + sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', + relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, + relayfileMounted: true, + relayfileMountPath: '/workspace', + providerId: 'agent37' as const, + })); + const errors: string[] = []; const program = new Command(); program.exitOverride(); registerFleetCommands(program, { sdk: { - createAgentRelay: vi.fn() as never, + createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, createWorkspaceRelay: vi.fn(() => ({ - workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + workspace: { + info: vi.fn(async () => ({ id: 'rw_abc' })), + register: vi.fn(async () => ({ token: 'at_live_isolated_launcher' })), + release: vi.fn(async () => ({ released: true, deleted: true })), + }, })) as never, createWorkspace: vi.fn() as never, log: vi.fn(), - error: vi.fn(), + error: (...args: unknown[]) => errors.push(args.join(' ')), exit: (() => { throw new Error('__exit__'); }) as never, }, - ensureCloudFleetSandbox: vi.fn(async () => { - throw new CloudFleetSandboxProvisionError('Cloud did not prove requested provider e2b.', { - cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', - sandboxId: 'sandbox-e2b', - nodeName: 'e2b-codex', - providerId: 'e2b', - outcomeUnknown: true, - }); + ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'flag', + origin: 'test', + workspaceId: 'rw_abc', }), + persistWorkspaceRelaycastTarget: () => true, deleteCloudFleetSandbox, createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, @@ -935,9 +1181,547 @@ describe('fleet command support', () => { 'codex', '--sandbox', '--sandbox-provider', - 'e2b', - '--sandbox-name', - 'e2b-codex', + 'agent37', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(errors.join('\n')).toContain('dispatch failed'); + expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( + expect.objectContaining({ + workloadProfile: 'long-running-agent', + }) + ); + expect(deleteCloudFleetSandbox).toHaveBeenCalledWith({ + cloudWorkspaceId: 'cloud-workspace', + sandboxId: 'sandbox-1', + providerId: 'agent37', + }); + }); + + it('preserves legacy custom sandbox names without sending a sandbox identity', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', undefined); + const placement = { + spawn: vi.fn(async () => ({ + invocationId: 'inv_legacy_sandbox', + node: { name: 'custom-node' }, + })), + }; + const register = vi.fn(async () => ({ token: 'at_live_launcher' })); + const release = vi.fn(async () => ({ released: true, deleted: true })); + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })), register, release }, + })); + const persistWorkspaceRelaycastTarget = vi.fn(() => true); + const ensureCloudFleetSandbox = vi.fn(async () => ({ + outcome: 'provisioned' as const, + providerId: 'e2b' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-legacy', + nodeName: 'custom-node', + sandboxId: 'legacy-public-sandbox', + providerSandboxId: 'legacy-provider-sandbox', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + relayfileMountPath: '/workspace', + })); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, + createWorkspaceRelay: createWorkspaceRelay as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'flag', + origin: 'test', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-name', + 'custom-node', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + + const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0]; + expect(persistWorkspaceRelaycastTarget).not.toHaveBeenCalled(); + expect(createWorkspaceRelay).toHaveBeenCalledWith({ + workspaceKey: 'rk_live_test', + token: undefined, + baseUrl: undefined, + ignorePersistedRelaycastTarget: true, + }); + expect(ensureInput).toMatchObject({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + forceProvision: true, + workloadProfile: 'long-running-agent', + name: 'custom-node', + }); + expect(ensureInput).not.toHaveProperty('sandboxId'); + }); + + it('rejects an explicit workspace ID that conflicts with the captured selection', async () => { + const ensureCloudFleetSandbox = vi.fn(); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn() as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: 'test', + workspaceId: 'rw_captured', + }), + persistWorkspaceRelaycastTarget: vi.fn(() => true), + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--workspace-id', + 'rw_other', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + expect(ensureCloudFleetSandbox).not.toHaveBeenCalled(); + }); + + it('accepts an explicit workspace ID matching the captured selection', async () => { + const ensureCloudFleetSandbox = vi.fn(async () => ({ + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-generated', + nodeName: 'generated-node', + sandboxId: 'generated-public-sandbox', + relayWorkspaceId: 'rw_captured', + relayfileMounted: true, + })); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn(() => ({ messaging: { placement: { spawn: vi.fn() } } })) as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { + register: vi.fn(async () => ({ token: 'at_live_launcher' })), + release: vi.fn(async () => ({ released: true, deleted: true })), + }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: 'test', + workspaceId: 'rw_captured', + }), + persistWorkspaceRelaycastTarget: () => true, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--no-sandbox-relayfile', + '--workspace-id', + 'rw_captured', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( + expect.objectContaining({ workspaceId: 'rw_captured' }) + ); + }); + + it('generates a stable identity and matching name when neither option is supplied', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', undefined); + const events: string[] = []; + const ensureCloudFleetSandbox = vi + .fn(async () => ({ + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-generated', + nodeName: 'generated-node', + sandboxId: 'generated-public-sandbox', + providerSandboxId: 'generated-provider-sandbox', + relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, + relayfileMounted: true, + relayfileMountPath: '/workspace', + })) + .mockImplementationOnce(async () => { + events.push('ensure'); + return { + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-generated', + nodeName: 'generated-node', + sandboxId: 'generated-public-sandbox', + providerSandboxId: 'generated-provider-sandbox', + relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, + relayfileMounted: true, + relayfileMountPath: '/workspace', + }; + }); + const createWorkspaceRelay = vi.fn(() => { + events.push('workspace-relay'); + return { + workspace: { + info: vi.fn(async () => ({ id: 'rw_abc' })), + register: vi.fn(async () => ({ token: 'at_live_launcher' })), + release: vi.fn(async () => ({ released: true, deleted: true })), + }, + }; + }); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn(() => ({ + messaging: { placement: { spawn: vi.fn(async () => ({ invocationId: 'inv_generated' })) } }, + })) as never, + createWorkspaceRelay: createWorkspaceRelay as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'flag', + origin: 'test', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + + const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0]; + expect(events.slice(0, 2)).toEqual(['ensure', 'workspace-relay']); + expect(createWorkspaceRelay).not.toHaveBeenCalledWith( + expect.objectContaining({ ignorePersistedRelaycastTarget: true }) + ); + expect(ensureInput?.sandboxId).toMatch( + /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ + ); + expect(ensureInput?.name).toBe(`fleet-sandbox-${ensureInput?.sandboxId?.slice('sbx_'.length)}`); + }); + + it('rejects a replay sandbox name that does not match its identity', async () => { + const ensureCloudFleetSandbox = vi.fn(); + const errors: string[] = []; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: (...args: unknown[]) => errors.push(args.join(' ')), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-id', + REPLAY_SANDBOX_ID, + '--sandbox-name', + 'custom-node', + '--workspace-id', + 'rw_abc', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(ensureCloudFleetSandbox).not.toHaveBeenCalled(); + expect(errors.join('\n')).toContain('--sandbox-name'); + expect(errors.join('\n')).toContain('when --sandbox-id is supplied'); + }); + + it('rejects an invalid replay sandbox ID before provisioning', async () => { + const ensureCloudFleetSandbox = vi.fn(); + const errors: string[] = []; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: (...args: unknown[]) => errors.push(args.join(' ')), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-id', + 'sandbox-1', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(ensureCloudFleetSandbox).not.toHaveBeenCalled(); + expect(errors.join('\n')).toContain('--sandbox-id must match lowercase sbx_'); + }); + + it('preserves the stable sandbox ID for replay when Cloud reports an unknown outcome', async () => { + const warnings: string[] = []; + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => { + throw new CloudFleetSandboxProvisionError('malformed response', { + cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + sandboxId: REPLAY_SANDBOX_ID, + nodeName: REPLAY_SANDBOX_NAME, + outcomeUnknown: true, + }); + }), + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: (...args: unknown[]) => warnings.push(args.join(' ')), + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-id', + REPLAY_SANDBOX_ID, + '--sandbox-name', + REPLAY_SANDBOX_NAME, + '--workspace-id', + 'rw_abc', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); + expect(warnings.join('\n')).toContain(`check Cloud Fleet for node '${REPLAY_SANDBOX_NAME}'`); + expect(warnings.join('\n')).toContain(`--sandbox-id '${REPLAY_SANDBOX_ID}'`); + }); + + it('pins cleanup to the requested E2B provider for a known post-provision failure', async () => { + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => { + throw new CloudFleetSandboxProvisionError('Cloud did not prove requested provider e2b.', { + cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + sandboxId: 'sandbox-e2b', + nodeName: 'e2b-codex', + providerId: 'e2b', + }); + }), + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-provider', + 'e2b', '--name', 'sandbox-worker', '--task', @@ -958,7 +1742,7 @@ describe('fleet command support', () => { }); }); - it('identifies the requested node when the provisioning response is interrupted', async () => { + it('keeps the caller sandbox ID for replay guidance after an identifier-less server failure', async () => { const warnings: string[] = []; const deleteCloudFleetSandbox = vi.fn(); const program = new Command(); @@ -980,6 +1764,7 @@ describe('fleet command support', () => { throw new CloudFleetSandboxProvisionError('request interrupted', { cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', nodeName: 'daytona-codex', + providerId: 'e2b', outcomeUnknown: true, }); }), @@ -997,8 +1782,12 @@ describe('fleet command support', () => { 'spawn', 'codex', '--sandbox', + '--sandbox-provider', + 'e2b', + '--sandbox-id', + REPLAY_SANDBOX_ID, '--sandbox-name', - 'daytona-codex', + REPLAY_SANDBOX_NAME, '--name', 'sandbox-worker', '--task', @@ -1014,6 +1803,7 @@ describe('fleet command support', () => { expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); expect(warnings.join('\n')).toContain("check Cloud Fleet for node 'daytona-codex'"); + expect(warnings.join('\n')).toContain(`--sandbox-id '${REPLAY_SANDBOX_ID}'`); }); it('warns when an unmounted sandbox cannot be cleaned up automatically', async () => { @@ -1040,10 +1830,19 @@ describe('fleet command support', () => { nodeId: 'node-1', nodeName: 'daytona-codex', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, relayfileMounted: false, providerId: 'agent37' as const, })), + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'flag', + origin: 'test', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, deleteCloudFleetSandbox: vi.fn(async () => Promise.reject(new Error('delete failed'))), createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, @@ -1058,6 +1857,8 @@ describe('fleet command support', () => { 'spawn', 'codex', '--sandbox', + '--sandbox-provider', + 'agent37', '--name', 'sandbox-worker', '--task', diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 2eebd392ee..0844ca22e9 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -28,9 +28,10 @@ import { redactSecrets } from '../lib/redact.js'; import { attributableReleaseReason } from '../lib/release-reason.js'; import { resolveAgentToken, - resolveBaseUrl, - resolveWorkspaceKey, + resolveWorkspaceSelection, + persistWorkspaceRelaycastTarget, resolveWorkspaceKeyWithSource, + resolveWorkspaceTransport, type SdkClientOptions, } from '../lib/sdk-client.js'; import { @@ -47,11 +48,15 @@ const SERVE_REPLACEMENT_MESSAGE = "for Cloud-managed nodes run 'relay cloud enroll --token ' first."; const FLEET_CLIS = new Set(['claude', 'codex', 'gemini', 'aider', 'goose', 'grok', 'opencode']); +const CLOUD_SANDBOX_ID_PATTERN = + /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; export interface FleetCommandDependencies { core: CoreDependencies; sdk: SdkCommandDeps; createFleetWorkspaceClient: (options: SdkClientOptions) => RelayWorkspaceThinClient; + resolveWorkspaceSelection: typeof resolveWorkspaceSelection; + persistWorkspaceRelaycastTarget: typeof persistWorkspaceRelaycastTarget; ensureCloudFleetSandbox: typeof ensureCloudFleetSandbox; deleteCloudFleetSandbox: typeof deleteCloudFleetSandbox; log: (...args: unknown[]) => void; @@ -66,11 +71,12 @@ function withFleetDefaults(overrides: Partial = {}): F return { core, sdk, - createFleetWorkspaceClient: (options) => - createWorkspaceClient({ - workspaceKey: resolveWorkspaceKey(options), - baseUrl: resolveBaseUrl(options), - }), + createFleetWorkspaceClient: (options) => { + const { workspaceKey, baseUrl } = resolveWorkspaceTransport(options); + return createWorkspaceClient({ workspaceKey, baseUrl }); + }, + resolveWorkspaceSelection, + persistWorkspaceRelaycastTarget, ensureCloudFleetSandbox, deleteCloudFleetSandbox, log: (...args: unknown[]) => console.log(...args), @@ -161,8 +167,13 @@ export function registerFleetCommands( '--sandbox', 'Provision a fresh Cloud sandbox node, mount this Relayfile workspace, and spawn there' ) - .option('--sandbox-name ', 'Name for the provisioned sandbox fleet node') - .option('--sandbox-provider ', 'Sandbox provider: daytona or e2b') + .option( + '--sandbox-name ', + 'Explicit sandbox node name (custom unless --sandbox-id requires matching fleet-sandbox-)' + ) + .option('--sandbox-id ', 'Reuse a caller-declared sbx_ identity for an exact replay') + .option('--workspace-id ', 'Explicit Relay workspace identity required for sandbox provisioning') + .option('--sandbox-provider ', 'Sandbox provider: daytona, e2b, or agent37') .option( '--sandbox-relayfile-path ', 'Mount only these Relayfile subtrees (each path must end in /**)' @@ -196,15 +207,22 @@ export function registerFleetCommands( let targetNode = optionalText(options.targetNode, 'Target node') ?? optionalText(options.node, 'Node'); const useSandbox = options.sandbox === true; const sandboxName = optionalText(options.sandboxName, 'Sandbox name'); + const sandboxIdOption = optionalText(options.sandboxId, 'Sandbox ID'); + const explicitWorkspaceId = optionalText(options.workspaceId, 'Workspace ID'); + if (sandboxIdOption !== undefined && !CLOUD_SANDBOX_ID_PATTERN.test(sandboxIdOption)) { + throw new Error('--sandbox-id must match lowercase sbx_ using an RFC 4122 UUID.'); + } const sandboxProviderText = optionalText(options.sandboxProvider, 'Sandbox provider'); const sandboxProvider: CloudFleetSandboxProviderId | undefined = sandboxProviderText === undefined ? undefined - : sandboxProviderText === 'daytona' || sandboxProviderText === 'e2b' + : sandboxProviderText === 'daytona' || + sandboxProviderText === 'e2b' || + sandboxProviderText === 'agent37' ? sandboxProviderText : undefined; if (sandboxProviderText !== undefined && sandboxProvider === undefined) { - throw new Error('--sandbox-provider must be daytona or e2b.'); + throw new Error('--sandbox-provider must be daytona, e2b, or agent37.'); } const mountSandboxRelayfile = options.sandboxRelayfile !== false; const sandboxRelayfilePaths = optionalTextList(options.sandboxRelayfilePath, 'Sandbox Relayfile path'); @@ -214,6 +232,12 @@ export function registerFleetCommands( if (!useSandbox && sandboxName) { throw new Error('--sandbox-name requires --sandbox.'); } + if (!useSandbox && sandboxIdOption) { + throw new Error('--sandbox-id requires --sandbox.'); + } + if (!useSandbox && explicitWorkspaceId) { + throw new Error('--workspace-id requires --sandbox.'); + } if (!useSandbox && sandboxProvider) { throw new Error('--sandbox-provider requires --sandbox.'); } @@ -247,14 +271,60 @@ export function registerFleetCommands( let sandbox: EnsureCloudFleetSandboxResult | undefined; let workspaceRelay: ReturnType | undefined; + let relaycastClientOptions = clientOptions; + let legacyWorkspaceClientOptions = clientOptions; if (useSandbox) { - workspaceRelay = deps.sdk.createWorkspaceRelay(clientOptions); - const workspaceInfo = await workspaceRelay.workspace.info(); - const relayWorkspaceId = workspaceInfo.id?.trim(); + // Cloud must be the first network authority for a sandbox invocation. + // A canonical Relaycast info call would both leak the workspace key and + // make it impossible to prove that Cloud's isolated target is the one + // subsequently used for registration and dispatch. + const workspaceSelection = deps.resolveWorkspaceSelection(clientOptions); + legacyWorkspaceClientOptions = { + ...clientOptions, + ...(sandboxProvider === 'agent37' ? {} : { ignorePersistedRelaycastTarget: true }), + }; + let relayWorkspaceId = explicitWorkspaceId ?? workspaceSelection?.workspaceId?.trim(); + // Legacy providers remain backward compatible: they may resolve the + // workspace from canonical Relaycast. Agent37 may not, because even a + // read there mutates rate-limit/presence accounting on the shared + // service and defeats the zero-shared-traffic canary proof. + if (!relayWorkspaceId && sandboxProvider === undefined) { + throw new Error( + 'Sandbox provisioning without --sandbox-provider requires a persisted Relay workspace identity; run `relay workspace pin` or pass --workspace-id.' + ); + } + if (!relayWorkspaceId && sandboxProvider !== undefined && sandboxProvider !== 'agent37') { + workspaceRelay = deps.sdk.createWorkspaceRelay(legacyWorkspaceClientOptions); + const workspaceInfo = await workspaceRelay.workspace.info(); + relayWorkspaceId = workspaceInfo.id?.trim(); + } if (!relayWorkspaceId) { - throw new Error('The current Relay workspace did not report an ID for Cloud provisioning.'); + throw new Error( + sandboxProvider === 'agent37' + ? 'Agent37 sandbox provisioning requires a persisted Relay workspace identity; run `relay workspace pin` or pass --workspace-id.' + : 'The current Relay workspace did not report an ID for Cloud provisioning.' + ); + } + const sandboxId = sandboxIdOption ?? (sandboxName === undefined ? `sbx_${randomUUID()}` : undefined); + const deterministicSandboxName = + sandboxId === undefined ? undefined : `fleet-sandbox-${sandboxId.slice('sbx_'.length)}`; + if ( + sandboxIdOption !== undefined && + sandboxName !== undefined && + sandboxName !== deterministicSandboxName + ) { + throw new Error( + `--sandbox-name must be '${deterministicSandboxName}' when --sandbox-id is supplied; custom names cannot preserve the one-to-one sandbox identity.` + ); + } + const effectiveSandboxName = deterministicSandboxName ?? sandboxName; + if ( + explicitWorkspaceId !== undefined && + workspaceSelection?.workspaceId !== undefined && + explicitWorkspaceId !== workspaceSelection.workspaceId.trim() + ) { + throw new Error('--workspace-id does not match the captured workspace identity.'); } - const requestedSandboxName = sandboxName ?? `fleet-sandbox-${randomUUID().slice(0, 8)}`; try { sandbox = await deps.ensureCloudFleetSandbox({ workspaceId: relayWorkspaceId, @@ -262,14 +332,27 @@ export function registerFleetCommands( maxAgents: 1, mountRelayfile: mountSandboxRelayfile, ...(sandboxRelayfilePaths === undefined ? {} : { relayfilePaths: sandboxRelayfilePaths }), + ...(sandboxId === undefined ? {} : { sandboxId }), forceProvision: true, ...(sandboxProvider === undefined ? {} : { providerId: sandboxProvider }), workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, - name: requestedSandboxName, + ...(effectiveSandboxName === undefined ? {} : { name: effectiveSandboxName }), }); } catch (error) { - if (error instanceof CloudFleetSandboxProvisionError && error.cloudWorkspaceId && error.sandboxId) { + if (error instanceof CloudFleetSandboxProvisionError && error.outcomeUnknown) { + deps.warn( + `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ + error.nodeName ?? effectiveSandboxName ?? 'the requested sandbox' + }'${ + sandboxId === undefined ? '' : ` before retrying with --sandbox-id '${sandboxId}'` + } so a sandbox is not left running.` + ); + } else if ( + error instanceof CloudFleetSandboxProvisionError && + error.cloudWorkspaceId && + error.sandboxId + ) { await deps .deleteCloudFleetSandbox({ cloudWorkspaceId: error.cloudWorkspaceId, @@ -283,15 +366,77 @@ export function registerFleetCommands( }` ); }); - } else if (error instanceof CloudFleetSandboxProvisionError && error.outcomeUnknown) { - deps.warn( - `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ - error.nodeName ?? requestedSandboxName - }' before retrying so a sandbox is not left running.` - ); } throw error; } + if (sandbox.outcome !== 'provisioning_timeout' && sandbox.relaycastTarget) { + // When Cloud returns a closed, server-owned target, apply it for any + // provider and outcome before registration, spawn, or launcher release. + // Rebuild both credentials and origin before any registration, spawn, + // or launcher release, then prove the authenticated client sees the + // exact workspace Cloud returned. + try { + const target = sandbox.relaycastTarget; + if ( + (sandboxProvider === 'agent37' && target.route !== 'agent37-isolated') || + target.workspaceId.trim() !== relayWorkspaceId.trim() || + (sandbox.outcome === 'provisioned' && + sandbox.relayWorkspaceId.trim() !== relayWorkspaceId.trim()) + ) { + throw new Error( + sandboxProvider === 'agent37' && target.route !== 'agent37-isolated' + ? 'Explicit Agent37 provisioning requires the isolated Agent37 Relaycast target.' + : 'Cloud returned a Relaycast target for a different workspace.' + ); + } + relaycastClientOptions = { + ...clientOptions, + workspaceKey: target.relaycastApiKey, + baseUrl: target.baseUrl, + }; + workspaceRelay = deps.sdk.createWorkspaceRelay(relaycastClientOptions); + const postEnsureWorkspace = await workspaceRelay.workspace.info(); + const postEnsureWorkspaceId = postEnsureWorkspace.id?.trim(); + if ( + !postEnsureWorkspaceId || + (sandbox.outcome === 'provisioned' && + postEnsureWorkspaceId !== sandbox.relayWorkspaceId.trim()) || + postEnsureWorkspaceId !== target.workspaceId.trim() + ) { + throw new Error( + 'Cloud returned a Relaycast workspace that could not be verified on the selected gateway.' + ); + } + if (!deps.persistWorkspaceRelaycastTarget(workspaceSelection, target)) { + throw new Error( + 'Cloud returned a Relaycast target, but no durable project session is available for follow-up attach.' + ); + } + } catch (error) { + if (sandbox.outcome === 'provisioned') { + await deps + .deleteCloudFleetSandbox({ + cloudWorkspaceId: sandbox.cloudWorkspaceId, + sandboxId: sandbox.sandboxId, + ...(sandbox.providerId === undefined ? {} : { providerId: sandbox.providerId }), + }) + .catch((cleanupError) => { + deps.warn( + `Relaycast workspace verification failed and sandbox cleanup also failed: ${ + cleanupError instanceof Error ? cleanupError.message : String(cleanupError) + }` + ); + }); + } + throw error; + } + } else if (sandbox.outcome !== 'provisioning_timeout') { + // Older non-Agent37 Cloud responses can omit a target. In that + // compatibility case, keep every subsequent client on the canonical + // workspace selection; a stale persisted Agent37 target must not + // leak into registration, dispatch, or launcher release. + relaycastClientOptions = legacyWorkspaceClientOptions; + } if (sandbox.outcome === 'provisioning_timeout') { await deps .deleteCloudFleetSandbox({ @@ -340,9 +485,18 @@ export function registerFleetCommands( if (targetNode) { let launcherName: string | undefined; try { - let agentToken = resolveAgentToken(clientOptions); + // Agent tokens are scoped to a Relaycast deployment. Never replay a + // canonical token after Cloud has selected the isolated shard; mint + // a temporary launcher on the validated target instead. + // A sandbox dispatch always mints a launcher on the transport Cloud + // selected (or the canonical compatibility transport when an older + // non-Agent37 response omitted the target). Ambient agent tokens do + // not carry enough provenance to prove they belong to that transport. + let agentToken = sandbox ? undefined : resolveAgentToken(clientOptions); if (!agentToken) { - workspaceRelay ??= deps.sdk.createWorkspaceRelay(clientOptions); + workspaceRelay ??= deps.sdk.createWorkspaceRelay( + sandbox?.relaycastTarget ? relaycastClientOptions : legacyWorkspaceClientOptions + ); const pendingLauncherName = `fleet-spawn-launcher-${randomUUID().slice(0, 8)}`; const launcher = await workspaceRelay.workspace.register( { @@ -358,7 +512,7 @@ export function registerFleetCommands( } } - const relay = deps.sdk.createAgentRelay({ ...clientOptions, token: agentToken }); + const relay = deps.sdk.createAgentRelay({ ...relaycastClientOptions, token: agentToken }); // Placement alone only proves the node accepted the dispatch. A node // running an obsolete broker advertises `spawn:` capacity, acks // the invocation and launches nothing, which is indistinguishable from @@ -381,13 +535,28 @@ export function registerFleetCommands( ...(sessionRef ? { session_ref: sessionRef } : {}), }, }); + const printableSandbox = + (sandbox?.outcome === 'provisioned' || sandbox?.outcome === 'reused') && sandbox.relaycastTarget + ? { + ...sandbox, + relaycastTarget: { + route: sandbox.relaycastTarget.route, + baseUrl: sandbox.relaycastTarget.baseUrl, + workspaceId: sandbox.relaycastTarget.workspaceId, + }, + } + : sandbox; printJson(deps.sdk, { ...(sandbox ? { - sandbox, + sandbox: printableSandbox, attachCommand: `agent-relay node agent attach ${shellQuote(name)} ` + - `--node ${shellQuote(targetNode)} --mode drive`, + `--node ${shellQuote(targetNode)} --mode drive` + + ((sandbox.outcome === 'provisioned' || sandbox.outcome === 'reused') && + sandbox.relaycastTarget + ? ` --base-url ${shellQuote(sandbox.relaycastTarget.baseUrl)}` + : ''), } : {}), invocation, diff --git a/packages/cli/src/cli/commands/integration.ts b/packages/cli/src/cli/commands/integration.ts index 2172dff128..9ee581272c 100644 --- a/packages/cli/src/cli/commands/integration.ts +++ b/packages/cli/src/cli/commands/integration.ts @@ -29,7 +29,7 @@ import { assertRelayfileVersion, type RelayfileClientOptions, } from '@relayfile/client'; -import { resolveBaseUrl, resolveWorkspaceKey } from '../lib/sdk-client.js'; +import { resolveBaseUrl, resolveWorkspaceKey, resolveWorkspaceTransport } from '../lib/sdk-client.js'; // Re-export the version gate so existing tests importing it from this module // (and any callers) keep working after it moved to the published client package. @@ -473,8 +473,14 @@ async function createRelayfileInboundTarget( const options = sdkOptionsFromOpts(commandOpts); const authOptions = local && !explicitWorkspaceKey(commandOpts) ? localRetryOptions(options, local) : options; - const workspaceKey = resolveWorkspaceKey(authOptions); - const baseUrl = resolveInboundTargetBaseUrl(options); + // A local broker session may expose its loopback broker URL. That URL is not + // the Relaycast control-plane origin for inbound-target provisioning, so pair + // its workspace selector with the caller's Relaycast URL (or canonical). + const { workspaceKey, baseUrl: selectedBaseUrl } = resolveWorkspaceTransport({ + ...authOptions, + baseUrl: options.baseUrl, + }); + const baseUrl = resolveInboundTargetBaseUrl(selectedBaseUrl); const response = await fetch(new URL('/v1/integrations/relayfile/inbound-target', baseUrl), { method: 'POST', headers: { @@ -525,8 +531,8 @@ function parseRelayfileInboundTargetResponse(body: unknown): { url: string; secr return { url: parsedUrl.toString(), secret }; } -function resolveInboundTargetBaseUrl(options: SdkClientOptions): string { - const baseUrl = resolveBaseUrl(options) ?? 'https://cast.agentrelay.com'; +function resolveInboundTargetBaseUrl(selectedBaseUrl: string | undefined): string { + const baseUrl = selectedBaseUrl ?? 'https://cast.agentrelay.com'; const parsed = new URL(baseUrl); if (parsed.protocol !== 'https:') { throw new Error('Inbound relayfile target provisioning requires an https Relaycast base URL.'); diff --git a/packages/cli/src/cli/commands/local-agent.test.ts b/packages/cli/src/cli/commands/local-agent.test.ts index eb4f8959c4..8e827dc12d 100644 --- a/packages/cli/src/cli/commands/local-agent.test.ts +++ b/packages/cli/src/cli/commands/local-agent.test.ts @@ -171,6 +171,29 @@ describe('local agent subtree', () => { ); }); + it('attach --node forwards an explicit base URL for an isolated fleet workspace', async () => { + const { program, attachNode } = harness(); + await program.parseAsync( + [ + 'local', + 'agent', + 'attach', + 'sandbox-worker', + '--node', + 'agent37-codex', + '--base-url', + 'https://agent37-cast.agentrelay.com', + ], + { from: 'user' } + ); + expect(attachNode).toHaveBeenCalledWith( + 'sandbox-worker', + 'view', + 'agent37-codex', + expect.objectContaining({ baseUrl: 'https://agent37-cast.agentrelay.com' }) + ); + }); + it('attach --node without --workspace-key leaves the precedence ladder to resolve it', async () => { const { program, attachNode } = harness(); await program.parseAsync(['local', 'agent', 'attach', 'lead', '--node', 'sf-mini'], { from: 'user' }); @@ -228,6 +251,69 @@ describe('local agent subtree', () => { expect(JSON.stringify([...log.mock.calls, ...error.mock.calls])).not.toContain('rjt_live_one_time'); }); + it('attach --node redeems an isolated-shard join ticket at the explicit base URL', async () => { + const { program, attachNode, redeemJoinTicket, persistWorkspaceSession } = harness(); + await program.parseAsync( + [ + 'local', + 'agent', + 'attach', + 'lead', + '--node', + 'agent37-codex', + '--join-ticket', + 'rjt_live_one_time', + '--base-url', + 'https://agent37-cast.agentrelay.com', + ], + { from: 'user' } + ); + + expect(redeemJoinTicket).toHaveBeenCalledWith( + expect.objectContaining({ + ticket: 'rjt_live_one_time', + node: 'agent37-codex', + baseUrl: 'https://agent37-cast.agentrelay.com', + }) + ); + expect(persistWorkspaceSession).toHaveBeenCalledWith( + expect.objectContaining({ + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_redeemed', + }) + ); + expect(attachNode).toHaveBeenCalledWith( + 'lead', + 'view', + 'agent37-codex', + expect.objectContaining({ baseUrl: 'https://agent37-cast.agentrelay.com' }) + ); + }); + + it('rejects an untrusted join-ticket base URL before redemption', async () => { + const { program, redeemJoinTicket, attachNode, error } = harness(); + await program.parseAsync( + [ + 'local', + 'agent', + 'attach', + 'lead', + '--node', + 'agent37-codex', + '--join-ticket', + 'rjt_live_one_time', + '--base-url', + 'https://evil.example.com', + ], + { from: 'user' } + ); + + expect(redeemJoinTicket).not.toHaveBeenCalled(); + expect(attachNode).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledWith(expect.stringContaining('trusted Relaycast origin')); + }); + it.each(['expired', 'invalid'])( 'attach --node reports a clear %s join-ticket error instead of falling through to workspace resolution', async (reason) => { diff --git a/packages/cli/src/cli/commands/local-agent.ts b/packages/cli/src/cli/commands/local-agent.ts index 05bb9f682f..9f3ae3599b 100644 --- a/packages/cli/src/cli/commands/local-agent.ts +++ b/packages/cli/src/cli/commands/local-agent.ts @@ -1,5 +1,6 @@ import type { Command } from 'commander'; +import { AGENT37_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; import { HarnessDriverClient } from '@agent-relay/harness-driver'; import type { InboundDeliveryMode, ListAgent, PendingRelayMessage } from '@agent-relay/harness-driver'; import type { HarnessRuntime } from '@agent-relay/harnesses'; @@ -12,7 +13,7 @@ import type { AttachMode } from '../lib/attach-mode.js'; import { attachNative, isNativeHarness, type NativeAttachOptions } from '../lib/attach-native.js'; import { attachPassthrough } from '../lib/attach-passthrough.js'; import { attachRemoteNode, type RemoteNodeAttachOptions } from '../lib/attach-remote-node.js'; -import { startFleetNodeAttachProxy } from '../lib/attach-fleet-node.js'; +import { startFleetNodeAttachProxy, validateFleetAttachBaseUrl } from '../lib/attach-fleet-node.js'; import { attachView } from '../lib/attach-view.js'; import { createBackpressureAwareWriter } from '../lib/attach.js'; import { @@ -88,6 +89,7 @@ export function runAttach(name: string, mode: AttachMode, options: NativeAttachO * rather than accepted and ignored. */ export type FleetNodeAttachCliOptions = Pick & { + baseUrl?: string; workspaceKey?: string; }; @@ -106,6 +108,7 @@ export async function attachFleetNode( agent: name, node, mode, + ...(options.baseUrl === undefined ? {} : { baseUrl: options.baseUrl }), ...(options.workspaceKey === undefined ? {} : { workspaceKey: options.workspaceKey }), }); const jsonWriter = options.json ? createBackpressureAwareWriter(process.stdout) : undefined; @@ -278,19 +281,27 @@ function resolveAttachCredentialSelection( async function redeemAndPersistAttachCredential( deps: LocalAgentDependencies, - options: { ticket: string; node: string; agent: string; mode: AttachMode } + options: { ticket: string; node: string; agent: string; mode: AttachMode; baseUrl?: string } ): Promise { const redeemed = await deps.redeemJoinTicket({ ticket: options.ticket, node: options.node, agent: options.agent, mode: options.mode, + ...(options.baseUrl === undefined ? {} : { baseUrl: options.baseUrl }), env: deps.env, fetch: deps.fetch, }); const persisted = deps.persistWorkspaceSession({ workspaceKey: redeemed.workspaceKey, workspaceId: redeemed.workspaceId, + ...(options.baseUrl + ? { + relaycastRoute: options.baseUrl === AGENT37_RELAYCAST_ORIGIN ? 'agent37-isolated' : 'canonical', + relaycastBaseUrl: options.baseUrl, + relaycastApiKey: redeemed.workspaceKey, + } + : {}), projectRoot: deps.cwd(), }); const warning = describeClearedEnrollment(persisted); @@ -845,6 +856,10 @@ export function registerLocalAgentCommands( .argument('', 'Agent name') .option('--mode ', 'drive | view | passthrough', 'view') .option('--node ', 'Canonical authenticated fleet-node terminal attach (physical or Daytona)') + .option( + '--base-url ', + 'Relaycast API base URL for --node (defaults to the selected workspace route)' + ) .option('--ssh-host ', 'SSH host fallback for a physical fleet node') .option('--broker-url ', 'Broker base URL (overrides RELAY_BROKER_URL and connection.json)') .option('--api-key ', 'Broker API key (overrides RELAY_BROKER_API_KEY and connection.json)') @@ -907,6 +922,10 @@ export function registerLocalAgentCommands( } try { let attachWorkspaceKey = credential.workspaceKey; + const validatedBaseUrl = + credential.joinTicket && typeof options.baseUrl === 'string' + ? validateFleetAttachBaseUrl(options.baseUrl) + : undefined; if (credential.joinTicket) { // Pass the redeemed key explicitly into the first attach. Merely // writing the project pin would leave this process vulnerable to @@ -916,9 +935,11 @@ export function registerLocalAgentCommands( node, agent: name, mode, + baseUrl: validatedBaseUrl, }); } const code = await deps.attachNode(name, mode, node, { + baseUrl: options.baseUrl as string | undefined, workspaceKey: attachWorkspaceKey, json: options.json as boolean | undefined, reasoning: options.reasoning as boolean | undefined, diff --git a/packages/cli/src/cli/commands/observer.test.ts b/packages/cli/src/cli/commands/observer.test.ts index 65ca1c6009..9723d1a82f 100644 --- a/packages/cli/src/cli/commands/observer.test.ts +++ b/packages/cli/src/cli/commands/observer.test.ts @@ -1,8 +1,12 @@ import { Command } from 'commander'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { registerObserverCommands, type ObserverCommandDependencies } from './observer.js'; import { observerUrl, resolveObserverBaseUrl } from '../lib/observer-url.js'; +import { writeProjectWorkspaceKey } from '../lib/project-workspace-key.js'; class ExitSignal extends Error { constructor(public readonly code: number) { @@ -94,6 +98,33 @@ describe('agent-relay observer', () => { expect(logs.join('\n')).not.toContain(WORKSPACE_KEY); }); + it('mints through the persisted credential and origin as one transport pair', async () => { + const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-observer-project-')); + vi.stubEnv('AGENT_RELAY_PROJECT', projectRoot); + writeProjectWorkspaceKey(path.join(projectRoot, '.agentworkforce/relay'), WORKSPACE_KEY, { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_isolated_observer', + }); + const { program, createObserverToken } = setup(); + + try { + await program.parseAsync( + ['observer', '--workspace-key', WORKSPACE_KEY, '--base-url', 'https://agent37-cast.agentrelay.com/'], + { from: 'user' } + ); + + const [call] = createObserverToken.mock.calls as unknown as [[Record]]; + expect(call[0]).toMatchObject({ + workspaceKey: 'rk_live_isolated_observer', + baseUrl: 'https://agent37-cast.agentrelay.com', + }); + } finally { + fs.rmSync(projectRoot, { recursive: true, force: true }); + } + }); + it('narrows to channels and includes DMs when asked', async () => { const { program, createObserverToken } = setup(); diff --git a/packages/cli/src/cli/commands/observer.ts b/packages/cli/src/cli/commands/observer.ts index d76ad6315a..233c6c65f3 100644 --- a/packages/cli/src/cli/commands/observer.ts +++ b/packages/cli/src/cli/commands/observer.ts @@ -25,7 +25,7 @@ import { resolveObserverBaseUrl, } from '../lib/observer-url.js'; import { printJson, runSdk, withSdkDefaults, type SdkCommandDeps } from '../lib/sdk-command.js'; -import { resolveBaseUrl, resolveWorkspaceKey } from '../lib/sdk-client.js'; +import { resolveWorkspaceTransport } from '../lib/sdk-client.js'; const MAX_CHANNEL_FILTERS = 50; @@ -100,11 +100,12 @@ function parseChannels(value: string): string[] { * `undefined` and an absent key differently. */ function connection(options: Record): { workspaceKey: string; baseUrl?: string } { - const baseUrl = resolveBaseUrl({ baseUrl: options.baseUrl as string | undefined }); + const { workspaceKey, baseUrl } = resolveWorkspaceTransport({ + workspaceKey: options.workspaceKey as string | undefined, + baseUrl: options.baseUrl as string | undefined, + }); return { - workspaceKey: resolveWorkspaceKey({ - workspaceKey: options.workspaceKey as string | undefined, - }), + workspaceKey, ...(baseUrl ? { baseUrl } : {}), }; } diff --git a/packages/cli/src/cli/lib/attach-fleet-node.test.ts b/packages/cli/src/cli/lib/attach-fleet-node.test.ts index 252894fc61..050b38751a 100644 --- a/packages/cli/src/cli/lib/attach-fleet-node.test.ts +++ b/packages/cli/src/cli/lib/attach-fleet-node.test.ts @@ -6,11 +6,15 @@ * under test is the actual runtime wiring rather than a mocked stand-in. */ import type { AddressInfo } from 'node:net'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { WebSocket as WsClient, WebSocketServer, type WebSocket as WsSocket } from 'ws'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { startFleetNodeAttachProxy, type FleetNodeAttachProxy } from './attach-fleet-node.js'; +import { writeProjectWorkspaceKey } from './project-workspace-key.js'; const SESSION_ID = 'session-under-test'; const RESUME_TOKEN = 'resume-token'; @@ -161,7 +165,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-transient', node: 'node-transient', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, sessionRequest: { @@ -217,7 +221,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-slow-healthy', node: 'node-slow-healthy', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, }); @@ -262,7 +266,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-slow-dead', node: 'node-slow-dead', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, }).catch((error: unknown) => error); @@ -306,7 +310,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-budget-expired', node: 'node-budget-expired', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, sessionRequest: { @@ -351,7 +355,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-dead', node: 'node-dead', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, sessionRequest: { @@ -370,7 +374,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { expect((rejected as Error).message).toContain('resolved node id unavailable'); expect((rejected as Error).message).toContain('attempts 5 (retried 4 times)'); expect((rejected as Error).message).toContain( - 'endpoint "https://fake.example/v1/nodes/node-dead/terminal/sessions"' + 'endpoint "https://cast.agentrelay.com/v1/nodes/node-dead/terminal/sessions"' ); }); @@ -385,7 +389,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-missing-node', node: 'node-missing', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, sessionRequest: { sleep: async () => undefined }, @@ -409,7 +413,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-unknown-completion', node: 'node-unknown-completion', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, sessionRequest: { sleep: async () => undefined }, @@ -436,7 +440,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-malformed', node: 'node-malformed', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, sessionRequest: { sleep: async () => undefined }, @@ -467,7 +471,7 @@ describe('startFleetNodeAttachProxy delivery-mode PUT lifecycle', () => { agent: 'agent-a', node: 'node-a', mode: 'drive', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -508,7 +512,7 @@ describe('startFleetNodeAttachProxy delivery-mode PUT lifecycle', () => { agent: 'agent-readiness', node: 'node-readiness', mode: 'drive', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -576,7 +580,7 @@ describe('startFleetNodeAttachProxy delivery-mode PUT lifecycle', () => { agent: 'agent-b', node: 'node-b', mode: 'drive', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -617,7 +621,7 @@ describe('startFleetNodeAttachProxy delivery-mode PUT lifecycle', () => { agent: 'agent-c', node: 'node-c', mode: 'drive', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -653,7 +657,7 @@ describe('startFleetNodeAttachProxy delivery-mode PUT lifecycle', () => { agent: 'agent-d', node: 'node-d', mode: 'drive', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -704,7 +708,7 @@ describe('startFleetNodeAttachProxy view target lifecycle', () => { agent: 'view-delayed-upgrade', node: 'node-delayed-upgrade', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), reconnectDelay: { handshakeTimeoutMs: 10_000, readyTimeoutMs: 4_000 }, @@ -737,7 +741,7 @@ describe('startFleetNodeAttachProxy view target lifecycle', () => { agent: 'view-backoff-snapshot', node: 'node-backoff-snapshot', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), reconnectDelay: { @@ -772,7 +776,7 @@ describe('startFleetNodeAttachProxy view target lifecycle', () => { agent: 'view-target', node: 'view-node', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -826,7 +830,7 @@ describe('startFleetNodeAttachProxy view target lifecycle', () => { agent: 'view-reconnect', node: 'node-reconnect', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), reconnectDelay: { initialMs: 1, maxMs: 1 }, @@ -877,7 +881,7 @@ describe('startFleetNodeAttachProxy view target lifecycle', () => { // path, including a multi-byte UTF-8 boundary. node: `node-exhaust-${'é'.repeat(100)}`, mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), reconnectDelay: { initialMs: 1, maxMs: 1 }, @@ -917,7 +921,7 @@ describe('startFleetNodeAttachProxy view target lifecycle', () => { agent: 'view-stalled-resume', node: 'node-stalled-resume', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), reconnectDelay: { @@ -1002,7 +1006,7 @@ describe('startFleetNodeAttachProxy readiness-gate status mapping', () => { agent, node: 'node-mapping', mode: 'drive', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remoteUrl), }); @@ -1121,9 +1125,12 @@ describe('startFleetNodeAttachProxy workspace-key precedence', () => { function capturingTicketFetch(remoteUrl: string): { fetch: typeof globalThis.fetch; authorization: () => string | undefined; + requestUrl: () => string | undefined; } { let seen: string | undefined; - const fetchFn = (async (_url: string, init?: RequestInit) => { + let requestedUrl: string | undefined; + const fetchFn = (async (url: string, init?: RequestInit) => { + requestedUrl = url; const headers = (init?.headers ?? {}) as Record; seen = headers.Authorization; return { @@ -1139,7 +1146,7 @@ describe('startFleetNodeAttachProxy workspace-key precedence', () => { }), } as unknown as Response; }) as unknown as typeof globalThis.fetch; - return { fetch: fetchFn, authorization: () => seen }; + return { fetch: fetchFn, authorization: () => seen, requestUrl: () => requestedUrl }; } it('presents an explicit workspace key ahead of the ambient environment', async () => { @@ -1150,7 +1157,7 @@ describe('startFleetNodeAttachProxy workspace-key precedence', () => { agent: 'agent-e', node: 'node-e', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'rk_live_explicit', env: { RELAY_WORKSPACE_KEY: 'rk_live_ambient' }, fetch: ticket.fetch, @@ -1160,6 +1167,123 @@ describe('startFleetNodeAttachProxy workspace-key precedence', () => { expect(ticket.authorization()).toBe('Bearer rk_live_explicit'); }); + it('uses an explicit isolated base URL instead of the canonical fallback', async () => { + const remote = await startFakeRemote(); + cleanup.push(remote.close); + const ticket = capturingTicketFetch(remote.url); + const proxy = await startFleetNodeAttachProxy({ + agent: 'sandbox-worker', + node: 'agent37-codex', + mode: 'view', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceKey: 'rk_live_agent37_target', + env: { RELAY_WORKSPACE_KEY: 'rk_live_ambient' }, + fetch: ticket.fetch, + }); + cleanup.push(proxy.close); + + expect(ticket.requestUrl()).toBe( + 'https://agent37-cast.agentrelay.com/v1/nodes/agent37-codex/terminal/sessions' + ); + expect(ticket.requestUrl()).not.toBe( + 'https://cast.agentrelay.com/v1/nodes/agent37-codex/terminal/sessions' + ); + }); + + it('binds a matching explicit canonical selector to its persisted isolated key and origin', async () => { + const remote = await startFakeRemote(); + cleanup.push(remote.close); + const ticket = capturingTicketFetch(remote.url); + const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-attach-project-')); + const priorProject = process.env.AGENT_RELAY_PROJECT; + process.env.AGENT_RELAY_PROJECT = projectRoot; + writeProjectWorkspaceKey(path.join(projectRoot, '.agentworkforce/relay'), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_isolated', + }); + + try { + const proxy = await startFleetNodeAttachProxy({ + agent: 'sandbox-worker', + node: 'agent37-codex', + mode: 'view', + baseUrl: 'https://agent37-cast.agentrelay.com/', + workspaceKey: 'rk_live_canonical', + env: {}, + fetch: ticket.fetch, + }); + cleanup.push(proxy.close); + + expect(ticket.authorization()).toBe('Bearer rk_live_isolated'); + expect(ticket.requestUrl()).toBe( + 'https://agent37-cast.agentrelay.com/v1/nodes/agent37-codex/terminal/sessions' + ); + } finally { + if (priorProject === undefined) delete process.env.AGENT_RELAY_PROJECT; + else process.env.AGENT_RELAY_PROJECT = priorProject; + fs.rmSync(projectRoot, { recursive: true, force: true }); + } + }); + + it('does not inherit a persisted route when an explicit key selects a different workspace', async () => { + const remote = await startFakeRemote(); + cleanup.push(remote.close); + const ticket = capturingTicketFetch(remote.url); + const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-attach-project-')); + const priorProject = process.env.AGENT_RELAY_PROJECT; + process.env.AGENT_RELAY_PROJECT = projectRoot; + writeProjectWorkspaceKey(path.join(projectRoot, '.agentworkforce/relay'), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_isolated', + }); + + try { + const proxy = await startFleetNodeAttachProxy({ + agent: 'other-worker', + node: 'other-node', + mode: 'view', + workspaceKey: 'rk_live_other', + env: {}, + fetch: ticket.fetch, + }); + cleanup.push(proxy.close); + + expect(ticket.authorization()).toBe('Bearer rk_live_other'); + expect(ticket.requestUrl()).toBe('https://cast.agentrelay.com/v1/nodes/other-node/terminal/sessions'); + } finally { + if (priorProject === undefined) delete process.env.AGENT_RELAY_PROJECT; + else process.env.AGENT_RELAY_PROJECT = priorProject; + fs.rmSync(projectRoot, { recursive: true, force: true }); + } + }); + + it.each([ + 'https://evil.example', + 'http://cast.agentrelay.com', + 'https://cast.agentrelay.com.attacker.example', + 'https://cast.agentrelay.com/path', + ])('rejects an untrusted explicit base URL before sending the workspace key', async (baseUrl) => { + const fetch = vi.fn(); + + await expect( + startFleetNodeAttachProxy({ + agent: 'agent-attacker', + node: 'node-attacker', + mode: 'view', + baseUrl, + workspaceKey: 'rk_live_must_not_leave_process', + env: {}, + fetch, + }) + ).rejects.toThrow(/trusted Relaycast origin/); + + expect(fetch).not.toHaveBeenCalled(); + }); + it('falls back to the environment when no explicit key is supplied', async () => { const remote = await startFakeRemote(); cleanup.push(remote.close); @@ -1168,7 +1292,7 @@ describe('startFleetNodeAttachProxy workspace-key precedence', () => { agent: 'agent-f', node: 'node-f', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', env: { RELAY_WORKSPACE_KEY: 'rk_live_ambient' }, fetch: ticket.fetch, }); @@ -1212,7 +1336,7 @@ describe('startFleetNodeAttachProxy flush route', () => { // changing delivery mode, so it must not have to seize the single drive // slot from whoever is attached. mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -1263,7 +1387,7 @@ describe('startFleetNodeAttachProxy flush route', () => { agent: 'agent-missing', node: 'node-remote', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -1304,7 +1428,7 @@ describe('startFleetNodeAttachProxy flush route', () => { agent: 'agent-noid', node: 'node-remote', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -1361,7 +1485,7 @@ describe('startFleetNodeAttachProxy flush route', () => { agent: 'agent-cross', node: 'node-remote', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); diff --git a/packages/cli/src/cli/lib/attach-fleet-node.ts b/packages/cli/src/cli/lib/attach-fleet-node.ts index a6375fd40a..634b9aaaed 100644 --- a/packages/cli/src/cli/lib/attach-fleet-node.ts +++ b/packages/cli/src/cli/lib/attach-fleet-node.ts @@ -15,9 +15,10 @@ import { randomBytes } from 'node:crypto'; import WebSocket, { WebSocketServer } from 'ws'; +import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; import type { AttachMode } from './attach-mode.js'; import { collectWithRetry } from './collect-with-retry.js'; -import { resolveBaseUrl, resolveWorkspaceKey } from './sdk-client.js'; +import { resolveWorkspaceTransport } from './sdk-client.js'; const MAX_BUFFERED_BYTES = 1024 * 1024; const MAX_WEBSOCKET_CLOSE_REASON_BYTES = 123; @@ -114,6 +115,30 @@ export class FleetNodeAttachError extends Error { } } +const TRUSTED_RELAYCAST_ORIGINS = new Set([CANONICAL_RELAYCAST_ORIGIN, AGENT37_RELAYCAST_ORIGIN]); + +export function validateFleetAttachBaseUrl(value: string): string { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new FleetNodeAttachError('Fleet node attach requires a trusted Relaycast origin.'); + } + if ( + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') || + !TRUSTED_RELAYCAST_ORIGINS.has(parsed.origin) + ) { + throw new FleetNodeAttachError('Fleet node attach requires a trusted Relaycast origin.'); + } + return parsed.origin; +} + class TerminalSessionAttemptError extends FleetNodeAttachError { constructor( message: string, @@ -290,11 +315,12 @@ export async function startFleetNodeAttachProxy( ): Promise { const env = options.env ?? process.env; const fetchFn = options.fetch ?? globalThis.fetch; - const workspaceKey = options.workspaceKey ?? resolveWorkspaceKey({ env }); - const baseUrl = (options.baseUrl ?? resolveBaseUrl({ env }) ?? 'https://cast.agentrelay.com').replace( - /\/+$/, - '' - ); + const { workspaceKey, baseUrl: requestedBaseUrl } = resolveWorkspaceTransport({ + workspaceKey: options.workspaceKey, + baseUrl: options.baseUrl, + env, + }); + const baseUrl = validateFleetAttachBaseUrl(requestedBaseUrl ?? CANONICAL_RELAYCAST_ORIGIN); const nodePath = safeNodePath(options.node); const sessionEndpoint = `${baseUrl}/v1/nodes/${nodePath}/terminal/sessions`; const sessionRequestTimeoutMs = options.sessionRequest?.timeoutMs ?? SESSION_REQUEST_TIMEOUT_MS; diff --git a/packages/cli/src/cli/lib/broker-lifecycle.test.ts b/packages/cli/src/cli/lib/broker-lifecycle.test.ts index a83a1e4662..19972961f8 100644 --- a/packages/cli/src/cli/lib/broker-lifecycle.test.ts +++ b/packages/cli/src/cli/lib/broker-lifecycle.test.ts @@ -701,6 +701,46 @@ describe('runUpCommand workspace precedence', () => { expect(readPin(dataDir)).toMatchObject({ workspaceKey: 'rk_test', workspaceId: 'rw_test' }); }); + it('retains the Relaycast target when the broker keeps the pinned workspace', async () => { + const { deps, dataDir } = createUpHarness(); + writeRepositoryPin(dataDir, { + workspaceKey: 'rk_test', + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + + await runUpCommand({}, deps); + + expect(readPin(dataDir)).toMatchObject({ + workspaceKey: 'rk_test', + workspaceId: 'rw_test', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + }); + + it('clears the old Relaycast target when the broker changes workspace', async () => { + const { deps, dataDir, createRelay } = createUpHarness(); + writeRepositoryPin(dataDir, { + workspaceKey: 'rk_old', + workspaceId: 'rw_old', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + createRelay.mockImplementationOnce(async () => ({ + spawn: vi.fn(async () => undefined), + getStatus: vi.fn(async () => ({})), + shutdown: vi.fn(async () => undefined), + workspaceKey: 'rk_new', + workspaceId: 'rw_new', + })); + + await runUpCommand({}, deps); + + expect(readPin(dataDir)).toEqual({ workspaceKey: 'rk_new', workspaceId: 'rw_new' }); + }); + it('never prints workspace key material while reporting the winning source', async () => { const { deps, dataDir, home, log, warn, error } = createUpHarness(); setWorkspaceKey('global', 'rk_global', { AGENT_RELAY_HOME: home }); diff --git a/packages/cli/src/cli/lib/broker-lifecycle.ts b/packages/cli/src/cli/lib/broker-lifecycle.ts index 4fba55ae80..1bfe0a6375 100644 --- a/packages/cli/src/cli/lib/broker-lifecycle.ts +++ b/packages/cli/src/cli/lib/broker-lifecycle.ts @@ -30,7 +30,7 @@ import { startReflexCapture, type RunningReflexCapture } from './reflex-capture. import { readProjectWorkspaceSession, resolveWorkspaceSelection, - writeProjectWorkspaceKey, + writeProjectWorkspaceKeyPreservingSession, type ProjectWorkspaceSession, type WorkspaceSelection, } from './project-workspace-key.js'; @@ -1953,12 +1953,16 @@ export async function runUpCommand(options: UpOptions, deps: CoreDependencies): // workspace. Persistence must never abort startup, so a write failure is // swallowed. try { - writeProjectWorkspaceKey(projectWorkspaceKeyDataDir, relay.workspaceKey ?? undefined, { + const relayWorkspaceKey = relay.workspaceKey ?? undefined; + const sameWorkspace = resumedProjectSession?.workspaceKey === relayWorkspaceKey; + writeProjectWorkspaceKeyPreservingSession(projectWorkspaceKeyDataDir, relayWorkspaceKey, { enrolledNodeId: deps.env.AGENT_RELAY_ENROLLED_NODE_ID ?? resumedProjectSession?.enrolledNodeId, // Recording the resolved workspace id lets the NEXT start detect a // conflicting source (a stored enrollment in another workspace) before // the broker comes up, instead of after agents land in the wrong place. - workspaceId: relay.workspaceId ?? resumedProjectSession?.workspaceId, + ...((relay.workspaceId ?? (sameWorkspace ? resumedProjectSession?.workspaceId : undefined)) + ? { workspaceId: relay.workspaceId ?? resumedProjectSession?.workspaceId } + : {}), }); } catch { // best-effort: a broker that came up should stay up even if the key file diff --git a/packages/cli/src/cli/lib/enrollment-pin.test.ts b/packages/cli/src/cli/lib/enrollment-pin.test.ts index 2be08d64a4..b4a74cb2a7 100644 --- a/packages/cli/src/cli/lib/enrollment-pin.test.ts +++ b/packages/cli/src/cli/lib/enrollment-pin.test.ts @@ -24,7 +24,11 @@ afterEach(() => { describe('linkEnrolledNodeToProjectPin', () => { it('records the enrolled node on a pin that has none', () => { const dataDir = projectDataDir(); - writeProjectWorkspaceKey(dataDir, 'rk_live_pinned'); + writeProjectWorkspaceKey(dataDir, 'rk_live_pinned', { + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); const result = linkEnrolledNodeToProjectPin({ nodeId: 'node_abc', projectDataDir: dataDir }); @@ -32,6 +36,9 @@ describe('linkEnrolledNodeToProjectPin', () => { expect(readProjectWorkspaceSession(dataDir)).toEqual({ workspaceKey: 'rk_live_pinned', enrolledNodeId: 'node_abc', + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', }); }); diff --git a/packages/cli/src/cli/lib/enrollment-pin.ts b/packages/cli/src/cli/lib/enrollment-pin.ts index 3641298952..3c2898c963 100644 --- a/packages/cli/src/cli/lib/enrollment-pin.ts +++ b/packages/cli/src/cli/lib/enrollment-pin.ts @@ -3,7 +3,7 @@ import { getProjectPaths } from '@agent-relay/config'; import { projectWorkspaceKeyPath, readProjectWorkspaceSession, - writeProjectWorkspaceKey, + writeProjectWorkspaceKeyPreservingSession, } from './project-workspace-key.js'; /** Outcome of reconciling a fresh enrollment against the project workspace pin. */ @@ -63,6 +63,6 @@ export function linkEnrolledNodeToProjectPin( return { status: 'conflict', nodeId, pinnedNodeId: session.enrolledNodeId, pinPath }; } - writeProjectWorkspaceKey(dataDir, session.workspaceKey, { enrolledNodeId: nodeId }); + writeProjectWorkspaceKeyPreservingSession(dataDir, session.workspaceKey, { enrolledNodeId: nodeId }); return { status: 'linked', nodeId, pinPath }; } diff --git a/packages/cli/src/cli/lib/project-workspace-key.ts b/packages/cli/src/cli/lib/project-workspace-key.ts index 5d97af820e..3c40b044bb 100644 --- a/packages/cli/src/cli/lib/project-workspace-key.ts +++ b/packages/cli/src/cli/lib/project-workspace-key.ts @@ -7,7 +7,9 @@ export { resolveActiveWorkspaceSelection, resolveWorkspaceSelection, writeProjectWorkspaceKey, + writeProjectWorkspaceKeyPreservingSession, type ProjectWorkspaceSession, + type ProjectWorkspaceSessionMetadata, type WorkspaceKeyFileSystem, type WorkspaceSelection, } from '@agent-relay/cloud/workspace-key'; diff --git a/packages/cli/src/cli/lib/sdk-client.test.ts b/packages/cli/src/cli/lib/sdk-client.test.ts index 65e9474535..6f329ad5d8 100644 --- a/packages/cli/src/cli/lib/sdk-client.test.ts +++ b/packages/cli/src/cli/lib/sdk-client.test.ts @@ -6,13 +6,16 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { createAgentRelay, + persistWorkspaceRelaycastTarget, resolveAgentToken, resolveBaseUrl, resolveWorkspaceKey, resolveWorkspaceKeyWithSource, + resolveWorkspaceSelection, + resolveWorkspaceTransport, } from './sdk-client.js'; import { setWorkspaceKey } from './workspace-store.js'; -import { writeProjectWorkspaceKey } from './project-workspace-key.js'; +import { readProjectWorkspaceSession, writeProjectWorkspaceKey } from './project-workspace-key.js'; let dir: string; let projectRoot: string; @@ -123,4 +126,288 @@ describe('sdk client option resolution', () => { expect(JSON.stringify(relay)).not.toContain('rk_live_project_owner_secret'); expect(JSON.stringify(relay)).not.toContain('at_live_participant_scoped'); }); + + it('does not inherit a persisted gateway when an explicit agent token is supplied', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + + const relay = createAgentRelay({ token: 'at_live_explicit' }) as unknown as { + messagingOptions: { baseUrl?: string }; + workspaceKey?: string; + }; + expect(relay.messagingOptions.baseUrl).toBeUndefined(); + expect(relay.workspaceKey).toBeUndefined(); + + const canonicalRelay = createAgentRelay({ + token: 'at_live_explicit', + baseUrl: 'https://cast.agentrelay.com', + }) as unknown as { messagingOptions: { baseUrl?: string } }; + expect(canonicalRelay.messagingOptions.baseUrl).toBe('https://cast.agentrelay.com'); + }); + + it('does not inherit a persisted gateway when an environment agent token is supplied', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + + const relay = createAgentRelay({ + env: { RELAY_AGENT_TOKEN: 'at_live_environment' }, + }) as unknown as { + messagingOptions: { baseUrl?: string }; + workspaceKey?: string; + }; + expect(relay.messagingOptions.baseUrl).toBeUndefined(); + expect(relay.workspaceKey).toBeUndefined(); + }); + + it('durably records an isolated target while preserving the enrolled node session', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + enrolledNodeId: 'node_1', + }); + const selection = resolveWorkspaceSelection({ env: { AGENT_RELAY_HOME: dir } }); + expect( + persistWorkspaceRelaycastTarget(selection, { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_agent37', + }) + ).toBe(true); + expect(readProjectWorkspaceSession(projectDataDir())).toEqual({ + workspaceKey: 'rk_live_canonical', + workspaceId: 'rw_abc', + enrolledNodeId: 'node_1', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + + const replayOptions = { + workspaceKey: 'rk_live_canonical', + env: { AGENT_RELAY_HOME: dir }, + }; + expect(resolveWorkspaceSelection(replayOptions)).toMatchObject({ + key: 'rk_live_canonical', + source: 'flag', + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + expect(resolveWorkspaceKey(replayOptions)).toBe('rk_live_agent37'); + expect(resolveBaseUrl(replayOptions)).toBe('https://agent37-cast.agentrelay.com'); + }); + + it.each(['flag', 'env'] as const)( + 'creates a fresh project target pin for an unpinned %s selection', + (source) => { + const selection = resolveWorkspaceSelection({ + ...(source === 'flag' ? { workspaceKey: 'rk_live_fresh' } : {}), + env: { + AGENT_RELAY_HOME: dir, + ...(source === 'env' ? { RELAY_WORKSPACE_KEY: 'rk_live_fresh' } : {}), + }, + }); + expect(selection).toMatchObject({ + key: 'rk_live_fresh', + source, + projectDataDir: projectDataDir(), + }); + + expect( + persistWorkspaceRelaycastTarget(selection, { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_fresh', + relaycastApiKey: 'rk_live_fresh_agent37', + }) + ).toBe(true); + expect(readProjectWorkspaceSession(projectDataDir())).toEqual({ + workspaceKey: 'rk_live_fresh', + workspaceId: 'rw_fresh', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_fresh_agent37', + }); + } + ); + + it('creates a fresh project target pin for an active machine-store selection', () => { + setWorkspaceKey('ops', 'rk_live_store', { AGENT_RELAY_HOME: dir }); + const selection = resolveWorkspaceSelection({ env: { AGENT_RELAY_HOME: dir } }); + expect(selection).toMatchObject({ + key: 'rk_live_store', + source: 'store', + projectDataDir: projectDataDir(), + projectSessionPresent: false, + }); + + expect( + persistWorkspaceRelaycastTarget(selection, { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_store', + relaycastApiKey: 'rk_live_store_agent37', + }) + ).toBe(true); + expect(readProjectWorkspaceSession(projectDataDir())).toEqual({ + workspaceKey: 'rk_live_store', + workspaceId: 'rw_store', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_store_agent37', + }); + }); + + it('refuses to overwrite a project session rebound after workspace selection', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_original', { workspaceId: 'rw_original' }); + const selection = resolveWorkspaceSelection({ env: { AGENT_RELAY_HOME: dir } }); + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_rebound', { + workspaceId: 'rw_rebound', + enrolledNodeId: 'node_rebound', + }); + + expect( + persistWorkspaceRelaycastTarget(selection, { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_original', + relaycastApiKey: 'rk_live_original_agent37', + }) + ).toBe(false); + expect(readProjectWorkspaceSession(projectDataDir())).toEqual({ + workspaceKey: 'rk_live_rebound', + workspaceId: 'rw_rebound', + enrolledNodeId: 'node_rebound', + }); + }); + + it('does not replace a project pin created while a fresh explicit selection is in flight', () => { + const selection = resolveWorkspaceSelection({ + workspaceKey: 'rk_live_fresh', + env: { AGENT_RELAY_HOME: dir }, + }); + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_concurrent', { + workspaceId: 'rw_concurrent', + enrolledNodeId: 'node_concurrent', + }); + + expect( + persistWorkspaceRelaycastTarget(selection, { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_fresh', + relaycastApiKey: 'rk_live_fresh_agent37', + }) + ).toBe(false); + expect(readProjectWorkspaceSession(projectDataDir())).toEqual({ + workspaceKey: 'rk_live_concurrent', + workspaceId: 'rw_concurrent', + enrolledNodeId: 'node_concurrent', + }); + }); + + it('refuses to overwrite a target changed after workspace selection', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { workspaceId: 'rw_abc' }); + const selection = resolveWorkspaceSelection({ env: { AGENT_RELAY_HOME: dir } }); + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'canonical', + relaycastBaseUrl: 'https://cast.agentrelay.com', + relaycastApiKey: 'rk_live_newer_route', + }); + + expect( + persistWorkspaceRelaycastTarget(selection, { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_stale_route', + }) + ).toBe(false); + expect(readProjectWorkspaceSession(projectDataDir())?.relaycastApiKey).toBe('rk_live_newer_route'); + }); + + it('keeps legacy persisted targets usable when no separate Relaycast key exists', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_legacy_agent37', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + + const options = { env: { AGENT_RELAY_HOME: dir } }; + expect(resolveWorkspaceKey(options)).toBe('rk_live_legacy_agent37'); + expect(resolveBaseUrl(options)).toBe('https://agent37-cast.agentrelay.com'); + }); + + it('rejects a separate Relaycast key without a complete persisted route', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_agent37', + }); + + const options = { env: { AGENT_RELAY_HOME: dir } }; + expect(() => resolveWorkspaceKey(options)).toThrow(/persisted Relaycast workspace route is incomplete/); + expect(() => resolveBaseUrl(options)).toThrow(/persisted Relaycast workspace route is incomplete/); + }); + + it('rejects a persisted route that is not the exact server-owned origin', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_agent37', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://evil.example', + }); + + expect(() => resolveBaseUrl({ env: { AGENT_RELAY_HOME: dir } })).toThrow(/not trusted/); + }); + + it('normalizes an equivalent requested trailing slash against the persisted route', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + + expect( + resolveWorkspaceTransport({ + baseUrl: 'https://agent37-cast.agentrelay.com/', + env: { AGENT_RELAY_HOME: dir }, + }) + ).toEqual({ + workspaceKey: 'rk_live_agent37', + baseUrl: 'https://agent37-cast.agentrelay.com', + source: 'project', + }); + }); + + it.each([ + 'https://agent37-cast.agentrelay.com/path', + 'https://agent37-cast.agentrelay.com?query=1', + 'https://agent37-cast.agentrelay.com#fragment', + 'https://user:pass@agent37-cast.agentrelay.com', + 'https://agent37-cast.agentrelay.com:443', + 'https://agent37-cast.agentrelay.com:444', + 'https://agent37-cast.agentrelay.com/%2e%2e', + 'https://agent37-cast.agentrelay.com.attacker.example', + ])('rejects an unsafe requested URL before pairing it with a persisted route-scoped key', (baseUrl) => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + + expect(() => resolveWorkspaceTransport({ baseUrl, env: { AGENT_RELAY_HOME: dir } })).toThrow( + /trusted origin|does not match/ + ); + }); }); diff --git a/packages/cli/src/cli/lib/sdk-client.ts b/packages/cli/src/cli/lib/sdk-client.ts index ae1e2b216a..6188b29ae5 100644 --- a/packages/cli/src/cli/lib/sdk-client.ts +++ b/packages/cli/src/cli/lib/sdk-client.ts @@ -1,6 +1,11 @@ +import path from 'node:path'; + import { AgentRelay, type AgentRelayAgent } from '@agent-relay/sdk'; +import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; import { - resolveWorkspaceKeyWithSource as resolveCloudWorkspaceKeyWithSource, + resolveWorkspaceSelection as resolveCloudWorkspaceSelection, + writeProjectWorkspaceTargetIfSelectionCurrent, + type WorkspaceSelection, type WorkspaceKeySource, } from '@agent-relay/cloud/workspace-key'; @@ -10,6 +15,8 @@ export interface SdkClientOptions { token?: string; baseUrl?: string; env?: NodeJS.ProcessEnv; + /** Use the canonical gateway instead of a persisted server-selected route. */ + ignorePersistedRelaycastTarget?: boolean; } function env(options: SdkClientOptions): NodeJS.ProcessEnv { @@ -23,6 +30,21 @@ function trimOrUndefined(value: string | undefined): string | undefined { /** Where a resolved workspace key came from, in precedence order. */ export type { WorkspaceKeySource }; +export type { WorkspaceSelection }; + +export type WorkspaceTransport = { + workspaceKey: string; + baseUrl?: string; + source: WorkspaceKeySource; +}; + +/** Resolve the selected key and any previously persisted Relay workspace identity. */ +export function resolveWorkspaceSelection(options: SdkClientOptions = {}): WorkspaceSelection | undefined { + return resolveCloudWorkspaceSelection({ + workspaceKey: options.workspaceKey, + env: env(options), + }); +} /** * Resolve the workspace key and report which source it came from. Precedence: @@ -35,14 +57,8 @@ export function resolveWorkspaceKeyWithSource(options: SdkClientOptions = {}): { key: string; source: WorkspaceKeySource; } { - const resolved = resolveCloudWorkspaceKeyWithSource({ - workspaceKey: options.workspaceKey, - env: env(options), - }); - if (resolved) return resolved; - throw new Error( - 'No workspace key found. Pass --workspace-key, set RELAY_WORKSPACE_KEY, or run `relay workspace set_key `.' - ); + const transport = resolveWorkspaceTransport(options); + return { key: transport.workspaceKey, source: transport.source }; } export function resolveWorkspaceKey(options: SdkClientOptions = {}): string { @@ -50,7 +66,130 @@ export function resolveWorkspaceKey(options: SdkClientOptions = {}): string { } export function resolveBaseUrl(options: SdkClientOptions = {}): string | undefined { - return trimOrUndefined(options.baseUrl) ?? trimOrUndefined(env(options).RELAY_BASE_URL); + const selection = selectionForTransport(options); + return resolveBaseUrlForSelection(selection, options); +} + +function selectionForTransport(options: SdkClientOptions): WorkspaceSelection | undefined { + const selection = resolveWorkspaceSelection(options); + if (!selection || !options.ignorePersistedRelaycastTarget) return selection; + const { + relaycastRoute: _relaycastRoute, + relaycastBaseUrl: _relaycastBaseUrl, + relaycastApiKey: _relaycastApiKey, + ...canonicalSelection + } = selection; + return canonicalSelection; +} + +function resolveBaseUrlForSelection( + selection: WorkspaceSelection | undefined, + options: SdkClientOptions +): string | undefined { + const persisted = validatePersistedRelaycastBaseUrl(selection); + const requested = trimOrUndefined(options.baseUrl) ?? trimOrUndefined(env(options).RELAY_BASE_URL); + if (persisted && requested) { + let parsed: URL; + try { + parsed = new URL(requested); + } catch { + throw new Error('The requested Relaycast base URL is invalid.'); + } + const authority = /^https:\/\/([^/?#]+)/i.exec(requested)?.[1] ?? ''; + if ( + !/^https:\/\/[^/?#]+\/?$/i.test(requested) || + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + /:\d+$/.test(authority) || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') + ) { + throw new Error('The requested Relaycast base URL is not a trusted origin.'); + } + if (parsed.origin !== persisted) { + throw new Error('The requested Relaycast base URL does not match the persisted workspace route.'); + } + } + return persisted ?? requested; +} + +/** Resolve one credential/origin pair from one workspace selection. */ +export function resolveWorkspaceTransport(options: SdkClientOptions = {}): WorkspaceTransport { + const selection = selectionForTransport(options); + if (!selection) { + throw new Error( + 'No workspace key found. Pass --workspace-key, set RELAY_WORKSPACE_KEY, or run `relay workspace set_key `.' + ); + } + const baseUrl = resolveBaseUrlForSelection(selection, options); + return { + workspaceKey: trimOrUndefined(selection.relaycastApiKey) ?? selection.key, + ...(baseUrl ? { baseUrl } : {}), + source: selection.source, + }; +} + +function validatePersistedRelaycastBaseUrl(selection: WorkspaceSelection | undefined): string | undefined { + const baseUrl = trimOrUndefined(selection?.relaycastBaseUrl); + const route = selection?.relaycastRoute; + const relaycastApiKey = trimOrUndefined(selection?.relaycastApiKey); + if (!baseUrl && !route && !relaycastApiKey) return undefined; + if (!baseUrl || !route) { + throw new Error('The persisted Relaycast workspace route is incomplete.'); + } + let parsed: URL; + try { + parsed = new URL(baseUrl); + } catch { + throw new Error('The persisted Relaycast workspace route is invalid.'); + } + const expectedOrigin = + route === 'canonical' + ? CANONICAL_RELAYCAST_ORIGIN + : route === 'agent37-isolated' + ? AGENT37_RELAYCAST_ORIGIN + : undefined; + if ( + !expectedOrigin || + parsed.origin !== expectedOrigin || + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') + ) { + throw new Error('The persisted Relaycast workspace route is not trusted.'); + } + return parsed.origin; +} + +/** Persist a server-selected target only while the captured project selection is still current. */ +export function persistWorkspaceRelaycastTarget( + selection: WorkspaceSelection | undefined, + target: { + route: 'canonical' | 'agent37-isolated'; + baseUrl: string; + workspaceId: string; + relaycastApiKey: string; + } +): boolean { + if (!selection) return false; + const selectionWithProjectDir = selection as WorkspaceSelection & { projectDataDir?: string }; + const dataDir = + selectionWithProjectDir?.projectDataDir ?? + (selection?.source === 'project' && selection.origin ? path.dirname(selection.origin) : undefined); + if (!dataDir) return false; + return writeProjectWorkspaceTargetIfSelectionCurrent(dataDir, selection, { + workspaceId: target.workspaceId, + relaycastRoute: target.route, + relaycastBaseUrl: target.baseUrl, + relaycastApiKey: target.relaycastApiKey, + }); } export function resolveAgentToken(options: SdkClientOptions = {}): string | undefined { @@ -59,7 +198,8 @@ export function resolveAgentToken(options: SdkClientOptions = {}): string | unde /** Workspace-scoped client (no agent token). */ export function createWorkspaceRelay(options: SdkClientOptions = {}): AgentRelay { - return new AgentRelay({ workspaceKey: resolveWorkspaceKey(options), baseUrl: resolveBaseUrl(options) }); + const { workspaceKey, baseUrl } = resolveWorkspaceTransport(options); + return new AgentRelay({ workspaceKey, baseUrl }); } /** @@ -76,11 +216,16 @@ export function createAgentRelay(options: SdkClientOptions = {}): AgentRelayAgen if (token) { return new AgentRelay({ agentToken: token, - baseUrl: resolveBaseUrl(options), + // An agent token is already scoped by the caller, whether supplied by a + // flag or RELAY_AGENT_TOKEN. Do not let a persisted project route + // silently select a different gateway; only an explicit/ambient base URL + // may choose the token's origin. + baseUrl: resolveBaseUrl({ + ...options, + ignorePersistedRelaycastTarget: true, + }), }); } - return new AgentRelay({ - workspaceKey: resolveWorkspaceKey(options), - baseUrl: resolveBaseUrl(options), - }); + const { workspaceKey, baseUrl } = resolveWorkspaceTransport(options); + return new AgentRelay({ workspaceKey, baseUrl }); } diff --git a/packages/cli/src/cli/lib/workspace-session.test.ts b/packages/cli/src/cli/lib/workspace-session.test.ts index 9691a535ae..7535553f34 100644 --- a/packages/cli/src/cli/lib/workspace-session.test.ts +++ b/packages/cli/src/cli/lib/workspace-session.test.ts @@ -155,7 +155,12 @@ describe('workspace session persistence', () => { const root = tempRoot(); const projectDataDir = path.join(root, 'project', '.agentworkforce', 'relay'); const env = isolatedEnv(root); - writeProjectWorkspaceKey(projectDataDir, 'rk_live_enrolled', { enrolledNodeId: 'node_abc' }); + writeProjectWorkspaceKey(projectDataDir, 'rk_live_enrolled', { + enrolledNodeId: 'node_abc', + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); const result = persistWorkspaceSession({ workspaceKey: 'rk_live_enrolled', @@ -169,10 +174,65 @@ describe('workspace session persistence', () => { expect(readProjectWorkspaceSession(projectDataDir)).toEqual({ workspaceKey: 'rk_live_enrolled', enrolledNodeId: 'node_abc', + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', }); expect(result.clearedEnrolledNodeId).toBeUndefined(); }); + it('persists a validated Relaycast route for later workspace selection', () => { + const root = tempRoot(); + const projectDataDir = path.join(root, 'project', '.agentworkforce', 'relay'); + const env = isolatedEnv(root); + + persistWorkspaceSession({ + workspaceKey: 'rk_live_agent37', + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + projectDataDir, + env, + }); + + expect(readProjectWorkspaceSession(projectDataDir)).toEqual({ + workspaceKey: 'rk_live_agent37', + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + }); + + it('replaces a stale route credential when re-persisting an explicit route', () => { + const root = tempRoot(); + const projectDataDir = path.join(root, 'project', '.agentworkforce', 'relay'); + const env = isolatedEnv(root); + writeProjectWorkspaceKey(projectDataDir, 'rk_live_redeemed', { + workspaceId: 'rw_redeemed', + relaycastRoute: 'canonical', + relaycastBaseUrl: 'https://cast.agentrelay.com', + relaycastApiKey: 'rk_live_stale_canonical', + }); + + persistWorkspaceSession({ + workspaceKey: 'rk_live_redeemed', + workspaceId: 'rw_redeemed', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_redeemed', + projectDataDir, + env, + }); + + expect(readProjectWorkspaceSession(projectDataDir)).toEqual({ + workspaceKey: 'rk_live_redeemed', + workspaceId: 'rw_redeemed', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_redeemed', + }); + }); + it('clears the enrolled Fleet node id when moving to a different workspace', () => { const root = tempRoot(); const projectDataDir = path.join(root, 'project', '.agentworkforce', 'relay'); @@ -227,6 +287,8 @@ describe('workspace session persistence', () => { writeProjectWorkspaceKey(projectDataDir, 'rk_live_old', { enrolledNodeId: 'node_old', workspaceId: 'rw_old', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', }); pinProjectWorkspaceSession({ workspaceKey: 'rk_live_default', projectDataDir, env }); diff --git a/packages/cli/src/cli/lib/workspace-session.ts b/packages/cli/src/cli/lib/workspace-session.ts index 4818ce1d4d..65def6f681 100644 --- a/packages/cli/src/cli/lib/workspace-session.ts +++ b/packages/cli/src/cli/lib/workspace-session.ts @@ -1,7 +1,11 @@ import { getProjectPaths } from '@agent-relay/config'; import { resolveWorkspaceKeyWithSource } from '@agent-relay/cloud/workspace-key'; -import { readProjectWorkspaceSession, writeProjectWorkspaceKey } from './project-workspace-key.js'; +import { + readProjectWorkspaceSession, + writeProjectWorkspaceKey, + writeProjectWorkspaceKeyPreservingSession, +} from './project-workspace-key.js'; import { setWorkspaceKey, switchWorkspace, validateWorkspaceName } from './workspace-store.js'; export interface WorkspaceSessionOptions { @@ -14,6 +18,10 @@ export interface PersistWorkspaceSessionOptions extends WorkspaceSessionOptions workspaceKey: string; /** Canonical Relaycast workspace id, when the credential issuer supplies it. */ workspaceId?: string; + relaycastRoute?: 'canonical' | 'agent37-isolated'; + relaycastBaseUrl?: string; + /** Route-scoped Relaycast credential paired with the persisted route. */ + relaycastApiKey?: string; /** Named sessions are also stored and selected in the machine-global workspace store. */ name?: string; } @@ -99,10 +107,22 @@ export function persistWorkspaceSession( // for an ordinary switch/join/create that happens to stay on the same key. const keepsWorkspace = existing?.workspaceKey === workspaceKey; const enrolledNodeId = keepsWorkspace ? existing?.enrolledNodeId : undefined; - writeProjectWorkspaceKey(projectDataDir, workspaceKey, { - ...(enrolledNodeId ? { enrolledNodeId } : {}), - ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), - }); + if (keepsWorkspace) { + writeProjectWorkspaceKeyPreservingSession(projectDataDir, workspaceKey, { + ...(enrolledNodeId ? { enrolledNodeId } : {}), + ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), + ...(options.relaycastRoute ? { relaycastRoute: options.relaycastRoute } : {}), + ...(options.relaycastBaseUrl ? { relaycastBaseUrl: options.relaycastBaseUrl } : {}), + ...(options.relaycastApiKey ? { relaycastApiKey: options.relaycastApiKey } : {}), + }); + } else { + writeProjectWorkspaceKey(projectDataDir, workspaceKey, { + ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), + ...(options.relaycastRoute ? { relaycastRoute: options.relaycastRoute } : {}), + ...(options.relaycastBaseUrl ? { relaycastBaseUrl: options.relaycastBaseUrl } : {}), + ...(options.relaycastApiKey ? { relaycastApiKey: options.relaycastApiKey } : {}), + }); + } if (name) { setWorkspaceKey(name, workspaceKey, options.env); diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index c9348c0174..4dc9b15cdf 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -11,9 +11,12 @@ vi.mock('./auth.js', () => ({ })); import { + AGENT37_RELAYCAST_ORIGIN, + CANONICAL_RELAYCAST_ORIGIN, CloudFleetSandboxProvisionError, deleteCloudFleetSandbox, ensureCloudFleetSandbox, + normalizeRelaycastTarget, } from './fleet-sandbox.js'; const auth = { @@ -24,6 +27,24 @@ const auth = { }; const refreshedAuth = { ...auth, accessToken: 'refreshed' }; const CLOUD_WORKSPACE_ID = '50587328-441d-4acb-b8f3-dbe1b3c5de99'; +const SANDBOX_ID = 'sbx_123e4567-e89b-42d3-a456-426614174000'; +const SANDBOX_NAME = 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000'; +const RELAYCAST_TARGET = { + route: 'agent37-isolated' as const, + baseUrl: AGENT37_RELAYCAST_ORIGIN, + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_agent37', +}; +const CANONICAL_RELAYCAST_TARGET = { + route: 'canonical' as const, + baseUrl: CANONICAL_RELAYCAST_ORIGIN, + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_canonical', +}; +const NON_AGENT_PROVIDER_IDS = ['daytona', 'e2b', 'vercel', 'freestyle', 'microsandbox'] as const; +const PROVIDER_OUTCOME_MATRIX = NON_AGENT_PROVIDER_IDS.flatMap((providerId) => + (['provisioned', 'reused'] as const).map((outcome) => ({ providerId, outcome })) +); describe('Cloud fleet sandbox client', () => { beforeEach(() => { @@ -35,6 +56,115 @@ describe('Cloud fleet sandbox client', () => { vi.restoreAllMocks(); }); + it('normalizes a closed Agent37 Relaycast target', () => { + expect( + normalizeRelaycastTarget({ ...RELAYCAST_TARGET, baseUrl: `${AGENT37_RELAYCAST_ORIGIN}/` }) + ).toEqual(RELAYCAST_TARGET); + expect( + normalizeRelaycastTarget({ + route: 'canonical', + baseUrl: CANONICAL_RELAYCAST_ORIGIN, + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_canonical', + }) + ).toMatchObject({ route: 'canonical', baseUrl: CANONICAL_RELAYCAST_ORIGIN }); + }); + + it.each([ + { route: 'agent37-isolated', baseUrl: 'https://evil.example' }, + { route: 'canonical', baseUrl: AGENT37_RELAYCAST_ORIGIN }, + { route: 'agent37-isolated', baseUrl: 'https://agent37-cast.agentrelay.com/path' }, + { route: 'agent37-isolated', baseUrl: 'https://user:pass@agent37-cast.agentrelay.com' }, + ])('rejects a route mapped to an untrusted Relaycast origin', (target) => { + expect(() => normalizeRelaycastTarget({ ...RELAYCAST_TARGET, ...target })).toThrow(/relaycast/i); + }); + + it.each([ + { ...RELAYCAST_TARGET, relaycastApiKey: 'rk_test_not_live' }, + { ...RELAYCAST_TARGET, workspaceId: '' }, + { ...RELAYCAST_TARGET, relaycastApiKey: undefined }, + ])('rejects an incomplete Relaycast target', (target) => { + expect(() => normalizeRelaycastTarget(target)).toThrow(/Relaycast|workspace/); + }); + + it('rejects the legacy generic API key field', () => { + expect(() => + normalizeRelaycastTarget({ + route: RELAYCAST_TARGET.route, + baseUrl: RELAYCAST_TARGET.baseUrl, + workspaceId: RELAYCAST_TARGET.workspaceId, + apiKey: RELAYCAST_TARGET.relaycastApiKey, + }) + ).toThrow(/API key/); + }); + + it('rejects a provisioned response with an untrusted server-owned Relaycast route', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-1', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relaycastTarget: { ...RELAYCAST_TARGET, baseUrl: 'https://evil.example' }, + relayfileMounted: true, + }), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + workloadProfile: 'long-running-agent', + }) + ).rejects.toThrow(/untrusted relaycastTarget.baseUrl/); + }); + + it('rejects a canonical target for an explicitly requested Agent37 provider', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-1', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relaycastTarget: { + route: 'canonical', + baseUrl: CANONICAL_RELAYCAST_ORIGIN, + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_canonical', + }, + relayfileMounted: false, + }), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId: 'agent37', + mountRelayfile: false, + }) + ).rejects.toThrow(/mapped Agent37 to a non-isolated/); + }); + it('resolves the unified workspace and provisions a ready mounted sandbox', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -46,9 +176,11 @@ describe('Cloud fleet sandbox client', () => { { outcome: 'provisioned', nodeId: 'node-1', - nodeName: 'daytona-codex', - sandboxId: 'sandbox-1', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, relayfileMountPath: '/workspace', providerId: 'agent37', @@ -60,7 +192,8 @@ describe('Cloud fleet sandbox client', () => { const result = await ensureCloudFleetSandbox({ workspaceId: 'rw_abc', - name: 'daytona-codex', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, requiredCapability: 'spawn:codex', maxAgents: 1, mountRelayfile: true, @@ -81,7 +214,8 @@ describe('Cloud fleet sandbox client', () => { expect(ensureCall?.[1]).toBe('/api/v1/fleet/nodes/sandbox/ensure'); expect(JSON.parse(String(ensureCall?.[2]?.body))).toEqual({ workspaceId: CLOUD_WORKSPACE_ID, - name: 'daytona-codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, requiredCapability: 'spawn:codex', maxAgents: 1, mountRelayfile: true, @@ -93,15 +227,465 @@ describe('Cloud fleet sandbox client', () => { outcome: 'provisioned', cloudWorkspaceId: CLOUD_WORKSPACE_ID, nodeId: 'node-1', - nodeName: 'daytona-codex', - sandboxId: 'sandbox-1', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, relayfileMountPath: '/workspace', providerId: 'agent37', }); }); + it('rejects an invalid sandbox identity before contacting Cloud', async () => { + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: 'sandbox-1', + forceProvision: true, + }) + ).rejects.toThrow('sandboxId must match lowercase sbx_'); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: 'sbx_123E4567-e89b-42d3-a456-426614174000', + forceProvision: true, + }) + ).rejects.toThrow('sandboxId must match lowercase sbx_'); + + expect(mocks.ensureCloudSession).not.toHaveBeenCalled(); + expect(mocks.authorizedApiFetch).not.toHaveBeenCalled(); + }); + + it('accepts a legacy non-Agent37 provisioned response without a Relaycast target', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeId: 'node-legacy', + nodeName: 'legacy-custom-node', + sandboxId: 'legacy-public-sandbox', + providerSandboxId: 'legacy-provider-sandbox', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + }, + { status: 201 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + name: 'legacy-custom-node', + workloadProfile: 'long-running-agent', + }) + ).resolves.toEqual( + expect.objectContaining({ + outcome: 'provisioned', + nodeName: 'legacy-custom-node', + relayWorkspaceId: 'rw_abc', + }) + ); + + const ensureBody = JSON.parse(String(mocks.authorizedApiFetch.mock.calls[1]?.[2]?.body)); + expect(ensureBody).toMatchObject({ + name: 'legacy-custom-node', + workloadProfile: 'long-running-agent', + }); + expect(ensureBody).not.toHaveProperty('sandboxId'); + }); + + it.each(['provisioned', 'provisioning_timeout'] as const)( + 'accepts an older Cloud %s response without providerSandboxId when the exact public identity matches', + async (outcome) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + outcome === 'provisioned' + ? { + outcome, + nodeId: 'node-legacy', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + providerId: 'agent37', + } + : { + outcome, + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + waitedMs: 90_000, + providerId: 'agent37', + }, + { status: outcome === 'provisioned' ? 201 : 202 } + ), + auth, + }); + + const result = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }); + + expect(result).toMatchObject({ + outcome, + sandboxId: SANDBOX_ID, + nodeName: SANDBOX_NAME, + providerId: 'agent37', + }); + expect(result).not.toHaveProperty('providerSandboxId'); + } + ); + + it('requires a deterministic name whenever a sandbox identity is supplied', async () => { + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: 'custom-name', + forceProvision: true, + workloadProfile: 'long-running-agent', + }) + ).rejects.toThrow(SANDBOX_NAME); + + expect(mocks.ensureCloudSession).not.toHaveBeenCalled(); + expect(mocks.authorizedApiFetch).not.toHaveBeenCalled(); + }); + + it.each(['provisioned', 'provisioning_timeout'] as const)( + 'fails closed when Cloud echoes a different sandbox identity for %s', + async (outcome) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + outcome === 'provisioned' + ? { + outcome, + nodeId: 'node-other', + nodeName: SANDBOX_NAME, + sandboxId: 'sbx_223e4567-e89b-42d3-a456-426614174000', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + } + : { + outcome, + sandboxId: 'sbx_223e4567-e89b-42d3-a456-426614174000', + relayWorkspaceId: 'rw_abc', + nodeName: SANDBOX_NAME, + waitedMs: 90_000, + }, + { status: outcome === 'provisioned' ? 201 : 202 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + outcomeUnknown: true, + sandboxId: SANDBOX_ID, + }); + expect(String(error)).toContain(`instead of requested sandboxId ${SANDBOX_ID}`); + } + ); + + it.each(['provisioned', 'provisioning_timeout'] as const)( + 'fails closed when Cloud echoes a different deterministic node name for %s', + async (outcome) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + outcome === 'provisioned' + ? { + outcome, + nodeId: 'node-other', + nodeName: 'fleet-sandbox-other', + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + } + : { + outcome, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + nodeName: 'fleet-sandbox-other', + waitedMs: 90_000, + }, + { status: outcome === 'provisioned' ? 201 : 202 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + outcomeUnknown: true, + sandboxId: SANDBOX_ID, + }); + expect(String(error)).toContain(`instead of requested nodeName ${SANDBOX_NAME}`); + } + ); + + it.each([ + ['malformed success', { outcome: 'provisioned' }], + ['unknown success outcome', { outcome: 'future_cloud_outcome', nodeName: SANDBOX_NAME }], + ])('rejects a mismatched sandbox identity before parsing a %s response', async (_case, responseFields) => { + const returnedSandboxId = 'sbx_223e4567-e89b-42d3-a456-426614174000'; + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ ...responseFields, sandboxId: returnedSandboxId }, { status: 201 }), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + outcomeUnknown: true, + sandboxId: SANDBOX_ID, + }); + expect(String(error)).toContain(`instead of requested sandboxId ${SANDBOX_ID}`); + }); + + it('does not trust a mismatched sandbox identity from a non-OK response for cleanup', async () => { + const returnedSandboxId = 'sbx_223e4567-e89b-42d3-a456-426614174000'; + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + error: 'provider rejected request', + nodeName: 'untrusted-node-name', + sandboxId: returnedSandboxId, + providerSandboxId: 'untrusted-provider-sandbox', + providerId: 'agent37', + }, + { status: 502 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + outcomeUnknown: true, + sandboxId: undefined, + providerId: undefined, + }); + expect(String(error)).toContain(`instead of requested sandboxId ${SANDBOX_ID}`); + }); + + it('does not expose a matching sandbox identity from a non-OK response for cleanup', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + error: 'provider request failed after allocation', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-sandbox-1', + providerId: 'agent37', + }, + { status: 502 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + outcomeUnknown: true, + sandboxId: SANDBOX_ID, + providerId: undefined, + }); + }); + + it.each([500, 502, 503, 504])( + 'marks an identifier-less server failure (%s) unknown without exposing a cleanup ID', + async (status) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'provider timed out after allocation' }, { status }), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + providerId: 'e2b', + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + providerId: 'e2b', + outcomeUnknown: true, + sandboxId: SANDBOX_ID, + }); + expect(String(error)).toContain('provider timed out after allocation'); + } + ); + + it('keeps a proven client error ordinary when a stable sandbox ID was supplied', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'invalid capability' }, { status: 422 }), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + }).catch((caught: unknown) => caught); + + expect(error).not.toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ message: expect.stringContaining('invalid capability') }); + }); + + it('does not expose a matching sandbox identity from a malformed success for cleanup', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + }, + { status: 201 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + outcomeUnknown: true, + sandboxId: SANDBOX_ID, + providerId: undefined, + }); + }); + it('forwards a repos list into the ensure request body when the caller opts in', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -115,7 +699,9 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'jit-repos-node', sandboxId: 'sandbox-9', + providerSandboxId: 'provider-sandbox-9', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -152,7 +738,9 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'scoped-reviewer', sandboxId: 'sandbox-scoped', + providerSandboxId: 'provider-sandbox-scoped', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -203,7 +791,9 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-e2b', nodeName: 'e2b-reviewer', sandboxId: 'sandbox-e2b', + providerSandboxId: 'provider-sandbox-e2b', relayWorkspaceId: 'rw_abc', + relaycastTarget: CANONICAL_RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -242,6 +832,7 @@ describe('Cloud fleet sandbox client', () => { status: 'online', activeAgents: 0, maxAgents: 1, + relaycastTarget: CANONICAL_RELAYCAST_TARGET, }), auth, }); @@ -260,6 +851,246 @@ describe('Cloud fleet sandbox client', () => { ); }); + it.each(PROVIDER_OUTCOME_MATRIX)( + 'accepts the canonical Relaycast target for the $providerId $outcome result', + async ({ providerId, outcome }) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + outcome === 'provisioned' + ? { + outcome, + providerId, + nodeId: `node-${providerId}`, + nodeName: `${providerId}-reviewer`, + sandboxId: `sandbox-${providerId}`, + providerSandboxId: `provider-sandbox-${providerId}`, + relayWorkspaceId: 'rw_abc', + relaycastTarget: CANONICAL_RELAYCAST_TARGET, + relayfileMounted: true, + } + : { + outcome, + providerId, + nodeId: `node-${providerId}`, + nodeName: `${providerId}-reviewer`, + status: 'online', + activeAgents: 0, + maxAgents: 1, + relaycastTarget: CANONICAL_RELAYCAST_TARGET, + }, + { status: outcome === 'provisioned' ? 201 : 200 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId, + }) + ).resolves.toEqual( + expect.objectContaining({ + outcome, + providerId, + relaycastTarget: CANONICAL_RELAYCAST_TARGET, + }) + ); + } + ); + + it.each(PROVIDER_OUTCOME_MATRIX)( + 'accepts a legacy $providerId $outcome result without a Relaycast target', + async ({ providerId, outcome }) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + outcome === 'provisioned' + ? { + outcome, + providerId, + nodeId: `node-${providerId}`, + nodeName: `${providerId}-reviewer`, + sandboxId: `sandbox-${providerId}`, + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + } + : { + outcome, + providerId, + nodeId: `node-${providerId}`, + nodeName: `${providerId}-reviewer`, + status: 'online', + activeAgents: 0, + maxAgents: 1, + }, + { status: outcome === 'provisioned' ? 201 : 200 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId, + }) + ).resolves.toEqual( + expect.not.objectContaining({ + relaycastTarget: expect.anything(), + }) + ); + } + ); + + it.each(PROVIDER_OUTCOME_MATRIX)( + 'rejects an Agent37 Relaycast target for the $providerId $outcome result', + async ({ providerId, outcome }) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + outcome === 'provisioned' + ? { + outcome, + providerId, + nodeId: `node-${providerId}`, + nodeName: `${providerId}-reviewer`, + sandboxId: `sandbox-${providerId}`, + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + } + : { + outcome, + providerId, + nodeId: `node-${providerId}`, + nodeName: `${providerId}-reviewer`, + status: 'online', + activeAgents: 0, + maxAgents: 1, + relaycastTarget: RELAYCAST_TARGET, + }, + { status: outcome === 'provisioned' ? 201 : 200 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId, + }) + ).rejects.toThrow(/non-canonical Relaycast target/); + } + ); + + it('requires and validates the isolated target for a reused Agent37 sandbox', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ + outcome: 'reused', + providerId: 'agent37', + nodeId: 'node-agent37', + nodeName: 'agent37-reviewer', + status: 'online', + activeAgents: 0, + maxAgents: 1, + relaycastTarget: RELAYCAST_TARGET, + }), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId: 'agent37', + mountRelayfile: false, + }) + ).resolves.toEqual(expect.objectContaining({ outcome: 'reused', relaycastTarget: RELAYCAST_TARGET })); + }); + + it('rejects a reused Agent37 response that omits its isolated target', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ + outcome: 'reused', + providerId: 'agent37', + nodeId: 'node-agent37', + nodeName: 'agent37-reviewer', + status: 'online', + activeAgents: 0, + maxAgents: 1, + }), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId: 'agent37', + }) + ).rejects.toThrow(/missing the Agent37 Relaycast target/); + }); + + it('rejects a provisioned Agent37 response that omits its isolated target', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-agent37', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + }, + { status: 201 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId: 'agent37', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }) + ).rejects.toThrow(/missing the Agent37 Relaycast target/); + }); + it('keeps router-selected responses forward compatible when no exact provider was requested', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -274,7 +1105,9 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-future', nodeName: 'future-reviewer', sandboxId: 'sandbox-future', + providerSandboxId: 'provider-sandbox-future', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -295,7 +1128,7 @@ describe('Cloud fleet sandbox client', () => { ); }); - it('rejects and preserves cleanup identity when Cloud cannot prove the requested provider', async () => { + it('rejects without exposing cleanup identity when Cloud cannot prove the requested provider', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), @@ -309,7 +1142,9 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'wrong-provider', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -325,9 +1160,9 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ - sandboxId: 'sandbox-1', - nodeName: 'wrong-provider', - providerId: 'daytona', + sandboxId: undefined, + nodeName: undefined, + providerId: 'e2b', outcomeUnknown: true, }); expect(String(error)).toContain('instead of requested provider e2b'); @@ -337,7 +1172,7 @@ describe('Cloud fleet sandbox client', () => { ['missing', undefined], ['invalid', 'modal'], ])( - 'preserves the requested provider for cleanup when a %s provider proof arrives on a 201 response', + 'rejects without cleanup identity when a %s provider proof arrives on a 201 response', async (_case, providerId) => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -351,6 +1186,7 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-e2b', nodeName: 'e2b-reviewer', sandboxId: 'sandbox-e2b', + providerSandboxId: 'provider-sandbox-e2b', relayWorkspaceId: 'rw_abc', relayfileMounted: true, ...(providerId === undefined ? {} : { providerId }), @@ -369,8 +1205,8 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ cloudWorkspaceId: CLOUD_WORKSPACE_ID, - sandboxId: 'sandbox-e2b', - nodeName: 'e2b-reviewer', + sandboxId: undefined, + nodeName: undefined, providerId: 'e2b', outcomeUnknown: true, }); @@ -381,7 +1217,7 @@ describe('Cloud fleet sandbox client', () => { ['missing', undefined], ['invalid', 'modal'], ])( - 'preserves the requested provider for cleanup when a %s provider proof arrives on a 202 timeout response', + 'rejects without cleanup identity when a %s provider proof arrives on a 202 timeout response', async (_case, providerId) => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -393,6 +1229,7 @@ describe('Cloud fleet sandbox client', () => { { outcome: 'provisioning_timeout', sandboxId: 'sandbox-e2b', + providerSandboxId: 'provider-sandbox-e2b', relayWorkspaceId: 'rw_abc', nodeName: 'e2b-reviewer', waitedMs: 90_000, @@ -412,8 +1249,8 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ cloudWorkspaceId: CLOUD_WORKSPACE_ID, - sandboxId: 'sandbox-e2b', - nodeName: 'e2b-reviewer', + sandboxId: undefined, + nodeName: undefined, providerId: 'e2b', outcomeUnknown: true, }); @@ -424,7 +1261,7 @@ describe('Cloud fleet sandbox client', () => { ['missing', undefined], ['invalid', 'modal'], ])( - 'preserves the requested provider for cleanup when a %s provider proof arrives on a non-OK response', + 'rejects without cleanup identity when a %s provider proof arrives on a non-OK response', async (_case, providerId) => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -437,6 +1274,7 @@ describe('Cloud fleet sandbox client', () => { error: 'provider rejected request', nodeName: 'e2b-reviewer', sandboxId: 'sandbox-e2b', + providerSandboxId: 'provider-sandbox-e2b', ...(providerId === undefined ? {} : { providerId }), }, { status: 502 } @@ -453,9 +1291,10 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ cloudWorkspaceId: CLOUD_WORKSPACE_ID, - sandboxId: 'sandbox-e2b', - nodeName: 'e2b-reviewer', + sandboxId: undefined, + nodeName: undefined, providerId: 'e2b', + outcomeUnknown: true, }); } ); @@ -473,7 +1312,9 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'bare-node', sandboxId: 'sandbox-10', + providerSandboxId: 'provider-sandbox-10', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -504,7 +1345,9 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'bare-node', sandboxId: 'sandbox-11', + providerSandboxId: 'provider-sandbox-11', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -534,10 +1377,10 @@ describe('Cloud fleet sandbox client', () => { { outcome: 'provisioning_timeout', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', nodeName: 'daytona-codex', waitedMs: 90_000, - providerId: 'agent37', }, { status: 202 } ), @@ -552,6 +1395,7 @@ describe('Cloud fleet sandbox client', () => { ).resolves.toMatchObject({ outcome: 'provisioning_timeout', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', nodeName: 'daytona-codex', waitedMs: 90_000, }); @@ -578,7 +1422,9 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'daytona-codex', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: CANONICAL_RELAYCAST_TARGET, relayfileMounted: true, providerId: 'daytona', }, @@ -617,7 +1463,9 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'daytona-codex', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: CANONICAL_RELAYCAST_TARGET, relayfileMounted: true, providerId: 'daytona', }, @@ -664,7 +1512,7 @@ describe('Cloud fleet sandbox client', () => { expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(1); }); - it('preserves the sandbox identity from a malformed successful response', async () => { + it('does not expose an unrequested sandbox identity from a malformed successful response', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), @@ -676,6 +1524,7 @@ describe('Cloud fleet sandbox client', () => { outcome: 'provisioned', nodeName: 'daytona-codex', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, }, @@ -692,10 +1541,36 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ cloudWorkspaceId: CLOUD_WORKSPACE_ID, - sandboxId: 'sandbox-1', - nodeName: 'daytona-codex', + sandboxId: undefined, + nodeName: undefined, + outcomeUnknown: true, + }); + }); + + it('does not copy a caller sandbox identity into an unknown-response error', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockRejectedValueOnce(new Error('request timed out')); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, outcomeUnknown: true, }); + expect(error).toMatchObject({ sandboxId: SANDBOX_ID }); }); it('rejects a timeout response that omits waitedMs', async () => { @@ -709,6 +1584,7 @@ describe('Cloud fleet sandbox client', () => { { outcome: 'provisioning_timeout', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', nodeName: 'daytona-codex', }, diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 51f6702b8c..9a5a426867 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -5,6 +5,16 @@ import { defaultApiUrl } from './types.js'; type JsonRecord = Record; const CLOUD_WORKSPACE_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const CLOUD_SANDBOX_ID_PATTERN = + /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +/** + * Cloud may hand back the gateway route owned by a provisioned sandbox. This + * is deliberately an exact-origin allowlist: a route is control-plane input, + * not a caller-controlled SDK override. + */ +export const CANONICAL_RELAYCAST_ORIGIN = 'https://cast.agentrelay.com'; +export const AGENT37_RELAYCAST_ORIGIN = 'https://agent37-cast.agentrelay.com'; +const TRUSTED_RELAYCAST_ORIGINS = new Set([CANONICAL_RELAYCAST_ORIGIN, AGENT37_RELAYCAST_ORIGIN]); const DEFAULT_RESOLUTION_TIMEOUT_MS = 120_000; // Mounted provisioning can spend up to 240s completing the initial Relayfile // sync, then up to 90s waiting for the enrolled node to report ready. Leave a @@ -60,9 +70,13 @@ export class CloudFleetSandboxProvisionError extends Error { } } +class CloudFleetSandboxIdentityMismatchError extends Error {} + export type EnsureCloudFleetSandboxInput = { /** Cloud UUID or unified rw_* workspace id. */ workspaceId: string; + /** Caller-declared one-time Cloud identity used to resume a cut-off provision. */ + sandboxId?: string; name?: string; requiredCapability: string; maxAgents?: number; @@ -110,7 +124,10 @@ export type CloudFleetSandboxReady = { nodeId: string; nodeName: string; sandboxId: string; + providerSandboxId?: string; relayWorkspaceId: string; + /** Closed server-owned Relaycast contract when Cloud returned one. Required for Agent37. */ + relaycastTarget?: CloudFleetRelaycastTarget; relayfileMounted: boolean; relayfileMountPath?: string; providerId?: CloudFleetSandboxProviderId; @@ -125,13 +142,17 @@ export type CloudFleetSandboxReused = { activeAgents: number | null; maxAgents: number | null; providerId?: CloudFleetSandboxProviderId; + /** Closed server-owned Relaycast contract when Cloud returned one. Required for Agent37. */ + relaycastTarget?: CloudFleetRelaycastTarget; }; export type CloudFleetSandboxProvisioningTimeout = { outcome: 'provisioning_timeout'; cloudWorkspaceId: string; sandboxId: string; + providerSandboxId?: string; relayWorkspaceId: string; + relaycastTarget?: CloudFleetRelaycastTarget; nodeName: string; waitedMs: number; providerId?: CloudFleetSandboxProviderId; @@ -148,6 +169,15 @@ export type DeleteCloudFleetSandboxInput = { providerId?: CloudFleetSandboxProviderId; }; +export type CloudFleetRelaycastRoute = 'canonical' | 'agent37-isolated'; + +export type CloudFleetRelaycastTarget = { + route: CloudFleetRelaycastRoute; + baseUrl: string; + workspaceId: string; + relaycastApiKey: string; +}; + function isObject(value: unknown): value is JsonRecord { return value !== null && typeof value === 'object' && !Array.isArray(value); } @@ -157,6 +187,56 @@ function readString(payload: JsonRecord, key: string): string | undefined { return typeof value === 'string' && value.trim() ? value.trim() : undefined; } +function normalizeRelaycastOrigin(value: unknown, field: string): string { + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`Cloud fleet sandbox response has an invalid ${field}.`); + } + let parsed: URL; + try { + parsed = new URL(value.trim()); + } catch { + throw new Error(`Cloud fleet sandbox response has an invalid ${field}.`); + } + if ( + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') || + !TRUSTED_RELAYCAST_ORIGINS.has(parsed.origin) + ) { + throw new Error(`Cloud fleet sandbox response has an untrusted ${field}.`); + } + return parsed.origin; +} + +/** Validate Cloud's closed Relaycast route, identity, and scoped credential contract. */ +export function normalizeRelaycastTarget(value: unknown): CloudFleetRelaycastTarget { + if (!isObject(value)) { + throw new Error('Cloud fleet sandbox response is missing relaycastTarget.'); + } + const route = readString(value, 'route'); + if (route !== 'canonical' && route !== 'agent37-isolated') { + throw new Error('Cloud fleet sandbox response has an unknown Relaycast route.'); + } + const baseUrl = normalizeRelaycastOrigin(value.baseUrl, 'relaycastTarget.baseUrl'); + const expectedOrigin = route === 'canonical' ? CANONICAL_RELAYCAST_ORIGIN : AGENT37_RELAYCAST_ORIGIN; + if (baseUrl !== expectedOrigin) { + throw new Error('Cloud fleet sandbox response mapped Relaycast route to the wrong origin.'); + } + const workspaceId = readString(value, 'workspaceId'); + if (!workspaceId) { + throw new Error('Cloud fleet sandbox response is missing relaycastTarget.workspaceId.'); + } + const relaycastApiKey = readString(value, 'relaycastApiKey'); + if (!relaycastApiKey || !/^rk_live_[A-Za-z0-9_-]+$/.test(relaycastApiKey)) { + throw new Error('Cloud fleet sandbox response has an invalid Relaycast API key.'); + } + return { route, baseUrl, workspaceId, relaycastApiKey }; +} + function readNumber(payload: JsonRecord, key: string): number | undefined { const value = payload[key]; return typeof value === 'number' && Number.isFinite(value) ? value : undefined; @@ -168,6 +248,28 @@ function requiredNumber(payload: JsonRecord, key: string, context: string): numb return value; } +function assertProviderRelaycastTarget( + providerId: CloudFleetSandboxProviderId | undefined, + target: CloudFleetRelaycastTarget | undefined +): void { + if (providerId === 'agent37') { + if (!target) { + throw new Error('Cloud fleet sandbox response is missing the Agent37 Relaycast target.'); + } + if (target.route !== 'agent37-isolated' || target.baseUrl !== AGENT37_RELAYCAST_ORIGIN) { + throw new Error('Cloud fleet sandbox response mapped Agent37 to a non-isolated Relaycast target.'); + } + return; + } + if (providerId !== undefined && target) { + if (target.route !== 'canonical' || target.baseUrl !== CANONICAL_RELAYCAST_ORIGIN) { + throw new Error( + `Cloud fleet sandbox response mapped ${providerId} to a non-canonical Relaycast target.` + ); + } + } +} + function boundedSignal(options: CloudFleetSandboxRequestOptions, defaultTimeoutMs: number): AbortSignal { const timeoutMs = options.timeoutMs ?? defaultTimeoutMs; if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { @@ -214,6 +316,48 @@ function requiredString(payload: JsonRecord, key: string, context: string): stri return value; } +function validateSandboxIdentity(input: EnsureCloudFleetSandboxInput): { + sandboxId?: string; + name?: string; +} { + if (input.sandboxId !== undefined && typeof input.sandboxId !== 'string') { + throw new Error('Cloud fleet sandbox sandboxId must be a string.'); + } + if (input.name !== undefined && typeof input.name !== 'string') { + throw new Error('Cloud fleet sandbox name must be a string.'); + } + const sandboxId = input.sandboxId?.trim(); + const name = input.name?.trim(); + if (input.sandboxId !== undefined && (!sandboxId || !CLOUD_SANDBOX_ID_PATTERN.test(sandboxId))) { + throw new Error('Cloud fleet sandbox sandboxId must match lowercase sbx_ using an RFC 4122 UUID.'); + } + if (sandboxId !== undefined && input.forceProvision !== true) { + throw new Error('Cloud fleet sandbox sandboxId requires forceProvision: true.'); + } + if (sandboxId !== undefined && !name) { + throw new Error('Cloud fleet sandbox sandboxId requires a node name.'); + } + + const longRunning = + input.workloadProfile === 'long-running-agent' || input.workloadProfile === 'standard-long-running-agent'; + if (longRunning && sandboxId !== undefined) { + if (input.forceProvision !== true) { + throw new Error('Long-running Cloud fleet sandbox requests require forceProvision: true.'); + } + const expectedName = `fleet-sandbox-${sandboxId.slice('sbx_'.length)}`; + if (name !== expectedName) { + throw new Error( + `Long-running Cloud fleet sandbox requests require name '${expectedName}' to preserve the one-to-one sandbox identity.` + ); + } + } + + return { + ...(sandboxId === undefined ? {} : { sandboxId }), + ...(name === undefined ? {} : { name }), + }; +} + async function resolveCloudWorkspaceId( workspaceId: string, auth: Awaited>['auth'], @@ -256,25 +400,36 @@ function readProviderId( return undefined; } -function cleanupProviderId( - payload: JsonRecord, - requestedProviderId?: CloudFleetSandboxProviderId -): CloudFleetSandboxProviderId | undefined { - const payloadProviderId = readString(payload, 'providerId'); - return payloadProviderId && - CLOUD_FLEET_SANDBOX_PROVIDER_IDS.includes(payloadProviderId as CloudFleetSandboxProviderId) - ? (payloadProviderId as CloudFleetSandboxProviderId) - : requestedProviderId; +function assertExpectedSandboxIdentity(payload: JsonRecord, expectedSandboxId: string): void { + const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); + if (sandboxId !== expectedSandboxId) { + throw new CloudFleetSandboxIdentityMismatchError( + `Cloud returned sandboxId ${sandboxId} instead of requested sandboxId ${expectedSandboxId}.` + ); + } } function normalizeEnsureResult( payload: unknown, cloudWorkspaceId: string, + expectedSandboxId?: string, + expectedNodeName?: string, requestedProviderId?: CloudFleetSandboxProviderId ): EnsureCloudFleetSandboxResult { if (!isObject(payload)) throw new Error('Cloud fleet sandbox response was not valid JSON.'); + // A caller-declared identity is the cleanup authority. Validate it before + // reading any other response field so malformed and future outcomes cannot + // make an untrusted public ID eligible for automatic deletion. + if (expectedSandboxId !== undefined) { + assertExpectedSandboxIdentity(payload, expectedSandboxId); + } const outcome = readString(payload, 'outcome'); const nodeName = requiredString(payload, 'nodeName', 'Cloud fleet sandbox'); + if (expectedSandboxId !== undefined && expectedNodeName !== undefined && nodeName !== expectedNodeName) { + throw new CloudFleetSandboxIdentityMismatchError( + `Cloud returned nodeName ${nodeName} instead of requested nodeName ${expectedNodeName}.` + ); + } const providerId = readProviderId(payload, requestedProviderId !== undefined); if (requestedProviderId !== undefined && providerId !== requestedProviderId) { throw new Error( @@ -288,13 +443,24 @@ function normalizeEnsureResult( if (typeof payload.relayfileMounted !== 'boolean') { throw new Error('Cloud fleet sandbox response is missing relayfileMounted.'); } + const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); + const providerSandboxId = readString(payload, 'providerSandboxId'); + const relayWorkspaceId = requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'); + const relaycastTarget = + payload.relaycastTarget === undefined ? undefined : normalizeRelaycastTarget(payload.relaycastTarget); + if (relaycastTarget !== undefined && relaycastTarget.workspaceId !== relayWorkspaceId) { + throw new Error('Cloud fleet sandbox response has mismatched Relaycast workspace identities.'); + } + assertProviderRelaycastTarget(providerId, relaycastTarget); return { outcome, cloudWorkspaceId, nodeId: requiredString(payload, 'nodeId', 'Cloud fleet sandbox'), nodeName, - sandboxId: requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'), - relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), + sandboxId, + ...(providerSandboxId === undefined ? {} : { providerSandboxId }), + relayWorkspaceId, + ...(relaycastTarget === undefined ? {} : { relaycastTarget }), relayfileMounted: payload.relayfileMounted, ...(providerId === undefined ? {} : { providerId }), ...(readString(payload, 'relayfileMountPath') @@ -304,6 +470,9 @@ function normalizeEnsureResult( } if (outcome === 'reused') { + const relaycastTarget = + payload.relaycastTarget === undefined ? undefined : normalizeRelaycastTarget(payload.relaycastTarget); + assertProviderRelaycastTarget(providerId, relaycastTarget); return { outcome, cloudWorkspaceId, @@ -313,15 +482,26 @@ function normalizeEnsureResult( activeAgents: readNumber(payload, 'activeAgents') ?? null, maxAgents: readNumber(payload, 'maxAgents') ?? null, ...(providerId === undefined ? {} : { providerId }), + ...(relaycastTarget === undefined ? {} : { relaycastTarget }), }; } if (outcome === 'provisioning_timeout') { + const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); + const providerSandboxId = readString(payload, 'providerSandboxId'); + const relayWorkspaceId = requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'); + const relaycastTarget = + payload.relaycastTarget === undefined ? undefined : normalizeRelaycastTarget(payload.relaycastTarget); + if (relaycastTarget !== undefined && relaycastTarget.workspaceId !== relayWorkspaceId) { + throw new Error('Cloud fleet sandbox response has mismatched Relaycast workspace identities.'); + } return { outcome, cloudWorkspaceId, - sandboxId: requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'), - relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), + sandboxId, + ...(providerSandboxId === undefined ? {} : { providerSandboxId }), + relayWorkspaceId, + ...(relaycastTarget === undefined ? {} : { relaycastTarget }), nodeName, waitedMs: requiredNumber(payload, 'waitedMs', 'Cloud fleet sandbox'), ...(providerId === undefined ? {} : { providerId }), @@ -340,6 +520,7 @@ export async function ensureCloudFleetSandbox( const requiredCapability = input.requiredCapability.trim(); if (!workspaceId) throw new Error('A workspace ID is required to provision a fleet sandbox.'); if (!requiredCapability) throw new Error('A spawn capability is required to provision a fleet sandbox.'); + const sandboxIdentity = validateSandboxIdentity(input); if (input.relayfilePaths !== undefined && input.relayfilePaths.length === 0) { throw new Error('At least one Relayfile subtree path is required when relayfilePaths is provided.'); } @@ -362,7 +543,8 @@ export async function ensureCloudFleetSandbox( body: JSON.stringify({ workspaceId: resolved.cloudWorkspaceId, requiredCapability, - ...(input.name ? { name: input.name } : {}), + ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), + ...(sandboxIdentity.name === undefined ? {} : { name: sandboxIdentity.name }), ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), ...(input.relayfilePaths === undefined ? {} : { relayfilePaths: [...input.relayfilePaths] }), @@ -384,6 +566,7 @@ export async function ensureCloudFleetSandbox( ), { cloudWorkspaceId: resolved.cloudWorkspaceId, + ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(input.name ? { nodeName: input.name } : {}), ...(input.providerId ? { providerId: input.providerId } : {}), outcomeUnknown: true, @@ -393,34 +576,66 @@ export async function ensureCloudFleetSandbox( } const payload = await readJson(response); if (!response.ok) { + const returnedSandboxId = isObject(payload) ? readString(payload, 'sandboxId') : undefined; + if ( + sandboxIdentity.sandboxId !== undefined && + returnedSandboxId !== undefined && + returnedSandboxId !== sandboxIdentity.sandboxId + ) { + const mismatch = new CloudFleetSandboxIdentityMismatchError( + `Cloud returned sandboxId ${returnedSandboxId} instead of requested sandboxId ${sandboxIdentity.sandboxId}.` + ); + throw new CloudFleetSandboxProvisionError(mismatch.message, { + cloudWorkspaceId: resolved.cloudWorkspaceId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), + outcomeUnknown: true, + cause: mismatch, + }); + } const error = endpointError('provision the fleet sandbox', response, payload); + // Gateway/server failures can arrive after Cloud accepted the ensure + // request but before it could return an identity. Keep every 5xx failure + // replayable as an unknown outcome, even for legacy custom-name callers; + // never copy an unverified response ID into cleanup authority. + if (response.status >= 500) { + throw new CloudFleetSandboxProvisionError(error.message, { + cloudWorkspaceId: resolved.cloudWorkspaceId, + ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), + outcomeUnknown: true, + cause: error, + }); + } if (isObject(payload) && readString(payload, 'sandboxId')) { - const providerId = cleanupProviderId(payload, input.providerId); throw new CloudFleetSandboxProvisionError(error.message, { cloudWorkspaceId: resolved.cloudWorkspaceId, - sandboxId: readString(payload, 'sandboxId'), - nodeName: readString(payload, 'nodeName') ?? input.name, - ...(providerId === undefined ? {} : { providerId }), + ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), + outcomeUnknown: true, cause: error, }); } throw error; } try { - return normalizeEnsureResult(payload, resolved.cloudWorkspaceId, input.providerId); + return normalizeEnsureResult( + payload, + resolved.cloudWorkspaceId, + sandboxIdentity.sandboxId, + sandboxIdentity.name, + input.providerId + ); } catch (error) { - const providerId = isObject(payload) ? cleanupProviderId(payload, input.providerId) : input.providerId; throw new CloudFleetSandboxProvisionError( error instanceof Error ? error.message : 'Cloud fleet sandbox response was invalid.', { cloudWorkspaceId: resolved.cloudWorkspaceId, - ...(isObject(payload) && readString(payload, 'sandboxId') - ? { sandboxId: readString(payload, 'sandboxId') } - : {}), - ...(isObject(payload) && (readString(payload, 'nodeName') ?? input.name) - ? { nodeName: readString(payload, 'nodeName') ?? input.name } - : {}), - ...(providerId === undefined ? {} : { providerId }), + ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), outcomeUnknown: true, cause: error, } diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 2b5972facc..a110409687 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -81,6 +81,9 @@ export { ensureCloudFleetSandbox, deleteCloudFleetSandbox, CloudFleetSandboxProvisionError, + normalizeRelaycastTarget, + CANONICAL_RELAYCAST_ORIGIN, + AGENT37_RELAYCAST_ORIGIN, type EnsureCloudFleetSandboxInput, type EnsureCloudFleetSandboxResult, type CloudFleetSandboxReady, @@ -90,6 +93,8 @@ export { type CloudFleetSandboxWorkloadProfile, type DeleteCloudFleetSandboxInput, type CloudFleetSandboxRequestOptions, + type CloudFleetRelaycastRoute, + type CloudFleetRelaycastTarget, } from './fleet-sandbox.js'; export { @@ -155,6 +160,7 @@ export { resolveWorkspaceKeyWithSource, resolveWorkspaceSelection, writeProjectWorkspaceKey, + writeProjectWorkspaceTargetIfSelectionCurrent, type ProjectWorkspaceSession, type ResolveWorkspaceKeyOptions, type WorkspaceKeyFileSystem, diff --git a/packages/cloud/src/project-workspace-key.test.ts b/packages/cloud/src/project-workspace-key.test.ts index 29a5c1b4fa..ea894f1334 100644 --- a/packages/cloud/src/project-workspace-key.test.ts +++ b/packages/cloud/src/project-workspace-key.test.ts @@ -2,7 +2,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { projectWorkspaceKeyPath, @@ -36,6 +36,72 @@ describe('project workspace key resolution', () => { expect(fs.statSync(projectWorkspaceKeyPath(dataDir)).mode & 0o777).toBe(0o600); }); + it('does not remove a replacement lock when the original holder finishes', () => { + const lockDir = `${projectWorkspaceKeyPath(dataDir)}.lock`; + let replaced = false; + const originalRename = fs.renameSync.bind(fs); + const rename = vi.spyOn(fs, 'renameSync').mockImplementation((source, destination) => { + originalRename(source, destination); + if (!replaced && destination === projectWorkspaceKeyPath(dataDir)) { + replaced = true; + fs.rmSync(lockDir, { recursive: true, force: true }); + fs.mkdirSync(lockDir, { mode: 0o700 }); + fs.writeFileSync(path.join(lockDir, 'replacement-owner'), '', { mode: 0o600, flag: 'wx' }); + } + }); + + try { + writeProjectWorkspaceKey(dataDir, 'rk_project'); + } finally { + rename.mockRestore(); + } + + expect(fs.existsSync(path.join(lockDir, 'replacement-owner'))).toBe(true); + }); + + it('does not reclaim a stale lock whose recorded owner is still alive', () => { + const lockDir = `${projectWorkspaceKeyPath(dataDir)}.lock`; + const token = 'live-owner-token'; + fs.mkdirSync(lockDir, { recursive: true, mode: 0o700 }); + fs.writeFileSync(path.join(lockDir, token), JSON.stringify({ version: 1, pid: process.pid, token }), { + mode: 0o600, + flag: 'wx', + }); + const staleAt = new Date(Date.now() - 60_000); + fs.utimesSync(lockDir, staleAt, staleAt); + const kill = vi.spyOn(process, 'kill').mockImplementation(() => true); + + try { + expect(() => writeProjectWorkspaceKey(dataDir, 'rk_project')).toThrow(/Timed out waiting/); + expect(fs.existsSync(path.join(lockDir, token))).toBe(true); + expect(kill).toHaveBeenCalledWith(process.pid, 0); + } finally { + kill.mockRestore(); + fs.rmSync(lockDir, { recursive: true, force: true }); + } + }); + + it('preserves a callback error when lock cleanup also fails', () => { + const originalRename = fs.renameSync.bind(fs); + const rename = vi.spyOn(fs, 'renameSync').mockImplementation((source, destination) => { + originalRename(source, destination); + throw new Error('callback failed'); + }); + const rmdir = vi.spyOn(fs, 'rmdirSync').mockImplementation(() => { + throw new Error('cleanup failed'); + }); + const report = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + expect(() => writeProjectWorkspaceKey(dataDir, 'rk_project')).toThrow('callback failed'); + expect(report).toHaveBeenCalledWith(expect.stringContaining('Failed to release'), expect.any(Error)); + } finally { + rename.mockRestore(); + rmdir.mockRestore(); + report.mockRestore(); + } + }); + it('round-trips an enrolled Fleet identity and clears it on an explicit workspace change', () => { writeProjectWorkspaceKey(dataDir, 'rk_enrolled', { enrolledNodeId: ' node_1 ' }); expect(readProjectWorkspaceSession(dataDir)).toEqual({ @@ -87,6 +153,18 @@ describe('project workspace key resolution', () => { }); }); + it('records the absent project session snapshot for an active-store selection', () => { + const env = { AGENT_RELAY_HOME: home }; + setWorkspaceKey('global', 'rk_global', env); + + expect(resolveWorkspaceSelection({ projectDataDir: dataDir, env })).toMatchObject({ + key: 'rk_global', + source: 'store', + projectDataDir: dataDir, + projectSessionPresent: false, + }); + }); + it('returns undefined when no workspace source exists', () => { expect( resolveWorkspaceKeyWithSource({ projectDataDir: dataDir, env: { AGENT_RELAY_HOME: home } }) @@ -106,6 +184,58 @@ describe('workspace precedence ladder diagnostics', () => { ).toBe('rw_pinned'); }); + it('carries a persisted Relaycast target when an explicit key matches the project pin', () => { + writeProjectWorkspaceKey(dataDir, 'rk_canonical', { + workspaceId: 'rw_pinned', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_agent37', + }); + + expect( + resolveWorkspaceSelection({ + workspaceKey: 'rk_canonical', + projectDataDir: dataDir, + env: { AGENT_RELAY_HOME: home }, + }) + ).toMatchObject({ + key: 'rk_canonical', + source: 'flag', + workspaceId: 'rw_pinned', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_agent37', + }); + }); + + it.each(['flag', 'env'] as const)( + 'exposes the empty project directory for a fresh %s selection', + (source) => { + const selection = resolveWorkspaceSelection({ + ...(source === 'flag' ? { workspaceKey: 'rk_fresh' } : {}), + projectDataDir: dataDir, + env: { + AGENT_RELAY_HOME: home, + ...(source === 'env' ? { RELAY_WORKSPACE_KEY: 'rk_fresh' } : {}), + }, + }); + + expect(selection).toMatchObject({ key: 'rk_fresh', source, projectDataDir: dataDir }); + } + ); + + it('does not expose a project directory when an explicit selection conflicts with its pin', () => { + writeProjectWorkspaceKey(dataDir, 'rk_project'); + + expect( + resolveWorkspaceSelection({ + workspaceKey: 'rk_other', + projectDataDir: dataDir, + env: { AGENT_RELAY_HOME: home }, + }) + ).not.toHaveProperty('projectDataDir'); + }); + it('names each source without leaking key material', () => { const env = { AGENT_RELAY_HOME: home, AGENT_RELAY_WORKSPACE_KEY: 'rk_env' }; setWorkspaceKey('global', 'rk_global', env); diff --git a/packages/cloud/src/project-workspace-key.ts b/packages/cloud/src/project-workspace-key.ts index cf39845ecf..6dbee8fe0c 100644 --- a/packages/cloud/src/project-workspace-key.ts +++ b/packages/cloud/src/project-workspace-key.ts @@ -7,6 +7,18 @@ import { getProjectPaths } from '@agent-relay/config'; import { readWorkspaceStore, workspaceStorePath } from './workspace-store.js'; const PROJECT_WORKSPACE_KEY_FILENAME = 'workspace-key.json'; +const PROJECT_WORKSPACE_LOCK_SUFFIX = '.lock'; +const PROJECT_WORKSPACE_LOCK_TIMEOUT_MS = 2_000; +const PROJECT_WORKSPACE_LOCK_STALE_MS = 30_000; +const PROJECT_WORKSPACE_LOCK_RETRY_MS = 10; +const PROJECT_WORKSPACE_LOCK_WAIT = new Int32Array(new SharedArrayBuffer(4)); +const PROJECT_WORKSPACE_LOCK_OWNER_VERSION = 1; + +interface ProjectWorkspaceLockOwner { + version: number; + pid: number; + token: string; +} /** Workspace-key environment aliases, highest precedence first. */ const WORKSPACE_KEY_ENV_VARS = ['RELAY_WORKSPACE_KEY', 'AGENT_RELAY_WORKSPACE_KEY', 'RELAY_API_KEY'] as const; @@ -21,8 +33,15 @@ export interface ProjectWorkspaceSession { * source (a stored Fleet enrollment, say) points at a different workspace. */ workspaceId?: string; + /** Last server-selected Relaycast route for follow-up commands in this session. */ + relaycastRoute?: 'canonical' | 'agent37-isolated'; + relaycastBaseUrl?: string; + /** Route-scoped transport credential; the canonical Cloud workspace key remains `workspaceKey`. */ + relaycastApiKey?: string; } +export type ProjectWorkspaceSessionMetadata = Omit; + export type WorkspaceKeySource = 'flag' | 'env' | 'project' | 'store'; export interface ResolveWorkspaceKeyOptions { @@ -53,6 +72,13 @@ export interface WorkspaceSelection { origin: string; /** Workspace id this selection is known to address, when previously recorded. */ workspaceId?: string; + relaycastRoute?: 'canonical' | 'agent37-isolated'; + relaycastBaseUrl?: string; + relaycastApiKey?: string; + /** Project session directory that can durably carry a server-selected target. */ + projectDataDir?: string; + /** Whether that project session existed when this selection was captured. */ + projectSessionPresent?: boolean; } /** Absolute path to the workspace key recorded by `agent-relay node up`. */ @@ -80,10 +106,19 @@ export function readProjectWorkspaceSession( if (!workspaceKey) return undefined; const enrolledNodeId = trimOrUndefined(parsed.enrolledNodeId); const workspaceId = trimOrUndefined(parsed.workspaceId); + const relaycastRoute = + parsed.relaycastRoute === 'canonical' || parsed.relaycastRoute === 'agent37-isolated' + ? parsed.relaycastRoute + : undefined; + const relaycastBaseUrl = trimOrUndefined(parsed.relaycastBaseUrl); + const relaycastApiKey = trimOrUndefined(parsed.relaycastApiKey); return { workspaceKey, ...(enrolledNodeId ? { enrolledNodeId } : {}), ...(workspaceId ? { workspaceId } : {}), + ...(relaycastRoute ? { relaycastRoute } : {}), + ...(relaycastBaseUrl ? { relaycastBaseUrl } : {}), + ...(relaycastApiKey ? { relaycastApiKey } : {}), }; } catch { return undefined; @@ -97,12 +132,31 @@ export function readProjectWorkspaceSession( export function writeProjectWorkspaceKey( dataDir: string, workspaceKey: string | undefined, - options: { enrolledNodeId?: string; workspaceId?: string } = {} + options: { + enrolledNodeId?: string; + workspaceId?: string; + relaycastRoute?: 'canonical' | 'agent37-isolated'; + relaycastBaseUrl?: string; + relaycastApiKey?: string; + } = {} +): void { + const key = trimOrUndefined(workspaceKey); + if (!key) return; + withProjectWorkspaceKeyLock(dataDir, () => writeProjectWorkspaceKeyUnlocked(dataDir, key, options)); +} + +function writeProjectWorkspaceKeyUnlocked( + dataDir: string, + workspaceKey: string, + options: ProjectWorkspaceSessionMetadata = {} ): void { const key = trimOrUndefined(workspaceKey); if (!key) return; const enrolledNodeId = trimOrUndefined(options.enrolledNodeId); const workspaceId = trimOrUndefined(options.workspaceId); + const relaycastRoute = options.relaycastRoute; + const relaycastBaseUrl = trimOrUndefined(options.relaycastBaseUrl); + const relaycastApiKey = trimOrUndefined(options.relaycastApiKey); fs.mkdirSync(dataDir, { recursive: true, mode: 0o700 }); const file = projectWorkspaceKeyPath(dataDir); // Worker threads share a PID, so include a per-write nonce as well as the PID. @@ -112,6 +166,9 @@ export function writeProjectWorkspaceKey( workspaceKey: key, ...(enrolledNodeId ? { enrolledNodeId } : {}), ...(workspaceId ? { workspaceId } : {}), + ...(relaycastRoute ? { relaycastRoute } : {}), + ...(relaycastBaseUrl ? { relaycastBaseUrl } : {}), + ...(relaycastApiKey ? { relaycastApiKey } : {}), } satisfies ProjectWorkspaceSession, null, 2 @@ -141,6 +198,227 @@ export function writeProjectWorkspaceKey( } } +function withProjectWorkspaceKeyLock(dataDir: string, callback: () => T): T { + fs.mkdirSync(dataDir, { recursive: true, mode: 0o700 }); + const lockDir = `${projectWorkspaceKeyPath(dataDir)}${PROJECT_WORKSPACE_LOCK_SUFFIX}`; + const ownerToken = randomUUID(); + const ownerPath = path.join(lockDir, ownerToken); + const startedAt = Date.now(); + while (true) { + try { + fs.mkdirSync(lockDir, { mode: 0o700 }); + try { + // The token is a child of this lock directory, so a stale holder can + // release only its own marker even if another writer has already + // removed and reacquired the directory. + fs.writeFileSync( + ownerPath, + JSON.stringify({ + version: PROJECT_WORKSPACE_LOCK_OWNER_VERSION, + pid: process.pid, + token: ownerToken, + }), + { mode: 0o600, flag: 'wx' } + ); + } catch (error) { + // mkdir succeeded, but this invocation never acquired a usable lock. + // Clean up only the directory it just created before propagating. + fs.rmSync(ownerPath, { force: true }); + try { + fs.rmdirSync(lockDir); + } catch (cleanupError) { + if ((cleanupError as NodeJS.ErrnoException).code !== 'ENOENT') throw cleanupError; + } + throw error; + } + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + try { + if (Date.now() - fs.statSync(lockDir).mtimeMs >= PROJECT_WORKSPACE_LOCK_STALE_MS) { + const observedLock = inspectProjectWorkspaceLock(lockDir); + if (!observedLock.ownerIsAlive) { + // Reclaim only the exact marker entries observed in this lock. A + // replacement writer can add a different marker concurrently; the + // non-recursive rmdir then leaves that replacement lock untouched. + for (const entry of observedLock.entries) { + fs.rmSync(path.join(lockDir, entry), { force: true }); + } + try { + fs.rmdirSync(lockDir); + } catch (error) { + if ( + (error as NodeJS.ErrnoException).code !== 'ENOENT' && + (error as NodeJS.ErrnoException).code !== 'ENOTEMPTY' + ) { + throw error; + } + } + continue; + } + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue; + throw error; + } + if (Date.now() - startedAt >= PROJECT_WORKSPACE_LOCK_TIMEOUT_MS) { + throw new Error(`Timed out waiting for the project workspace lock at ${lockDir}.`); + } + Atomics.wait(PROJECT_WORKSPACE_LOCK_WAIT, 0, 0, PROJECT_WORKSPACE_LOCK_RETRY_MS); + } + let callbackFailed = false; + try { + return callback(); + } catch (error) { + callbackFailed = true; + throw error; + } finally { + // Remove our marker first. If the lock was declared stale and replaced + // while the callback was running, the replacement marker is different; + // rmdir then safely leaves the replacement lock in place. + if (fs.existsSync(ownerPath)) { + fs.rmSync(ownerPath, { force: true }); + try { + fs.rmdirSync(lockDir); + } catch (error) { + if ( + (error as NodeJS.ErrnoException).code !== 'ENOENT' && + (error as NodeJS.ErrnoException).code !== 'ENOTEMPTY' + ) { + if (callbackFailed) { + console.error(`Failed to release the project workspace lock at ${lockDir}.`, error); + } else { + throw error; + } + } + } + } + } +} + +function inspectProjectWorkspaceLock(lockDir: string): { + entries: string[]; + ownerIsAlive: boolean; +} { + const entries = fs.readdirSync(lockDir); + let sawLiveOwner = false; + for (const entry of entries) { + let owner: Partial; + try { + owner = JSON.parse( + fs.readFileSync(path.join(lockDir, entry), 'utf8') + ) as Partial; + } catch { + continue; + } + const pid = owner.pid; + if ( + owner.version !== PROJECT_WORKSPACE_LOCK_OWNER_VERSION || + typeof pid !== 'number' || + !Number.isInteger(pid) || + pid <= 0 || + typeof owner.token !== 'string' || + owner.token !== entry + ) { + continue; + } + try { + process.kill(pid, 0); + sawLiveOwner = true; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + // EPERM means the process exists but is not signalable. Treat all + // errors other than ESRCH conservatively as alive. + if (code !== 'ESRCH') sawLiveOwner = true; + } + } + return { entries, ownerIsAlive: sawLiveOwner }; +} + +/** Atomically persist a Relaycast target only if the captured project selection is still current. */ +export function writeProjectWorkspaceTargetIfSelectionCurrent( + dataDir: string, + selection: WorkspaceSelection, + target: Required< + Pick + > +): boolean { + return withProjectWorkspaceKeyLock(dataDir, () => { + const current = readProjectWorkspaceSession(dataDir); + if (selection.projectSessionPresent === false && current) return false; + if ( + current && + (current.workspaceKey !== selection.key || + current.workspaceId !== selection.workspaceId || + current.relaycastRoute !== selection.relaycastRoute || + current.relaycastBaseUrl !== selection.relaycastBaseUrl || + current.relaycastApiKey !== selection.relaycastApiKey) + ) { + return false; + } + if (!current && (selection.projectSessionPresent === true || selection.source === 'project')) { + return false; + } + writeProjectWorkspaceKeyUnlocked(dataDir, selection.key, { + ...(current?.enrolledNodeId ? { enrolledNodeId: current.enrolledNodeId } : {}), + workspaceId: target.workspaceId, + relaycastRoute: target.relaycastRoute, + relaycastBaseUrl: target.relaycastBaseUrl, + relaycastApiKey: target.relaycastApiKey, + }); + return true; + }); +} + +/** + * Rewrite a project session while retaining server-selected metadata when the + * workspace key is unchanged. A changed key is an intentional rebind, so no + * metadata from the old workspace is carried across. + * + * The low-level writer above deliberately replaces the complete record. This + * helper is for callers that update one part of an existing session (for + * example, linking an enrolled node) and must not accidentally discard the + * Relaycast target or workspace identity recorded alongside the key. + */ +export function writeProjectWorkspaceKeyPreservingSession( + dataDir: string, + workspaceKey: string | undefined, + options: ProjectWorkspaceSessionMetadata = {} +): void { + const key = trimOrUndefined(workspaceKey); + if (!key) return; + + withProjectWorkspaceKeyLock(dataDir, () => { + const existing = readProjectWorkspaceSession(dataDir); + const sameWorkspace = existing?.workspaceKey === key; + const retained: ProjectWorkspaceSessionMetadata = sameWorkspace + ? { + ...(existing?.enrolledNodeId ? { enrolledNodeId: existing.enrolledNodeId } : {}), + ...(existing?.workspaceId ? { workspaceId: existing.workspaceId } : {}), + ...(existing?.relaycastRoute ? { relaycastRoute: existing.relaycastRoute } : {}), + ...(existing?.relaycastBaseUrl ? { relaycastBaseUrl: existing.relaycastBaseUrl } : {}), + ...(existing?.relaycastApiKey ? { relaycastApiKey: existing.relaycastApiKey } : {}), + } + : {}; + + writeProjectWorkspaceKeyUnlocked(dataDir, key, { + ...retained, + ...(trimOrUndefined(options.enrolledNodeId) + ? { enrolledNodeId: trimOrUndefined(options.enrolledNodeId) } + : {}), + ...(trimOrUndefined(options.workspaceId) ? { workspaceId: trimOrUndefined(options.workspaceId) } : {}), + ...(options.relaycastRoute ? { relaycastRoute: options.relaycastRoute } : {}), + ...(trimOrUndefined(options.relaycastBaseUrl) + ? { relaycastBaseUrl: trimOrUndefined(options.relaycastBaseUrl) } + : {}), + ...(trimOrUndefined(options.relaycastApiKey) + ? { relaycastApiKey: trimOrUndefined(options.relaycastApiKey) } + : {}), + }); + }); +} + /** * Resolve which Relay workspace this process addresses. * @@ -163,26 +441,70 @@ export function resolveWorkspaceSelection( options: ResolveWorkspaceKeyOptions = {} ): WorkspaceSelection | undefined { const env = options.env ?? process.env; + const dataDir = options.projectDataDir ?? projectDataDir(options.projectRoot); + const project = dataDir ? readProjectWorkspaceSession(dataDir, options.fileSystem ?? fs) : undefined; const flag = trimOrUndefined(options.workspaceKey); - if (flag) return { key: flag, source: 'flag', origin: '--workspace-key' }; + if (flag) { + return { + key: flag, + source: 'flag', + origin: '--workspace-key', + ...(project?.workspaceKey === flag && project.workspaceId ? { workspaceId: project.workspaceId } : {}), + ...(project?.workspaceKey === flag && project.relaycastRoute + ? { relaycastRoute: project.relaycastRoute } + : {}), + ...(project?.workspaceKey === flag && project.relaycastBaseUrl + ? { relaycastBaseUrl: project.relaycastBaseUrl } + : {}), + ...(project?.workspaceKey === flag && project.relaycastApiKey + ? { relaycastApiKey: project.relaycastApiKey } + : {}), + ...((!project || project.workspaceKey === flag) && dataDir + ? { projectDataDir: dataDir, projectSessionPresent: project !== undefined } + : {}), + }; + } for (const name of WORKSPACE_KEY_ENV_VARS) { const envKey = trimOrUndefined(env[name]); - if (envKey) return { key: envKey, source: 'env', origin: `$${name}` }; + if (envKey) { + return { + key: envKey, + source: 'env', + origin: `$${name}`, + ...(project?.workspaceKey === envKey && project.workspaceId + ? { workspaceId: project.workspaceId } + : {}), + ...(project?.workspaceKey === envKey && project.relaycastRoute + ? { relaycastRoute: project.relaycastRoute } + : {}), + ...(project?.workspaceKey === envKey && project.relaycastBaseUrl + ? { relaycastBaseUrl: project.relaycastBaseUrl } + : {}), + ...(project?.workspaceKey === envKey && project.relaycastApiKey + ? { relaycastApiKey: project.relaycastApiKey } + : {}), + ...((!project || project.workspaceKey === envKey) && dataDir + ? { projectDataDir: dataDir, projectSessionPresent: project !== undefined } + : {}), + }; + } } - const dataDir = options.projectDataDir ?? projectDataDir(options.projectRoot); - const project = dataDir ? readProjectWorkspaceSession(dataDir, options.fileSystem ?? fs) : undefined; if (project) { return { key: project.workspaceKey, source: 'project', origin: projectWorkspaceKeyPath(dataDir as string), ...(project.workspaceId ? { workspaceId: project.workspaceId } : {}), + ...(project.relaycastRoute ? { relaycastRoute: project.relaycastRoute } : {}), + ...(project.relaycastBaseUrl ? { relaycastBaseUrl: project.relaycastBaseUrl } : {}), + ...(project.relaycastApiKey ? { relaycastApiKey: project.relaycastApiKey } : {}), + ...(dataDir ? { projectDataDir: dataDir, projectSessionPresent: true } : {}), }; } - return resolveActiveWorkspaceSelection(env); + return resolveActiveWorkspaceSelection(env, dataDir); } /** @@ -194,7 +516,8 @@ export function resolveWorkspaceSelection( * It is never correct to consult this ahead of steps 1–3. */ export function resolveActiveWorkspaceSelection( - env: NodeJS.ProcessEnv = process.env + env: NodeJS.ProcessEnv = process.env, + projectDataDir?: string ): WorkspaceSelection | undefined { const store = readWorkspaceStore(env); const activeName = trimOrUndefined(store.active); @@ -204,6 +527,7 @@ export function resolveActiveWorkspaceSelection( key: storeKey, source: 'store', origin: `${workspaceStorePath(env)} (active: "${activeName}")`, + ...(projectDataDir ? { projectDataDir, projectSessionPresent: false } : {}), } : undefined; } diff --git a/packages/cloud/src/workspace-key.ts b/packages/cloud/src/workspace-key.ts index f102295835..79d0761e11 100644 --- a/packages/cloud/src/workspace-key.ts +++ b/packages/cloud/src/workspace-key.ts @@ -7,7 +7,10 @@ export { resolveWorkspaceKeyWithSource, resolveWorkspaceSelection, writeProjectWorkspaceKey, + writeProjectWorkspaceKeyPreservingSession, + writeProjectWorkspaceTargetIfSelectionCurrent, type ProjectWorkspaceSession, + type ProjectWorkspaceSessionMetadata, type ResolveWorkspaceKeyOptions, type WorkspaceKeyFileSystem, type WorkspaceKeySource, diff --git a/scripts/ci-standalone-smoke.sh b/scripts/ci-standalone-smoke.sh index 56a797c777..607e6d2c6c 100755 --- a/scripts/ci-standalone-smoke.sh +++ b/scripts/ci-standalone-smoke.sh @@ -6,23 +6,31 @@ if [ "$#" -ne 2 ]; then exit 2 fi -if [[ -z "${RELAY_WORKSPACE_KEY:-}" || "${RELAY_WORKSPACE_KEY:-}" =~ ^[[:space:]]+$ ]]; then - echo "ERROR: RELAY_WORKSPACE_KEY must name the dedicated standalone-smoke CI workspace." >&2 - echo "Refusing to start without it because node up would create an undeletable throwaway workspace." >&2 - exit 2 -fi +# The standalone smoke is deliberately pinned to the trusted hosted Relaycast +# engine. Do not make this caller-selectable: the workspace key created below +# is scoped to this origin and must never be sent to an arbitrary endpoint. +TRUSTED_RELAY_BASE_URL="https://cast.agentrelay.com" # The broker gives a multi-workspace session higher precedence than the single -# key. This smoke intentionally exercises one dedicated workspace, so do not +# key. This smoke intentionally exercises one ephemeral workspace, so do not # let an ambient developer/runner session silently replace the CI credential. -unset RELAY_WORKSPACES_JSON +unset RELAY_WORKSPACES_JSON RELAY_WORKSPACE_KEY AGENT_RELAY_WORKSPACE_KEY RELAY_API_KEY \ + RELAYCAST_BASE_URL RELAY_AGENT_TOKEN RELAY_WORKSPACE # Startup can legitimately consume the broker's 40-second aggregate Relaycast # handshake budget on a loaded macOS runner. Keep the outer supervisor at # least ten seconds above that bound so an override cannot reintroduce the race # this smoke is meant to catch. MIN_STARTUP_TIMEOUT_SECONDS=50 -MAX_STARTUP_TIMEOUT_SECONDS=86400 +# Keep every accepted startup override inside the ephemeral workspace lease, +# with a full minute left for shutdown and deletion verification. +WORKSPACE_LEASE_SECONDS=300 +MAX_STARTUP_TIMEOUT_SECONDS=240 +CURL_CONNECT_TIMEOUT_SECONDS=10 +CURL_MAX_TIME_SECONDS=60 +CLEANUP_VERIFY_MAX_ATTEMPTS=3 +CLEANUP_VERIFY_MAX_TIME_SECONDS=10 +CLEANUP_VERIFY_FALLBACK_DELAY_SECONDS=2 STARTUP_TIMEOUT_SECONDS="${AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS:-60}" if ! [[ "$STARTUP_TIMEOUT_SECONDS" =~ ^[1-9][0-9]{0,4}$ ]]; then echo "ERROR: AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS must be a base-10 integer between ${MIN_STARTUP_TIMEOUT_SECONDS}s and ${MAX_STARTUP_TIMEOUT_SECONDS}s without leading zeros." >&2 @@ -74,28 +82,110 @@ validate_binary "BROKER" "$BROKER_BIN" TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/agent-relay-standalone-smoke.XXXXXX")" HOME_DIR="$TMP_ROOT/home" PROJECT_DIR="$TMP_ROOT/project" +WORKSPACE_RESPONSE="$TMP_ROOT/workspace-response.json" +DELETE_RESPONSE="$TMP_ROOT/delete-response.json" +VERIFY_HEADERS="$TMP_ROOT/verify-headers.txt" mkdir -p "$HOME_DIR" "$PROJECT_DIR" CLEANUP_STARTED=false +WORKSPACE_KEY="" +WORKSPACE_ID="" cleanup() { if [ "$CLEANUP_STARTED" = true ]; then - return + return 0 fi CLEANUP_STARTED=true + local cleanup_status=0 # Disarm before entering the cleanup subshell. Some Bash exit paths can # otherwise inherit this EXIT trap and recursively run node down again. trap - EXIT ( cd "$PROJECT_DIR" HOME="$HOME_DIR" \ + RELAY_BASE_URL="$TRUSTED_RELAY_BASE_URL" \ AGENT_RELAY_BIN="$BROKER_BIN" \ AGENT_RELAY_SKIP_UPDATE_CHECK=1 \ AGENT_RELAY_STARTUP_DEBUG=1 \ AGENT_RELAY_TELEMETRY_DISABLED=1 \ + RELAY_WORKSPACE_KEY="$WORKSPACE_KEY" \ "$CLI_BIN" node down --force --timeout 5000 >/dev/null 2>&1 || true ) + if [ -n "$WORKSPACE_KEY" ]; then + local delete_status delete_error_code delete_error_code_raw verify_status verify_attempt verify_delay + delete_status="$(curl --silent --show-error --output "$DELETE_RESPONSE" --write-out '%{http_code}' \ + --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CURL_MAX_TIME_SECONDS" \ + --request DELETE \ + --header "Authorization: Bearer $WORKSPACE_KEY" \ + "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" + delete_error_code_raw="$(jq -er '.error.code // .code // empty' "$DELETE_RESPONSE" 2>/dev/null || true)" + delete_error_code="" + case "$delete_error_code_raw" in + internal_error|workspace_storage_unavailable|database_overloaded|file_storage_delete_unsupported) + delete_error_code="$delete_error_code_raw" + ;; + ?*) delete_error_code="other" ;; + esac + if [ "$delete_status" = "200" ] || [ "$delete_status" = "204" ]; then + # The engine only returns success after its atomic deletion batch commits. + # A second database read would add an unrelated availability dependency. + echo "Ephemeral workspace deletion verified" + else + verify_attempt=1 + while [ "$verify_attempt" -le "$CLEANUP_VERIFY_MAX_ATTEMPTS" ]; do + verify_status="$(curl --silent --show-error --output /dev/null --dump-header "$VERIFY_HEADERS" \ + --write-out '%{http_code}' \ + --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CLEANUP_VERIFY_MAX_TIME_SECONDS" \ + --request GET \ + --header "Authorization: Bearer $WORKSPACE_KEY" \ + "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" + case "$verify_status" in + 401) break ;; + 000|429|5??) + if [ "$verify_attempt" -lt "$CLEANUP_VERIFY_MAX_ATTEMPTS" ]; then + # Relaycast advertises integer Retry-After values from 2–8s for + # database overload. Accept only that bounded vocabulary; a + # malformed, date-form, or excessive value falls back to 2s. + verify_delay="$(awk ' + { + line = $0 + gsub(/\r/, "", line) + } + tolower(line) ~ /^retry-after:[[:space:]]*/ { + sub(/^[^:]*:[[:space:]]*/, "", line) + value = line + } + END { print value } + ' "$VERIFY_HEADERS" 2>/dev/null || true)" + case "$verify_delay" in + 0|1|2|3|4|5|6|7|8) ;; + *) verify_delay="$CLEANUP_VERIFY_FALLBACK_DELAY_SECONDS" ;; + esac + sleep "$verify_delay" + verify_attempt=$((verify_attempt + 1)) + continue + fi + ;; + esac + break + done + # A 5xx response can be lost after the database commit. The workspace key + # was proven valid by the lifecycle above, so 401 from the same key is the + # authoritative absence check. Transient verification reads get a bounded + # retry window; any readable or still-unverifiable state remains a hard + # failure. + if [ "$verify_status" = "401" ]; then + echo "Ephemeral workspace deletion verified after ambiguous DELETE HTTP ${delete_status:-unknown}${delete_error_code:+, error code $delete_error_code}" + else + echo "ERROR: ephemeral workspace cleanup returned HTTP ${delete_status:-unknown}${delete_error_code:+, error code $delete_error_code}." >&2 + echo "ERROR: ephemeral workspace deletion was not proved (follow-up HTTP ${verify_status:-unknown})." >&2 + echo "The workspace id is ${WORKSPACE_ID:-unknown}; remove it manually from the trusted smoke shard." >&2 + cleanup_status=1 + fi + fi + fi rm -rf "$TMP_ROOT" + return "$cleanup_status" } trap cleanup EXIT @@ -104,6 +194,8 @@ run_cli() { ( cd "$PROJECT_DIR" HOME="$HOME_DIR" \ + RELAY_BASE_URL="$TRUSTED_RELAY_BASE_URL" \ + RELAY_WORKSPACE_KEY="$WORKSPACE_KEY" \ AGENT_RELAY_BIN="$BROKER_BIN" \ AGENT_RELAY_SKIP_UPDATE_CHECK=1 \ AGENT_RELAY_STARTUP_DEBUG=1 \ @@ -112,6 +204,48 @@ run_cli() { ) } +if ! command -v curl >/dev/null 2>&1; then + echo "ERROR: curl is required to create and delete the ephemeral smoke workspace." >&2 + exit 2 +fi +if ! command -v jq >/dev/null 2>&1; then + echo "ERROR: jq is required to parse the ephemeral smoke workspace response." >&2 + exit 2 +fi + +WORKSPACE_NAME="relay-standalone-smoke-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-$$" +CREATE_STATUS="$(curl --silent --show-error --output "$WORKSPACE_RESPONSE" --write-out '%{http_code}' \ + --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CURL_MAX_TIME_SECONDS" \ + --request POST \ + --header 'Content-Type: application/json' \ + --data "$(jq -cn --arg name "$WORKSPACE_NAME" --argjson expires "$WORKSPACE_LEASE_SECONDS" '{name: $name, expires_in_seconds: $expires}')" \ + "$TRUSTED_RELAY_BASE_URL/v1/workspaces" 2>/dev/null || true)" +# Mask the key before extracting or using any other response field. Never log +# the response body: it contains the administrative workspace credential. +WORKSPACE_KEY="$(jq -er '.data.api_key // .api_key // empty' "$WORKSPACE_RESPONSE" 2>/dev/null || true)" +CREATE_ERROR_CODE="$(jq -er '.error.code // .code // empty' "$WORKSPACE_RESPONSE" 2>/dev/null || true)" +if [ -n "$WORKSPACE_KEY" ]; then + printf '::add-mask::%s\n' "$WORKSPACE_KEY" +fi +if [ -z "$WORKSPACE_KEY" ]; then + echo "ERROR: ephemeral smoke workspace response did not contain an API key (HTTP ${CREATE_STATUS:-unknown}${CREATE_ERROR_CODE:+, error code ${CREATE_ERROR_CODE}})." >&2 + exit 1 +fi +if [[ ! "$WORKSPACE_KEY" =~ ^rk_live_[A-Za-z0-9_-]+$ ]]; then + echo "ERROR: ephemeral smoke workspace response contained an invalid API key scheme." >&2 + exit 1 +fi +WORKSPACE_ID="$(jq -er '.data.workspace_id // .workspace_id // empty' "$WORKSPACE_RESPONSE" 2>/dev/null || true)" +if [ -z "$WORKSPACE_ID" ]; then + echo "ERROR: ephemeral smoke workspace response did not contain a workspace id." >&2 + exit 1 +fi +if [ "$CREATE_STATUS" != "200" ] && [ "$CREATE_STATUS" != "201" ]; then + echo "ERROR: ephemeral smoke workspace creation returned HTTP ${CREATE_STATUS:-unknown}." >&2 + exit 1 +fi +echo "Ephemeral workspace created on trusted smoke shard (id ${WORKSPACE_ID})" + print_output_excerpt() { local output="$1" local total_lines @@ -240,4 +374,8 @@ if printf '%s\n' "$UP_OUTPUT" | grep -q 'Broker already running for this project exit 1 fi +if ! cleanup; then + echo "Standalone smoke lifecycle passed but ephemeral workspace cleanup was not proved" >&2 + exit 1 +fi echo "Standalone smoke passed" diff --git a/scripts/pr-proof/process-runner.mjs b/scripts/pr-proof/process-runner.mjs index 44a95e32f5..7028d806c4 100644 --- a/scripts/pr-proof/process-runner.mjs +++ b/scripts/pr-proof/process-runner.mjs @@ -104,6 +104,14 @@ export function runBoundedProcess(command, args, options = {}) { const stdoutDecoder = new StringDecoder('utf8'); const stderrDecoder = new StringDecoder('utf8'); + // Transforms run before capture, live output, and callbacks. A transform + // may retain a small streaming boundary and is called once more with + // `final=true` after the decoder has been flushed. + const transformOutput = (stream, text, final = false) => { + if (typeof options.transformChunk !== 'function') return text; + return options.transformChunk(text, stream, final) ?? ''; + }; + const forceKill = () => { if (forced) return; forced = true; @@ -130,6 +138,21 @@ export function runBoundedProcess(command, args, options = {}) { options.signal?.removeEventListener('abort', abortHandler); }; + const fail = (error) => { + if (settled) return; + settled = true; + cleanup(); + try { + forceKill(); + } catch { + // Preserve the transform/spawn failure as the rejection reason. The + // parent-side pipes still must close even if process signaling fails. + child.stdout.destroy(); + child.stderr.destroy(); + } + reject(error); + }; + const writeLiveOutput = (stream, text) => { if (options.echo === false || !text) return; if (liveOutputBytes >= maximumLiveOutput) { @@ -152,46 +175,57 @@ export function runBoundedProcess(command, args, options = {}) { } }; + const consumeOutput = (stream, text, final = false) => { + const transformed = transformOutput(stream, text, final); + if (!transformed) return; + if (stream === 'stdout') { + stdout = appendBounded(stdout, transformed, maximum); + options.onStdout?.(transformed); + writeLiveOutput(process.stdout, transformed); + } else { + stderr = appendBounded(stderr, transformed, maximum); + options.onStderr?.(transformed); + writeLiveOutput(process.stderr, transformed); + } + }; + child.stdout.on('data', (chunk) => { - const text = stdoutDecoder.write(chunk); - if (!text) return; - stdout = appendBounded(stdout, text, maximum); - options.onStdout?.(text); - writeLiveOutput(process.stdout, text); + try { + consumeOutput('stdout', stdoutDecoder.write(chunk)); + } catch (error) { + fail(error); + } }); child.stderr.on('data', (chunk) => { - const text = stderrDecoder.write(chunk); - if (!text) return; - stderr = appendBounded(stderr, text, maximum); - options.onStderr?.(text); - writeLiveOutput(process.stderr, text); - }); - child.on('error', (error) => { - if (settled) return; - settled = true; - cleanup(); - reject(error); + try { + consumeOutput('stderr', stderrDecoder.write(chunk)); + } catch (error) { + fail(error); + } }); + child.on('error', fail); child.on('close', (code, signal) => { if (settled) return; - settled = true; // The process-group leader can exit after SIGTERM while a descendant // with detached stdio remains alive. Force-kill the group before // clearing the grace timer so that descendant cannot escape cleanup. if (timedOut || aborted) forceKill(); cleanup(); - const stdoutTail = stdoutDecoder.end(); - const stderrTail = stderrDecoder.end(); - stdout = appendBounded(stdout, stdoutTail, maximum); - stderr = appendBounded(stderr, stderrTail, maximum); - if (stdoutTail) { - options.onStdout?.(stdoutTail); - writeLiveOutput(process.stdout, stdoutTail); - } - if (stderrTail) { - options.onStderr?.(stderrTail); - writeLiveOutput(process.stderr, stderrTail); + try { + const stdoutTail = stdoutDecoder.end(); + const stderrTail = stderrDecoder.end(); + consumeOutput('stdout', stdoutTail); + consumeOutput('stderr', stderrTail); + consumeOutput('stdout', '', true); + consumeOutput('stderr', '', true); + } catch (error) { + // A descendant can keep running after the process-group leader closes + // its inherited pipes. Use the same terminal cleanup path as a + // transform failure observed during a data event. + fail(error); + return; } + settled = true; resolve({ exitCode: code ?? 1, signal, stdout, stderr, timedOut, aborted }); }); }); diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 00cd83d22f..903290a043 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -9,6 +9,14 @@ import { runBoundedProcess } from './process-runner.mjs'; const TERMINAL_SUCCESS = new Set(['completed', 'succeeded', 'success']); const TERMINAL_FAILURE = new Set(['failed', 'cancelled', 'canceled', 'timed_out', 'error']); +const CLOUD_RUN_STATUSES = new Set([ + 'pending', + 'queued', + 'launching', + 'running', + ...TERMINAL_SUCCESS, + ...TERMINAL_FAILURE, +]); const LEGACY_REFRESHABLE_AUTH_KEYS = [ 'CLOUD_API_ACCESS_TOKEN', 'CLOUD_API_REFRESH_TOKEN', @@ -17,12 +25,55 @@ const LEGACY_REFRESHABLE_AUTH_KEYS = [ ]; const MAX_CAPTURE_BYTES = 2 * 1024 * 1024; const MAX_LIVE_OUTPUT_BYTES = 256 * 1024; +const MAX_DIAGNOSTIC_BYTES = 64 * 1024; +const MIN_FINAL_MASK_FRAGMENT_LENGTH = 4; const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60_000; const PREPARED_RUN_ID_MARKER = 'AGENT_RELAY_CLOUD_PREPARED_RUN_ID='; const RUN_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; +const DIAGNOSTIC_TOKEN_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/; +const ISO_TIMESTAMP_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/; +const LIVE_CREDENTIAL_PREFIXES = [ + 'rk_live_', + 'rjt_live_', + 'at_live_', + 'nt_live_', + 'ot_live_', + 'cld_at_', + 'rth_at_', + 'ocl_node_enr_', + 'br_', + 'github_pat_', + 'ghp_', + 'gho_', + 'ghu_', + 'ghs_', + 'ghr_', +]; +const LIVE_CREDENTIAL_PREFIX_RE = new RegExp( + `(${LIVE_CREDENTIAL_PREFIXES.map((prefix) => prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|')})`, + 'g' +); +const STATUS_DIAGNOSTIC_FIELDS = [ + 'runId', + 'status', + 'sandboxId', + 'dispatchType', + 'relayflowVersion', + 'createdAt', + 'updatedAt', +]; +const STATUS_FAILURE_DIAGNOSTIC_FIELDS = ['phase', 'code', 'dispatchType', 'sandboxId', 'occurredAt']; -function run(command, args, options = {}) { - return runBoundedProcess(command, args, { +export async function run(command, args, options = {}) { + const diagnosticSecretValues = options.diagnosticSecretValues ?? []; + // Keep a redaction boundary per pipe. A partial credential suffix from + // stdout must never be prepended to the next stderr chunk (or vice versa). + // Each pipe is finalized independently, so an actually benign suffix can be + // released once that originating stream closes. + const outputRedactors = createCommandOutputRedactors(diagnosticSecretValues, { + maskPendingOnFinal: true, + }); + const result = await runBoundedProcess(command, args, { env: options.env, echo: !options.quiet, maxCaptureBytes: MAX_CAPTURE_BYTES, @@ -31,7 +82,34 @@ function run(command, args, options = {}) { signal: options.signal, onStdout: options.onStdout, onStderr: options.onStderr, + transformChunk: (text, stream, final) => outputRedactors[stream].push(text, final), }); + + return maskCapturedCommandOutput(result, outputRedactors); +} + +/** Create independent streaming redactors for subprocess stdout and stderr. */ +export function createCommandOutputRedactors(secretValues = [], { maskPendingOnFinal = false } = {}) { + return { + stdout: createCredentialRedactor(secretValues, { maskPendingOnFinal }), + stderr: createCredentialRedactor(secretValues, { maskPendingOnFinal }), + }; +} + +export function maskCapturedCommandOutput(result, outputRedactor) { + if (outputRedactor && typeof outputRedactor.requiresCapturedOutputMask === 'function') { + return outputRedactor.requiresCapturedOutputMask() + ? { ...result, stdout: result.stdout ? '[redacted]' : '', stderr: result.stderr ? '[redacted]' : '' } + : result; + } + const maskedStreams = Object.entries(outputRedactor ?? {}) + .filter(([, redactor]) => redactor?.requiresCapturedOutputMask?.()) + .map(([stream]) => stream); + if (maskedStreams.length === 0) return result; + return { + ...result, + ...Object.fromEntries(maskedStreams.map((stream) => [stream, result[stream] ? '[redacted]' : ''])), + }; } export function boundedDuration(value, { fallback, minimum, maximum, label }) { @@ -43,24 +121,396 @@ export function boundedDuration(value, { fallback, minimum, maximum, label }) { return parsed; } -function parseJsonOutput(output, label) { +export function parseJsonOutput(output, label) { try { return JSON.parse(output); } catch { const first = output.indexOf('{'); const last = output.lastIndexOf('}'); - if (first >= 0 && last > first) return JSON.parse(output.slice(first, last + 1)); + if (first >= 0 && last > first) { + try { + return JSON.parse(output.slice(first, last + 1)); + } catch { + // Fall through to the fixed error below without exposing payload excerpts. + } + } throw new Error(`${label} did not return JSON`); } } +export function boundedDiagnostic(value) { + const text = String(value ?? ''); + const bytes = Buffer.from(text, 'utf8'); + if (bytes.length <= MAX_DIAGNOSTIC_BYTES) return text; + + const marker = '\n[... diagnostic output truncated ...]'; + const tailBudget = MAX_DIAGNOSTIC_BYTES - Buffer.byteLength(marker, 'utf8'); + let tail = bytes.subarray(bytes.length - tailBudget).toString('utf8'); + // A byte slice can begin in the middle of a multi-byte code point. Removing + // the replacement character (or another leading code point if needed) keeps + // the final diagnostic, including its marker, within the byte contract. + while (Buffer.byteLength(tail, 'utf8') > tailBudget) tail = tail.slice(1); + return `${tail}${marker}`; +} + +function longestSuffixThatStartsSecret(value, secrets) { + const maximum = Math.min(value.length, Math.max(...secrets.map((secret) => secret.length), 0) - 1); + for (let length = maximum; length > 0; length -= 1) { + const suffix = value.slice(value.length - length); + if (secrets.some((secret) => secret.startsWith(suffix))) return length; + } + return 0; +} + +function longestSuffixThatMatchesSecret(value, secrets) { + const maximum = Math.min(value.length, Math.max(...secrets.map((secret) => secret.length), 0) - 1); + for (let length = maximum; length > 0; length -= 1) { + const suffix = value.slice(value.length - length); + if (secrets.some((secret) => secret.startsWith(suffix) || secret.endsWith(suffix))) return length; + } + return 0; +} + +function createCredentialPrefixRedactor(maskPendingOnFinal = false) { + let pending = ''; + let active = null; + let maskedPendingOnFinal = false; + + return { + push(value, final = false) { + const input = pending + String(value ?? ''); + pending = ''; + let output = ''; + let index = 0; + + while (index < input.length) { + if (active) { + if (!active.emitted) { + if (/[A-Za-z0-9_%-]/.test(input[index])) { + output += `${active.prefix}…`; + active.emitted = true; + } else { + output += active.prefix; + active = null; + continue; + } + } + while (index < input.length && /[A-Za-z0-9_%.%-]/.test(input[index])) index += 1; + if (index === input.length) { + if (final) active = null; + break; + } + active = null; + continue; + } + + LIVE_CREDENTIAL_PREFIX_RE.lastIndex = index; + const match = LIVE_CREDENTIAL_PREFIX_RE.exec(input); + if (match) { + const prefixIndex = match.index; + const prefix = match[0]; + output += input.slice(index, prefixIndex); + index = prefixIndex; + active = { prefix, emitted: false }; + index += prefix.length; + continue; + } + + const suffixLength = + final && !maskPendingOnFinal + ? 0 + : longestSuffixThatStartsSecret(input.slice(index), LIVE_CREDENTIAL_PREFIXES); + const end = input.length - suffixLength; + output += input.slice(index, end); + if (suffixLength > 0) { + pending = input.slice(end); + } + break; + } + + if (final && active && !active.emitted) { + output += maskPendingOnFinal ? '[redacted]' : active.prefix; + } + if (final) { + active = null; + if (pending) { + output += + maskPendingOnFinal && pending.length >= MIN_FINAL_MASK_FRAGMENT_LENGTH ? '[redacted]' : pending; + pending = ''; + } + } + return output; + }, + maskedPendingOnFinal() { + return maskedPendingOnFinal; + }, + }; +} + +function createConfiguredSecretRedactor(secretValues, maskPendingOnFinal = false) { + const secrets = [...new Set(secretValues.filter((value) => typeof value === 'string' && value))]; + let pending = ''; + let maskedPendingOnFinal = false; + + return { + push(value, final = false) { + if (secrets.length === 0) return String(value ?? ''); + const input = pending + String(value ?? ''); + pending = ''; + let output = ''; + let index = 0; + while (index < input.length) { + let secret; + let secretIndex = -1; + for (const candidate of secrets) { + const candidateIndex = input.indexOf(candidate, index); + if ( + candidateIndex !== -1 && + (secretIndex === -1 || + candidateIndex < secretIndex || + (candidateIndex === secretIndex && candidate.length > (secret?.length ?? 0))) + ) { + secret = candidate; + secretIndex = candidateIndex; + } + } + if (secret !== undefined) { + output += input.slice(index, secretIndex); + output += '[redacted]'; + index = secretIndex + secret.length; + continue; + } + const suffixLength = + final && !maskPendingOnFinal ? 0 : longestSuffixThatMatchesSecret(input.slice(index), secrets); + const end = input.length - suffixLength; + output += input.slice(index, end); + if (suffixLength > 0) { + pending = input.slice(end); + } + break; + } + if (final) { + output += + pending && maskPendingOnFinal && pending.length >= MIN_FINAL_MASK_FRAGMENT_LENGTH + ? '[redacted]' + : pending; + pending = ''; + } + return output; + }, + maskedPendingOnFinal() { + return maskedPendingOnFinal; + }, + }; +} + +/** Redact credentials across subprocess chunks before bounded capture. */ +export function createCredentialRedactor(secretValues = [], { maskPendingOnFinal = false } = {}) { + const prefixRedactor = createCredentialPrefixRedactor(maskPendingOnFinal); + const secretRedactor = createConfiguredSecretRedactor(secretValues, maskPendingOnFinal); + return { + push(value, final = false) { + const prefixed = prefixRedactor.push(value, final); + return secretRedactor.push(prefixed, final); + }, + requiresCapturedOutputMask() { + return prefixRedactor.maskedPendingOnFinal() || secretRedactor.maskedPendingOnFinal(); + }, + }; +} + +export function sanitizeCloudCommandOutput(value, secretValues = []) { + return createCredentialRedactor(secretValues).push(value, true); +} + +function structuralDiagnosticValue(field, value, secretValues) { + const sanitized = sanitizeCloudCommandOutput(value, secretValues); + if (field === 'status') return recognizedCloudRunStatus(sanitized); + if (field === 'runId' || field === 'sandboxId') { + return RUN_ID_RE.test(sanitized) ? sanitized : null; + } + if (field === 'relayflowVersion') { + return sanitized === 'v1' || sanitized === 'v2' ? sanitized : null; + } + if (field === 'createdAt' || field === 'updatedAt' || field === 'occurredAt') { + return ISO_TIMESTAMP_RE.test(sanitized) && Number.isFinite(Date.parse(sanitized)) ? sanitized : null; + } + return DIAGNOSTIC_TOKEN_RE.test(sanitized) ? sanitized : null; +} + +function diagnosticRecord(value, secretValues) { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const diagnostic = {}; + for (const field of STATUS_DIAGNOSTIC_FIELDS) { + if (typeof value[field] === 'string') { + const structuralValue = structuralDiagnosticValue(field, value[field], secretValues); + if (structuralValue) diagnostic[field] = structuralValue; + } + } + if (value.failure && typeof value.failure === 'object' && !Array.isArray(value.failure)) { + const failure = {}; + for (const field of STATUS_FAILURE_DIAGNOSTIC_FIELDS) { + if (typeof value.failure[field] === 'string') { + const structuralValue = structuralDiagnosticValue(field, value.failure[field], secretValues); + if (structuralValue) failure[field] = structuralValue; + } + } + if (Object.keys(failure).length > 0) diagnostic.failure = failure; + } + return diagnostic; +} + +/** + * Reduce `cloud status --json` to the structural fields useful for triage. + * Workflow source, result payloads, nested errors, and cause chains are never + * copied because they can contain arbitrary workflow-provided credentials. + */ +export function sanitizeCloudStatusDiagnostic(output, secretValues = []) { + const text = String(output ?? '').trim(); + if (!text) return ''; + try { + const payload = parseJsonOutput(text, 'Cloud status diagnostic'); + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + return ''; + } + const source = + typeof payload.status === 'string' + ? payload + : payload.run && typeof payload.run === 'object' && !Array.isArray(payload.run) + ? payload.run + : payload.workflowRun && + typeof payload.workflowRun === 'object' && + !Array.isArray(payload.workflowRun) + ? payload.workflowRun + : payload; + const diagnostic = diagnosticRecord(source, secretValues); + return boundedDiagnostic( + diagnostic && Object.keys(diagnostic).length > 0 + ? JSON.stringify(diagnostic) + : '' + ); + } catch { + if (text.includes('{') || text.includes('}')) { + return ''; + } + return ''; + } +} + +export function formatCloudRunDiagnostics({ + runId, + terminalStatus, + lastStatusOutput, + statusPollFailures, + logs, + diagnosticSecretValues = [], +}) { + const logOutput = `${logs?.stdout ?? ''}${logs?.stderr ?? ''}`; + return [ + 'Cloud RelayFlow diagnostics', + `run_id=${sanitizeCloudCommandOutput(runId, diagnosticSecretValues)}`, + `terminal_status=${sanitizeCloudCommandOutput(terminalStatus ?? 'unknown', diagnosticSecretValues)}`, + `status_poll_failures=${statusPollFailures}`, + `last_status_response=${ + sanitizeCloudStatusDiagnostic(lastStatusOutput, diagnosticSecretValues) || '' + }`, + `cloud_logs_exit_code=${logs?.exitCode ?? 'unknown'}`, + `cloud_logs_timed_out=${logs?.timedOut === true}`, + `cloud_logs_output=${logOutput ? 'present' : 'empty'}`, + '', + ].join('\n'); +} + +export function formatCloudRunArtifact(input) { + return ( + formatCloudRunDiagnostics(input) + + sanitizeCloudCommandOutput( + `${input.logs?.stdout ?? ''}${input.logs?.stderr ?? ''}`, + input.diagnosticSecretValues + ) + ); +} + +async function writeStatusPollDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, + terminalStatus, + logsTimedOut, + diagnosticSecretValues = [], +}) { + await mkdir(path.dirname(logsPath), { recursive: true }); + await writeFile( + logsPath, + formatCloudRunDiagnostics({ + runId, + terminalStatus, + lastStatusOutput, + statusPollFailures, + logs: { stdout: '', stderr: '', exitCode: 'unknown', timedOut: logsTimedOut }, + diagnosticSecretValues, + }) + ); +} + +export async function writeStatusPollTimeoutDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, + diagnosticSecretValues = [], +}) { + await writeStatusPollDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, + terminalStatus: 'status_poll_timeout', + logsTimedOut: true, + diagnosticSecretValues, + }); +} + +export async function writeStatusPollDeadlineDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, + diagnosticSecretValues = [], +}) { + await writeStatusPollDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, + terminalStatus: 'status_poll_deadline_exceeded', + logsTimedOut: false, + diagnosticSecretValues, + }); +} + +export function recognizedCloudRunStatus(value) { + if (typeof value !== 'string') return null; + const status = value.toLowerCase(); + return CLOUD_RUN_STATUSES.has(status) ? status : null; +} + function statusFrom(payload) { for (const candidate of [payload.status, payload.run?.status, payload.workflowRun?.status]) { - if (typeof candidate === 'string') return candidate.toLowerCase(); + if (typeof candidate === 'string') return recognizedCloudRunStatus(candidate); } throw new Error('Cloud status response did not contain a status'); } +export function recognizedCloudStatusFromOutput(output) { + try { + return statusFrom(parseJsonOutput(output, 'Cloud status')); + } catch { + return null; + } +} + function requiredCredential(env, name) { const value = env[name]?.trim(); if (!value) throw new Error(`${name} is required`); @@ -117,7 +567,7 @@ export function createCliApiKeyEnvironment(env = process.env) { const cliEnv = { ...env, CLOUD_API_URL: apiUrl, CLOUD_API_KEY: apiKey }; for (const key of LEGACY_REFRESHABLE_AUTH_KEYS) delete cliEnv[key]; - return { cliEnv }; + return { cliEnv, diagnosticSecretValues: [apiKey] }; } export async function main() { @@ -149,6 +599,8 @@ export async function main() { let shuttingDown = false; let activeCommandController = null; let launchProgressError = null; + let lastStatusOutput = ''; + let statusPollFailures = 0; const notePreparedRunId = (preparedRunId) => { try { @@ -173,7 +625,11 @@ export async function main() { const controller = new AbortController(); activeCommandController = controller; try { - return await run(command, args, { ...options, signal: controller.signal }); + return await run(command, args, { + ...options, + diagnosticSecretValues: auth.diagnosticSecretValues, + signal: controller.signal, + }); } finally { if (activeCommandController === controller) activeCommandController = null; } @@ -182,14 +638,25 @@ export async function main() { const cancelRemote = async (reason) => { if (!runId || terminal) return; cancelPromise ??= (async () => { - console.warn(`Cancelling Cloud RelayFlow run ${runId} (${reason})`); + console.warn( + `Cancelling Cloud RelayFlow run ${sanitizeCloudCommandOutput( + runId, + auth.diagnosticSecretValues + )} (${reason})` + ); const result = await run(cli, ['cloud', 'cancel', runId, '--json'], { env: auth.cliEnv, quiet: true, timeoutMs: commandTimeoutMs, + diagnosticSecretValues: auth.diagnosticSecretValues, }); if (result.exitCode !== 0 || result.timedOut) { - console.warn(`Cloud cancellation failed with exit ${result.exitCode}: ${result.stderr.trim()}`); + console.warn( + `Cloud cancellation failed with exit ${result.exitCode}: ${sanitizeCloudCommandOutput( + result.stderr.trim(), + auth.diagnosticSecretValues + )}` + ); } })(); await cancelPromise; @@ -200,7 +667,9 @@ export async function main() { shuttingDown = true; activeCommandController?.abort(); void (async () => { - await cancelRemote(signal).catch((error) => console.warn(error.message)); + await cancelRemote(signal).catch((error) => + console.warn(sanitizeCloudCommandOutput(error.message, auth.diagnosticSecretValues)) + ); process.exit(signal === 'SIGINT' ? 130 : 143); })(); }; @@ -227,7 +696,7 @@ export async function main() { ); } if (launch.exitCode !== 0) { - process.stderr.write(launch.stderr); + process.stderr.write(sanitizeCloudCommandOutput(launch.stderr, auth.diagnosticSecretValues)); throw new Error(`Cloud workflow submission failed with exit ${launch.exitCode}`); } const launchPayload = parseJsonOutput(launch.stdout, 'Cloud run'); @@ -239,7 +708,7 @@ export async function main() { throw new Error(`Cloud prepare/run ID mismatch: ${runId} != ${launchedRunId}`); } runId = launchedRunId; - console.log(`Cloud RelayFlow run: ${runId}`); + console.log(`Cloud RelayFlow run: ${sanitizeCloudCommandOutput(runId, auth.diagnosticSecretValues)}`); if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `run_id=${runId}\n`); const deadline = Date.now() + timeoutMs; @@ -252,13 +721,36 @@ export async function main() { timeoutMs: commandTimeoutMs, }); if (statusResult.timedOut) { + statusPollFailures += 1; + lastStatusOutput = statusResult.stderr.trim() || statusResult.stdout.trim(); + await writeStatusPollTimeoutDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, + diagnosticSecretValues: auth.diagnosticSecretValues, + }); throw new Error(`Cloud status command timed out for run ${runId}`); } if (statusResult.exitCode !== 0) { - console.warn(`Cloud status poll failed (${statusResult.exitCode}); retrying`); + statusPollFailures += 1; + lastStatusOutput = statusResult.stderr.trim() || statusResult.stdout.trim(); + console.warn( + `Cloud status poll failed (${statusResult.exitCode}); retrying${ + lastStatusOutput + ? `: ${sanitizeCloudStatusDiagnostic(lastStatusOutput, auth.diagnosticSecretValues)}` + : '' + }` + ); + continue; + } + lastStatusOutput = statusResult.stdout.trim(); + const status = recognizedCloudStatusFromOutput(statusResult.stdout); + if (!status) { + statusPollFailures += 1; + console.warn('Cloud RelayFlow status: '); continue; } - const status = statusFrom(parseJsonOutput(statusResult.stdout, 'Cloud status')); console.log(`Cloud RelayFlow status: ${status}`); if (TERMINAL_SUCCESS.has(status) || TERMINAL_FAILURE.has(status)) { terminalStatus = status; @@ -267,6 +759,13 @@ export async function main() { } } if (!terminalStatus) { + await writeStatusPollDeadlineDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, + diagnosticSecretValues: auth.diagnosticSecretValues, + }); await cancelRemote('deadline exceeded'); terminal = true; throw new Error(`Cloud RelayFlow exceeded ${timeoutMs}ms`); @@ -278,9 +777,24 @@ export async function main() { quiet: true, timeoutMs: commandTimeoutMs, }); - await writeFile(logsPath, logs.stdout + logs.stderr); - if (logs.stdout) process.stdout.write(logs.stdout); - if (logs.stderr) process.stderr.write(logs.stderr); + await writeFile( + logsPath, + formatCloudRunArtifact({ + runId, + terminalStatus, + lastStatusOutput, + statusPollFailures, + logs, + diagnosticSecretValues: auth.diagnosticSecretValues, + }) + ); + const sanitizedLogs = sanitizeCloudCommandOutput( + `${logs.stdout ?? ''}${logs.stderr ?? ''}`, + auth.diagnosticSecretValues + ); + if (sanitizedLogs) { + process.stdout.write(sanitizedLogs); + } if (logs.timedOut) throw new Error(`Cloud log retrieval timed out for run ${runId}`); if (logs.exitCode !== 0) throw new Error(`Cloud log retrieval failed with exit ${logs.exitCode}`); @@ -290,7 +804,10 @@ export async function main() { if (process.env.GITHUB_STEP_SUMMARY) { await appendFile( process.env.GITHUB_STEP_SUMMARY, - `\n- Cloud run: \`${runId}\`\n- Cloud status: **${terminalStatus}**\n` + `\n- Cloud run: \`${sanitizeCloudCommandOutput( + runId, + auth.diagnosticSecretValues + )}\`\n- Cloud status: **${terminalStatus}**\n` ); } } finally { @@ -298,13 +815,15 @@ export async function main() { process.removeListener('SIGINT', signalHandler); process.removeListener('SIGTERM', signalHandler); if (runId && !terminal) - await cancelRemote('dispatcher exiting').catch((error) => console.warn(error.message)); + await cancelRemote('dispatcher exiting').catch((error) => + console.warn(sanitizeCloudCommandOutput(error.message, auth.diagnosticSecretValues)) + ); } } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { main().catch((error) => { - console.error(error.message); + console.error(sanitizeCloudCommandOutput(error.message, [process.env.CLOUD_API_KEY])); process.exitCode = 1; }); } diff --git a/scripts/pr-proof/run-cloud.test.mjs b/scripts/pr-proof/run-cloud.test.mjs new file mode 100644 index 0000000000..aa3ad9063c --- /dev/null +++ b/scripts/pr-proof/run-cloud.test.mjs @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { createCommandOutputRedactors, run } from './run-cloud.mjs'; + +describe('cloud command output redaction', () => { + it('keeps partial credential suffixes on their originating stream', () => { + const redactors = createCommandOutputRedactors(); + + assert.equal(redactors.stdout.push('stdout: rk_live_', false), 'stdout: '); + assert.equal(redactors.stderr.push('stderr: unrelated', false), 'stderr: unrelated'); + assert.equal(redactors.stdout.push('secret-value', true), 'rk_live_…'); + }); + + it('does not redact or drop a benign trailing prefix fragment at stream end', () => { + const redactors = createCommandOutputRedactors(); + + assert.equal(redactors.stdout.push('finished with br', false), 'finished with '); + assert.equal(redactors.stdout.push('', true), 'br'); + }); + + for (let boundary = 1; boundary < 'custom-secret-value'.length; boundary += 1) { + it(`redacts a configured secret split at boundary ${boundary}`, () => { + const secret = 'custom-secret-value'; + const redactors = createCommandOutputRedactors([secret], { + maskPendingOnFinal: true, + }); + + assert.equal(redactors.stdout.push(secret.slice(0, boundary), false), ''); + const tail = redactors.stdout.push(secret.slice(boundary), true); + assert.equal(tail, '[redacted]'); + assert.equal(redactors.stdout.requiresCapturedOutputMask(), false); + }); + } + + for (const fragment of ['e', 'ue', 'lue']) { + it(`does not wipe a stream for the benign ${fragment.length}-character secret fragment`, async () => { + const output = `finished with ${fragment}`; + const result = await run(process.execPath, ['-e', `process.stdout.write(${JSON.stringify(output)})`], { + diagnosticSecretValues: ['custom-secret-value'], + }); + + assert.equal(result.stdout, output); + }); + } + + it('masks configured secrets split across streams before echo and capture', async () => { + const result = await run( + process.execPath, + ['-e', "process.stdout.write('custom-'); process.stderr.write('secret-value')"], + { diagnosticSecretValues: ['custom-secret-value'] } + ); + + assert.equal(result.stdout, '[redacted]'); + assert.equal(result.stderr, '[redacted]'); + assert.doesNotMatch(result.stdout, /custom-/); + assert.doesNotMatch(result.stderr, /secret-value/); + }); +}); diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index f3f107b892..ea8af193bd 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -44,12 +44,27 @@ import { } from '../../scripts/pr-proof/prepare.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. import { + boundedDiagnostic, boundedDuration, + createCommandOutputRedactors, createPreparedRunProgressParser, + createCredentialRedactor, createCliApiKeyEnvironment, + formatCloudRunArtifact, + formatCloudRunDiagnostics, + maskCapturedCommandOutput, + parseJsonOutput, preparedRunIdFromOutput, + recognizedCloudStatusFromOutput, + recognizedCloudRunStatus, + sanitizeCloudCommandOutput, + sanitizeCloudStatusDiagnostic, + writeStatusPollDeadlineDiagnostics, + writeStatusPollTimeoutDiagnostics, } from '../../scripts/pr-proof/run-cloud.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. +import { runBoundedProcess } from '../../scripts/pr-proof/process-runner.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. import { openVerifiedBrokerExecutable, probeLandlockedProcessSupport, @@ -438,6 +453,420 @@ describe('Cloud dispatcher API key lifecycle', () => { ); }); + it('retains terminal status diagnostics when Cloud logs are empty', () => { + const diagnostics = formatCloudRunDiagnostics({ + runId: 'cloud-run-123', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed","error":"step timeout"}', + statusPollFailures: 2, + logs: { stdout: '', stderr: '', exitCode: 0, timedOut: false }, + }); + + expect(diagnostics).toContain('run_id=cloud-run-123'); + expect(diagnostics).toContain('terminal_status=failed'); + expect(diagnostics).toContain('status_poll_failures=2'); + expect(diagnostics).not.toContain('step timeout'); + expect(diagnostics).toContain('cloud_logs_output=empty'); + }); + + it('allowlists status diagnostics without retaining nested workflow output', () => { + const rawStatus = JSON.stringify({ + runId: 'cloud-run-123', + status: 'failed', + updatedAt: '2026-09-08T10:00:00.000Z', + workflow: 'return process.env.SECRET', + error: 'top-level-error-must-not-survive ci-key', + message: 'top-level-message-must-not-survive', + dispatchType: 'arbitrary free text must not survive', + result: { + error: { + token: 'rk_live_0123456789abcdef', + detail: 'nested-result-must-not-survive', + }, + }, + failure: { + phase: 'launch', + code: 'workflow_launch_failed', + message: 'failure-message-must-not-survive ci-key rk_live_0123456789abcdef', + sandboxId: 'sandbox id with arbitrary free text', + causeChain: ['nested-cause-must-not-survive'], + }, + }); + const diagnostic = sanitizeCloudStatusDiagnostic(`status response follows\n${rawStatus}`, ['ci-key']); + + expect(JSON.parse(diagnostic)).toEqual({ + runId: 'cloud-run-123', + status: 'failed', + updatedAt: '2026-09-08T10:00:00.000Z', + failure: { + phase: 'launch', + code: 'workflow_launch_failed', + }, + }); + expect(diagnostic).not.toContain('nested-result-must-not-survive'); + expect(diagnostic).not.toContain('top-level-error-must-not-survive'); + expect(diagnostic).not.toContain('top-level-message-must-not-survive'); + expect(diagnostic).not.toContain('failure-message-must-not-survive'); + expect(diagnostic).not.toContain('arbitrary free text'); + expect(diagnostic).not.toContain('nested-cause-must-not-survive'); + expect(diagnostic).not.toContain('0123456789abcdef'); + + const persisted = formatCloudRunDiagnostics({ + runId: 'cloud-run-123', + terminalStatus: 'failed', + lastStatusOutput: rawStatus, + statusPollFailures: 0, + logs: { stdout: '', stderr: '', exitCode: 0, timedOut: false }, + diagnosticSecretValues: ['ci-key'], + }); + expect(persisted).toContain('workflow_launch_failed'); + expect(persisted).not.toContain('nested-result-must-not-survive'); + expect(persisted).not.toContain('top-level-error-must-not-survive'); + expect(persisted).not.toContain('top-level-message-must-not-survive'); + expect(persisted).not.toContain('failure-message-must-not-survive'); + expect(persisted).not.toContain('arbitrary free text'); + expect(persisted).not.toContain('ci-key'); + }); + + it('omits non-JSON status errors even when they contain configured secrets', () => { + const diagnostic = sanitizeCloudStatusDiagnostic('Status request failed: upstream rejected short-key', [ + 'short-key', + ]); + + expect(diagnostic).toBe(''); + expect(diagnostic).not.toContain('short-key'); + }); + + it('accepts only known Cloud run statuses', () => { + expect(recognizedCloudRunStatus('RUNNING')).toBe('running'); + expect(recognizedCloudRunStatus('failed')).toBe('failed'); + expect(recognizedCloudRunStatus('running-opaque-secret')).toBeNull(); + expect(recognizedCloudRunStatus(null)).toBeNull(); + }); + + it('treats malformed and status-less successful poll output as retryable', () => { + expect(recognizedCloudStatusFromOutput('{"status":unknown-secret}')).toBeNull(); + expect(recognizedCloudStatusFromOutput('{"runId":"still-running"}')).toBeNull(); + expect(recognizedCloudStatusFromOutput('{"workflowRun":{"status":"running"}}')).toBe('running'); + }); + + it('redacts configured and recognized credentials from raw command output and artifacts', () => { + const raw = 'stdout ci-api-key rk_live_0123456789abcdef'; + expect(sanitizeCloudCommandOutput(raw, ['ci-api-key'])).toBe('stdout [redacted] rk_live_…'); + const githubToken = 'github_pat_0123456789abcdef0123456789abcdef'; + expect(sanitizeCloudCommandOutput(`status ${githubToken}`)).toBe('status github_pat_…'); + + const artifact = formatCloudRunArtifact({ + runId: 'cloud-run-123', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed","error":"opaque-status-secret"}', + statusPollFailures: 0, + logs: { + stdout: `workflow output ci-api-key\n`, + stderr: 'failure rth_at_0123456789abcdef\n', + exitCode: 0, + timedOut: false, + }, + diagnosticSecretValues: ['ci-api-key'], + }); + + expect(artifact).toContain('workflow output [redacted]'); + expect(artifact).toContain('failure rth_at_…'); + expect(artifact).not.toContain('ci-api-key'); + expect(artifact).not.toContain('0123456789abcdef'); + expect(artifact).not.toContain('opaque-status-secret'); + }); + + it('redacts complete credentials before configured prefix secrets in console and artifacts', () => { + const credentialSuffix = '0123456789abcdef'; + const consoleOutput = sanitizeCloudCommandOutput(`launch rk_live_${credentialSuffix}`, ['rk_live_']); + expect(consoleOutput).toBe('launch [redacted]…'); + expect(consoleOutput).not.toContain(credentialSuffix); + + const artifact = formatCloudRunArtifact({ + runId: 'cloud-run-prefix-secret', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed"}', + statusPollFailures: 0, + logs: { + stdout: `workflow output rk_live_${credentialSuffix}\n`, + stderr: '', + exitCode: 0, + timedOut: false, + }, + diagnosticSecretValues: ['rk_live_'], + }); + + expect(artifact).toContain('workflow output [redacted]…'); + expect(artifact).not.toContain(credentialSuffix); + }); + + it('redacts credentials reconstructed across captured stdout and stderr', () => { + const configuredArtifact = formatCloudRunArtifact({ + runId: 'cloud-run-split-secret', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed"}', + statusPollFailures: 0, + logs: { stdout: 'split-', stderr: 'secret', exitCode: 1, timedOut: false }, + diagnosticSecretValues: ['split-secret'], + }); + expect(configuredArtifact).toContain('[redacted]'); + expect(configuredArtifact).not.toContain('split-secret'); + + const prefixedArtifact = formatCloudRunArtifact({ + runId: 'cloud-run-split-prefix', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed"}', + statusPollFailures: 0, + logs: { stdout: 'rk_', stderr: 'live_token', exitCode: 1, timedOut: false }, + }); + expect(prefixedArtifact).toContain('rk_live_…'); + expect(prefixedArtifact).not.toContain('rk_live_token'); + }); + + it('keeps credential fragments attached to their originating output stream', async () => { + const captureWithRedaction = async (code: string, secretValues: string[]) => { + const redactors = createCommandOutputRedactors(secretValues); + const capture = await runBoundedProcess(process.execPath, ['-e', code], { + echo: false, + transformChunk: (text, stream, final) => redactors[stream].push(text, final), + }); + return maskCapturedCommandOutput(capture, redactors); + }; + + const configuredCapture = await captureWithRedaction( + "process.stdout.write('secret'); setTimeout(() => process.stderr.write('split-'), 25)", + ['split-secret'] + ); + const configuredArtifact = formatCloudRunArtifact({ + runId: 'cloud-run-redaction-a', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed"}', + statusPollFailures: 0, + logs: configuredCapture, + diagnosticSecretValues: ['split-secret'], + }); + + expect(configuredCapture.stdout).toBe('secret'); + expect(configuredCapture.stderr).toBe('split-'); + expect(configuredArtifact).not.toContain('split-secret'); + + const prefixCapture = await captureWithRedaction( + "process.stdout.write('live_token'); setTimeout(() => process.stderr.write('rk_'), 25)", + [] + ); + const prefixArtifact = formatCloudRunArtifact({ + runId: 'cloud-run-redaction-b', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed"}', + statusPollFailures: 0, + logs: prefixCapture, + diagnosticSecretValues: [], + }); + + expect(prefixCapture.stdout).toBe('live_token'); + expect(prefixCapture.stderr).toBe('rk_'); + expect(prefixArtifact).not.toContain('rk_live_token'); + + const benignTrailingPrefix = createCommandOutputRedactors(); + expect( + benignTrailingPrefix.stdout.push('finished with br', false) + benignTrailingPrefix.stdout.push('', true) + ).toBe('finished with br'); + }); + + it('redacts credential prefixes and configured secrets across subprocess chunk boundaries', async () => { + const redactor = createCredentialRedactor(['split-secret']); + const sanitized = + redactor.push('prefix rk_', false) + + redactor.push('live_0123456789 split-', false) + + redactor.push('secret tail', true); + expect(sanitized).toBe('prefix rk_live_… [redacted] tail'); + + const longCredentialLength = 200_000; + const capture = await runBoundedProcess( + process.execPath, + ['-e', `process.stdout.write('head rk_live_' + 'a'.repeat(${longCredentialLength}) + ' tail')`], + { + echo: false, + maxCaptureBytes: 128, + maxLiveOutputBytes: 128, + transformChunk: (text, stream, final) => redactor.push(text, final), + } + ); + + expect(capture.stdout).toBe('head rk_live_… tail'); + expect(capture.stdout).not.toContain('a'.repeat(longCredentialLength)); + + const captureLimit = 2 * 1024 * 1024; + const boundaryCredential = '0123456789abcdef'.repeat(128); + const boundaryRedactor = createCredentialRedactor(); + const boundaryCapture = await runBoundedProcess( + process.execPath, + [ + '-e', + [ + `process.stdout.write('x'.repeat(${captureLimit - 'rk_live_'.length}))`, + "process.stdout.write('rk_')", + "setImmediate(() => process.stdout.write('live_' + process.env.BOUNDARY_CREDENTIAL + ' tail'))", + ].join(';'), + ], + { + echo: false, + env: { ...process.env, BOUNDARY_CREDENTIAL: boundaryCredential }, + maxCaptureBytes: captureLimit, + maxLiveOutputBytes: 128, + transformChunk: (text, stream, final) => { + // Force the credential prefix to cross a redactor boundary even if + // the OS coalesces the two writes into one pipe chunk. + const prefixIndex = text.indexOf('rk_live_'); + if (prefixIndex >= 0) { + const split = prefixIndex + 'rk_'.length; + return ( + boundaryRedactor.push(text.slice(0, split), false) + + boundaryRedactor.push(text.slice(split), final) + ); + } + return boundaryRedactor.push(text, final); + }, + } + ); + + expect(Buffer.byteLength(boundaryCapture.stdout, 'utf8')).toBeLessThanOrEqual(captureLimit); + expect(boundaryCapture.stdout).toContain('rk_live_…'); + expect(boundaryCapture.stdout).not.toContain('0123456789abcdef'); + }); + + it('preserves large credential-free output without pathological rescanning', () => { + const cleanOutput = `head ${'ordinary-output '.repeat(20_000)}tail`; + expect(sanitizeCloudCommandOutput(cleanOutput, ['configured-secret'])).toBe(cleanOutput); + }); + + it('rejects and terminates when an output transform throws', async () => { + await expect( + runBoundedProcess(process.execPath, ['-e', "process.stdout.write('trigger')"], { + echo: false, + transformChunk: () => { + throw new Error('transform failed'); + }, + }) + ).rejects.toThrow('transform failed'); + }); + + it.skipIf(process.platform === 'win32' || !PS_PATH)( + 'kills same-group descendants when an output transform fails during final flush', + async () => { + const script = [ + "const { spawn } = require('node:child_process');", + "const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });", + "process.stdout.write(String(child.pid) + '\\n');", + 'child.unref();', + ].join(''); + let transformed = ''; + await expect( + runBoundedProcess(process.execPath, ['-e', script], { + echo: false, + transformChunk: (text, _stream, final) => { + if (final) throw new Error('final transform failed'); + transformed += text; + return text; + }, + }) + ).rejects.toThrow('final transform failed'); + + const descendantPid = Number(transformed.trim()); + expect(descendantPid).toBeGreaterThan(0); + const deadline = Date.now() + 2_000; + let running = true; + while (running && Date.now() < deadline) { + let processState = ''; + try { + processState = execFileSync(PS_PATH!, ['-o', 'stat=', '-p', String(descendantPid)], { + encoding: 'utf8', + }).trim(); + } catch (error) { + const status = (error as { status?: number }).status; + if (status !== 1) throw error; + } + running = processState.length > 0 && !processState.startsWith('Z'); + if (running) await new Promise((resolve) => setTimeout(resolve, 20)); + } + if (running) process.kill(descendantPid, 'SIGKILL'); + expect(running).toBe(false); + } + ); + + it('omits malformed JSON status payloads instead of falling back to raw output', () => { + const diagnostic = sanitizeCloudStatusDiagnostic( + '{"status":"failed","result":{"token":"unknown-secret"}' + ); + + expect(diagnostic).toBe(''); + expect(diagnostic).not.toContain('unknown-secret'); + }); + + it('uses a fixed error when malformed JSON contains unrecognized secrets', () => { + const malformed = 'status response follows\n{"status":"failed","token":unknown-secret}'; + + expect(() => parseJsonOutput(malformed, 'Cloud status')).toThrow( + new Error('Cloud status did not return JSON') + ); + }); + + it('persists bounded diagnostics when a Cloud status poll times out', async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'relay-pr-proof-status-timeout-')); + const logsPath = path.join(root, 'nested', 'cloud.log'); + try { + await writeStatusPollTimeoutDiagnostics({ + logsPath, + runId: 'cloud-run-timeout', + lastStatusOutput: '😀'.repeat(100_000), + statusPollFailures: 3, + }); + + const diagnostics = await readFile(logsPath, 'utf8'); + expect(diagnostics).toContain('run_id=cloud-run-timeout'); + expect(diagnostics).toContain('terminal_status=status_poll_timeout'); + expect(diagnostics).toContain('status_poll_failures=3'); + expect(diagnostics).toContain('cloud_logs_timed_out=true'); + expect(Buffer.byteLength(diagnostics, 'utf8')).toBeLessThanOrEqual(65 * 1024); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it.each([ + ['nonzero', 'Status request failed: unknown-secret'], + ['malformed', '{"status":unknown-secret}'], + ['status-less', '{"runId":"cloud-run-deadline","result":{"token":"unknown-secret"}}'], + ])('persists safe bounded diagnostics when %s polls exhaust the deadline', async (_case, output) => { + const root = await mkdtemp(path.join(os.tmpdir(), 'relay-pr-proof-status-deadline-')); + const logsPath = path.join(root, 'nested', 'cloud.log'); + try { + await writeStatusPollDeadlineDiagnostics({ + logsPath, + runId: 'cloud-run-deadline', + lastStatusOutput: output, + statusPollFailures: 4, + }); + + const diagnostics = await readFile(logsPath, 'utf8'); + expect(diagnostics).toContain('run_id=cloud-run-deadline'); + expect(diagnostics).toContain('terminal_status=status_poll_deadline_exceeded'); + expect(diagnostics).toContain('status_poll_failures=4'); + expect(diagnostics).not.toContain('unknown-secret'); + expect(Buffer.byteLength(diagnostics, 'utf8')).toBeLessThanOrEqual(65 * 1024); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it('bounds multibyte Cloud diagnostics by UTF-8 bytes', () => { + const diagnostics = boundedDiagnostic('😀'.repeat(100_000)); + + expect(Buffer.byteLength(diagnostics, 'utf8')).toBeLessThanOrEqual(64 * 1024); + expect(diagnostics).toContain('[... diagnostic output truncated ...]'); + }); + it('waits for a complete prepared-run progress line split across stderr chunks', () => { const runIds: string[] = []; const parser = createPreparedRunProgressParser((runId: string) => runIds.push(runId)); @@ -459,6 +888,7 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(auth.cliEnv.CLOUD_API_URL).toBe(credentialEnv.CLOUD_API_URL); expect(auth.cliEnv.CLOUD_API_KEY).toBe(credentialEnv.CLOUD_API_KEY); + expect(auth.diagnosticSecretValues).toEqual([credentialEnv.CLOUD_API_KEY]); expect(auth.cliEnv.CLOUD_API_ACCESS_TOKEN).toBeUndefined(); expect(auth.cliEnv.CLOUD_API_REFRESH_TOKEN).toBeUndefined(); expect(auth.cliEnv.CLOUD_API_ACCESS_TOKEN_EXPIRES_AT).toBeUndefined(); @@ -1723,6 +2153,15 @@ describe('trusted dispatcher source contract', () => { expect(source).toContain("requiredCredential(env, 'CLOUD_API_KEY')"); expect(source).not.toContain("path.join(authDir, 'cloud-auth.json')"); expect(source).not.toContain('CLOUD_API_REFRESH_TOKEN='); + expect(source).toContain('formatCloudRunArtifact({'); + expect(source).toContain('sanitizeCloudCommandOutput(launch.stderr'); + expect(source).toContain('const sanitizedLogs = sanitizeCloudCommandOutput('); + expect(source).toContain("`${logs.stdout ?? ''}${logs.stderr ?? ''}`"); + expect(source).toContain('sanitizeCloudCommandOutput(error.message'); + expect(source).toContain("console.warn('Cloud RelayFlow status: ')"); + expect(source).not.toContain('process.stderr.write(launch.stderr)'); + expect(source).not.toContain('process.stdout.write(logs.stdout)'); + expect(source).not.toContain('process.stderr.write(logs.stderr)'); }); it('emits the prepared Cloud run id before upload and final submission', async () => { diff --git a/tests/relayflows/cases/1656-long-running-agent37-sandbox/case.json b/tests/relayflows/cases/1656-long-running-agent37-sandbox/case.json index be05a4b080..5d19739175 100644 --- a/tests/relayflows/cases/1656-long-running-agent37-sandbox/case.json +++ b/tests/relayflows/cases/1656-long-running-agent37-sandbox/case.json @@ -2,7 +2,7 @@ "version": 1, "id": "1656-long-running-agent37-sandbox", "kind": "feature", - "title": "Route unpinned fleet sandboxes with long-running semantics and preserve Agent37 attribution", + "title": "Replay fleet sandbox identities with long-running semantics and preserve Agent37 attribution", "runner": { "command": ["node", "tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs"] }, @@ -10,11 +10,11 @@ "expected": { "base": { "outcome": "absent", - "signature": "long_running_profile_and_agent37_attribution_absent" + "signature": "long_running_profile_replay_identity_absent_agent37_attribution_preserved" }, "head": { "outcome": "fixed", - "signature": "long_running_profile_and_agent37_attribution_preserved" + "signature": "long_running_profile_replay_identity_and_agent37_attribution_preserved" } } } diff --git a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs index 0f330762ae..352c42c7df 100644 --- a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs +++ b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs @@ -1,14 +1,14 @@ /** * relay#1656 — `agent-relay fleet spawn --sandbox` must ask Cloud for - * long-running semantics and must clean up against the provider Cloud actually - * chose. + * long-running semantics with a one-to-one sandbox identity and must clean up + * against the provider Cloud actually chose. * * The claim has two halves and they live in different packages, so the probe * runs them as one chain rather than as two independent assertions: * * `fleet spawn --sandbox` (packages/cli) * -> ensureCloudFleetSandbox (packages/cloud, REAL) - * -> POST /fleet/nodes/sandbox/ensure <- workloadProfile observed here + * -> POST /fleet/nodes/sandbox/ensure <- identity/profile observed here * <- Cloud answers `providerId: agent37` * -> deleteCloudFleetSandbox (packages/cloud, REAL) * -> DELETE /fleet/nodes/sandbox/ <- providerId observed here @@ -29,13 +29,17 @@ * * The probe deliberately requests NO `--sandbox-provider`. That is the feature: * Cloud picks the provider, and `agent37` must survive back out into cleanup. - * Dispatch is then failed on purpose, because the cleanup call is what carries - * the attribution. + * After dispatch succeeds, the probe captures the CLI's JSON output and makes + * that output sink fail on purpose. This observes the normalized Cloud response + * and drives the same command into cleanup, where provider attribution is used. * - * Base: the CLI sends no workload profile, and `agent37` is not a provider the - * Cloud client will parse, so it is dropped and cleanup names no provider. - * Head: the profile reaches the ensure body and `agent37` reaches the delete - * body. + * Base: the CLI sends no sandbox identity, while the existing long-running + * profile and Cloud-selected `agent37` response and cleanup attribution are + * present. Head: an explicit replay + * `sbx_` identity, deterministic node name, and long-running profile + * reach the ensure body, while the exact public identity reaches cleanup and + * the separate physical provider sandbox ID is returned as evidence. Base does + * not know the replay flags, so it remains the negative arm. */ import { execFileSync, spawnSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; @@ -45,6 +49,9 @@ import process from 'node:process'; import { fileURLToPath } from 'node:url'; const CASE_ID = '1656-long-running-agent37-sandbox'; +const REPLAY_SANDBOX_ID = 'sbx_123e4567-e89b-42d3-a456-426614174000'; +const REPLAY_SANDBOX_NAME = 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000'; +const PROVIDER_SANDBOX_ID = 'provider-sandbox-relayflow'; // Separate budgets, both inside case.json's 900s per-arm deadline. The install // and the probe are very different jobs and a shared cap sizes neither: a // timeout on either is an INFRASTRUCTURE failure, which cannot report red or @@ -124,6 +131,8 @@ import { Command } from 'commander'; const mocks = vi.hoisted(() => ({ ensureCloudSession: vi.fn(), authorizedApiFetch: vi.fn(), + persistWorkspaceRelaycastTarget: vi.fn(() => true), + resolveWorkspaceSelection: vi.fn(() => ({ workspaceId: 'rw_relayflow' })), })); // The Cloud network boundary -- the only thing this probe stubs. Everything @@ -171,6 +180,9 @@ import { deleteCloudFleetSandbox, ensureCloudFleetSandbox } from './fleet-sandbo import { registerFleetCommands } from '../../cli/src/cli/commands/fleet.js'; const CLOUD_WORKSPACE_ID = '50587328-441d-4acb-b8f3-dbe1b3c5de99'; +const REPLAY_SANDBOX_ID = 'sbx_123e4567-e89b-42d3-a456-426614174000'; +const REPLAY_SANDBOX_NAME = 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000'; +const PROVIDER_SANDBOX_ID = 'provider-sandbox-relayflow'; const auth = { accessToken: 'relayflow-probe-access', refreshToken: 'relayflow-probe-refresh', @@ -178,11 +190,28 @@ const auth = { apiUrl: 'https://relayflow.invalid', }; -test('fleet spawn --sandbox reaches Cloud with a profile and cleans up by returned provider', async () => { +test('fleet spawn --sandbox replays an exact identity and cleans up by returned provider', async () => { const output = process.env.RELAY_PR1656_OBSERVATION_PATH; if (!output) throw new Error('Missing RELAY_PR1656_OBSERVATION_PATH.'); mocks.ensureCloudSession.mockResolvedValue({ auth, client: {} }); + const provisionedResponse = { + outcome: 'provisioned', + nodeId: 'node-relayflow', + nodeName: REPLAY_SANDBOX_NAME, + sandboxId: REPLAY_SANDBOX_ID, + providerSandboxId: PROVIDER_SANDBOX_ID, + relayWorkspaceId: 'rw_relayflow', + relaycastTarget: { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_relayflow', + relaycastApiKey: 'rk_live_relayflow_probe', + }, + relayfileMounted: true, + relayfileMountPath: '/workspace', + providerId: 'agent37', + }; mocks.authorizedApiFetch // 1. workspace resolution .mockResolvedValueOnce({ response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), auth }) @@ -190,47 +219,62 @@ test('fleet spawn --sandbox reaches Cloud with a profile and cleans up by return // never names a provider, which is the whole point of the feature. .mockResolvedValueOnce({ response: Response.json( - { - outcome: 'provisioned', - nodeId: 'node-relayflow', - nodeName: 'agent37-relayflow', - sandboxId: 'sandbox-relayflow', - relayWorkspaceId: 'rw_relayflow', - relayfileMounted: true, - relayfileMountPath: '/workspace', - providerId: 'agent37', - }, + provisionedResponse, { status: 201 } ), auth, }) - // 3. cleanup, triggered by the deliberate dispatch failure below + // 3. cleanup, triggered by the deliberate output-sink failure below .mockResolvedValueOnce({ - response: Response.json({ sandboxId: 'sandbox-relayflow', deleted: true }), + response: Response.json({ sandboxId: REPLAY_SANDBOX_ID, deleted: true }), auth, }); const errors: string[] = []; + // A progress line ahead of the JSON result proves the observation does not + // depend on the result being the first (or only) CLI output call. + const cliOutput: string[] = ['Preparing sandbox dispatch']; + const replayArgs = + process.env.RELAY_PR_PROOF_ARM === 'head' + ? ['--sandbox-id', REPLAY_SANDBOX_ID, '--sandbox-name', REPLAY_SANDBOX_NAME] + : []; const program = new Command(); program.exitOverride(); registerFleetCommands(program, { sdk: { - // Dispatch fails on purpose: the cleanup call is what carries provider - // attribution, and it only happens on this path. createAgentRelay: vi.fn(() => ({ messaging: { placement: { - spawn: vi.fn(async () => { - throw new Error('dispatch failed'); - }), + spawn: vi.fn(async () => ({ + invocationId: 'inv_relayflow', + node: { name: REPLAY_SANDBOX_NAME }, + })), }, }, })) as never, createWorkspaceRelay: vi.fn(() => ({ - workspace: { info: vi.fn(async () => ({ id: 'rw_relayflow' })) }, + workspace: { + info: vi.fn(async () => ({ id: 'rw_relayflow' })), + register: vi.fn(async () => ({ token: 'at_relayflow_launcher' })), + release: vi.fn(async () => undefined), + }, })) as never, createWorkspace: vi.fn() as never, - log: vi.fn(), + // Capture the real CLI serialization, then fail so this same invocation + // exercises cleanup without sourcing evidence from the network mock. + log: (...args: unknown[]) => { + const line = args.join(' '); + cliOutput.push(line); + let candidate: { sandbox?: { sandboxId?: unknown } } | null = null; + try { + candidate = JSON.parse(line); + } catch { + // Auxiliary CLI output is allowed; only the sandbox result drives cleanup. + } + if (candidate?.sandbox?.sandboxId === REPLAY_SANDBOX_ID) { + throw new Error('CLI output sink failed after capture'); + } + }, error: (...args: unknown[]) => errors.push(args.join(' ')), exit: (() => { throw new Error('__exit__'); @@ -238,6 +282,8 @@ test('fleet spawn --sandbox reaches Cloud with a profile and cleans up by return }, ensureCloudFleetSandbox, deleteCloudFleetSandbox, + resolveWorkspaceSelection: mocks.resolveWorkspaceSelection, + persistWorkspaceRelaycastTarget: mocks.persistWorkspaceRelaycastTarget, createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, warn: () => undefined, @@ -251,6 +297,7 @@ test('fleet spawn --sandbox reaches Cloud with a profile and cleans up by return 'spawn', 'codex', '--sandbox', + ...replayArgs, '--name', 'sandbox-worker', '--task', @@ -264,9 +311,24 @@ test('fleet spawn --sandbox reaches Cloud with a profile and cleans up by return ) ).rejects.toThrow('__exit__'); - // The command really ran and really failed dispatch; without this a probe - // that never reached the sandbox path could report a false base. - expect(errors.join('\n')).toContain('dispatch failed'); + // The command really reached successful dispatch and serialized its result; + // without this a probe that never reached the sandbox path could report a false base. + expect(errors.join('\n')).toContain('CLI output sink failed after capture'); + const sandboxResults = cliOutput.flatMap((line) => { + try { + const candidate = JSON.parse(line); + return candidate?.sandbox?.sandboxId === REPLAY_SANDBOX_ID ? [candidate] : []; + } catch { + return []; + } + }); + if (sandboxResults.length !== 1) { + throw new Error( + 'Expected exactly one serialized sandbox result, found ' + + String(sandboxResults.length) + + '.' + ); + } const ensureRequest = mocks.authorizedApiFetch.mock.calls[1]?.[2]; const deleteRequest = mocks.authorizedApiFetch.mock.calls[2]?.[2]; @@ -274,14 +336,26 @@ test('fleet spawn --sandbox reaches Cloud with a profile and cleans up by return expect(deleteRequest?.body).toEqual(expect.any(String)); const ensureBody = JSON.parse(ensureRequest.body); const deleteBody = JSON.parse(deleteRequest.body); + const [cliResult] = sandboxResults; await writeFile( output, JSON.stringify({ ensureWorkloadProfile: ensureBody.workloadProfile ?? null, + ensureSandboxId: ensureBody.sandboxId ?? null, + ensureForceProvision: ensureBody.forceProvision ?? null, + ensureName: ensureBody.name ?? null, ensureProviderId: ensureBody.providerId ?? null, + responseSandboxId: cliResult.sandbox?.sandboxId ?? null, + responseProviderSandboxId: cliResult.sandbox?.providerSandboxId ?? null, + responseProviderId: cliResult.sandbox?.providerId ?? null, + responseNodeName: cliResult.sandbox?.nodeName ?? null, + deleteSandboxId: decodeURIComponent( + String(mocks.authorizedApiFetch.mock.calls[2]?.[1] ?? '').split('/').pop() ?? '' + ) || null, deleteProviderId: deleteBody.providerId ?? null, - dispatchFailureObserved: errors.join('\n').includes('dispatch failed'), + relaycastTargetPersisted: mocks.persistWorkspaceRelaycastTarget.mock.calls.length === 1, + outputFailureObserved: errors.join('\n').includes('CLI output sink failed after capture'), }), 'utf8' ); @@ -312,8 +386,11 @@ try { ); const observation = JSON.parse(await readFile(observationPath, 'utf8')); - if (observation.dispatchFailureObserved !== true) { - throw new Error('The probe did not reach the sandbox dispatch path, so it observed nothing.'); + if (observation.outputFailureObserved !== true) { + throw new Error('The probe did not serialize the CLI sandbox result, so it observed nothing.'); + } + if (arm === 'head' && observation.relaycastTargetPersisted !== true) { + throw new Error("The head did not persist Cloud's Relaycast target before dispatch."); } // The command line named no provider on either arm. If this ever stops being // true the case is proving provider pinning, not capability routing. @@ -323,23 +400,48 @@ try { ); } - const baseObserved = observation.ensureWorkloadProfile === null && observation.deleteProviderId === null; + const baseObserved = + observation.ensureSandboxId === null && + observation.ensureWorkloadProfile === 'long-running-agent' && + observation.responseSandboxId === REPLAY_SANDBOX_ID && + observation.responseProviderSandboxId === null && + observation.responseProviderId === 'agent37' && + observation.responseNodeName === REPLAY_SANDBOX_NAME && + observation.deleteSandboxId === REPLAY_SANDBOX_ID && + observation.deleteProviderId === 'agent37'; + const sandboxIdentityObserved = + typeof observation.ensureSandboxId === 'string' && + /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( + observation.ensureSandboxId + ) && + observation.ensureName === `fleet-sandbox-${observation.ensureSandboxId.slice('sbx_'.length)}` && + observation.ensureForceProvision === true; + const replayIdentityObserved = + observation.ensureSandboxId === REPLAY_SANDBOX_ID && observation.ensureName === REPLAY_SANDBOX_NAME; const headObserved = - observation.ensureWorkloadProfile === 'long-running-agent' && observation.deleteProviderId === 'agent37'; + observation.ensureWorkloadProfile === 'long-running-agent' && + sandboxIdentityObserved && + replayIdentityObserved && + observation.responseSandboxId === REPLAY_SANDBOX_ID && + observation.responseProviderSandboxId === PROVIDER_SANDBOX_ID && + observation.responseProviderId === 'agent37' && + observation.responseNodeName === REPLAY_SANDBOX_NAME && + observation.deleteSandboxId === REPLAY_SANDBOX_ID && + observation.deleteProviderId === 'agent37'; let outcome; let signature; let details; if (baseObserved) { outcome = 'absent'; - signature = 'long_running_profile_and_agent37_attribution_absent'; + signature = 'long_running_profile_replay_identity_absent_agent37_attribution_preserved'; details = - 'fleet spawn --sandbox reached Cloud with no workload profile in the ensure request, and the agent37 provider Cloud returned was dropped, so cleanup named no provider.'; + "fleet spawn --sandbox reached Cloud with the existing long-running profile and Cloud's agent37 response, but without a caller-declared sandbox identity; the one-to-one replay identity and separately observed providerSandboxId are absent while agent37 cleanup attribution remains preserved."; } else if (headObserved) { outcome = 'fixed'; - signature = 'long_running_profile_and_agent37_attribution_preserved'; + signature = 'long_running_profile_replay_identity_and_agent37_attribution_preserved'; details = - "fleet spawn --sandbox reached Cloud with workloadProfile 'long-running-agent' in the ensure request without pinning a provider, and the agent37 provider Cloud returned was carried into the cleanup request."; + "fleet spawn --sandbox replayed the exact caller-declared sbx_/fleet-sandbox- identity with workloadProfile 'long-running-agent' and forceProvision true without pinning a provider; Cloud echoed the public identity, returned a separate providerSandboxId, and agent37 attribution reached cleanup by public sandbox ID."; } else { throw new Error(`Unexpected long-running Agent37 observation: ${JSON.stringify(observation)}.`); }