From b6c3bb157709eb83865c2eb302f9b0105186c9ae Mon Sep 17 00:00:00 2001 From: Relayflow Date: Fri, 18 Sep 2026 15:21:43 +0000 Subject: [PATCH 01/19] Expose fleet attach subpaths and forward sandbox readonly requests --- .../2026-09/traj_qdtkdvki0xoh/summary.md | 23 + .../2026-09/traj_qdtkdvki0xoh/trajectory.json | 54 + CHANGELOG.md | 8 +- package-lock.json | 9 +- packages/cli/README.md | 13 + packages/cli/src/cli/commands/fleet.test.ts | 38 + packages/cli/src/cli/commands/fleet.ts | 13 +- packages/cli/src/cli/lib/attach-fleet-node.ts | 1223 +-------------- packages/cli/src/cli/lib/attach-mode.ts | 2 +- .../cli/src/cli/lib/collect-with-retry.ts | 43 +- packages/cli/src/cli/lib/sdk-client.ts | 144 +- packages/cloud/package.json | 22 +- packages/cloud/src/attach-mode.ts | 1 + packages/cloud/src/attach.test.ts | 110 ++ packages/cloud/src/attach.ts | 1362 +++++++++++++++++ packages/cloud/src/collect-with-retry.ts | 42 + packages/cloud/src/fleet-sandbox.test.ts | 28 + packages/cloud/src/fleet-sandbox.ts | 21 + packages/cloud/src/fleet.ts | 10 + packages/cloud/src/public-entries.test.ts | 28 + packages/cloud/src/workspace-transport.ts | 160 ++ packages/sdk/package.json | 13 +- packages/sdk/src/attach.ts | 1 + packages/sdk/src/fleet.ts | 1 + packages/sdk/tsconfig.build.json | 4 +- summary.md | 33 + .../cases/sandbox-readonly-paths/case.json | 20 + .../cases/sandbox-readonly-paths/run.mjs | 273 ++++ 28 files changed, 2290 insertions(+), 1409 deletions(-) create mode 100644 .agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/summary.md create mode 100644 .agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/trajectory.json create mode 100644 packages/cloud/src/attach-mode.ts create mode 100644 packages/cloud/src/attach.test.ts create mode 100644 packages/cloud/src/attach.ts create mode 100644 packages/cloud/src/collect-with-retry.ts create mode 100644 packages/cloud/src/public-entries.test.ts create mode 100644 packages/cloud/src/workspace-transport.ts create mode 100644 packages/sdk/src/attach.ts create mode 100644 packages/sdk/src/fleet.ts create mode 100644 summary.md create mode 100644 tests/relayflows/cases/sandbox-readonly-paths/case.json create mode 100644 tests/relayflows/cases/sandbox-readonly-paths/run.mjs diff --git a/.agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/summary.md b/.agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/summary.md new file mode 100644 index 0000000000..b04d0f9c86 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/summary.md @@ -0,0 +1,23 @@ +# Trajectory: Publish fleet and attach subpaths and forward sandbox readonlyPaths + +> **Status:** ❌ Abandoned +> **Started:** September 18, 2026 at 03:11 PM +> **Completed:** September 18, 2026 at 03:21 PM + +--- + +## Key Decisions + +### Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics +- **Chose:** Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics +- **Reasoning:** Reviewed plan identifies Cloud as the consumer import target, no remote exit-code protocol, and no server mount handler in this repository. D4 running-agent orchestration remains pending user scope clarification. + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics: Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics +- Abandoned: Attach extraction and readonlyPaths client implementation ready; full spawnFleetSandbox orchestration awaits reviewed-plan.md D4 scope decision. Server chmod enforcement is outside this repository. diff --git a/.agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/trajectory.json b/.agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/trajectory.json new file mode 100644 index 0000000000..7b06f12566 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/trajectory.json @@ -0,0 +1,54 @@ +{ + "id": "traj_qdtkdvki0xoh", + "version": 1, + "task": { + "title": "Publish fleet and attach subpaths and forward sandbox readonlyPaths" + }, + "status": "abandoned", + "startedAt": "2026-09-18T15:11:05.520Z", + "completedAt": "2026-09-18T15:21:12.959Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-09-18T15:20:29.731Z" + } + ], + "chapters": [ + { + "id": "chap_jnv5kvm9t42d", + "title": "Work", + "agentName": "default", + "startedAt": "2026-09-18T15:20:29.731Z", + "endedAt": "2026-09-18T15:21:12.959Z", + "events": [ + { + "ts": 1789744829731, + "type": "decision", + "content": "Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics: Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics", + "raw": { + "question": "Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics", + "chosen": "Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics", + "alternatives": [], + "reasoning": "Reviewed plan identifies Cloud as the consumer import target, no remote exit-code protocol, and no server mount handler in this repository. D4 running-agent orchestration remains pending user scope clarification." + }, + "significance": "high" + }, + { + "ts": 1789744872960, + "type": "note", + "content": "Abandoned: Attach extraction and readonlyPaths client implementation ready; full spawnFleetSandbox orchestration awaits reviewed-plan.md D4 scope decision. Server chmod enforcement is outside this repository.", + "significance": "high" + } + ] + } + ], + "commits": [], + "filesChanged": [], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "c97830275c0849f8ae1ed7fc1f347a5198124b11", + "endRef": "c97830275c0849f8ae1ed7fc1f347a5198124b11" + } +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index a0c4a641a4..6fe86ce505 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,13 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Minor] + +### Added + +- `@agent-relay/cloud/attach` and `@agent-relay/sdk/attach` expose fleet terminal attachment with a private local stdio socket and completion promise. +- `@agent-relay/cloud/fleet` and `@agent-relay/sdk/fleet` expose existing sandbox ensure and deletion primitives. +- Fleet sandbox ensure and `fleet spawn --sandbox-readonly-path` forward explicit read-only Relayfile subtree requests to Cloud for server-side enforcement. ## [12.2.4] - 2026-09-18 diff --git a/package-lock.json b/package-lock.json index e2264cad8a..e986031aaf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14080,12 +14080,14 @@ "@agent-relay/config": "12.2.3", "@aws-sdk/client-s3": "3.1020.0", "ignore": "^7.0.5", - "tar": "^7.5.21" + "tar": "^7.5.21", + "ws": "^8.18.3" }, "devDependencies": { "@types/node": "^22.19.3", "@types/ssh2": "^1.15.5", - "vitest": "^4.1.0" + "vitest": "^4.1.0", + "@types/ws": "^8.18.1" }, "engines": { "node": ">=22.0.0" @@ -14205,7 +14207,8 @@ "dependencies": { "@relaycast/sdk": "^8.0.7", "@relaycast/types": "^8.0.7", - "zod": "^4.4.3" + "zod": "^4.4.3", + "@agent-relay/cloud": "12.2.4" }, "devDependencies": { "@types/node": "^22.13.10" diff --git a/packages/cli/README.md b/packages/cli/README.md index 44e019ca06..73cfae0e63 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -395,6 +395,19 @@ agent-relay fleet spawn claude \ --task 'Review the live draft under /workspace/live-review/run-123' ``` +Use `--sandbox-readonly-path '/reference/**'` to request a read-only Relayfile +subtree. It requires `--sandbox` with Relayfile enabled and accepts explicit +`/path/**` subtrees (no traversal or other wildcards). This client forwards the +request; the deployed Cloud ensure handler must implement chmod enforcement. + +The ESM `@agent-relay/cloud/attach` and `@agent-relay/sdk/attach` entries expose +`startFleetNodeAttachProxy({ nodeId, mode })`. An omitted agent is discovered +only when the node has exactly one agent. `socketPath` accepts one raw stdio +connection; `close()` removes it. `finished` reports an inferred status (0 for +terminal closure or detach, 1 for transport failure), because Relaycast does +not currently transmit the remote harness exit code. Existing CLI broker +transport fields remain available for compatibility. + `--session-ref` is a real CLI resume, not a logical collaboration label. Pass the actual Claude session ID or Codex thread ID and target its origin node. `--cwd` must name an absolute directory that exists on the selected node. The diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index e1414923d8..27ac2882b5 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -983,6 +983,8 @@ describe('fleet command support', () => { REPLAY_SANDBOX_NAME, '--sandbox-relayfile-path', '/live-review/run-123/**', + '--sandbox-readonly-path', + '/live-review/run-123/reference/**', '--name', 'sandbox-worker', '--task', @@ -999,6 +1001,7 @@ describe('fleet command support', () => { maxAgents: 1, mountRelayfile: true, relayfilePaths: ['/live-review/run-123/**'], + readonlyPaths: ['/live-review/run-123/reference/**'], sandboxId: REPLAY_SANDBOX_ID, forceProvision: true, providerId: 'agent37', @@ -3962,3 +3965,38 @@ describe('fleet command support', () => { expect(errors.join('\n')).toMatch(/relay cloud enroll/); }); }); + +describe('sandbox read-only path guards', () => { + it.each([[], ['--sandbox', '--no-sandbox-relayfile']])( + 'rejects unavailable mounts (%j)', + async (...flags) => { + const ensureCloudFleetSandbox = vi.fn(); + const error = vi.fn(); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + ensureCloudFleetSandbox, + sdk: { error, exit: vi.fn() as never }, + }); + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'claude', + ...flags, + '--sandbox-readonly-path', + '/reference/**', + '--name', + 'worker', + '--task', + 'review', + ], + { from: 'user' } + ); + expect(error).toHaveBeenCalledWith( + expect.stringContaining('--sandbox-readonly-path requires --sandbox') + ); + expect(ensureCloudFleetSandbox).not.toHaveBeenCalled(); + } + ); +}); diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 50ab4f2f3f..c0eb0f554f 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -6,13 +6,13 @@ import { findProjectRoot } from '@agent-relay/config'; import { CloudFleetSandboxProvisionError, deleteCloudFleetSandbox, - ensureCloudFleetSandbox, materializeCloudRelayfileRepository, resolveWorkspaceByKey, type CloudFleetSandboxProviderId, type CloudRelayfileRepositoryMaterialization, type EnsureCloudFleetSandboxResult, } from '@agent-relay/cloud'; +import { ensureCloudFleetSandbox } from '@agent-relay/sdk/fleet'; import { HarnessDriverClient } from '@agent-relay/harness-driver'; import { createWorkspaceClient, @@ -331,6 +331,10 @@ export function registerFleetCommands( '--sandbox-relayfile-path ', 'Mount only these Relayfile subtrees (each path must end in /**)' ) + .option( + '--sandbox-readonly-path ', + 'Read-only Relayfile subtrees (/path/**); requires Cloud enforcement' + ) .option('--no-sandbox-relayfile', 'Provision the sandbox without mounting Relayfile') .option('--channel ', 'Channel for the worker to join') .option('--persona ', 'Worker persona (automatic placement)') @@ -397,6 +401,12 @@ export function registerFleetCommands( throw new Error('--sandbox-provider must be daytona, e2b, or agent37.'); } const mountSandboxRelayfile = options.sandboxRelayfile !== false; + const sandboxReadonlyPaths = optionalTextList(options.sandboxReadonlyPath, 'Sandbox read-only path'); + if (sandboxReadonlyPaths && (!useSandbox || !mountSandboxRelayfile)) { + throw new Error( + '--sandbox-readonly-path requires --sandbox and cannot be combined with --no-sandbox-relayfile.' + ); + } const sandboxRelayfilePaths = optionalTextList(options.sandboxRelayfilePath, 'Sandbox Relayfile path'); if (useSandbox && targetNode) { throw new Error('--sandbox cannot be combined with --node or --target-node.'); @@ -591,6 +601,7 @@ export function registerFleetCommands( requiredCapability: `spawn:${cli}`, maxAgents: 1, mountRelayfile: mountSandboxRelayfile, + ...(sandboxReadonlyPaths === undefined ? {} : { readonlyPaths: sandboxReadonlyPaths }), ...(liveRepository ? { relayfilePaths: liveRelayfileMountPaths(liveRepository, sandboxRelayfilePaths) } : sandboxRelayfilePaths === undefined diff --git a/packages/cli/src/cli/lib/attach-fleet-node.ts b/packages/cli/src/cli/lib/attach-fleet-node.ts index 634b9aaaed..71c417be17 100644 --- a/packages/cli/src/cli/lib/attach-fleet-node.ts +++ b/packages/cli/src/cli/lib/attach-fleet-node.ts @@ -1,1222 +1 @@ -/** - * Ticketed fleet-node attach adapter. - * - * The established attach clients intentionally continue to speak the local - * broker HTTP/WebSocket contract. This short-lived loopback adapter maps that - * contract onto Relaycast's authenticated terminal session, so view/drive and - * passthrough retain their behaviour without exposing a remote broker listener - * or copying a broker API key off a physical or Daytona node. - */ - -import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; -import { once } from 'node:events'; -import { Buffer } from 'node:buffer'; -import { randomBytes } from 'node:crypto'; - -import WebSocket, { WebSocketServer } from 'ws'; - -import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; -import type { AttachMode } from './attach-mode.js'; -import { collectWithRetry } from './collect-with-retry.js'; -import { resolveWorkspaceTransport } from './sdk-client.js'; - -const MAX_BUFFERED_BYTES = 1024 * 1024; -const MAX_WEBSOCKET_CLOSE_REASON_BYTES = 123; -// Use the same finite 30s request window as the broker's Relaycast HTTP calls. -// Terminal-session creation can exceed the shorter startup-handshake latency -// under load, and this POST cannot be replayed safely after an ambiguous client -// timeout because allocation may have completed server-side. -const SESSION_REQUEST_TIMEOUT_MS = 30_000; -// Immediate structured reachability failures still receive the complete -// five-attempt/31.2s retry schedule, while slow responses cannot multiply the -// per-attempt timeout into a roughly three-minute CLI hang. -const SESSION_REQUEST_TOTAL_TIMEOUT_MS = 90_000; -const SESSION_REQUEST_RETRIES = 4; -// The fleet node publishes liveness every 12s. With four retries, the helper's -// deterministic delays are 6s, 7.2s, 8.4s, and 9.6s: 31.2s total. That spans -// more than two heartbeat intervals and matches the established terminal -// transport's bounded recovery window instead of exhausting every retry inside -// the same stale control-plane read. -const SESSION_REQUEST_RETRY_DELAY_MS = 6_000; -const TERMINAL_CONNECT_TIMEOUT_MS = 10_000; -const INITIAL_RECONNECT_DELAY_MS = 500; -const MAX_RECONNECT_DELAY_MS = 30_000; -// Six delays (0.5s + 1s + 2s + 4s + 8s + 16s) cover the node terminal -// transport's independent 30s maximum reconnect backoff without leaving this -// client unbounded. Five attempts previously stopped after only 15.5s. -const MAX_RECONNECT_ATTEMPTS = 6; -// Bounds the acknowledgement for a delivery-mode command after readiness. -// The command is not replayable across reconnects, so this is intentionally a -// separate post-readiness phase rather than another full reconnect window. -const DELIVERY_MODE_TIMEOUT_MS = 10_000; -// The caller starts its HTTP deadline before the loopback handler starts its -// readiness timer. Leave a small response-delivery margin so the proxy's -// actionable readiness error wins that race. -const LOOPBACK_REQUEST_TIMEOUT_MARGIN_MS = 1_000; - -type FleetSessionResponse = { - ok?: boolean; - data?: { - session_id?: string; - terminal_url?: string; - resume_token?: string; - }; - error?: { code?: string; message?: string }; -}; - -type TerminalFrame = Record & { type?: string; session_id?: string }; - -type TerminalReadiness = { - generation: number; - settled: boolean; - promise: Promise; - resolve: () => void; - reject: (error: Error) => void; -}; - -export interface FleetNodeAttachOptions { - agent: string; - node: string; - mode: AttachMode; - env?: NodeJS.ProcessEnv; - baseUrl?: string; - workspaceKey?: string; - fetch?: typeof globalThis.fetch; - /** Deterministic test seam for the bounded session-request retry delay. */ - sessionRequest?: { - timeoutMs?: number; - totalTimeoutMs?: number; - sleep?: (ms: number) => Promise; - }; - /** Deterministic test seam for established-session reconnect timing. */ - reconnectDelay?: { - initialMs?: number; - maxMs?: number; - handshakeTimeoutMs?: number; - readyTimeoutMs?: number; - beforeReadyTimeoutTerminate?: (socket: WebSocket) => void; - }; -} - -export interface FleetNodeAttachProxy { - brokerUrl: string; - apiKey: string; - requestTimeoutMs: number; - close(): Promise; -} - -export class FleetNodeAttachError extends Error { - constructor( - message: string, - readonly code?: string - ) { - super(message); - this.name = 'FleetNodeAttachError'; - } -} - -const TRUSTED_RELAYCAST_ORIGINS = new Set([CANONICAL_RELAYCAST_ORIGIN, AGENT37_RELAYCAST_ORIGIN]); - -export function validateFleetAttachBaseUrl(value: string): string { - let parsed: URL; - try { - parsed = new URL(value); - } catch { - throw new FleetNodeAttachError('Fleet node attach requires a trusted Relaycast origin.'); - } - if ( - parsed.protocol !== 'https:' || - parsed.username || - parsed.password || - parsed.port || - parsed.search || - parsed.hash || - (parsed.pathname !== '' && parsed.pathname !== '/') || - !TRUSTED_RELAYCAST_ORIGINS.has(parsed.origin) - ) { - throw new FleetNodeAttachError('Fleet node attach requires a trusted Relaycast origin.'); - } - return parsed.origin; -} - -class TerminalSessionAttemptError extends FleetNodeAttachError { - constructor( - message: string, - code: string | undefined, - readonly status: number | undefined, - readonly retryable: boolean, - readonly completionUnknown: boolean - ) { - super(message, code); - } -} - -/** - * Canonical HTTP status for a terminal failure code. - * - * `agent_not_found` must stay 404: {@link switchInboundDeliveryModeOrAbort} - * only emits the "no agent named X" message and the cross-node placement hint - * on a 404, so collapsing it into 503 replaces actionable guidance with an - * opaque unreachable-node error. `unsupported_runtime` stays 409. Everything - * else is a transport-level failure and reports 503. - */ -function terminalErrorStatus(code: string | undefined): number { - if (code === 'agent_not_found') return 404; - if (code === 'unsupported_runtime') return 409; - return 503; -} - -function json(response: ServerResponse, status: number, payload: unknown): void { - response.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - response.end(JSON.stringify(payload)); -} - -function readBody(request: IncomingMessage): Promise> { - return new Promise((resolve) => { - let settled = false; - const finish = (body: Record) => { - if (settled) return; - settled = true; - resolve(body); - }; - let body = ''; - request.setEncoding('utf8'); - request.on('data', (chunk: string) => { - body += chunk; - }); - request.on('end', () => { - try { - const parsed = JSON.parse(body) as unknown; - finish( - parsed && typeof parsed === 'object' && !Array.isArray(parsed) - ? (parsed as Record) - : {} - ); - } catch { - finish({}); - } - }); - request.on('error', () => finish({})); - request.on('aborted', () => finish({})); - }); -} - -function asWsUrl(value: string): string { - const lower = value.toLowerCase(); - if (lower.startsWith('https://')) return 'wss://' + value.slice(8); - if (lower.startsWith('http://')) return 'ws://' + value.slice(7); - return value; -} - -function safeNodePath(node: string): string { - const trimmed = node.trim().replace(/^#/, ''); - if (!trimmed) throw new FleetNodeAttachError('Error: --node requires a node name or id.', 'invalid_node'); - return encodeURIComponent(trimmed); -} - -function parseFrame(data: WebSocket.RawData): TerminalFrame | null { - try { - const parsed = JSON.parse(rawDataToString(data)) as unknown; - return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as TerminalFrame) : null; - } catch { - return null; - } -} - -function rawDataToString(data: WebSocket.RawData): string { - if (Buffer.isBuffer(data)) return data.toString('utf8'); - if (data instanceof ArrayBuffer) return Buffer.from(data).toString('utf8'); - if (Array.isArray(data)) return Buffer.concat(data).toString('utf8'); - return String(data); -} - -function diagnosticEndpoint(value: string): string { - try { - const url = new URL(value); - url.username = ''; - url.password = ''; - url.search = ''; - url.hash = ''; - return url.toString(); - } catch { - return '(invalid endpoint)'; - } -} - -function diagnosticEndpointOrigin(value: string): string { - try { - return new URL(value).origin; - } catch { - return '(invalid endpoint)'; - } -} - -function boundedWebSocketCloseReason(reason: string): string { - const encoded = Buffer.from(reason, 'utf8'); - if (encoded.length <= MAX_WEBSOCKET_CLOSE_REASON_BYTES) return reason; - - const suffix = Buffer.from('…', 'utf8'); - let end = MAX_WEBSOCKET_CLOSE_REASON_BYTES - suffix.length; - // Do not cut through a UTF-8 continuation sequence. Excluding the leading - // byte at this boundary also excludes the incomplete code point. - while (end > 0 && (encoded[end] & 0xc0) === 0x80) end -= 1; - return Buffer.concat([encoded.subarray(0, end), suffix]).toString('utf8'); -} - -function diagnosticValue(value: string): string { - // JSON quoting keeps node names and upstream text from injecting terminal - // control characters into the operator-facing error. - return JSON.stringify(value); -} - -function resolvedNodeIdFromTerminalUrl(value: string): string | undefined { - try { - const match = /^\/v1\/nodes\/([^/]+)\/terminal\/connect$/.exec(new URL(value).pathname); - return match?.[1] ? decodeURIComponent(match[1]) : undefined; - } catch { - return undefined; - } -} - -function retryDelayBudgetMs(attempts: number, initialMs: number, maxMs: number): number { - let total = 0; - for (let attempt = 0; attempt < attempts; attempt += 1) { - total += Math.min(initialMs * 2 ** attempt, maxMs); - } - return total; -} - -function terminalSessionFailureSummary(error: TerminalSessionAttemptError): string { - if (error.code === 'node_not_found') return 'Control-plane node lookup found no matching record'; - if (error.code === 'node_unreachable') { - return /no terminal transport/i.test(error.message) - ? 'The node record was found, but its terminal transport was unavailable' - : 'The control plane classified the node as unreachable'; - } - if (error.code === 'terminal_session_unavailable') { - return 'The control plane could not allocate a terminal session'; - } - if (error.code === 'control_plane_timeout') return 'Control-plane terminal-session lookup timed out'; - if (error.code === 'control_plane_unavailable') return 'Control-plane terminal-session lookup failed'; - return 'The terminal-session request was rejected'; -} - -function isRetryableTerminalSessionFailure(code: string | undefined): boolean { - // These structured responses are emitted before a session is returned, so - // retrying cannot duplicate a successful allocation. A fetch timeout, - // network failure, or unclassified 5xx is different: this POST may already - // have completed server-side, and retrying it could create a second session. - return code === 'node_unreachable' || code === 'terminal_session_unavailable'; -} - -/** Start a broker-compatible loopback proxy for one remote terminal session. */ -export async function startFleetNodeAttachProxy( - options: FleetNodeAttachOptions -): Promise { - const env = options.env ?? process.env; - const fetchFn = options.fetch ?? globalThis.fetch; - const { workspaceKey, baseUrl: requestedBaseUrl } = resolveWorkspaceTransport({ - workspaceKey: options.workspaceKey, - baseUrl: options.baseUrl, - env, - }); - const baseUrl = validateFleetAttachBaseUrl(requestedBaseUrl ?? CANONICAL_RELAYCAST_ORIGIN); - const nodePath = safeNodePath(options.node); - const sessionEndpoint = `${baseUrl}/v1/nodes/${nodePath}/terminal/sessions`; - const sessionRequestTimeoutMs = options.sessionRequest?.timeoutMs ?? SESSION_REQUEST_TIMEOUT_MS; - const sessionRequestTotalTimeoutMs = - options.sessionRequest?.totalTimeoutMs ?? SESSION_REQUEST_TOTAL_TIMEOUT_MS; - const sessionRequestDeadline = Date.now() + sessionRequestTotalTimeoutMs; - const sessionRequestSleep = - options.sessionRequest?.sleep ?? - ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); - let sessionRequestAttempts = 0; - let sessionRequestBudgetExhaustedBetweenAttempts = false; - let lastSessionError: TerminalSessionAttemptError | undefined; - const sessionResult = await collectWithRetry( - 'terminal session request', - async () => { - const remainingRequestBudgetMs = sessionRequestDeadline - Date.now(); - if (remainingRequestBudgetMs <= 0) { - sessionRequestBudgetExhaustedBetweenAttempts = true; - const exhausted = - lastSessionError === undefined - ? new TerminalSessionAttemptError( - 'overall terminal-session request deadline exhausted', - 'control_plane_timeout', - undefined, - false, - false - ) - : new TerminalSessionAttemptError( - lastSessionError.message, - lastSessionError.code, - lastSessionError.status, - false, - lastSessionError.completionUnknown - ); - lastSessionError = exhausted; - throw exhausted; - } - sessionRequestAttempts += 1; - const controller = new AbortController(); - let timedOut = false; - const attemptTimeoutMs = Math.min(sessionRequestTimeoutMs, remainingRequestBudgetMs); - const timeout = setTimeout(() => { - timedOut = true; - controller.abort(); - }, attemptTimeoutMs); - let ticketResponse: Response; - let ticketPayload: FleetSessionResponse; - try { - ticketResponse = await fetchFn(sessionEndpoint, { - method: 'POST', - headers: { Authorization: `Bearer ${workspaceKey}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ agent: options.agent, mode: options.mode }), - signal: controller.signal, - }); - const parsedPayload = (await ticketResponse.json()) as unknown; - ticketPayload = - parsedPayload && typeof parsedPayload === 'object' && !Array.isArray(parsedPayload) - ? (parsedPayload as FleetSessionResponse) - : {}; - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - lastSessionError = new TerminalSessionAttemptError( - timedOut ? 'request exceeded its deadline' : detail, - timedOut ? 'control_plane_timeout' : 'control_plane_unavailable', - undefined, - false, - true - ); - throw lastSessionError; - } finally { - clearTimeout(timeout); - } - - const terminalUrl = ticketPayload.data?.terminal_url; - const sessionId = ticketPayload.data?.session_id; - const resumeToken = ticketPayload.data?.resume_token; - if (!ticketResponse.ok || !terminalUrl || !sessionId || !resumeToken) { - const code = ticketPayload.error?.code; - const message = - ticketPayload.error?.message ?? `terminal session request failed (HTTP ${ticketResponse.status})`; - lastSessionError = new TerminalSessionAttemptError( - message, - code, - ticketResponse.status, - isRetryableTerminalSessionFailure(code), - false - ); - throw lastSessionError; - } - return { terminalUrl, sessionId, resumeToken }; - }, - { - retries: SESSION_REQUEST_RETRIES, - baseDelayMs: SESSION_REQUEST_RETRY_DELAY_MS, - sleep: async (delayMs) => { - const remainingRequestBudgetMs = sessionRequestDeadline - Date.now(); - if (remainingRequestBudgetMs <= 0) return; - await sessionRequestSleep(Math.min(delayMs, remainingRequestBudgetMs)); - }, - shouldRetry: (error) => error instanceof TerminalSessionAttemptError && error.retryable, - } - ); - if (!sessionResult.ok) { - const failure = lastSessionError; - const status = failure?.status === undefined ? '' : ` HTTP ${failure.status};`; - const code = failure?.code === undefined ? '' : ` code ${failure.code};`; - const retryNote = sessionRequestBudgetExhaustedBetweenAttempts - ? sessionRequestAttempts > 1 - ? `retried ${sessionRequestAttempts - 1} time${sessionRequestAttempts === 2 ? '' : 's'};` + - ' overall budget exhausted before the next attempt' - : 'not retried because the overall budget was exhausted before the next attempt' - : sessionRequestAttempts > 1 - ? `retried ${sessionRequestAttempts - 1} time${sessionRequestAttempts === 2 ? '' : 's'}` + - (failure?.completionUnknown - ? '; final POST not retried because it may have completed server-side' - : '') - : failure?.completionUnknown - ? 'not retried because the POST may have completed server-side' - : 'not retried because the failure was terminal'; - const upstream = failure?.message ? ` Upstream message ${diagnosticValue(failure.message)}.` : ''; - throw new FleetNodeAttachError( - `Error: ${failure ? terminalSessionFailureSummary(failure) : 'Terminal-session request failed'}.` + - `${upstream} Node ref ${diagnosticValue(options.node.trim())}, resolved node id unavailable (session creation did not complete);` + - ` endpoint ${diagnosticValue(diagnosticEndpoint(sessionEndpoint))};${status}${code}` + - ` timeout ${sessionRequestTimeoutMs}ms per attempt; overall budget ${sessionRequestTotalTimeoutMs}ms;` + - ` attempts ${sessionRequestAttempts} (${retryNote}).`, - failure?.code - ); - } - const { terminalUrl, sessionId, resumeToken } = sessionResult.value; - const resolvedNodeId = resolvedNodeIdFromTerminalUrl(terminalUrl); - const remoteEndpoint = diagnosticEndpoint(terminalUrl); - const reconnectInitialDelayMs = options.reconnectDelay?.initialMs ?? INITIAL_RECONNECT_DELAY_MS; - const reconnectMaxDelayMs = options.reconnectDelay?.maxMs ?? MAX_RECONNECT_DELAY_MS; - const terminalHandshakeTimeoutMs = - options.reconnectDelay?.handshakeTimeoutMs ?? TERMINAL_CONNECT_TIMEOUT_MS; - const terminalReadyTimeoutMs = options.reconnectDelay?.readyTimeoutMs ?? TERMINAL_CONNECT_TIMEOUT_MS; - // A readiness-gated local request follows activeReadiness across reconnect - // generations. Its own deadline therefore has to cover the same complete, - // finite recovery path: every backoff plus every handshake/readiness pair. - const terminalWaitTimeoutMs = - retryDelayBudgetMs(MAX_RECONNECT_ATTEMPTS, reconnectInitialDelayMs, reconnectMaxDelayMs) + - MAX_RECONNECT_ATTEMPTS * (terminalHandshakeTimeoutMs + terminalReadyTimeoutMs); - - let connectionGeneration = 0; - const createReadiness = (): TerminalReadiness => { - let resolve!: () => void; - let reject!: (error: Error) => void; - const promise = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise; - reject = rejectPromise; - }); - // A failure can land before a snapshot request attaches its waiter. Keep - // the rejection observable while avoiding an unhandled-rejection warning. - void promise.catch(() => undefined); - return { generation: ++connectionGeneration, settled: false, promise, resolve, reject }; - }; - let activeReadiness = createReadiness(); - const resolveReadiness = (readiness: TerminalReadiness) => { - if (readiness.settled) return; - readiness.settled = true; - readiness.resolve(); - }; - const rejectReadiness = (readiness: TerminalReadiness, error: Error) => { - if (readiness.settled) return; - readiness.settled = true; - readiness.reject(error); - }; - const waitForCurrentReadiness = async (): Promise => { - for (;;) { - const readiness = activeReadiness; - await readiness.promise; - if (readiness === activeReadiness) return; - } - }; - /** - * Await the live readiness generation through both the WebSocket handshake - * and the post-open terminal.ready allowance. - * Every handler that must not run before `terminal.ready` — snapshot, - * delivery-mode PUT, resize — goes through this one helper so the - * timer/clearTimeout/settle logic cannot diverge between copies. - * - * Rejects with the underlying {@link FleetNodeAttachError} when the terminal - * failed (preserving its `code`), or a plain `Error` carrying - * `timeoutMessage` when the wait expired. - */ - const waitForTerminalReady = (timeoutMessage: string): Promise => - new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error(timeoutMessage)), terminalWaitTimeoutMs); - void waitForCurrentReadiness().then( - () => { - clearTimeout(timer); - resolve(); - }, - (error: Error) => { - clearTimeout(timer); - reject(error); - } - ); - }); - const snapshot: { screen: string; rows: number; cols: number; offset: number } = { - screen: '', - rows: 24, - cols: 80, - offset: 0, - }; - const eventSockets = new Set(); - const inputSockets = new Set(); - const outputHistory: Array<{ chunk: string; offset?: number }> = []; - let outputHistoryBytes = 0; - let remote: WebSocket | undefined; - let stopped = false; - let terminalEnded = false; - let terminalEverReady = false; - let reconnecting = false; - let reconnectAttempts = 0; - let reconnectTimer: ReturnType | undefined; - const terminalReadinessTimers = new Set>(); - /** Locally-tracked delivery mode, kept in sync with each broker reply. */ - let loopbackDeliveryMode: 'manual_flush' | 'auto_inject' = - options.mode === 'drive' ? 'manual_flush' : 'auto_inject'; - type DeliveryModeResult = { mode: string; flushed: number; matched: boolean; revision: string }; - type FlushResult = { - flushed: number; - dead_lettered: number; - held: number; - blocked_reason: string | null; - }; - let pendingFlush: { - requestId: string; - resolve: (result: FlushResult) => void; - reject: (error: Error) => void; - timer: ReturnType; - } | null = null; - /** At most one in-flight delivery-mode PUT at a time. */ - let pendingDeliveryMode: { - requestId: string; - resolve: (result: DeliveryModeResult) => void; - reject: (err: Error) => void; - timer: ReturnType; - } | null = null; - const loopbackApiKey = randomBytes(32).toString('base64url'); - const loopbackAuthorized = (headers: IncomingMessage['headers']) => - headers.authorization === `Bearer ${loopbackApiKey}` || headers['x-api-key'] === loopbackApiKey; - - const server = createServer(async (request, response) => { - if (!loopbackAuthorized(request.headers)) { - json(response, 401, { - error: { code: 'unauthorized', message: 'loopback terminal token is required' }, - }); - return; - } - const path = new URL(request.url ?? '/', 'http://127.0.0.1').pathname; - if (request.method === 'GET' && path === `/api/spawned/${encodeURIComponent(options.agent)}/snapshot`) { - try { - await waitForTerminalReady('terminal snapshot timed out'); - } catch (error) { - const terminalError = error instanceof FleetNodeAttachError ? error : undefined; - json(response, terminalErrorStatus(terminalError?.code), { - error: { - code: terminalError?.code ?? 'snapshot_unavailable', - message: - terminalError?.message ?? (error instanceof Error ? error.message : 'snapshot unavailable'), - }, - }); - return; - } - json(response, 200, { format: 'ansi', ...snapshot }); - return; - } - const name = encodeURIComponent(options.agent); - if (path === `/api/spawned/${name}/flush` && request.method === 'POST') { - // Forward the flush to the remote broker over the terminal websocket. - // Without this the `--node` form of `node agent message flush` reached - // only the LOCAL broker's worker registry and returned agent_not_found - // for a name that `node agent list` and `attach` both resolve. - try { - await waitForTerminalReady('terminal connection timed out'); - } catch (error) { - const terminalError = error instanceof FleetNodeAttachError ? error : undefined; - json(response, terminalErrorStatus(terminalError?.code), { - error: { - code: terminalError?.code ?? 'node_unreachable', - message: - terminalError?.message ?? (error instanceof Error ? error.message : 'terminal unavailable'), - }, - }); - return; - } - if (!remote || remote.readyState !== WebSocket.OPEN) { - json(response, 503, { - error: { code: 'node_unreachable', message: 'terminal transport is not connected' }, - }); - return; - } - if (pendingFlush) { - json(response, 503, { - error: { code: 'flush_conflict', message: 'a flush request is already in flight' }, - }); - return; - } - const requestId = randomBytes(8).toString('hex'); - const result = await new Promise((resolve) => { - const timer = setTimeout(() => { - pendingFlush = null; - resolve(new FleetNodeAttachError('flush request timed out', 'flush_timeout')); - }, DELIVERY_MODE_TIMEOUT_MS); - pendingFlush = { - requestId, - resolve: (r) => resolve(r), - reject: (e) => resolve(e), - timer, - }; - remote!.send( - JSON.stringify({ - type: 'terminal.flush_pending', - session_id: sessionId, - request_id: requestId, - }) - ); - }); - if (result instanceof Error) { - const errCode = - result instanceof FleetNodeAttachError ? (result.code ?? 'flush_failed') : 'flush_failed'; - json(response, terminalErrorStatus(errCode), { - error: { code: errCode, message: result.message }, - }); - return; - } - json(response, 200, result); - return; - } - if (path === `/api/spawned/${name}/delivery-mode`) { - if (request.method === 'GET') { - json(response, 200, { mode: loopbackDeliveryMode }); - return; - } - // PUT — forward the request to the remote broker via the terminal WS and - // await the broker's real reply. This is the path that was previously a - // static stub returning manual_flush, causing drive attach to fail with - // "broker remained in manual_flush mode". - const body = await readBody(request); - const requestedMode = - body.mode === 'auto_inject' ? 'auto_inject' : body.mode === 'manual_flush' ? 'manual_flush' : null; - if (requestedMode === null) { - json(response, 400, { - error: { code: 'invalid_mode', message: `unsupported delivery mode '${String(body.mode)}'` }, - }); - return; - } - // Drive attach changes delivery mode before it requests the initial - // snapshot. Gate the PUT on terminal.ready so a fast local caller does - // not lose a race with the remote websocket handshake and receive the - // misleading "terminal transport is not connected" failure. - try { - await waitForTerminalReady('terminal connection timed out'); - } catch (error) { - // Preserve the canonical status mapping here too: a readiness failure - // carrying `agent_not_found` has to reach the preflight as a 404 or - // the operator loses the cross-node placement hint that tells them - // which machine to run the attach on. - const terminalError = error instanceof FleetNodeAttachError ? error : undefined; - json(response, terminalErrorStatus(terminalError?.code), { - error: { - code: terminalError?.code ?? 'node_unreachable', - message: - terminalError?.message ?? (error instanceof Error ? error.message : 'terminal unavailable'), - }, - }); - return; - } - if (!remote || remote.readyState !== WebSocket.OPEN) { - json(response, 503, { - error: { code: 'node_unreachable', message: 'terminal transport is not connected' }, - }); - return; - } - if (pendingDeliveryMode) { - json(response, 503, { - error: { code: 'delivery_mode_conflict', message: 'a delivery mode request is already in flight' }, - }); - return; - } - const requestId = randomBytes(8).toString('hex'); - const result = await new Promise((resolve) => { - const timer = setTimeout(() => { - pendingDeliveryMode = null; - resolve(new FleetNodeAttachError('delivery mode request timed out', 'delivery_mode_timeout')); - }, DELIVERY_MODE_TIMEOUT_MS); - pendingDeliveryMode = { - requestId, - resolve: (r) => resolve(r), - reject: (e) => resolve(e), - timer, - }; - const frame: Record = { - type: 'terminal.set_delivery_mode', - session_id: sessionId, - mode: requestedMode, - request_id: requestId, - }; - if (typeof body.expected_mode === 'string') frame.expected_mode = body.expected_mode; - if (typeof body.expected_revision === 'string') frame.expected_revision = body.expected_revision; - remote!.send(JSON.stringify(frame)); - }); - if (result instanceof Error) { - const errCode = - result instanceof FleetNodeAttachError - ? (result.code ?? 'delivery_mode_failed') - : 'delivery_mode_failed'; - // Same canonical mapping as the readiness gate: 404 for - // agent_not_found so the attach preflight can produce the no-agent or - // cross-node placement error, 409 for unsupported_runtime, 503 for - // everything else. - json(response, terminalErrorStatus(errCode), { - error: { code: errCode, message: result.message }, - }); - return; - } - loopbackDeliveryMode = result.mode === 'manual_flush' ? 'manual_flush' : 'auto_inject'; - json(response, 200, { - mode: result.mode, - flushed: result.flushed, - matched: result.matched, - revision: result.revision, - }); - return; - } - if (request.method === 'GET' && path === `/api/spawned/${name}/pending`) { - json(response, 200, { pending: [] }); - return; - } - if (request.method === 'POST' && path === `/api/spawned/${name}/flush`) { - json(response, 200, { flushed: 0 }); - return; - } - if (request.method === 'GET' && path === '/api/spawned') { - json(response, 200, { agents: [{ name: options.agent, workerPid: 1 }] }); - return; - } - if (request.method === 'POST' && path === `/api/resize/${name}`) { - const body = await readBody(request); - if (body.release === true) { - json(response, 200, { name: options.agent, released: true }); - return; - } - const rows = typeof body.rows === 'number' ? body.rows : 0; - const cols = typeof body.cols === 'number' ? body.cols : 0; - if (!Number.isInteger(rows) || !Number.isInteger(cols) || rows < 1 || cols < 1) { - json(response, 400, { - error: { code: 'invalid_dimensions', message: 'rows and cols must be positive integers' }, - }); - return; - } - try { - await waitForTerminalReady('terminal resize timed out'); - } catch (error) { - json(response, 503, { - error: { - code: 'session_not_ready', - message: error instanceof Error ? error.message : 'terminal session is not ready', - }, - }); - return; - } - if (!remote || remote.readyState !== WebSocket.OPEN || remote.bufferedAmount > MAX_BUFFERED_BYTES) { - json(response, 503, { - error: { code: 'node_unreachable', message: 'terminal transport is unavailable' }, - }); - return; - } - remote.send(JSON.stringify({ type: 'terminal.resize', session_id: sessionId, rows, cols })); - json(response, 200, { name: options.agent, rows, cols, applied: true }); - return; - } - json(response, 404, { error: { code: 'not_found', message: 'loopback terminal endpoint not found' } }); - }); - const websocketServer = new WebSocketServer({ noServer: true }); - - const closeSocket = (socket: WebSocket, code: number, reason: string) => { - try { - socket.close(code, boundedWebSocketCloseReason(reason)); - } catch { - /* connection already gone */ - } - }; - const broadcast = (sockets: Set, payload: unknown): boolean => { - const encoded = JSON.stringify(payload); - let accepted = false; - for (const socket of sockets) { - if (socket.readyState !== WebSocket.OPEN) continue; - if (socket.bufferedAmount > MAX_BUFFERED_BYTES) { - closeSocket(socket, 1013, 'loopback client backpressure exceeded'); - sockets.delete(socket); - continue; - } - try { - socket.send(encoded); - accepted = true; - } catch { - sockets.delete(socket); - } - } - return accepted; - }; - const workerStreamEvent = (chunk: string, offset?: number) => ({ - kind: 'worker_stream', - name: options.agent, - stream: 'stdout', - chunk, - ...(offset === undefined ? {} : { offset }), - }); - const retainOutput = (chunk: string, offset: number | undefined): boolean => { - const bytes = Buffer.byteLength(chunk, 'utf8'); - if (outputHistoryBytes + bytes > MAX_BUFFERED_BYTES) return false; - outputHistory.push({ chunk, ...(offset === undefined ? {} : { offset }) }); - outputHistoryBytes += bytes; - return true; - }; - - websocketServer.on('connection', (socket, request) => { - const path = new URL(request.url ?? '/', 'http://127.0.0.1').pathname; - if (path === '/ws') { - eventSockets.add(socket); - socket.on('close', () => eventSockets.delete(socket)); - let replayed = 0; - for (const event of outputHistory) { - if (socket.readyState !== WebSocket.OPEN || socket.bufferedAmount > MAX_BUFFERED_BYTES) break; - try { - socket.send(JSON.stringify(workerStreamEvent(event.chunk, event.offset))); - } catch { - break; - } - replayed += 1; - } - if (replayed > 0) { - const sentBytes = outputHistory - .slice(0, replayed) - .reduce((total, event) => total + Buffer.byteLength(event.chunk, 'utf8'), 0); - outputHistory.splice(0, replayed); - outputHistoryBytes -= sentBytes; - } - return; - } - if (path === `/api/input/${encodeURIComponent(options.agent)}/stream`) { - inputSockets.add(socket); - socket.on('close', () => inputSockets.delete(socket)); - socket.send(JSON.stringify({ type: 'pty_input_ready', name: options.agent })); - socket.on('message', (data) => { - if (!remote || remote.readyState !== WebSocket.OPEN || remote.bufferedAmount > MAX_BUFFERED_BYTES) { - broadcast(inputSockets, { - type: 'pty_input_error', - code: 'node_unreachable', - message: 'terminal transport is unavailable', - }); - return; - } - const raw = rawDataToString(data); - remote.send( - JSON.stringify({ - type: 'terminal.input', - session_id: sessionId, - data_base64: Buffer.from(raw, 'utf8').toString('base64'), - }) - ); - }); - return; - } - closeSocket(socket, 1008, 'unknown loopback endpoint'); - }); - server.on('upgrade', (request, socket, head) => { - if (!loopbackAuthorized(request.headers)) { - socket.write('HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n'); - socket.destroy(); - return; - } - websocketServer.handleUpgrade(request, socket, head, (client) => - websocketServer.emit('connection', client, request) - ); - }); - server.listen(0, '127.0.0.1'); - await once(server, 'listening'); - const address = server.address(); - if (!address || typeof address === 'string') - throw new FleetNodeAttachError( - 'Error: could not allocate loopback terminal listener.', - 'loopback_unavailable' - ); - - const resumeUrl = new URL(terminalUrl); - resumeUrl.searchParams.delete('ticket'); - resumeUrl.searchParams.set('session_id', sessionId); - resumeUrl.searchParams.set('resume', resumeToken); - /** Reject and clear any in-flight delivery-mode PUT, if one is pending. */ - const rejectPendingDeliveryMode = (error: FleetNodeAttachError) => { - if (pendingFlush) { - const flush = pendingFlush; - pendingFlush = null; - clearTimeout(flush.timer); - // A flush never changes delivery mode, so it must not inherit the - // delivery-mode disconnect wording. An operator debugging a failed flush - // would otherwise be told the "delivery-mode change" was interrupted — - // pointing at an operation their command never performed. - const flushError = - error.code === 'delivery_mode_disconnected' - ? new FleetNodeAttachError( - 'terminal transport disconnected while the flush was in flight', - 'flush_disconnected' - ) - : error; - flush.reject(flushError); - } - if (!pendingDeliveryMode) return; - const pending = pendingDeliveryMode; - pendingDeliveryMode = null; - clearTimeout(pending.timer); - pending.reject(error); - }; - const endTerminal = (error: FleetNodeAttachError, eventCloseReason = error.message) => { - if (terminalEnded) return; - terminalEnded = true; - if (reconnectTimer) { - clearTimeout(reconnectTimer); - reconnectTimer = undefined; - } - for (const timer of terminalReadinessTimers) clearTimeout(timer); - terminalReadinessTimers.clear(); - rejectPendingDeliveryMode(error); - const activeRemote = remote; - remote = undefined; - rejectReadiness(activeReadiness, error); - broadcast(inputSockets, { type: 'pty_input_error', code: error.code, message: error.message }); - for (const socket of eventSockets) closeSocket(socket, 1011, eventCloseReason); - if (activeRemote && activeRemote.readyState !== WebSocket.CLOSED) { - try { - activeRemote.terminate(); - } catch { - // The socket may have closed between the state check and terminate. - } - } - }; - const failRemote = (message: string, eventCloseReason?: string) => { - endTerminal(new FleetNodeAttachError(message, 'node_unreachable'), eventCloseReason); - }; - const connect = (url: string, readiness: TerminalReadiness) => { - if (stopped || terminalEnded) return; - const socket = new WebSocket(asWsUrl(url), { handshakeTimeout: terminalHandshakeTimeoutMs }); - remote = socket; - let readinessTimer: ReturnType | undefined; - let readinessExpired = false; - const clearReadinessTimer = () => { - if (!readinessTimer) return; - clearTimeout(readinessTimer); - terminalReadinessTimers.delete(readinessTimer); - readinessTimer = undefined; - }; - socket.on('open', () => { - // `handshakeTimeout` independently bounds the HTTP upgrade. Start the - // terminal.ready allowance only after that upgrade succeeds so a slow - // but valid handshake cannot consume the readiness window. - readinessTimer = setTimeout(() => { - const expiredTimer = readinessTimer; - readinessTimer = undefined; - if (expiredTimer) terminalReadinessTimers.delete(expiredTimer); - if (remote !== socket || stopped || terminalEnded || readiness.settled) return; - // Mark this generation stale before terminating. The ws receiver may - // still deliver data already buffered on the socket while close is - // propagating; none of it may restore readiness or reset retry state. - readinessExpired = true; - if (!terminalEverReady) { - failRemote( - `terminal transport connected but did not become ready (node ref ${diagnosticValue(options.node.trim())},` + - ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint)},` + - ` readiness timeout ${terminalReadyTimeoutMs}ms, attempts 1; not retried because no terminal session became ready)` - ); - return; - } - // A successful WebSocket upgrade is not sufficient: Relaycast may - // accept a resume lane that never produces terminal.ready. Terminating - // it drives the same bounded close/retry path as a transport failure. - options.reconnectDelay?.beforeReadyTimeoutTerminate?.(socket); - socket.terminate(); - }, terminalReadyTimeoutMs); - terminalReadinessTimers.add(readinessTimer); - }); - socket.on('message', (data) => { - // A late frame from a transport superseded during reconnect must never - // overwrite the fresh snapshot or end the replacement session. - if (remote !== socket || stopped || terminalEnded || readinessExpired) return; - const frame = parseFrame(data); - if (!frame || frame.session_id !== sessionId) return; - if (frame.type === 'terminal.ready') { - clearReadinessTimer(); - snapshot.screen = typeof frame.screen === 'string' ? frame.screen : ''; - snapshot.rows = typeof frame.rows === 'number' ? frame.rows : 24; - snapshot.cols = typeof frame.cols === 'number' ? frame.cols : 80; - snapshot.offset = typeof frame.offset === 'number' ? frame.offset : 0; - // Seed loopbackDeliveryMode from the broker's actual state so that - // detach restores the correct mode even when the worker started in a - // different mode than our local inference at line 214. - if (frame.delivery_mode === 'manual_flush' || frame.delivery_mode === 'auto_inject') { - loopbackDeliveryMode = frame.delivery_mode; - } - terminalEverReady = true; - reconnectAttempts = 0; - if (readiness === activeReadiness) { - resolveReadiness(readiness); - // A reconnect gets a fresh ANSI grid but existing local `/ws` - // consumers have already performed their initial HTTP snapshot. - // Re-emit this screen without an offset so they repaint instead of - // retaining a stale pre-reconnect terminal image. - if (readiness.generation > 1 && snapshot.screen) { - broadcast(eventSockets, workerStreamEvent(snapshot.screen)); - } - } - } else if (frame.type === 'terminal.output' && typeof frame.chunk === 'string') { - const offset = typeof frame.offset === 'number' ? frame.offset : undefined; - if (!broadcast(eventSockets, workerStreamEvent(frame.chunk, offset))) { - if (!retainOutput(frame.chunk, offset)) { - endTerminal( - new FleetNodeAttachError( - 'terminal output exceeded the bounded loopback buffer', - 'output_backpressure' - ) - ); - } - } - } else if (frame.type === 'terminal.input_ack') { - broadcast(inputSockets, { - type: 'pty_input_ack', - name: options.agent, - bytes_written: typeof frame.bytes_written === 'number' ? frame.bytes_written : 0, - }); - } else if (frame.type === 'terminal.delivery_mode') { - const frameRid = typeof frame.request_id === 'string' ? frame.request_id : undefined; - if (pendingDeliveryMode && (frameRid === undefined || frameRid === pendingDeliveryMode.requestId)) { - const pending = pendingDeliveryMode; - pendingDeliveryMode = null; - clearTimeout(pending.timer); - pending.resolve({ - mode: typeof frame.mode === 'string' ? frame.mode : 'auto_inject', - flushed: typeof frame.flushed === 'number' ? frame.flushed : 0, - matched: typeof frame.matched === 'boolean' ? frame.matched : true, - revision: typeof frame.revision === 'string' ? frame.revision : '1', - }); - } - } else if (frame.type === 'terminal.flush_pending') { - const frameRid = typeof frame.request_id === 'string' ? frame.request_id : undefined; - // Exact match only. The proxy always sends a request_id, and unlike - // delivery-mode there is no older-broker reply shape to stay - // compatible with, so a reply without one is not ours — accepting it - // would resolve the caller's flush with an unrelated result. - if (pendingFlush && frameRid === pendingFlush.requestId) { - const pending = pendingFlush; - pendingFlush = null; - clearTimeout(pending.timer); - pending.resolve({ - flushed: typeof frame.flushed === 'number' ? frame.flushed : 0, - dead_lettered: typeof frame.dead_lettered === 'number' ? frame.dead_lettered : 0, - held: typeof frame.held === 'number' ? frame.held : 0, - blocked_reason: typeof frame.blocked_reason === 'string' ? frame.blocked_reason : null, - }); - } - } else if (frame.type === 'terminal.error') { - const message = typeof frame.message === 'string' ? frame.message : 'remote terminal failed'; - const code = typeof frame.code === 'string' ? frame.code : 'terminal_error'; - const frameRid = typeof frame.request_id === 'string' ? frame.request_id : undefined; - // Route the error to the pending delivery-mode request only when the - // request_id matches (or the broker sent no request_id at all — older - // broker compat). An unrelated session-level error must not cancel a - // live delivery-mode PUT and vice-versa. - if (pendingFlush && frameRid !== undefined && frameRid === pendingFlush.requestId) { - const pending = pendingFlush; - pendingFlush = null; - clearTimeout(pending.timer); - pending.reject(new FleetNodeAttachError(message, code)); - } else if ( - pendingDeliveryMode && - (frameRid === undefined || frameRid === pendingDeliveryMode.requestId) - ) { - const pending = pendingDeliveryMode; - pendingDeliveryMode = null; - clearTimeout(pending.timer); - pending.reject(new FleetNodeAttachError(message, code)); - } else if (!frameRid && readiness === activeReadiness && !readiness.settled) { - endTerminal(new FleetNodeAttachError(message, code)); - } else if (!frameRid) { - broadcast(inputSockets, { type: 'pty_input_error', code, message }); - } - } else if (frame.type === 'terminal.closed') { - endTerminal(new FleetNodeAttachError('remote terminal session closed', 'terminal_closed')); - } - }); - socket.on('error', () => { - readinessExpired = true; - clearReadinessTimer(); - // Initial connection failure has no terminal state worth preserving. - // Fail promptly with the canonical unavailable-node error instead of - // letting the HTTP snapshot timeout mask it. Once Ready has been seen, - // the close handler retains the bounded resume/backoff behaviour. - if (remote === socket && readiness === activeReadiness && !readiness.settled && !terminalEverReady) { - failRemote( - `terminal transport could not connect to the fleet node (node ref ${diagnosticValue(options.node.trim())},` + - ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint)},` + - ` handshake budget ${terminalHandshakeTimeoutMs}ms, attempts 1; not retried because no terminal session became ready)` - ); - } - }); - socket.on('close', () => { - readinessExpired = true; - clearReadinessTimer(); - if (remote !== socket || stopped || terminalEnded || reconnecting) return; - // Relaycast drops the old lane's terminal session state on disconnect, - // so a set_delivery_mode frame already sent on this dying socket is - // lost and will never get a reply on the replacement socket — even - // once reconnect succeeds. Fail the pending PUT fast with a retryable - // error instead of leaving it to hang out the full timeout. - rejectPendingDeliveryMode( - new FleetNodeAttachError( - 'terminal transport disconnected while the delivery-mode change was in flight', - 'delivery_mode_disconnected' - ) - ); - // Any waiter that observed the prior connection must retry against the - // fresh generation instead of receiving its stale resolved snapshot. - resolveReadiness(readiness); - const nextReadiness = createReadiness(); - activeReadiness = nextReadiness; - if (reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) { - const backoffBudgetMs = retryDelayBudgetMs( - MAX_RECONNECT_ATTEMPTS, - reconnectInitialDelayMs, - reconnectMaxDelayMs - ); - const message = - `terminal transport could not reconnect to the fleet node (node ref ${diagnosticValue(options.node.trim())},` + - ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint)},` + - ` handshake timeout ${terminalHandshakeTimeoutMs}ms, readiness timeout ${terminalReadyTimeoutMs}ms,` + - ` attempts ${reconnectAttempts},` + - ` backoff budget ${backoffBudgetMs}ms)`; - failRemote( - message, - `terminal reconnect failed; attempts=${reconnectAttempts}; budget=${backoffBudgetMs}ms;` + - ` endpoint=${diagnosticValue(diagnosticEndpointOrigin(terminalUrl))};` + - ` node=${diagnosticValue(resolvedNodeId ?? options.node.trim())}` - ); - return; - } - reconnecting = true; - const delay = Math.min(reconnectInitialDelayMs * 2 ** reconnectAttempts, reconnectMaxDelayMs); - reconnectAttempts += 1; - reconnectTimer = setTimeout(() => { - reconnectTimer = undefined; - reconnecting = false; - connect(resumeUrl.toString(), nextReadiness); - }, delay); - }); - }; - connect(terminalUrl, activeReadiness); - - return { - brokerUrl: `http://127.0.0.1:${address.port}`, - apiKey: loopbackApiKey, - requestTimeoutMs: terminalWaitTimeoutMs + DELIVERY_MODE_TIMEOUT_MS + LOOPBACK_REQUEST_TIMEOUT_MARGIN_MS, - async close() { - if (stopped) return; - stopped = true; - terminalEnded = true; - if (reconnectTimer) { - clearTimeout(reconnectTimer); - reconnectTimer = undefined; - } - for (const timer of terminalReadinessTimers) clearTimeout(timer); - terminalReadinessTimers.clear(); - rejectReadiness(activeReadiness, new FleetNodeAttachError('terminal attach closed', 'closed')); - rejectPendingDeliveryMode(new FleetNodeAttachError('terminal attach closed', 'closed')); - const activeRemote = remote; - remote = undefined; - if (activeRemote && activeRemote.readyState === WebSocket.OPEN) { - try { - activeRemote.send(JSON.stringify({ type: 'terminal.close', session_id: sessionId })); - } catch { - // Best effort; terminate below still prevents a late reconnect. - } - } - if (activeRemote && activeRemote.readyState !== WebSocket.CLOSED) { - try { - activeRemote.terminate(); - } catch { - // Socket is already gone. - } - } - for (const socket of [...eventSockets, ...inputSockets]) - closeSocket(socket, 1000, 'terminal attach closed'); - websocketServer.close(); - await new Promise((resolve) => server.close(() => resolve())); - }, - }; -} +export * from '@agent-relay/cloud/attach'; diff --git a/packages/cli/src/cli/lib/attach-mode.ts b/packages/cli/src/cli/lib/attach-mode.ts index b1107eecec..b10726bba6 100644 --- a/packages/cli/src/cli/lib/attach-mode.ts +++ b/packages/cli/src/cli/lib/attach-mode.ts @@ -1 +1 @@ -export type AttachMode = 'drive' | 'view' | 'passthrough'; +export type { AttachMode } from '@agent-relay/cloud/attach'; diff --git a/packages/cli/src/cli/lib/collect-with-retry.ts b/packages/cli/src/cli/lib/collect-with-retry.ts index fb726a2315..71c417be17 100644 --- a/packages/cli/src/cli/lib/collect-with-retry.ts +++ b/packages/cli/src/cli/lib/collect-with-retry.ts @@ -1,42 +1 @@ -export interface CollectWithRetryOptions { - retries?: number; - baseDelayMs?: number; - sleep?: (ms: number) => Promise; - /** Return false for terminal failures that must be surfaced immediately. */ - shouldRetry?: (error: unknown, attempt: number) => boolean; -} - -/** - * Run a bounded operation with the fleet CLI's small retry delay. - * - * The result records whether another attempt actually ran. Callers that need - * richer failure metadata can retain the thrown value in their operation; - * this helper deliberately keeps the existing fleet-list rendering contract. - */ -export async function collectWithRetry( - label: string, - fn: () => Promise, - options?: CollectWithRetryOptions -): Promise<{ ok: true; value: T; retried: boolean } | { ok: false; error: string; retried: boolean }> { - const retries = options?.retries ?? 1; - const baseDelay = options?.baseDelayMs ?? 500; - const sleep = options?.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); - let lastError: unknown; - let attempts = 0; - for (let attempt = 0; attempt <= retries; attempt += 1) { - attempts += 1; - try { - const value = await fn(); - return { ok: true, value, retried: attempt > 0 }; - } catch (error) { - lastError = error; - if (attempt >= retries || options?.shouldRetry?.(error, attempt + 1) === false) break; - // Keep the established fleet-list delay shape while making later - // attempts wait slightly longer, without introducing test randomness. - const delay = baseDelay + Math.floor(baseDelay * (attempt / (retries + 1))); - await sleep(delay); - } - } - const message = lastError instanceof Error ? lastError.message : String(lastError); - return { ok: false, error: `${label}: ${message}`, retried: attempts > 1 }; -} +export * from '@agent-relay/cloud/attach'; diff --git a/packages/cli/src/cli/lib/sdk-client.ts b/packages/cli/src/cli/lib/sdk-client.ts index fe9bf17169..42ea056dec 100644 --- a/packages/cli/src/cli/lib/sdk-client.ts +++ b/packages/cli/src/cli/lib/sdk-client.ts @@ -1,9 +1,18 @@ import path from 'node:path'; import { AgentRelay, type AgentRelayAgent } from '@agent-relay/sdk'; -import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; import { - resolveWorkspaceSelection as resolveCloudWorkspaceSelection, + resolveBaseUrl, + resolveWorkspaceSelection, + resolveWorkspaceTransport, +} from '@agent-relay/cloud/workspace-transport'; +export { + resolveBaseUrl, + resolveWorkspaceSelection, + resolveWorkspaceTransport, + type WorkspaceTransport, +} from '@agent-relay/cloud/workspace-transport'; +import { writeProjectWorkspaceTargetIfSelectionCurrent, type WorkspaceSelection, type WorkspaceKeySource, @@ -34,23 +43,6 @@ function trimOrUndefined(value: string | undefined): string | undefined { export type { WorkspaceKeySource }; export type { WorkspaceSelection }; -export type WorkspaceTransport = { - workspaceKey: string; - baseUrl?: string; - source: WorkspaceKeySource; -}; - -/** Resolve the selected key and any previously persisted Relay workspace identity. */ -export function resolveWorkspaceSelection(options: SdkClientOptions = {}): WorkspaceSelection | undefined { - const explicitProject = trimOrUndefined(env(options).AGENT_RELAY_PROJECT); - const projectRoot = explicitProject ? path.resolve(explicitProject) : options.projectRoot; - return resolveCloudWorkspaceSelection({ - workspaceKey: options.workspaceKey, - env: env(options), - ...(projectRoot ? { projectRoot } : {}), - }); -} - /** * Resolve the workspace key and report which source it came from. Precedence: * explicit flag → `RELAY_WORKSPACE_KEY`/`RELAY_API_KEY` env → the key the local @@ -70,120 +62,6 @@ export function resolveWorkspaceKey(options: SdkClientOptions = {}): string { return resolveWorkspaceKeyWithSource(options).key; } -export function resolveBaseUrl(options: SdkClientOptions = {}): string | undefined { - const selection = selectionForTransport(options); - return resolveBaseUrlForSelection(selection, options); -} - -function selectionForTransport(options: SdkClientOptions): WorkspaceSelection | undefined { - const selection = resolveWorkspaceSelection(options); - if (!selection || !options.ignorePersistedRelaycastTarget) return selection; - const { - relaycastRoute: _relaycastRoute, - relaycastBaseUrl: _relaycastBaseUrl, - relaycastApiKey: _relaycastApiKey, - relaycastApiKeyRef: _relaycastApiKeyRef, - ...canonicalSelection - } = selection; - return canonicalSelection; -} - -function resolveBaseUrlForSelection( - selection: WorkspaceSelection | undefined, - options: SdkClientOptions -): string | undefined { - const persisted = validatePersistedRelaycastBaseUrl(selection); - const requested = trimOrUndefined(options.baseUrl) ?? trimOrUndefined(env(options).RELAY_BASE_URL); - if (persisted && requested) { - let parsed: URL; - try { - parsed = new URL(requested); - } catch { - throw new Error('The requested Relaycast base URL is invalid.'); - } - const authority = /^https:\/\/([^/?#]+)/i.exec(requested)?.[1] ?? ''; - if ( - !/^https:\/\/[^/?#]+\/?$/i.test(requested) || - parsed.protocol !== 'https:' || - parsed.username || - parsed.password || - parsed.port || - /:\d+$/.test(authority) || - parsed.search || - parsed.hash || - (parsed.pathname !== '' && parsed.pathname !== '/') - ) { - throw new Error('The requested Relaycast base URL is not a trusted origin.'); - } - if (parsed.origin !== persisted) { - throw new Error('The requested Relaycast base URL does not match the persisted workspace route.'); - } - } - return persisted ?? requested; -} - -/** Resolve one credential/origin pair from one workspace selection. */ -export function resolveWorkspaceTransport(options: SdkClientOptions = {}): WorkspaceTransport { - const selection = selectionForTransport(options); - if (!selection) { - throw new Error( - 'No workspace key found. Pass --workspace-key, set RELAY_WORKSPACE_KEY, or run `relay workspace set_key `.' - ); - } - const baseUrl = resolveBaseUrlForSelection(selection, options); - // Project-session loading already validates the reference against the - // project/workspace/route/base tuple. Never re-read a ref here: doing so - // would let a tampered ref bypass that binding and pair an unrelated key - // with this route. - const routeCredential = trimOrUndefined(selection.relaycastApiKey); - if (selection.relaycastRoute === 'agent37-isolated' && !routeCredential) { - throw new Error( - 'The persisted isolated Relaycast credential is unavailable or mismatched; rerun the sandbox command to mint a fresh route.' - ); - } - return { - workspaceKey: routeCredential ?? selection.key, - ...(baseUrl ? { baseUrl } : {}), - source: selection.source, - }; -} - -function validatePersistedRelaycastBaseUrl(selection: WorkspaceSelection | undefined): string | undefined { - const baseUrl = trimOrUndefined(selection?.relaycastBaseUrl); - const route = selection?.relaycastRoute; - const relaycastApiKey = trimOrUndefined(selection?.relaycastApiKey); - if (!baseUrl && !route && !relaycastApiKey) return undefined; - if (!baseUrl || !route) { - throw new Error('The persisted Relaycast workspace route is incomplete.'); - } - let parsed: URL; - try { - parsed = new URL(baseUrl); - } catch { - throw new Error('The persisted Relaycast workspace route is invalid.'); - } - const expectedOrigin = - route === 'canonical' - ? CANONICAL_RELAYCAST_ORIGIN - : route === 'agent37-isolated' - ? AGENT37_RELAYCAST_ORIGIN - : undefined; - if ( - !expectedOrigin || - parsed.origin !== expectedOrigin || - parsed.protocol !== 'https:' || - parsed.username || - parsed.password || - parsed.port || - parsed.search || - parsed.hash || - (parsed.pathname !== '' && parsed.pathname !== '/') - ) { - throw new Error('The persisted Relaycast workspace route is not trusted.'); - } - return parsed.origin; -} - /** Persist a server-selected target only while the captured project selection is still current. */ export function persistWorkspaceRelaycastTarget( selection: WorkspaceSelection | undefined, diff --git a/packages/cloud/package.json b/packages/cloud/package.json index 54bafde6bc..8832733be2 100644 --- a/packages/cloud/package.json +++ b/packages/cloud/package.json @@ -4,7 +4,7 @@ "engines": { "node": ">=22.0.0" }, - "description": "Cloud SDK for Agent Relay — auth, workflow execution, and provider connections", + "description": "Cloud SDK for Agent Relay \u2014 auth, workflow execution, and provider connections", "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -49,7 +49,19 @@ "types": "./dist/workspace-key.d.ts", "import": "./dist/workspace-key.js" }, - "./package.json": "./package.json" + "./package.json": "./package.json", + "./attach": { + "types": "./dist/attach.d.ts", + "import": "./dist/attach.js" + }, + "./workspace-transport": { + "types": "./dist/workspace-transport.d.ts", + "import": "./dist/workspace-transport.js" + }, + "./fleet": { + "types": "./dist/fleet.d.ts", + "import": "./dist/fleet.js" + } }, "files": [ "dist", @@ -65,7 +77,8 @@ "@agent-relay/config": "12.2.4", "@aws-sdk/client-s3": "3.1020.0", "ignore": "^7.0.5", - "tar": "^7.5.21" + "tar": "^7.5.21", + "ws": "^8.18.3" }, "optionalDependencies": { "ssh2": "^1.17.0" @@ -73,7 +86,8 @@ "devDependencies": { "@types/node": "^22.19.3", "@types/ssh2": "^1.15.5", - "vitest": "^4.1.0" + "vitest": "^4.1.0", + "@types/ws": "^8.18.1" }, "publishConfig": { "access": "public" diff --git a/packages/cloud/src/attach-mode.ts b/packages/cloud/src/attach-mode.ts new file mode 100644 index 0000000000..b1107eecec --- /dev/null +++ b/packages/cloud/src/attach-mode.ts @@ -0,0 +1 @@ +export type AttachMode = 'drive' | 'view' | 'passthrough'; diff --git a/packages/cloud/src/attach.test.ts b/packages/cloud/src/attach.test.ts new file mode 100644 index 0000000000..53c58bb4bf --- /dev/null +++ b/packages/cloud/src/attach.test.ts @@ -0,0 +1,110 @@ +import { once } from 'node:events'; +import { connect, type AddressInfo, type Socket } from 'node:net'; +import { stat } from 'node:fs/promises'; +import { WebSocketServer, type WebSocket } from 'ws'; +import { afterEach, expect, it, vi } from 'vitest'; +import { startFleetNodeAttachProxy, type FleetNodeAttachProxy } from './attach.js'; + +let proxy: FleetNodeAttachProxy | undefined; +let server: WebSocketServer | undefined; +let local: Socket | undefined; +afterEach(async () => { + local?.destroy(); + await proxy?.close(); + proxy = undefined; + if (server) { + for (const client of server.clients) client.terminate(); + await new Promise((resolve) => server!.close(() => resolve())); + } + server = undefined; +}); + +async function setup(mode: 'drive' | 'view' = 'drive') { + server = new WebSocketServer({ host: '127.0.0.1', port: 0 }); + await once(server, 'listening'); + const remoteConnection = once(server, 'connection'); + const fetch = vi.fn(async (url: string | URL | Request, _init?: RequestInit) => + Response.json({ + ok: true, + data: String(url).endsWith('/agents') + ? [{ agentName: 'worker' }] + : { + session_id: 'session', + resume_token: 'resume', + terminal_url: `ws://127.0.0.1:${(server!.address() as AddressInfo).port}/terminal`, + }, + }) + ); + proxy = await startFleetNodeAttachProxy({ nodeId: 'node', mode, workspaceKey: 'rk_test', env: {}, fetch }); + const [remote] = (await remoteConnection) as [WebSocket]; + return { remote, fetch }; +} +function send(remote: WebSocket, type: string, data = {}) { + remote.send(JSON.stringify({ session_id: 'session', type, ...data })); +} + +it('discovers one agent and pipes raw bytes through a private socket, then removes it', async () => { + const { remote, fetch } = await setup(); + expect(JSON.parse(String(fetch.mock.calls[1]?.[1]?.body)).agent).toBe('worker'); + expect((await stat(proxy!.socketPath)).mode & 0o777).toBe(0o600); + local = connect(proxy!.socketPath); + await once(local, 'connect'); + send(remote, 'terminal.ready', { screen: 'hello' }); + expect((await once(local, 'data'))[0].toString()).toBe('hello'); + const frame = once(remote, 'message'); + const bytes = Buffer.from([0x00, 0x1b, 0xff, 0xc3, 0xa9]); + local.write(bytes); + expect(JSON.parse((await frame)[0].toString())).toMatchObject({ + type: 'terminal.input', + data_base64: bytes.toString('base64'), + }); + const output = once(local, 'data'); + send(remote, 'terminal.output', { chunk: 'world' }); + expect((await output)[0].toString()).toBe('world'); + const socketPath = proxy!.socketPath; + send(remote, 'terminal.closed'); + await expect(proxy!.finished).resolves.toBe(0); + await Promise.all([proxy!.close(), proxy!.close()]); + await expect(stat(socketPath)).rejects.toMatchObject({ code: 'ENOENT' }); +}); + +it('settles finished on explicit close before terminal.ready', async () => { + await setup(); + await proxy!.close(); + await expect(proxy!.finished).resolves.toBe(0); +}); + +it('resolves failure without an unhandled rejection', async () => { + const { remote } = await setup(); + send(remote, 'terminal.error', { code: 'node_unreachable', message: 'offline' }); + await expect(proxy!.finished).resolves.toBe(1); +}); + +it('view sockets cannot send terminal input and reject additional clients', async () => { + const { remote } = await setup('view'); + local = connect(proxy!.socketPath); + await once(local, 'connect'); + send(remote, 'terminal.ready', { screen: 'ready' }); + await once(local, 'data'); + const received = vi.fn(); + remote.on('message', received); + local.write('ignored'); + const extra = connect(proxy!.socketPath); + await once(extra, 'close'); + expect(received).not.toHaveBeenCalled(); +}); + +it.each([[], [{ agentName: 'a' }, { agentName: 'b' }]])( + 'rejects missing or ambiguous agents', + async (...agents) => { + await expect( + startFleetNodeAttachProxy({ + nodeId: 'node', + mode: 'drive', + workspaceKey: 'rk_test', + env: {}, + fetch: vi.fn(async () => Response.json({ data: agents.length === 0 ? [] : agents })), + }) + ).rejects.toMatchObject({ code: 'ambiguous_agent' }); + } +); diff --git a/packages/cloud/src/attach.ts b/packages/cloud/src/attach.ts new file mode 100644 index 0000000000..d0d1161996 --- /dev/null +++ b/packages/cloud/src/attach.ts @@ -0,0 +1,1362 @@ +/** + * Ticketed fleet-node attach adapter. + * + * The established attach clients intentionally continue to speak the local + * broker HTTP/WebSocket contract. This short-lived loopback adapter maps that + * contract onto Relaycast's authenticated terminal session, so view/drive and + * passthrough retain their behaviour without exposing a remote broker listener + * or copying a broker API key off a physical or Daytona node. + */ + +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { once } from 'node:events'; +import { Buffer } from 'node:buffer'; +import { randomBytes } from 'node:crypto'; +import { createServer as createSocketServer, type Socket } from 'node:net'; +import { mkdtemp, chmod, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import WebSocket, { WebSocketServer } from 'ws'; + +import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from './fleet-sandbox.js'; +import type { AttachMode } from './attach-mode.js'; +import { collectWithRetry } from './collect-with-retry.js'; +import { resolveWorkspaceTransport } from './workspace-transport.js'; + +const MAX_BUFFERED_BYTES = 1024 * 1024; +const MAX_WEBSOCKET_CLOSE_REASON_BYTES = 123; +// Use the same finite 30s request window as the broker's Relaycast HTTP calls. +// Terminal-session creation can exceed the shorter startup-handshake latency +// under load, and this POST cannot be replayed safely after an ambiguous client +// timeout because allocation may have completed server-side. +const SESSION_REQUEST_TIMEOUT_MS = 30_000; +// Immediate structured reachability failures still receive the complete +// five-attempt/31.2s retry schedule, while slow responses cannot multiply the +// per-attempt timeout into a roughly three-minute CLI hang. +const SESSION_REQUEST_TOTAL_TIMEOUT_MS = 90_000; +const SESSION_REQUEST_RETRIES = 4; +// The fleet node publishes liveness every 12s. With four retries, the helper's +// deterministic delays are 6s, 7.2s, 8.4s, and 9.6s: 31.2s total. That spans +// more than two heartbeat intervals and matches the established terminal +// transport's bounded recovery window instead of exhausting every retry inside +// the same stale control-plane read. +const SESSION_REQUEST_RETRY_DELAY_MS = 6_000; +const TERMINAL_CONNECT_TIMEOUT_MS = 10_000; +const INITIAL_RECONNECT_DELAY_MS = 500; +const MAX_RECONNECT_DELAY_MS = 30_000; +// Six delays (0.5s + 1s + 2s + 4s + 8s + 16s) cover the node terminal +// transport's independent 30s maximum reconnect backoff without leaving this +// client unbounded. Five attempts previously stopped after only 15.5s. +const MAX_RECONNECT_ATTEMPTS = 6; +// Bounds the acknowledgement for a delivery-mode command after readiness. +// The command is not replayable across reconnects, so this is intentionally a +// separate post-readiness phase rather than another full reconnect window. +const DELIVERY_MODE_TIMEOUT_MS = 10_000; +// The caller starts its HTTP deadline before the loopback handler starts its +// readiness timer. Leave a small response-delivery margin so the proxy's +// actionable readiness error wins that race. +const LOOPBACK_REQUEST_TIMEOUT_MARGIN_MS = 1_000; + +type FleetSessionResponse = { + ok?: boolean; + data?: { + session_id?: string; + terminal_url?: string; + resume_token?: string; + }; + error?: { code?: string; message?: string }; +}; + +type TerminalFrame = Record & { type?: string; session_id?: string }; + +type TerminalReadiness = { + generation: number; + settled: boolean; + promise: Promise; + resolve: () => void; + reject: (error: Error) => void; +}; + +export interface FleetNodeAttachOptions { + /** Defaults to the only agent on the node; ambiguous nodes require an explicit agent. */ + agent?: string; + node?: string; + nodeId?: string; + mode: AttachMode; + env?: NodeJS.ProcessEnv; + baseUrl?: string; + workspaceKey?: string; + fetch?: typeof globalThis.fetch; + /** Deterministic test seam for the bounded session-request retry delay. */ + sessionRequest?: { + timeoutMs?: number; + totalTimeoutMs?: number; + sleep?: (ms: number) => Promise; + }; + /** Deterministic test seam for established-session reconnect timing. */ + reconnectDelay?: { + initialMs?: number; + maxMs?: number; + handshakeTimeoutMs?: number; + readyTimeoutMs?: number; + beforeReadyTimeoutTerminate?: (socket: WebSocket) => void; + }; +} + +/** Internal compatibility transport for broker-based CLI attach clients. */ +export interface FleetNodeAttachTransport { + brokerUrl: string; + apiKey: string; + requestTimeoutMs: number; +} + +export interface FleetNodeAttachProxy extends FleetNodeAttachTransport { + /** Private UNIX byte-stream socket (named pipe on Windows). */ + socketPath: string; + /** Inferred status: 0 on terminal.closed or detach, 1 on transport failure. + * Relaycast does not currently transmit the remote process exit code. */ + finished: Promise; + close(): Promise; +} + +export class FleetNodeAttachError extends Error { + constructor( + message: string, + readonly code?: string + ) { + super(message); + this.name = 'FleetNodeAttachError'; + } +} + +const TRUSTED_RELAYCAST_ORIGINS = new Set([CANONICAL_RELAYCAST_ORIGIN, AGENT37_RELAYCAST_ORIGIN]); + +export function validateFleetAttachBaseUrl(value: string): string { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new FleetNodeAttachError('Fleet node attach requires a trusted Relaycast origin.'); + } + if ( + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') || + !TRUSTED_RELAYCAST_ORIGINS.has(parsed.origin) + ) { + throw new FleetNodeAttachError('Fleet node attach requires a trusted Relaycast origin.'); + } + return parsed.origin; +} + +class TerminalSessionAttemptError extends FleetNodeAttachError { + constructor( + message: string, + code: string | undefined, + readonly status: number | undefined, + readonly retryable: boolean, + readonly completionUnknown: boolean + ) { + super(message, code); + } +} + +/** + * Canonical HTTP status for a terminal failure code. + * + * `agent_not_found` must stay 404: {@link switchInboundDeliveryModeOrAbort} + * only emits the "no agent named X" message and the cross-node placement hint + * on a 404, so collapsing it into 503 replaces actionable guidance with an + * opaque unreachable-node error. `unsupported_runtime` stays 409. Everything + * else is a transport-level failure and reports 503. + */ +function terminalErrorStatus(code: string | undefined): number { + if (code === 'agent_not_found') return 404; + if (code === 'unsupported_runtime') return 409; + return 503; +} + +function json(response: ServerResponse, status: number, payload: unknown): void { + response.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + response.end(JSON.stringify(payload)); +} + +function readBody(request: IncomingMessage): Promise> { + return new Promise((resolve) => { + let settled = false; + const finish = (body: Record) => { + if (settled) return; + settled = true; + resolve(body); + }; + let body = ''; + request.setEncoding('utf8'); + request.on('data', (chunk: string) => { + body += chunk; + }); + request.on('end', () => { + try { + const parsed = JSON.parse(body) as unknown; + finish( + parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : {} + ); + } catch { + finish({}); + } + }); + request.on('error', () => finish({})); + request.on('aborted', () => finish({})); + }); +} + +function asWsUrl(value: string): string { + const lower = value.toLowerCase(); + if (lower.startsWith('https://')) return 'wss://' + value.slice(8); + if (lower.startsWith('http://')) return 'ws://' + value.slice(7); + return value; +} + +function safeNodePath(node: string): string { + const trimmed = node.trim().replace(/^#/, ''); + if (!trimmed) throw new FleetNodeAttachError('Error: --node requires a node name or id.', 'invalid_node'); + return encodeURIComponent(trimmed); +} + +function parseFrame(data: WebSocket.RawData): TerminalFrame | null { + try { + const parsed = JSON.parse(rawDataToString(data)) as unknown; + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as TerminalFrame) : null; + } catch { + return null; + } +} + +function rawDataToString(data: WebSocket.RawData): string { + if (Buffer.isBuffer(data)) return data.toString('utf8'); + if (data instanceof ArrayBuffer) return Buffer.from(data).toString('utf8'); + if (Array.isArray(data)) return Buffer.concat(data).toString('utf8'); + return String(data); +} + +function diagnosticEndpoint(value: string): string { + try { + const url = new URL(value); + url.username = ''; + url.password = ''; + url.search = ''; + url.hash = ''; + return url.toString(); + } catch { + return '(invalid endpoint)'; + } +} + +function diagnosticEndpointOrigin(value: string): string { + try { + return new URL(value).origin; + } catch { + return '(invalid endpoint)'; + } +} + +function boundedWebSocketCloseReason(reason: string): string { + const encoded = Buffer.from(reason, 'utf8'); + if (encoded.length <= MAX_WEBSOCKET_CLOSE_REASON_BYTES) return reason; + + const suffix = Buffer.from('…', 'utf8'); + let end = MAX_WEBSOCKET_CLOSE_REASON_BYTES - suffix.length; + // Do not cut through a UTF-8 continuation sequence. Excluding the leading + // byte at this boundary also excludes the incomplete code point. + while (end > 0 && (encoded[end] & 0xc0) === 0x80) end -= 1; + return Buffer.concat([encoded.subarray(0, end), suffix]).toString('utf8'); +} + +function diagnosticValue(value: string): string { + // JSON quoting keeps node names and upstream text from injecting terminal + // control characters into the operator-facing error. + return JSON.stringify(value); +} + +function resolvedNodeIdFromTerminalUrl(value: string): string | undefined { + try { + const match = /^\/v1\/nodes\/([^/]+)\/terminal\/connect$/.exec(new URL(value).pathname); + return match?.[1] ? decodeURIComponent(match[1]) : undefined; + } catch { + return undefined; + } +} + +function retryDelayBudgetMs(attempts: number, initialMs: number, maxMs: number): number { + let total = 0; + for (let attempt = 0; attempt < attempts; attempt += 1) { + total += Math.min(initialMs * 2 ** attempt, maxMs); + } + return total; +} + +function terminalSessionFailureSummary(error: TerminalSessionAttemptError): string { + if (error.code === 'node_not_found') return 'Control-plane node lookup found no matching record'; + if (error.code === 'node_unreachable') { + return /no terminal transport/i.test(error.message) + ? 'The node record was found, but its terminal transport was unavailable' + : 'The control plane classified the node as unreachable'; + } + if (error.code === 'terminal_session_unavailable') { + return 'The control plane could not allocate a terminal session'; + } + if (error.code === 'control_plane_timeout') return 'Control-plane terminal-session lookup timed out'; + if (error.code === 'control_plane_unavailable') return 'Control-plane terminal-session lookup failed'; + return 'The terminal-session request was rejected'; +} + +function isRetryableTerminalSessionFailure(code: string | undefined): boolean { + // These structured responses are emitted before a session is returned, so + // retrying cannot duplicate a successful allocation. A fetch timeout, + // network failure, or unclassified 5xx is different: this POST may already + // have completed server-side, and retrying it could create a second session. + return code === 'node_unreachable' || code === 'terminal_session_unavailable'; +} + +/** Start a broker-compatible loopback proxy for one remote terminal session. */ +export async function startFleetNodeAttachProxy( + input: FleetNodeAttachOptions +): Promise { + const options = { ...input, node: input.node ?? input.nodeId ?? '', agent: input.agent ?? '' }; + if (input.node && input.nodeId && input.node !== input.nodeId) { + throw new FleetNodeAttachError('node and nodeId must identify the same node.', 'invalid_node'); + } + const env = options.env ?? process.env; + const fetchFn = options.fetch ?? globalThis.fetch; + const { workspaceKey, baseUrl: requestedBaseUrl } = resolveWorkspaceTransport({ + workspaceKey: options.workspaceKey, + baseUrl: options.baseUrl, + env, + }); + const baseUrl = validateFleetAttachBaseUrl(requestedBaseUrl ?? CANONICAL_RELAYCAST_ORIGIN); + const nodePath = safeNodePath(options.node); + if (!options.agent) { + const response = await fetchFn(`${baseUrl}/v1/nodes/${nodePath}/agents`, { + headers: { Authorization: `Bearer ${workspaceKey}` }, + signal: AbortSignal.timeout(SESSION_REQUEST_TIMEOUT_MS), + }); + const payload = (await response.json()) as { data?: Array<{ agentName?: string }> }; + if (!response.ok || !Array.isArray(payload.data)) { + throw new FleetNodeAttachError('Could not discover the fleet node agent.', 'agent_discovery_failed'); + } + const names = [ + ...new Set( + payload.data + .map((agent) => agent.agentName) + .filter((name): name is string => typeof name === 'string' && !!name.trim()) + ), + ]; + if (names.length !== 1) { + throw new FleetNodeAttachError( + 'Specify agent explicitly unless the node has exactly one agent.', + 'ambiguous_agent' + ); + } + options.agent = names[0]; + } + const sessionEndpoint = `${baseUrl}/v1/nodes/${nodePath}/terminal/sessions`; + const sessionRequestTimeoutMs = options.sessionRequest?.timeoutMs ?? SESSION_REQUEST_TIMEOUT_MS; + const sessionRequestTotalTimeoutMs = + options.sessionRequest?.totalTimeoutMs ?? SESSION_REQUEST_TOTAL_TIMEOUT_MS; + const sessionRequestDeadline = Date.now() + sessionRequestTotalTimeoutMs; + const sessionRequestSleep = + options.sessionRequest?.sleep ?? + ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + let sessionRequestAttempts = 0; + let sessionRequestBudgetExhaustedBetweenAttempts = false; + let lastSessionError: TerminalSessionAttemptError | undefined; + const sessionResult = await collectWithRetry( + 'terminal session request', + async () => { + const remainingRequestBudgetMs = sessionRequestDeadline - Date.now(); + if (remainingRequestBudgetMs <= 0) { + sessionRequestBudgetExhaustedBetweenAttempts = true; + const exhausted = + lastSessionError === undefined + ? new TerminalSessionAttemptError( + 'overall terminal-session request deadline exhausted', + 'control_plane_timeout', + undefined, + false, + false + ) + : new TerminalSessionAttemptError( + lastSessionError.message, + lastSessionError.code, + lastSessionError.status, + false, + lastSessionError.completionUnknown + ); + lastSessionError = exhausted; + throw exhausted; + } + sessionRequestAttempts += 1; + const controller = new AbortController(); + let timedOut = false; + const attemptTimeoutMs = Math.min(sessionRequestTimeoutMs, remainingRequestBudgetMs); + const timeout = setTimeout(() => { + timedOut = true; + controller.abort(); + }, attemptTimeoutMs); + let ticketResponse: Response; + let ticketPayload: FleetSessionResponse; + try { + ticketResponse = await fetchFn(sessionEndpoint, { + method: 'POST', + headers: { Authorization: `Bearer ${workspaceKey}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ agent: options.agent, mode: options.mode }), + signal: controller.signal, + }); + const parsedPayload = (await ticketResponse.json()) as unknown; + ticketPayload = + parsedPayload && typeof parsedPayload === 'object' && !Array.isArray(parsedPayload) + ? (parsedPayload as FleetSessionResponse) + : {}; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + lastSessionError = new TerminalSessionAttemptError( + timedOut ? 'request exceeded its deadline' : detail, + timedOut ? 'control_plane_timeout' : 'control_plane_unavailable', + undefined, + false, + true + ); + throw lastSessionError; + } finally { + clearTimeout(timeout); + } + + const terminalUrl = ticketPayload.data?.terminal_url; + const sessionId = ticketPayload.data?.session_id; + const resumeToken = ticketPayload.data?.resume_token; + if (!ticketResponse.ok || !terminalUrl || !sessionId || !resumeToken) { + const code = ticketPayload.error?.code; + const message = + ticketPayload.error?.message ?? `terminal session request failed (HTTP ${ticketResponse.status})`; + lastSessionError = new TerminalSessionAttemptError( + message, + code, + ticketResponse.status, + isRetryableTerminalSessionFailure(code), + false + ); + throw lastSessionError; + } + return { terminalUrl, sessionId, resumeToken }; + }, + { + retries: SESSION_REQUEST_RETRIES, + baseDelayMs: SESSION_REQUEST_RETRY_DELAY_MS, + sleep: async (delayMs) => { + const remainingRequestBudgetMs = sessionRequestDeadline - Date.now(); + if (remainingRequestBudgetMs <= 0) return; + await sessionRequestSleep(Math.min(delayMs, remainingRequestBudgetMs)); + }, + shouldRetry: (error) => error instanceof TerminalSessionAttemptError && error.retryable, + } + ); + if (!sessionResult.ok) { + const failure = lastSessionError; + const status = failure?.status === undefined ? '' : ` HTTP ${failure.status};`; + const code = failure?.code === undefined ? '' : ` code ${failure.code};`; + const retryNote = sessionRequestBudgetExhaustedBetweenAttempts + ? sessionRequestAttempts > 1 + ? `retried ${sessionRequestAttempts - 1} time${sessionRequestAttempts === 2 ? '' : 's'};` + + ' overall budget exhausted before the next attempt' + : 'not retried because the overall budget was exhausted before the next attempt' + : sessionRequestAttempts > 1 + ? `retried ${sessionRequestAttempts - 1} time${sessionRequestAttempts === 2 ? '' : 's'}` + + (failure?.completionUnknown + ? '; final POST not retried because it may have completed server-side' + : '') + : failure?.completionUnknown + ? 'not retried because the POST may have completed server-side' + : 'not retried because the failure was terminal'; + const upstream = failure?.message ? ` Upstream message ${diagnosticValue(failure.message)}.` : ''; + throw new FleetNodeAttachError( + `Error: ${failure ? terminalSessionFailureSummary(failure) : 'Terminal-session request failed'}.` + + `${upstream} Node ref ${diagnosticValue(options.node.trim())}, resolved node id unavailable (session creation did not complete);` + + ` endpoint ${diagnosticValue(diagnosticEndpoint(sessionEndpoint))};${status}${code}` + + ` timeout ${sessionRequestTimeoutMs}ms per attempt; overall budget ${sessionRequestTotalTimeoutMs}ms;` + + ` attempts ${sessionRequestAttempts} (${retryNote}).`, + failure?.code + ); + } + const { terminalUrl, sessionId, resumeToken } = sessionResult.value; + const resolvedNodeId = resolvedNodeIdFromTerminalUrl(terminalUrl); + const remoteEndpoint = diagnosticEndpoint(terminalUrl); + const reconnectInitialDelayMs = options.reconnectDelay?.initialMs ?? INITIAL_RECONNECT_DELAY_MS; + const reconnectMaxDelayMs = options.reconnectDelay?.maxMs ?? MAX_RECONNECT_DELAY_MS; + const terminalHandshakeTimeoutMs = + options.reconnectDelay?.handshakeTimeoutMs ?? TERMINAL_CONNECT_TIMEOUT_MS; + const terminalReadyTimeoutMs = options.reconnectDelay?.readyTimeoutMs ?? TERMINAL_CONNECT_TIMEOUT_MS; + // A readiness-gated local request follows activeReadiness across reconnect + // generations. Its own deadline therefore has to cover the same complete, + // finite recovery path: every backoff plus every handshake/readiness pair. + const terminalWaitTimeoutMs = + retryDelayBudgetMs(MAX_RECONNECT_ATTEMPTS, reconnectInitialDelayMs, reconnectMaxDelayMs) + + MAX_RECONNECT_ATTEMPTS * (terminalHandshakeTimeoutMs + terminalReadyTimeoutMs); + + let connectionGeneration = 0; + const createReadiness = (): TerminalReadiness => { + let resolve!: () => void; + let reject!: (error: Error) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + // A failure can land before a snapshot request attaches its waiter. Keep + // the rejection observable while avoiding an unhandled-rejection warning. + void promise.catch(() => undefined); + return { generation: ++connectionGeneration, settled: false, promise, resolve, reject }; + }; + let activeReadiness = createReadiness(); + const resolveReadiness = (readiness: TerminalReadiness) => { + if (readiness.settled) return; + readiness.settled = true; + readiness.resolve(); + }; + const rejectReadiness = (readiness: TerminalReadiness, error: Error) => { + if (readiness.settled) return; + readiness.settled = true; + readiness.reject(error); + }; + const waitForCurrentReadiness = async (): Promise => { + for (;;) { + const readiness = activeReadiness; + await readiness.promise; + if (readiness === activeReadiness) return; + } + }; + /** + * Await the live readiness generation through both the WebSocket handshake + * and the post-open terminal.ready allowance. + * Every handler that must not run before `terminal.ready` — snapshot, + * delivery-mode PUT, resize — goes through this one helper so the + * timer/clearTimeout/settle logic cannot diverge between copies. + * + * Rejects with the underlying {@link FleetNodeAttachError} when the terminal + * failed (preserving its `code`), or a plain `Error` carrying + * `timeoutMessage` when the wait expired. + */ + const waitForTerminalReady = (timeoutMessage: string): Promise => + new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(timeoutMessage)), terminalWaitTimeoutMs); + void waitForCurrentReadiness().then( + () => { + clearTimeout(timer); + resolve(); + }, + (error: Error) => { + clearTimeout(timer); + reject(error); + } + ); + }); + const snapshot: { screen: string; rows: number; cols: number; offset: number } = { + screen: '', + rows: 24, + cols: 80, + offset: 0, + }; + const eventSockets = new Set(); + const inputSockets = new Set(); + const outputHistory: Array<{ chunk: string; offset?: number }> = []; + let outputHistoryBytes = 0; + let remote: WebSocket | undefined; + let rawSocket: Socket | undefined; + let rawConnected = false; + let rawReady = false; + let finish!: (code: number) => void; + const finished = new Promise((resolve) => { + finish = resolve; + }); + let closePromise: Promise | undefined; + let stopped = false; + let terminalEnded = false; + let terminalEverReady = false; + let reconnecting = false; + let reconnectAttempts = 0; + let reconnectTimer: ReturnType | undefined; + const terminalReadinessTimers = new Set>(); + /** Locally-tracked delivery mode, kept in sync with each broker reply. */ + let loopbackDeliveryMode: 'manual_flush' | 'auto_inject' = + options.mode === 'drive' ? 'manual_flush' : 'auto_inject'; + type DeliveryModeResult = { mode: string; flushed: number; matched: boolean; revision: string }; + type FlushResult = { + flushed: number; + dead_lettered: number; + held: number; + blocked_reason: string | null; + }; + let pendingFlush: { + requestId: string; + resolve: (result: FlushResult) => void; + reject: (error: Error) => void; + timer: ReturnType; + } | null = null; + /** At most one in-flight delivery-mode PUT at a time. */ + let pendingDeliveryMode: { + requestId: string; + resolve: (result: DeliveryModeResult) => void; + reject: (err: Error) => void; + timer: ReturnType; + } | null = null; + const loopbackApiKey = randomBytes(32).toString('base64url'); + const loopbackAuthorized = (headers: IncomingMessage['headers']) => + headers.authorization === `Bearer ${loopbackApiKey}` || headers['x-api-key'] === loopbackApiKey; + + const server = createServer(async (request, response) => { + if (!loopbackAuthorized(request.headers)) { + json(response, 401, { + error: { code: 'unauthorized', message: 'loopback terminal token is required' }, + }); + return; + } + const path = new URL(request.url ?? '/', 'http://127.0.0.1').pathname; + if (request.method === 'GET' && path === `/api/spawned/${encodeURIComponent(options.agent)}/snapshot`) { + try { + await waitForTerminalReady('terminal snapshot timed out'); + } catch (error) { + const terminalError = error instanceof FleetNodeAttachError ? error : undefined; + json(response, terminalErrorStatus(terminalError?.code), { + error: { + code: terminalError?.code ?? 'snapshot_unavailable', + message: + terminalError?.message ?? (error instanceof Error ? error.message : 'snapshot unavailable'), + }, + }); + return; + } + json(response, 200, { format: 'ansi', ...snapshot }); + return; + } + const name = encodeURIComponent(options.agent); + if (path === `/api/spawned/${name}/flush` && request.method === 'POST') { + // Forward the flush to the remote broker over the terminal websocket. + // Without this the `--node` form of `node agent message flush` reached + // only the LOCAL broker's worker registry and returned agent_not_found + // for a name that `node agent list` and `attach` both resolve. + try { + await waitForTerminalReady('terminal connection timed out'); + } catch (error) { + const terminalError = error instanceof FleetNodeAttachError ? error : undefined; + json(response, terminalErrorStatus(terminalError?.code), { + error: { + code: terminalError?.code ?? 'node_unreachable', + message: + terminalError?.message ?? (error instanceof Error ? error.message : 'terminal unavailable'), + }, + }); + return; + } + if (!remote || remote.readyState !== WebSocket.OPEN) { + json(response, 503, { + error: { code: 'node_unreachable', message: 'terminal transport is not connected' }, + }); + return; + } + if (pendingFlush) { + json(response, 503, { + error: { code: 'flush_conflict', message: 'a flush request is already in flight' }, + }); + return; + } + const requestId = randomBytes(8).toString('hex'); + const result = await new Promise((resolve) => { + const timer = setTimeout(() => { + pendingFlush = null; + resolve(new FleetNodeAttachError('flush request timed out', 'flush_timeout')); + }, DELIVERY_MODE_TIMEOUT_MS); + pendingFlush = { + requestId, + resolve: (r) => resolve(r), + reject: (e) => resolve(e), + timer, + }; + remote!.send( + JSON.stringify({ + type: 'terminal.flush_pending', + session_id: sessionId, + request_id: requestId, + }) + ); + }); + if (result instanceof Error) { + const errCode = + result instanceof FleetNodeAttachError ? (result.code ?? 'flush_failed') : 'flush_failed'; + json(response, terminalErrorStatus(errCode), { + error: { code: errCode, message: result.message }, + }); + return; + } + json(response, 200, result); + return; + } + if (path === `/api/spawned/${name}/delivery-mode`) { + if (request.method === 'GET') { + json(response, 200, { mode: loopbackDeliveryMode }); + return; + } + // PUT — forward the request to the remote broker via the terminal WS and + // await the broker's real reply. This is the path that was previously a + // static stub returning manual_flush, causing drive attach to fail with + // "broker remained in manual_flush mode". + const body = await readBody(request); + const requestedMode = + body.mode === 'auto_inject' ? 'auto_inject' : body.mode === 'manual_flush' ? 'manual_flush' : null; + if (requestedMode === null) { + json(response, 400, { + error: { code: 'invalid_mode', message: `unsupported delivery mode '${String(body.mode)}'` }, + }); + return; + } + // Drive attach changes delivery mode before it requests the initial + // snapshot. Gate the PUT on terminal.ready so a fast local caller does + // not lose a race with the remote websocket handshake and receive the + // misleading "terminal transport is not connected" failure. + try { + await waitForTerminalReady('terminal connection timed out'); + } catch (error) { + // Preserve the canonical status mapping here too: a readiness failure + // carrying `agent_not_found` has to reach the preflight as a 404 or + // the operator loses the cross-node placement hint that tells them + // which machine to run the attach on. + const terminalError = error instanceof FleetNodeAttachError ? error : undefined; + json(response, terminalErrorStatus(terminalError?.code), { + error: { + code: terminalError?.code ?? 'node_unreachable', + message: + terminalError?.message ?? (error instanceof Error ? error.message : 'terminal unavailable'), + }, + }); + return; + } + if (!remote || remote.readyState !== WebSocket.OPEN) { + json(response, 503, { + error: { code: 'node_unreachable', message: 'terminal transport is not connected' }, + }); + return; + } + if (pendingDeliveryMode) { + json(response, 503, { + error: { code: 'delivery_mode_conflict', message: 'a delivery mode request is already in flight' }, + }); + return; + } + const requestId = randomBytes(8).toString('hex'); + const result = await new Promise((resolve) => { + const timer = setTimeout(() => { + pendingDeliveryMode = null; + resolve(new FleetNodeAttachError('delivery mode request timed out', 'delivery_mode_timeout')); + }, DELIVERY_MODE_TIMEOUT_MS); + pendingDeliveryMode = { + requestId, + resolve: (r) => resolve(r), + reject: (e) => resolve(e), + timer, + }; + const frame: Record = { + type: 'terminal.set_delivery_mode', + session_id: sessionId, + mode: requestedMode, + request_id: requestId, + }; + if (typeof body.expected_mode === 'string') frame.expected_mode = body.expected_mode; + if (typeof body.expected_revision === 'string') frame.expected_revision = body.expected_revision; + remote!.send(JSON.stringify(frame)); + }); + if (result instanceof Error) { + const errCode = + result instanceof FleetNodeAttachError + ? (result.code ?? 'delivery_mode_failed') + : 'delivery_mode_failed'; + // Same canonical mapping as the readiness gate: 404 for + // agent_not_found so the attach preflight can produce the no-agent or + // cross-node placement error, 409 for unsupported_runtime, 503 for + // everything else. + json(response, terminalErrorStatus(errCode), { + error: { code: errCode, message: result.message }, + }); + return; + } + loopbackDeliveryMode = result.mode === 'manual_flush' ? 'manual_flush' : 'auto_inject'; + json(response, 200, { + mode: result.mode, + flushed: result.flushed, + matched: result.matched, + revision: result.revision, + }); + return; + } + if (request.method === 'GET' && path === `/api/spawned/${name}/pending`) { + json(response, 200, { pending: [] }); + return; + } + if (request.method === 'POST' && path === `/api/spawned/${name}/flush`) { + json(response, 200, { flushed: 0 }); + return; + } + if (request.method === 'GET' && path === '/api/spawned') { + json(response, 200, { agents: [{ name: options.agent, workerPid: 1 }] }); + return; + } + if (request.method === 'POST' && path === `/api/resize/${name}`) { + const body = await readBody(request); + if (body.release === true) { + json(response, 200, { name: options.agent, released: true }); + return; + } + const rows = typeof body.rows === 'number' ? body.rows : 0; + const cols = typeof body.cols === 'number' ? body.cols : 0; + if (!Number.isInteger(rows) || !Number.isInteger(cols) || rows < 1 || cols < 1) { + json(response, 400, { + error: { code: 'invalid_dimensions', message: 'rows and cols must be positive integers' }, + }); + return; + } + try { + await waitForTerminalReady('terminal resize timed out'); + } catch (error) { + json(response, 503, { + error: { + code: 'session_not_ready', + message: error instanceof Error ? error.message : 'terminal session is not ready', + }, + }); + return; + } + if (!remote || remote.readyState !== WebSocket.OPEN || remote.bufferedAmount > MAX_BUFFERED_BYTES) { + json(response, 503, { + error: { code: 'node_unreachable', message: 'terminal transport is unavailable' }, + }); + return; + } + remote.send(JSON.stringify({ type: 'terminal.resize', session_id: sessionId, rows, cols })); + json(response, 200, { name: options.agent, rows, cols, applied: true }); + return; + } + json(response, 404, { error: { code: 'not_found', message: 'loopback terminal endpoint not found' } }); + }); + const websocketServer = new WebSocketServer({ noServer: true }); + + const closeSocket = (socket: WebSocket, code: number, reason: string) => { + try { + socket.close(code, boundedWebSocketCloseReason(reason)); + } catch { + /* connection already gone */ + } + }; + const broadcast = (sockets: Set, payload: unknown): boolean => { + const encoded = JSON.stringify(payload); + let accepted = false; + for (const socket of sockets) { + if (socket.readyState !== WebSocket.OPEN) continue; + if (socket.bufferedAmount > MAX_BUFFERED_BYTES) { + closeSocket(socket, 1013, 'loopback client backpressure exceeded'); + sockets.delete(socket); + continue; + } + try { + socket.send(encoded); + accepted = true; + } catch { + sockets.delete(socket); + } + } + return accepted; + }; + const workerStreamEvent = (chunk: string, offset?: number) => ({ + kind: 'worker_stream', + name: options.agent, + stream: 'stdout', + chunk, + ...(offset === undefined ? {} : { offset }), + }); + const retainOutput = (chunk: string, offset: number | undefined): boolean => { + const bytes = Buffer.byteLength(chunk, 'utf8'); + if (outputHistoryBytes + bytes > MAX_BUFFERED_BYTES) return false; + outputHistory.push({ chunk, ...(offset === undefined ? {} : { offset }) }); + outputHistoryBytes += bytes; + return true; + }; + + websocketServer.on('connection', (socket, request) => { + const path = new URL(request.url ?? '/', 'http://127.0.0.1').pathname; + if (path === '/ws') { + eventSockets.add(socket); + socket.on('close', () => eventSockets.delete(socket)); + let replayed = 0; + for (const event of outputHistory) { + if (socket.readyState !== WebSocket.OPEN || socket.bufferedAmount > MAX_BUFFERED_BYTES) break; + try { + socket.send(JSON.stringify(workerStreamEvent(event.chunk, event.offset))); + } catch { + break; + } + replayed += 1; + } + if (replayed > 0) { + const sentBytes = outputHistory + .slice(0, replayed) + .reduce((total, event) => total + Buffer.byteLength(event.chunk, 'utf8'), 0); + outputHistory.splice(0, replayed); + outputHistoryBytes -= sentBytes; + } + return; + } + if (path === `/api/input/${encodeURIComponent(options.agent)}/stream`) { + inputSockets.add(socket); + socket.on('close', () => inputSockets.delete(socket)); + socket.send(JSON.stringify({ type: 'pty_input_ready', name: options.agent })); + socket.on('message', (data) => { + if (!remote || remote.readyState !== WebSocket.OPEN || remote.bufferedAmount > MAX_BUFFERED_BYTES) { + broadcast(inputSockets, { + type: 'pty_input_error', + code: 'node_unreachable', + message: 'terminal transport is unavailable', + }); + return; + } + const raw = rawDataToString(data); + remote.send( + JSON.stringify({ + type: 'terminal.input', + session_id: sessionId, + data_base64: Buffer.from(raw, 'utf8').toString('base64'), + }) + ); + }); + return; + } + closeSocket(socket, 1008, 'unknown loopback endpoint'); + }); + server.on('upgrade', (request, socket, head) => { + if (!loopbackAuthorized(request.headers)) { + socket.write('HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n'); + socket.destroy(); + return; + } + websocketServer.handleUpgrade(request, socket, head, (client) => + websocketServer.emit('connection', client, request) + ); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const address = server.address(); + if (!address || typeof address === 'string') + throw new FleetNodeAttachError( + 'Error: could not allocate loopback terminal listener.', + 'loopback_unavailable' + ); + + let socketDirectory: string | undefined; + let socketPath: string; + const rawServer = createSocketServer((socket) => { + if (rawConnected || stopped || terminalEnded) { + socket.destroy(); + return; + } + rawConnected = true; + rawSocket = socket; + socket.pause(); + socket.on('error', () => socket.destroy()); + socket.on('close', () => { + if (rawSocket === socket) rawSocket = undefined; + }); + socket.on('data', (data: Buffer) => { + if (options.mode === 'view') return; + if ( + !remote || + remote.readyState !== WebSocket.OPEN || + remote.bufferedAmount + data.length > MAX_BUFFERED_BYTES + ) { + endTerminal(new FleetNodeAttachError('terminal input transport unavailable', 'node_unreachable')); + return; + } + remote.send( + JSON.stringify({ + type: 'terminal.input', + session_id: sessionId, + data_base64: data.toString('base64'), + }) + ); + }); + void waitForTerminalReady('terminal socket readiness timed out').then( + () => { + if (socket.destroyed || stopped || terminalEnded) return; + if (snapshot.screen) socket.write(snapshot.screen); + for (const entry of outputHistory) socket.write(entry.chunk); + outputHistory.length = 0; + outputHistoryBytes = 0; + rawReady = true; + socket.resume(); + }, + () => socket.destroy() + ); + }); + try { + socketPath = + process.platform === 'win32' + ? `\\\\.\\pipe\\agent-relay-attach-${randomBytes(16).toString('hex')}` + : join((socketDirectory = await mkdtemp(join(tmpdir(), 'relay-attach-'))), 'stdio.sock'); + rawServer.listen(socketPath); + await once(rawServer, 'listening'); + if (socketDirectory) await chmod(socketPath, 0o600); + } catch (error) { + rawServer.close(); + websocketServer.close(); + await new Promise((resolve) => server.close(() => resolve())); + if (socketDirectory) await rm(socketDirectory, { recursive: true, force: true }); + throw error; + } + + const resumeUrl = new URL(terminalUrl); + resumeUrl.searchParams.delete('ticket'); + resumeUrl.searchParams.set('session_id', sessionId); + resumeUrl.searchParams.set('resume', resumeToken); + /** Reject and clear any in-flight delivery-mode PUT, if one is pending. */ + const rejectPendingDeliveryMode = (error: FleetNodeAttachError) => { + if (pendingFlush) { + const flush = pendingFlush; + pendingFlush = null; + clearTimeout(flush.timer); + // A flush never changes delivery mode, so it must not inherit the + // delivery-mode disconnect wording. An operator debugging a failed flush + // would otherwise be told the "delivery-mode change" was interrupted — + // pointing at an operation their command never performed. + const flushError = + error.code === 'delivery_mode_disconnected' + ? new FleetNodeAttachError( + 'terminal transport disconnected while the flush was in flight', + 'flush_disconnected' + ) + : error; + flush.reject(flushError); + } + if (!pendingDeliveryMode) return; + const pending = pendingDeliveryMode; + pendingDeliveryMode = null; + clearTimeout(pending.timer); + pending.reject(error); + }; + const endTerminal = (error: FleetNodeAttachError, eventCloseReason = error.message) => { + if (terminalEnded) return; + terminalEnded = true; + finish(error.code === 'terminal_closed' ? 0 : 1); + rawSocket?.end(); + if (reconnectTimer) { + clearTimeout(reconnectTimer); + reconnectTimer = undefined; + } + for (const timer of terminalReadinessTimers) clearTimeout(timer); + terminalReadinessTimers.clear(); + rejectPendingDeliveryMode(error); + const activeRemote = remote; + remote = undefined; + rejectReadiness(activeReadiness, error); + broadcast(inputSockets, { type: 'pty_input_error', code: error.code, message: error.message }); + for (const socket of eventSockets) closeSocket(socket, 1011, eventCloseReason); + if (activeRemote && activeRemote.readyState !== WebSocket.CLOSED) { + try { + activeRemote.terminate(); + } catch { + // The socket may have closed between the state check and terminate. + } + } + }; + const failRemote = (message: string, eventCloseReason?: string) => { + endTerminal(new FleetNodeAttachError(message, 'node_unreachable'), eventCloseReason); + }; + const connect = (url: string, readiness: TerminalReadiness) => { + if (stopped || terminalEnded) return; + const socket = new WebSocket(asWsUrl(url), { handshakeTimeout: terminalHandshakeTimeoutMs }); + remote = socket; + let readinessTimer: ReturnType | undefined; + let readinessExpired = false; + const clearReadinessTimer = () => { + if (!readinessTimer) return; + clearTimeout(readinessTimer); + terminalReadinessTimers.delete(readinessTimer); + readinessTimer = undefined; + }; + socket.on('open', () => { + // `handshakeTimeout` independently bounds the HTTP upgrade. Start the + // terminal.ready allowance only after that upgrade succeeds so a slow + // but valid handshake cannot consume the readiness window. + readinessTimer = setTimeout(() => { + const expiredTimer = readinessTimer; + readinessTimer = undefined; + if (expiredTimer) terminalReadinessTimers.delete(expiredTimer); + if (remote !== socket || stopped || terminalEnded || readiness.settled) return; + // Mark this generation stale before terminating. The ws receiver may + // still deliver data already buffered on the socket while close is + // propagating; none of it may restore readiness or reset retry state. + readinessExpired = true; + if (!terminalEverReady) { + failRemote( + `terminal transport connected but did not become ready (node ref ${diagnosticValue(options.node.trim())},` + + ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint)},` + + ` readiness timeout ${terminalReadyTimeoutMs}ms, attempts 1; not retried because no terminal session became ready)` + ); + return; + } + // A successful WebSocket upgrade is not sufficient: Relaycast may + // accept a resume lane that never produces terminal.ready. Terminating + // it drives the same bounded close/retry path as a transport failure. + options.reconnectDelay?.beforeReadyTimeoutTerminate?.(socket); + socket.terminate(); + }, terminalReadyTimeoutMs); + terminalReadinessTimers.add(readinessTimer); + }); + socket.on('message', (data) => { + // A late frame from a transport superseded during reconnect must never + // overwrite the fresh snapshot or end the replacement session. + if (remote !== socket || stopped || terminalEnded || readinessExpired) return; + const frame = parseFrame(data); + if (!frame || frame.session_id !== sessionId) return; + if (frame.type === 'terminal.ready') { + clearReadinessTimer(); + snapshot.screen = typeof frame.screen === 'string' ? frame.screen : ''; + snapshot.rows = typeof frame.rows === 'number' ? frame.rows : 24; + snapshot.cols = typeof frame.cols === 'number' ? frame.cols : 80; + snapshot.offset = typeof frame.offset === 'number' ? frame.offset : 0; + // Seed loopbackDeliveryMode from the broker's actual state so that + // detach restores the correct mode even when the worker started in a + // different mode than our local inference at line 214. + if (frame.delivery_mode === 'manual_flush' || frame.delivery_mode === 'auto_inject') { + loopbackDeliveryMode = frame.delivery_mode; + } + terminalEverReady = true; + reconnectAttempts = 0; + if (readiness === activeReadiness) { + resolveReadiness(readiness); + // A reconnect gets a fresh ANSI grid but existing local `/ws` + // consumers have already performed their initial HTTP snapshot. + // Re-emit this screen without an offset so they repaint instead of + // retaining a stale pre-reconnect terminal image. + if (readiness.generation > 1 && snapshot.screen) { + broadcast(eventSockets, workerStreamEvent(snapshot.screen)); + rawSocket?.write(snapshot.screen); + } + } + } else if (frame.type === 'terminal.output' && typeof frame.chunk === 'string') { + const offset = typeof frame.offset === 'number' ? frame.offset : undefined; + if (rawReady && rawSocket && !rawSocket.destroyed) { + if (rawSocket.writableLength + Buffer.byteLength(frame.chunk) > MAX_BUFFERED_BYTES) { + endTerminal( + new FleetNodeAttachError( + 'terminal output exceeded the bounded socket buffer', + 'output_backpressure' + ) + ); + return; + } + rawSocket.write(frame.chunk); + } + if (!broadcast(eventSockets, workerStreamEvent(frame.chunk, offset)) && !rawReady) { + if (!retainOutput(frame.chunk, offset)) { + endTerminal( + new FleetNodeAttachError( + 'terminal output exceeded the bounded loopback buffer', + 'output_backpressure' + ) + ); + } + } + } else if (frame.type === 'terminal.input_ack') { + broadcast(inputSockets, { + type: 'pty_input_ack', + name: options.agent, + bytes_written: typeof frame.bytes_written === 'number' ? frame.bytes_written : 0, + }); + } else if (frame.type === 'terminal.delivery_mode') { + const frameRid = typeof frame.request_id === 'string' ? frame.request_id : undefined; + if (pendingDeliveryMode && (frameRid === undefined || frameRid === pendingDeliveryMode.requestId)) { + const pending = pendingDeliveryMode; + pendingDeliveryMode = null; + clearTimeout(pending.timer); + pending.resolve({ + mode: typeof frame.mode === 'string' ? frame.mode : 'auto_inject', + flushed: typeof frame.flushed === 'number' ? frame.flushed : 0, + matched: typeof frame.matched === 'boolean' ? frame.matched : true, + revision: typeof frame.revision === 'string' ? frame.revision : '1', + }); + } + } else if (frame.type === 'terminal.flush_pending') { + const frameRid = typeof frame.request_id === 'string' ? frame.request_id : undefined; + // Exact match only. The proxy always sends a request_id, and unlike + // delivery-mode there is no older-broker reply shape to stay + // compatible with, so a reply without one is not ours — accepting it + // would resolve the caller's flush with an unrelated result. + if (pendingFlush && frameRid === pendingFlush.requestId) { + const pending = pendingFlush; + pendingFlush = null; + clearTimeout(pending.timer); + pending.resolve({ + flushed: typeof frame.flushed === 'number' ? frame.flushed : 0, + dead_lettered: typeof frame.dead_lettered === 'number' ? frame.dead_lettered : 0, + held: typeof frame.held === 'number' ? frame.held : 0, + blocked_reason: typeof frame.blocked_reason === 'string' ? frame.blocked_reason : null, + }); + } + } else if (frame.type === 'terminal.error') { + const message = typeof frame.message === 'string' ? frame.message : 'remote terminal failed'; + const code = typeof frame.code === 'string' ? frame.code : 'terminal_error'; + const frameRid = typeof frame.request_id === 'string' ? frame.request_id : undefined; + // Route the error to the pending delivery-mode request only when the + // request_id matches (or the broker sent no request_id at all — older + // broker compat). An unrelated session-level error must not cancel a + // live delivery-mode PUT and vice-versa. + if (pendingFlush && frameRid !== undefined && frameRid === pendingFlush.requestId) { + const pending = pendingFlush; + pendingFlush = null; + clearTimeout(pending.timer); + pending.reject(new FleetNodeAttachError(message, code)); + } else if ( + pendingDeliveryMode && + (frameRid === undefined || frameRid === pendingDeliveryMode.requestId) + ) { + const pending = pendingDeliveryMode; + pendingDeliveryMode = null; + clearTimeout(pending.timer); + pending.reject(new FleetNodeAttachError(message, code)); + } else if (!frameRid && readiness === activeReadiness && !readiness.settled) { + endTerminal(new FleetNodeAttachError(message, code)); + } else if (!frameRid) { + broadcast(inputSockets, { type: 'pty_input_error', code, message }); + } + } else if (frame.type === 'terminal.closed') { + endTerminal(new FleetNodeAttachError('remote terminal session closed', 'terminal_closed')); + } + }); + socket.on('error', () => { + readinessExpired = true; + clearReadinessTimer(); + // Initial connection failure has no terminal state worth preserving. + // Fail promptly with the canonical unavailable-node error instead of + // letting the HTTP snapshot timeout mask it. Once Ready has been seen, + // the close handler retains the bounded resume/backoff behaviour. + if (remote === socket && readiness === activeReadiness && !readiness.settled && !terminalEverReady) { + failRemote( + `terminal transport could not connect to the fleet node (node ref ${diagnosticValue(options.node.trim())},` + + ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint)},` + + ` handshake budget ${terminalHandshakeTimeoutMs}ms, attempts 1; not retried because no terminal session became ready)` + ); + } + }); + socket.on('close', () => { + readinessExpired = true; + clearReadinessTimer(); + if (remote !== socket || stopped || terminalEnded || reconnecting) return; + // Relaycast drops the old lane's terminal session state on disconnect, + // so a set_delivery_mode frame already sent on this dying socket is + // lost and will never get a reply on the replacement socket — even + // once reconnect succeeds. Fail the pending PUT fast with a retryable + // error instead of leaving it to hang out the full timeout. + rejectPendingDeliveryMode( + new FleetNodeAttachError( + 'terminal transport disconnected while the delivery-mode change was in flight', + 'delivery_mode_disconnected' + ) + ); + // Any waiter that observed the prior connection must retry against the + // fresh generation instead of receiving its stale resolved snapshot. + resolveReadiness(readiness); + const nextReadiness = createReadiness(); + activeReadiness = nextReadiness; + if (reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) { + const backoffBudgetMs = retryDelayBudgetMs( + MAX_RECONNECT_ATTEMPTS, + reconnectInitialDelayMs, + reconnectMaxDelayMs + ); + const message = + `terminal transport could not reconnect to the fleet node (node ref ${diagnosticValue(options.node.trim())},` + + ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint)},` + + ` handshake timeout ${terminalHandshakeTimeoutMs}ms, readiness timeout ${terminalReadyTimeoutMs}ms,` + + ` attempts ${reconnectAttempts},` + + ` backoff budget ${backoffBudgetMs}ms)`; + failRemote( + message, + `terminal reconnect failed; attempts=${reconnectAttempts}; budget=${backoffBudgetMs}ms;` + + ` endpoint=${diagnosticValue(diagnosticEndpointOrigin(terminalUrl))};` + + ` node=${diagnosticValue(resolvedNodeId ?? options.node.trim())}` + ); + return; + } + reconnecting = true; + const delay = Math.min(reconnectInitialDelayMs * 2 ** reconnectAttempts, reconnectMaxDelayMs); + reconnectAttempts += 1; + reconnectTimer = setTimeout(() => { + reconnectTimer = undefined; + reconnecting = false; + connect(resumeUrl.toString(), nextReadiness); + }, delay); + }); + }; + connect(terminalUrl, activeReadiness); + + return { + socketPath, + finished, + brokerUrl: `http://127.0.0.1:${address.port}`, + apiKey: loopbackApiKey, + requestTimeoutMs: terminalWaitTimeoutMs + DELIVERY_MODE_TIMEOUT_MS + LOOPBACK_REQUEST_TIMEOUT_MARGIN_MS, + close() { + return (closePromise ??= (async () => { + if (stopped) return; + stopped = true; + finish(0); + rawSocket?.destroy(); + terminalEnded = true; + if (reconnectTimer) { + clearTimeout(reconnectTimer); + reconnectTimer = undefined; + } + for (const timer of terminalReadinessTimers) clearTimeout(timer); + terminalReadinessTimers.clear(); + rejectReadiness(activeReadiness, new FleetNodeAttachError('terminal attach closed', 'closed')); + rejectPendingDeliveryMode(new FleetNodeAttachError('terminal attach closed', 'closed')); + const activeRemote = remote; + remote = undefined; + if (activeRemote && activeRemote.readyState === WebSocket.OPEN) { + try { + activeRemote.send(JSON.stringify({ type: 'terminal.close', session_id: sessionId })); + } catch { + // Best effort; terminate below still prevents a late reconnect. + } + } + if (activeRemote && activeRemote.readyState !== WebSocket.CLOSED) { + try { + activeRemote.terminate(); + } catch { + // Socket is already gone. + } + } + for (const socket of [...eventSockets, ...inputSockets]) + closeSocket(socket, 1000, 'terminal attach closed'); + websocketServer.close(); + await Promise.all([ + new Promise((resolve) => server.close(() => resolve())), + new Promise((resolve) => rawServer.close(() => resolve())), + ]); + if (socketDirectory) await rm(socketDirectory, { recursive: true, force: true }); + })()); + }, + }; +} + +export type { AttachMode } from './attach-mode.js'; +export { collectWithRetry } from './collect-with-retry.js'; diff --git a/packages/cloud/src/collect-with-retry.ts b/packages/cloud/src/collect-with-retry.ts new file mode 100644 index 0000000000..fb726a2315 --- /dev/null +++ b/packages/cloud/src/collect-with-retry.ts @@ -0,0 +1,42 @@ +export interface CollectWithRetryOptions { + retries?: number; + baseDelayMs?: number; + sleep?: (ms: number) => Promise; + /** Return false for terminal failures that must be surfaced immediately. */ + shouldRetry?: (error: unknown, attempt: number) => boolean; +} + +/** + * Run a bounded operation with the fleet CLI's small retry delay. + * + * The result records whether another attempt actually ran. Callers that need + * richer failure metadata can retain the thrown value in their operation; + * this helper deliberately keeps the existing fleet-list rendering contract. + */ +export async function collectWithRetry( + label: string, + fn: () => Promise, + options?: CollectWithRetryOptions +): Promise<{ ok: true; value: T; retried: boolean } | { ok: false; error: string; retried: boolean }> { + const retries = options?.retries ?? 1; + const baseDelay = options?.baseDelayMs ?? 500; + const sleep = options?.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); + let lastError: unknown; + let attempts = 0; + for (let attempt = 0; attempt <= retries; attempt += 1) { + attempts += 1; + try { + const value = await fn(); + return { ok: true, value, retried: attempt > 0 }; + } catch (error) { + lastError = error; + if (attempt >= retries || options?.shouldRetry?.(error, attempt + 1) === false) break; + // Keep the established fleet-list delay shape while making later + // attempts wait slightly longer, without introducing test randomness. + const delay = baseDelay + Math.floor(baseDelay * (attempt / (retries + 1))); + await sleep(delay); + } + } + const message = lastError instanceof Error ? lastError.message : String(lastError); + return { ok: false, error: `${label}: ${message}`, retried: attempts > 1 }; +} diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index d90e1800f4..242c1240f2 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -1114,6 +1114,7 @@ describe('Cloud fleet sandbox client', () => { requiredCapability: 'spawn:claude', forceProvision: true, relayfilePaths: ['/live-review/run-123/**'], + readonlyPaths: ['/live-review/run-123/reference/**'], }); const ensureCall = mocks.authorizedApiFetch.mock.calls[1]; @@ -1122,9 +1123,36 @@ describe('Cloud fleet sandbox client', () => { requiredCapability: 'spawn:claude', forceProvision: true, relayfilePaths: ['/live-review/run-123/**'], + readonlyPaths: ['/live-review/run-123/reference/**'], }); }); + it.each([[], ['/'], ['relative/**'], ['/foo'], ['/foo/../bar/**'], ['/foo/*/**']])( + 'rejects invalid readonlyPaths before authentication: %j', + async (...paths) => { + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:claude', + readonlyPaths: paths as string[], + }) + ).rejects.toThrow('readonlyPaths'); + expect(mocks.ensureCloudSession).not.toHaveBeenCalled(); + } + ); + + it('rejects readonly paths without a mount', async () => { + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:claude', + mountRelayfile: false, + readonlyPaths: ['/foo/**'], + }) + ).rejects.toThrow('requires mounting'); + expect(mocks.ensureCloudSession).not.toHaveBeenCalled(); + }); + it('rejects incomplete revision maps before Cloud authentication', async () => { await expect( ensureCloudFleetSandbox({ diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index a698a92a59..0f66de9ac4 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -119,6 +119,8 @@ export type EnsureCloudFleetSandboxInput = { * must use the explicit `/path/**` subtree form accepted by Cloud. */ relayfilePaths?: readonly string[]; + /** Read-only Relayfile subtrees in explicit `/path/**` form. Cloud owns chmod enforcement. */ + readonlyPaths?: readonly string[]; forceProvision?: boolean; /** Constrain provisioning to a provider that Cloud has enabled for routing. */ providerId?: CloudFleetSandboxProviderId; @@ -906,6 +908,24 @@ export async function ensureCloudFleetSandbox( if (input.relayfilePaths !== undefined && input.relayfilePaths.length === 0) { throw new Error('At least one Relayfile subtree path is required when relayfilePaths is provided.'); } + if (input.readonlyPaths !== undefined) { + if ( + input.readonlyPaths.length === 0 || + input.readonlyPaths.some( + (path) => + !/^\/(?:[^/*?\\\x00-\x1f]+\/)+\*\*$/.test(path) || + path + .slice(1, -3) + .split('/') + .some((segment) => segment === '.' || segment === '..') + ) + ) { + throw new Error( + 'readonlyPaths must contain at least one explicit /path/** subtree without traversal or wildcards.' + ); + } + if (input.mountRelayfile === false) throw new Error('readonlyPaths requires mounting Relayfile.'); + } validateRequestedRepos(input.repos); const repoRevisions = validateRepoRevisions(input.repos, input.repoRevisions); @@ -932,6 +952,7 @@ export async function ensureCloudFleetSandbox( ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), ...(input.relayfilePaths === undefined ? {} : { relayfilePaths: [...input.relayfilePaths] }), + ...(input.readonlyPaths === undefined ? {} : { readonlyPaths: [...input.readonlyPaths] }), ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), ...(input.providerId !== undefined ? { providerId: input.providerId } : {}), ...(input.workloadProfile !== undefined ? { workloadProfile: input.workloadProfile } : {}), diff --git a/packages/cloud/src/fleet.ts b/packages/cloud/src/fleet.ts index 36c67c5709..f34cc48366 100644 --- a/packages/cloud/src/fleet.ts +++ b/packages/cloud/src/fleet.ts @@ -382,3 +382,13 @@ export function resolveActiveFleetNodeEnrollment( `Multiple fleet node enrollments match; pass baseUrl and workspaceId to disambiguate. Candidates: ${candidates}.` ); } + +/** Fleet sandbox provisioning primitives. */ +export { + ensureCloudFleetSandbox, + deleteCloudFleetSandbox, + CloudFleetSandboxProvisionError, + type EnsureCloudFleetSandboxInput, + type EnsureCloudFleetSandboxResult, + type CloudFleetSandboxRequestOptions, +} from './fleet-sandbox.js'; diff --git a/packages/cloud/src/public-entries.test.ts b/packages/cloud/src/public-entries.test.ts new file mode 100644 index 0000000000..17063bf707 --- /dev/null +++ b/packages/cloud/src/public-entries.test.ts @@ -0,0 +1,28 @@ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { execFileSync } from 'node:child_process'; +import { expect, it } from 'vitest'; + +it('resolves the published Cloud and SDK fleet/attach ESM entries after build', () => { + for (const name of ['cloud', 'sdk']) { + const manifest = JSON.parse(readFileSync(new URL(`../../${name}/package.json`, import.meta.url), 'utf8')); + for (const subpath of ['fleet', 'attach']) { + const entry = manifest.exports[`./${subpath}`]; + expect(entry.types).toBe(`./dist/${subpath}.d.ts`); + expect(entry.import).toBe(`./dist/${subpath}.js`); + expect(readFileSync(new URL(`../../${name}/${entry.types}`, import.meta.url), 'utf8')).toBeTruthy(); + const exportedFunction = subpath === 'attach' ? 'startFleetNodeAttachProxy' : 'ensureCloudFleetSandbox'; + expect( + execFileSync( + process.execPath, + [ + '--input-type=module', + '-e', + `import { ${exportedFunction} } from '@agent-relay/${name}/${subpath}'; console.log(typeof ${exportedFunction});`, + ], + { cwd: fileURLToPath(new URL('../../..', import.meta.url)), encoding: 'utf8' } + ).trim() + ).toBe('function'); + } + } +}); diff --git a/packages/cloud/src/workspace-transport.ts b/packages/cloud/src/workspace-transport.ts new file mode 100644 index 0000000000..663091a53a --- /dev/null +++ b/packages/cloud/src/workspace-transport.ts @@ -0,0 +1,160 @@ +import path from 'node:path'; +import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from './fleet-sandbox.js'; +import { + resolveWorkspaceSelection as resolveCloudWorkspaceSelection, + type WorkspaceSelection, + type WorkspaceKeySource, +} from './workspace-key.js'; +/** Options shared by the SDK-backed (Relaycast) CLI command groups. */ +export interface WorkspaceTransportOptions { + workspaceKey?: string; + token?: string; + baseUrl?: string; + env?: NodeJS.ProcessEnv; + /** Explicit project root for nested invocations such as packages/web. */ + projectRoot?: string; + /** Use the canonical gateway instead of a persisted server-selected route. */ + ignorePersistedRelaycastTarget?: boolean; +} + +function env(options: WorkspaceTransportOptions): NodeJS.ProcessEnv { + return options.env ?? process.env; +} + +function trimOrUndefined(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +export type WorkspaceTransport = { + workspaceKey: string; + baseUrl?: string; + source: WorkspaceKeySource; +}; + +/** Resolve the selected key and any previously persisted Relay workspace identity. */ +export function resolveWorkspaceSelection( + options: WorkspaceTransportOptions = {} +): WorkspaceSelection | undefined { + const explicitProject = trimOrUndefined(env(options).AGENT_RELAY_PROJECT); + const projectRoot = explicitProject ? path.resolve(explicitProject) : options.projectRoot; + return resolveCloudWorkspaceSelection({ + workspaceKey: options.workspaceKey, + env: env(options), + ...(projectRoot ? { projectRoot } : {}), + }); +} + +export function resolveBaseUrl(options: WorkspaceTransportOptions = {}): string | undefined { + const selection = selectionForTransport(options); + return resolveBaseUrlForSelection(selection, options); +} + +function selectionForTransport(options: WorkspaceTransportOptions): WorkspaceSelection | undefined { + const selection = resolveWorkspaceSelection(options); + if (!selection || !options.ignorePersistedRelaycastTarget) return selection; + const { + relaycastRoute: _relaycastRoute, + relaycastBaseUrl: _relaycastBaseUrl, + relaycastApiKey: _relaycastApiKey, + relaycastApiKeyRef: _relaycastApiKeyRef, + ...canonicalSelection + } = selection; + return canonicalSelection; +} + +function resolveBaseUrlForSelection( + selection: WorkspaceSelection | undefined, + options: WorkspaceTransportOptions +): string | undefined { + const persisted = validatePersistedRelaycastBaseUrl(selection); + const requested = trimOrUndefined(options.baseUrl) ?? trimOrUndefined(env(options).RELAY_BASE_URL); + if (persisted && requested) { + let parsed: URL; + try { + parsed = new URL(requested); + } catch { + throw new Error('The requested Relaycast base URL is invalid.'); + } + const authority = /^https:\/\/([^/?#]+)/i.exec(requested)?.[1] ?? ''; + if ( + !/^https:\/\/[^/?#]+\/?$/i.test(requested) || + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + /:\d+$/.test(authority) || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') + ) { + throw new Error('The requested Relaycast base URL is not a trusted origin.'); + } + if (parsed.origin !== persisted) { + throw new Error('The requested Relaycast base URL does not match the persisted workspace route.'); + } + } + return persisted ?? requested; +} + +/** Resolve one credential/origin pair from one workspace selection. */ +export function resolveWorkspaceTransport(options: WorkspaceTransportOptions = {}): WorkspaceTransport { + const selection = selectionForTransport(options); + if (!selection) { + throw new Error( + 'No workspace key found. Pass --workspace-key, set RELAY_WORKSPACE_KEY, or run `relay workspace set_key `.' + ); + } + const baseUrl = resolveBaseUrlForSelection(selection, options); + // Project-session loading already validates the reference against the + // project/workspace/route/base tuple. Never re-read a ref here: doing so + // would let a tampered ref bypass that binding and pair an unrelated key + // with this route. + const routeCredential = trimOrUndefined(selection.relaycastApiKey); + if (selection.relaycastRoute === 'agent37-isolated' && !routeCredential) { + throw new Error( + 'The persisted isolated Relaycast credential is unavailable or mismatched; rerun the sandbox command to mint a fresh route.' + ); + } + return { + workspaceKey: routeCredential ?? selection.key, + ...(baseUrl ? { baseUrl } : {}), + source: selection.source, + }; +} + +function validatePersistedRelaycastBaseUrl(selection: WorkspaceSelection | undefined): string | undefined { + const baseUrl = trimOrUndefined(selection?.relaycastBaseUrl); + const route = selection?.relaycastRoute; + const relaycastApiKey = trimOrUndefined(selection?.relaycastApiKey); + if (!baseUrl && !route && !relaycastApiKey) return undefined; + if (!baseUrl || !route) { + throw new Error('The persisted Relaycast workspace route is incomplete.'); + } + let parsed: URL; + try { + parsed = new URL(baseUrl); + } catch { + throw new Error('The persisted Relaycast workspace route is invalid.'); + } + const expectedOrigin = + route === 'canonical' + ? CANONICAL_RELAYCAST_ORIGIN + : route === 'agent37-isolated' + ? AGENT37_RELAYCAST_ORIGIN + : undefined; + if ( + !expectedOrigin || + parsed.origin !== expectedOrigin || + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') + ) { + throw new Error('The persisted Relaycast workspace route is not trusted.'); + } + return parsed.origin; +} diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 01015b9469..e26d7757ed 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -37,6 +37,16 @@ "types": "./dist/capabilities.d.ts", "import": "./dist/capabilities.js", "default": "./dist/capabilities.js" + }, + "./attach": { + "types": "./dist/attach.d.ts", + "import": "./dist/attach.js", + "default": "./dist/attach.js" + }, + "./fleet": { + "types": "./dist/fleet.d.ts", + "import": "./dist/fleet.js", + "default": "./dist/fleet.js" } }, "files": [ @@ -64,6 +74,7 @@ "dependencies": { "@relaycast/sdk": "^8.0.7", "@relaycast/types": "^8.0.7", - "zod": "^4.4.3" + "zod": "^4.4.3", + "@agent-relay/cloud": "12.2.4" } } diff --git a/packages/sdk/src/attach.ts b/packages/sdk/src/attach.ts new file mode 100644 index 0000000000..71c417be17 --- /dev/null +++ b/packages/sdk/src/attach.ts @@ -0,0 +1 @@ +export * from '@agent-relay/cloud/attach'; diff --git a/packages/sdk/src/fleet.ts b/packages/sdk/src/fleet.ts new file mode 100644 index 0000000000..caf0e4fd8d --- /dev/null +++ b/packages/sdk/src/fleet.ts @@ -0,0 +1 @@ +export * from '@agent-relay/cloud/fleet'; diff --git a/packages/sdk/tsconfig.build.json b/packages/sdk/tsconfig.build.json index 0c970cfdd8..975f698853 100644 --- a/packages/sdk/tsconfig.build.json +++ b/packages/sdk/tsconfig.build.json @@ -28,7 +28,9 @@ "src/messaging/**/*.ts", "src/delivery/**/*.ts", "src/actions/**/*.ts", - "src/session/**/*.ts" + "src/session/**/*.ts", + "src/attach.ts", + "src/fleet.ts" ], "exclude": ["src/__tests__/**"] } diff --git a/summary.md b/summary.md new file mode 100644 index 0000000000..74f0c7cd68 --- /dev/null +++ b/summary.md @@ -0,0 +1,33 @@ +## Summary + +Draft implementation of the attach and read-only-request portions of the workforce#338 upstream work. **This does not yet unblock workforce#338.** The reviewed plan's D4 decision is pending: `spawnFleetSandbox` must either orchestrate a running agent or expose a narrower ensure-only contract. No placeholder `spawnFleetSandbox` is exported. + +- Move the attach adapter and workspace transport resolution into Cloud, retaining CLI re-exports and existing broker transport fields. Add Cloud and SDK `/attach` entries, nodeId/optional-agent discovery, a private single-client raw stdio socket, idempotent cleanup, and a completion promise. +- Expose existing provisioning primitives through Cloud and SDK `/fleet`; route the CLI's ensure import through the SDK entry. Preserve existing Cloud enrollment APIs. The running-agent spawn wrapper remains pending D4. +- Validate and forward `readonlyPaths` as explicit `/path/**` subtrees; expose `fleet spawn --sandbox-readonly-path` with mount guards. +- Add socket, request-validation, CLI, and built-entry regression coverage. Add an independent read-only-request RelayFlow proof; leave historical case 1630 unchanged. + +The proxy retains broker transport fields alongside socketPath/finished/close, as the reviewed plan recommends: workforce duck-types the three consumer fields and ignores extras. Completion is an **inferred** status (0 for normal terminal closure/detach, 1 for transport failure), not a remote harness exit code. SDK re-exports add a dependency on Cloud and its transitive dependencies. No package versions changed. + +## Remaining requirements + +- Resolve reviewed-plan.md D4 and implement/test `spawnFleetSandbox` registration, placement confirmation, input mapping, and teardown. +- Cloud's server-side ensure handler/mount builder is absent from this repository. Forwarding does not enforce chmod-444; server implementation and a live Daytona write-denial proof are still required. +- Exports follow the reviewed plan's existing ESM conventions. No CommonJS build or `require` condition was added. +- No live CLI sandbox-spawn or node-attach parity run was performed. + +## Validation + +- `npm run typecheck`: passed. +- `npm run build`: passed (Rust build skipped automatically because Cargo is unavailable). +- SDK suite: 182 passed; fleet suite: 43 passed. +- Cloud suite: 450 passed, 4 skipped after removing inherited Cloud authentication variables. +- Existing and new attach tests: 42 passed, using actual local WebSocket and UNIX socket servers. +- CLI regression and built Cloud/SDK entry-resolution checks passed; full CLI results pending final verification. + +## RelayFlow Proof + +- Change type: `feature` +- RelayFlow case: `sandbox-readonly-paths` + +This generated base/head probe verifies exact request forwarding and compiled CLI help. It does not provision a sandbox or prove chmod enforcement. diff --git a/tests/relayflows/cases/sandbox-readonly-paths/case.json b/tests/relayflows/cases/sandbox-readonly-paths/case.json new file mode 100644 index 0000000000..fd0d6c2997 --- /dev/null +++ b/tests/relayflows/cases/sandbox-readonly-paths/case.json @@ -0,0 +1,20 @@ +{ + "version": 1, + "id": "sandbox-readonly-paths", + "kind": "feature", + "title": "Forward read-only Relayfile subtree requests to Cloud", + "runner": { + "command": ["node", "tests/relayflows/cases/sandbox-readonly-paths/run.mjs"] + }, + "timeoutSeconds": 900, + "expected": { + "base": { + "outcome": "absent", + "signature": "sandbox_readonly_paths_not_forwarded" + }, + "head": { + "outcome": "fixed", + "signature": "sandbox_readonly_paths_forwarded" + } + } +} diff --git a/tests/relayflows/cases/sandbox-readonly-paths/run.mjs b/tests/relayflows/cases/sandbox-readonly-paths/run.mjs new file mode 100644 index 0000000000..896eded193 --- /dev/null +++ b/tests/relayflows/cases/sandbox-readonly-paths/run.mjs @@ -0,0 +1,273 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { lstat, mkdir, open, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const CASE_ID = 'sandbox-readonly-paths'; +const COMMAND_TIMEOUT_MS = 5 * 60 * 1000; +const targetDir = requiredDirectory('RELAY_PR_PROOF_TARGET_DIR'); +const harnessDir = requiredDirectory('RELAY_PR_PROOF_HARNESS_DIR'); +const resultPath = requiredValue('RELAY_PR_PROOF_RESULT_PATH'); +const arm = requiredValue('RELAY_PR_PROOF_ARM'); + +if (arm !== 'base' && arm !== 'head') { + throw new Error(`RELAY_PR_PROOF_ARM must be base or head, received ${JSON.stringify(arm)}.`); +} + +const expectedSha = + arm === 'base' ? process.env.RELAY_PR_PROOF_BASE_SHA : process.env.RELAY_PR_PROOF_HEAD_SHA; +if (!expectedSha) throw new Error(`Missing expected ${arm} SHA.`); +const targetSha = execFileSync('git', ['-C', targetDir, 'rev-parse', 'HEAD'], { + encoding: 'utf8', +}).trim(); +if (targetSha !== expectedSha) { + throw new Error(`Target checkout ${targetSha} does not match exact ${arm} SHA ${expectedSha}.`); +} + +const runnerPath = fileURLToPath(import.meta.url); +if (!isWithin(harnessDir, runnerPath)) { + throw new Error('The RelayFlow runner must execute from the exact-head harness checkout.'); +} + +const probePath = path.join( + targetDir, + 'packages/cloud/src/.relayflow-sandbox-readonly-paths.test.ts' +); +const probeObservationPath = path.join( + targetDir, + '.relayflow-sandbox-readonly-paths-observation.json' +); +const probeConfigPath = path.join( + targetDir, + '.relayflow-sandbox-readonly-paths.vitest.config.mjs' +); + +const probeSource = String.raw`import { afterEach, expect, test, vi } from 'vitest'; +import { writeFile } from 'node:fs/promises'; + +const mocks = vi.hoisted(() => ({ + ensureCloudSession: vi.fn(), + authorizedApiFetch: vi.fn(), +})); + +vi.mock('./auth.js', () => ({ + ensureCloudSession: mocks.ensureCloudSession, + authorizedApiFetch: mocks.authorizedApiFetch, +})); + +import { ensureCloudFleetSandbox } from './fleet-sandbox.js'; + +const auth = { + accessToken: 'relayflow-probe-access', + refreshToken: 'relayflow-probe-refresh', + accessTokenExpiresAt: '2099-01-01T00:00:00Z', + apiUrl: 'https://relayflow.invalid', +}; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +test('observes exact Relayfile subtree forwarding', async () => { + const observationPath = process.env.RELAY_READONLY_OBSERVATION_PATH; + if (!observationPath) throw new Error('Missing RELAY_READONLY_OBSERVATION_PATH.'); + + mocks.ensureCloudSession.mockResolvedValue({ auth, client: {} }); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99' }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeId: 'node-relayflow', + nodeName: 'scoped-relayflow', + sandboxId: 'sandbox-relayflow', + relayWorkspaceId: 'rw_relayflow', + relayfileMounted: true, + providerId: 'daytona', + providerSandboxId: '223e4567-e89b-42d3-a456-426614174000', + }, + { status: 201 } + ), + auth, + }); + + await ensureCloudFleetSandbox({ + workspaceId: 'rw_relayflow', + requiredCapability: 'spawn:claude', + mountRelayfile: true, + readonlyPaths: ['/live-review/run-123/**', '/live-review/run-123/reviews/**'], + }); + + const ensureRequest = mocks.authorizedApiFetch.mock.calls[1]?.[2]; + expect(ensureRequest?.body).toEqual(expect.any(String)); + const ensureBody = JSON.parse(ensureRequest.body); + + await writeFile( + observationPath, + JSON.stringify({ readonlyPaths: ensureBody.readonlyPaths ?? null }), + 'utf8' + ); +}); +`; + +const probeConfigSource = `export default { + test: { + environment: 'node', + include: ['packages/cloud/src/.relayflow-sandbox-readonly-paths.test.ts'], + setupFiles: [], + }, +};\n`; + +try { + run('npm', ['ci', '--ignore-scripts'], targetDir, 'workspace dependency installation'); + run('npm', ['run', 'build:session'], targetDir, 'session package build'); + run('npm', ['run', 'build:config'], targetDir, 'configuration package build'); + run('npm', ['run', 'build:cloud'], targetDir, 'Cloud package build'); + run('npm', ['run', 'build:utils'], targetDir, 'utilities package build'); + run('npm', ['run', 'build:policy'], targetDir, 'policy package build'); + run('npm', ['run', 'build:sdk'], targetDir, 'SDK package build'); + run('npm', ['run', 'build:harness-driver'], targetDir, 'harness driver package build'); + run('npm', ['run', 'build:harnesses'], targetDir, 'harnesses package build'); + run('npm', ['run', 'build:fleet'], targetDir, 'fleet package build'); + run('npm', ['run', 'build:cli-surface'], targetDir, 'CLI surface package build'); + run('npm', ['run', 'build:cli'], targetDir, 'CLI package build'); + + await writeGeneratedFile(probePath, probeSource); + await writeGeneratedFile(probeConfigPath, probeConfigSource); + run( + 'npm', + ['exec', '--', 'vitest', 'run', '--config', path.relative(targetDir, probeConfigPath)], + targetDir, + 'Relayfile subtree forwarding probe', + { RELAY_READONLY_OBSERVATION_PATH: probeObservationPath } + ); + + const observation = JSON.parse(await readFile(probeObservationPath, 'utf8')); + const cliHelp = runCapture( + process.execPath, + [path.join(targetDir, 'packages/cli/dist/cli/index.js'), 'fleet', 'spawn', '--help'], + targetDir, + 'compiled fleet spawn help' + ); + const cliHasScopedMountFlag = cliHelp.includes('--sandbox-readonly-path '); + const requestedPaths = ['/live-review/run-123/**', '/live-review/run-123/reviews/**']; + const baseObserved = observation.readonlyPaths === null && !cliHasScopedMountFlag; + const headObserved = + JSON.stringify(observation.readonlyPaths) === JSON.stringify(requestedPaths) && cliHasScopedMountFlag; + + let outcome; + let signature; + let details; + if (baseObserved) { + outcome = 'absent'; + signature = 'sandbox_readonly_paths_not_forwarded'; + details = + 'The base Cloud client dropped requested Relayfile subtrees and the fleet spawn CLI exposed no read-only-mount flag.'; + } else if (headObserved) { + outcome = 'fixed'; + signature = 'sandbox_readonly_paths_forwarded'; + details = + 'The head Cloud client forwarded the exact requested Relayfile subtrees and the fleet spawn CLI exposed the read-only-mount flag.'; + } else { + throw new Error( + `Unexpected read-only Relayfile mount observation: ${JSON.stringify({ + ...observation, + cliHasScopedMountFlag, + })}.` + ); + } + + await mkdir(path.dirname(resultPath), { recursive: true }); + await writeFile( + resultPath, + `${JSON.stringify({ version: 1, caseId: CASE_ID, arm, outcome, signature, details })}\n`, + 'utf8' + ); +} finally { + await rm(probePath, { force: true }); + await rm(probeConfigPath, { force: true }); + await rm(probeObservationPath, { force: true }); +} + +function requiredValue(name) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`Missing required environment variable ${name}.`); + return value; +} + +function requiredDirectory(name) { + return path.resolve(requiredValue(name)); +} + +function isWithin(directory, candidate) { + const relative = path.relative(directory, candidate); + return ( + relative === '' || + (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) + ); +} + +async function writeGeneratedFile(targetPath, source) { + try { + const existing = await lstat(targetPath); + if (!existing.isFile()) { + throw new Error(`Refusing to replace non-regular generated file ${targetPath}.`); + } + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + + const temporaryPath = `${targetPath}.tmp-${process.pid}-${randomUUID()}`; + try { + const handle = await open(temporaryPath, 'wx', 0o600); + try { + await handle.writeFile(source, 'utf8'); + } finally { + await handle.close(); + } + await rename(temporaryPath, targetPath); + } finally { + await rm(temporaryPath, { force: true }); + } +} + +function run(command, args, cwd, label, extraEnv = {}) { + const completed = spawnSync(command, args, { + cwd, + env: { ...process.env, ...extraEnv }, + stdio: ['ignore', 'inherit', 'inherit'], + timeout: COMMAND_TIMEOUT_MS, + }); + if (completed.error) throw new Error(`${label} could not start: ${completed.error.message}`); + if (completed.status !== 0) { + throw new Error( + `${label} failed with ${ + completed.signal ? `signal ${completed.signal}` : `exit code ${completed.status ?? 'unknown'}` + }.` + ); + } +} + +function runCapture(command, args, cwd, label, extraEnv = {}) { + const completed = spawnSync(command, args, { + cwd, + env: { ...process.env, ...extraEnv }, + encoding: 'utf8', + timeout: COMMAND_TIMEOUT_MS, + }); + if (completed.error) throw new Error(`${label} could not start: ${completed.error.message}`); + if (completed.status !== 0) { + throw new Error( + `${label} failed with ${ + completed.signal ? `signal ${completed.signal}` : `exit code ${completed.status ?? 'unknown'}` + }: ${completed.stderr?.slice(-2000) ?? ''}` + ); + } + return completed.stdout; +} From e9e44b2681b3642b8a3c7123db2a4f9f7d56001f Mon Sep 17 00:00:00 2001 From: Relayflow Date: Fri, 18 Sep 2026 15:23:19 +0000 Subject: [PATCH 02/19] Record package validation and base-head readonly proof --- summary.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/summary.md b/summary.md index 74f0c7cd68..e43f0d704f 100644 --- a/summary.md +++ b/summary.md @@ -23,7 +23,9 @@ The proxy retains broker transport fields alongside socketPath/finished/close, a - SDK suite: 182 passed; fleet suite: 43 passed. - Cloud suite: 450 passed, 4 skipped after removing inherited Cloud authentication variables. - Existing and new attach tests: 42 passed, using actual local WebSocket and UNIX socket servers. -- CLI regression and built Cloud/SDK entry-resolution checks passed; full CLI results pending final verification. +- CLI suite via `npx vitest run packages/cli/ --maxWorkers=2`: 1,652 passed, 29 skipped. `npm --prefix packages/cli test` is unavailable because that package has no test script. Tests ran without inherited Cloud/Relay credentials and with local Git file transport enabled for the temporary-repository fixture. +- Built Cloud/SDK entry-resolution regression: passed. +- New proof runner executed locally against base `c978302` and implementation `b6c3bb1`: base `absent`, head `fixed`. Both generated probes and compiled CLI-help checks passed. This was not a hosted Cloud or live Daytona run. ## RelayFlow Proof From b46ed1c60aa7e19304767776e6e90bcc71ebc3c8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 18 Sep 2026 15:24:34 +0000 Subject: [PATCH 03/19] style: auto-format with Prettier --- README.md | 11 +++++++++-- packages/cloud/package.json | 2 +- .../cases/sandbox-readonly-paths/run.mjs | 15 +++------------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 489ad51579..129efe2fa6 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@

TypeScript Tests Last commit npm version Downloads License

# Infrastructure for coding agents + Tired of copy/pasting from Claude Code into Slack so your coworker can paste it into their agent? Ever put an important rule in a skill or AGENTS.md, only for the agent to completely ignore it? @@ -13,34 +14,40 @@ Agent Relay is an open-source toolkit for problems like these. It gives engineer Use the pieces you need, or combine them to build workflows across agents, tools, people, and machines. ### Messaging + Claude/Codex/etc can talk directly through shared channels, threads, DMs, files, search, and real-time events. Agents can run on different machines and still coordinate in the same workspace. [Read the docs](https://agentrelay.com/docs/introduction) ### Integrations + GitHub, Linear, Notion, Slack, and other tools are exposed as a virtual filesystem. Agents use ls, cat, grep, and ordinary file writes to work with them. [Peep the open source repo](https://github.com/agentworkforce/relayfile) ### Shared Sessions + Capture coding agent sessions so your team and their agents can search previous work, decisions, and context. [How we capture sessions](https://github.com/agentworkforce/relayhistory)
[How we capture decisions](https://github.com/agentworkforce/trajectories) ### Flows -Turn instructions you hope an agent follows into workflows you can enforce. + +Turn instructions you hope an agent follows into workflows you can enforce. Define multi-step workflows in TypeScript with deterministic checks, required steps, and human gates. Put the rules that matter in code instead of relying on a skill or prompt to be remembered and followed. [Learn how write a flow](https://github.com/agentworkforce/flows) (or lets be honest, show your agent how) ## Getting Started -The easiest way to get started is to use [Agent Relay Cloud](https://agentrelay.com/flows). + +The easiest way to get started is to use [Agent Relay Cloud](https://agentrelay.com/flows). You don't need a credit card and you can explore all the pieces without setting up any infrastructure. ### Self Hosting + Agent Relay has self hosting options for each primitive. We're happy to help you set up the whole system on your environment, just reach out to our team hi(at)agentrelay.com and we'll walk you through it. ## License diff --git a/packages/cloud/package.json b/packages/cloud/package.json index 8832733be2..d6b63072ed 100644 --- a/packages/cloud/package.json +++ b/packages/cloud/package.json @@ -4,7 +4,7 @@ "engines": { "node": ">=22.0.0" }, - "description": "Cloud SDK for Agent Relay \u2014 auth, workflow execution, and provider connections", + "description": "Cloud SDK for Agent Relay — auth, workflow execution, and provider connections", "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/tests/relayflows/cases/sandbox-readonly-paths/run.mjs b/tests/relayflows/cases/sandbox-readonly-paths/run.mjs index 896eded193..693fdc8eea 100644 --- a/tests/relayflows/cases/sandbox-readonly-paths/run.mjs +++ b/tests/relayflows/cases/sandbox-readonly-paths/run.mjs @@ -31,18 +31,9 @@ if (!isWithin(harnessDir, runnerPath)) { throw new Error('The RelayFlow runner must execute from the exact-head harness checkout.'); } -const probePath = path.join( - targetDir, - 'packages/cloud/src/.relayflow-sandbox-readonly-paths.test.ts' -); -const probeObservationPath = path.join( - targetDir, - '.relayflow-sandbox-readonly-paths-observation.json' -); -const probeConfigPath = path.join( - targetDir, - '.relayflow-sandbox-readonly-paths.vitest.config.mjs' -); +const probePath = path.join(targetDir, 'packages/cloud/src/.relayflow-sandbox-readonly-paths.test.ts'); +const probeObservationPath = path.join(targetDir, '.relayflow-sandbox-readonly-paths-observation.json'); +const probeConfigPath = path.join(targetDir, '.relayflow-sandbox-readonly-paths.vitest.config.mjs'); const probeSource = String.raw`import { afterEach, expect, test, vi } from 'vitest'; import { writeFile } from 'node:fs/promises'; From d916282d9f0b281e992ebe2f47ceeac46eef4f7e Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 10:50:04 -0700 Subject: [PATCH 04/19] feat(sdk): spawnFleetSandbox starts a running agent in a Cloud sandbox @agent-relay/sdk/fleet now exports spawnFleetSandbox (D4 option 1): it provisions the sandbox, starts the agent harness, confirms the node the agent landed on, and returns a handle with attach() and an idempotent destroy(). agent-relay fleet spawn --sandbox drives the same function, so the CLI and the SDK contract cannot drift. - An agent that lands on a node other than the ensured sandbox fails closed: the agent is released and an owned sandbox is deleted. - A caller-declared sandboxId is retained by failure cleanup and destroy(). - A non-empty env is rejected rather than silently dropped. - persistWorkspaceRelaycastTarget moves to @agent-relay/cloud/workspace-transport (the CLI re-exports it). - attach tests send base64 terminal.ready screens, matching relay#1829. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- packages/cli/src/cli/commands/fleet.test.ts | 85 +++ packages/cli/src/cli/commands/fleet.ts | 485 +++---------- packages/cli/src/cli/lib/sdk-client.ts | 43 +- packages/cloud/src/attach.test.ts | 4 +- packages/cloud/src/fleet.ts | 6 + packages/cloud/src/workspace-transport.ts | 34 + packages/sdk/README.md | 31 + packages/sdk/package.json | 2 +- .../sdk/src/__tests__/fleet-spawn.test.ts | 241 +++++++ packages/sdk/src/fleet-spawn.ts | 647 ++++++++++++++++++ packages/sdk/src/fleet.ts | 1 + packages/sdk/tsconfig.build.json | 3 +- summary.md | 35 - 13 files changed, 1164 insertions(+), 453 deletions(-) create mode 100644 packages/sdk/src/__tests__/fleet-spawn.test.ts create mode 100644 packages/sdk/src/fleet-spawn.ts delete mode 100644 summary.md diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 1b0e112413..ab5e78a6cb 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1256,6 +1256,91 @@ describe('fleet command support', () => { }); }); + it('fleet spawn --sandbox rejects an agent that lands on another node and tears the sandbox down', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', undefined); + const placement = { + spawn: vi.fn(async () => ({ invocationId: 'inv_sandbox', node: { name: 'some-other-node' } })), + }; + const release = vi.fn(async () => ({ released: true, deleted: true })); + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { + info: vi.fn(async () => ({ id: 'rw_abc' })), + register: vi.fn(async () => ({ token: 'at_live_launcher' })), + release, + }, + })); + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const errors: string[] = []; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + resolveSandboxRepository: vi.fn(() => undefined), + sdk: { + createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, + createWorkspaceRelay: createWorkspaceRelay as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: (message: unknown) => errors.push(String(message)), + exit: vi.fn(() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => ({ + outcome: 'provisioned' as const, + providerId: 'e2b' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-1', + nodeName: 'e2b-codex', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + relayfileMountPath: '/workspace', + })), + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/tmp/agent-relay-test/workspace-key.json', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-provider', + 'e2b', + '--no-sandbox-relayfile', + '--name', + 'sandbox-worker', + '--task', + 'Wait for VERIFY', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + expect(errors.join('\n')).toContain("landed on node 'some-other-node', not sandbox node 'e2b-codex'"); + expect(release).toHaveBeenCalledWith( + expect.objectContaining({ name: 'sandbox-worker', deleteAgent: true }) + ); + expect(deleteCloudFleetSandbox).toHaveBeenCalledWith({ + cloudWorkspaceId: 'cloud-workspace', + sandboxId: 'sandbox-1', + providerId: 'e2b', + }); + }); + it('plain --sandbox materializes the inferred repository through Relayfile and starts in its live relative cwd', async () => { vi.stubEnv('RELAY_AGENT_TOKEN', undefined); const revision = '0123456789abcdef0123456789abcdef01234567'; diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 6b4302db8a..c63c090257 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -4,18 +4,14 @@ import path from 'node:path'; import { InvalidArgumentError, type Command } from 'commander'; import { findProjectRoot } from '@agent-relay/config'; import { - CloudFleetSandboxProvisionError, - DEV_CLOUD_API_URL, - DEV_RELAYCAST_ORIGIN, deleteCloudFleetSandbox, materializeCloudRelayfileRepository, resolveWorkspaceByKey, type CloudFleetSandboxProviderId, type CloudRelayfileRepositoryMaterialization, - type EnsureCloudFleetSandboxInput, type EnsureCloudFleetSandboxResult, } from '@agent-relay/cloud'; -import { ensureCloudFleetSandbox } from '@agent-relay/sdk/fleet'; +import { ensureCloudFleetSandbox, spawnFleetSandbox } from '@agent-relay/sdk/fleet'; import { HarnessDriverClient } from '@agent-relay/harness-driver'; import { createWorkspaceClient, @@ -500,7 +496,8 @@ export function registerFleetCommands( const clientOptions = sdkOptionsFromOpts(options); const name = requiredText(options.name, 'Worker name'); const task = requiredText(options.task, 'Task'); - let targetNode = optionalText(options.targetNode, 'Target node') ?? optionalText(options.node, 'Node'); + const targetNode = + optionalText(options.targetNode, 'Target node') ?? optionalText(options.node, 'Node'); const useSandbox = options.sandbox === true; // Explicit hosted credentials/transport and personas retain their legacy // automatic-placement contract. Ambient credentials and a persisted @@ -517,10 +514,6 @@ export function registerFleetCommands( const checkoutRepository = options.checkout === true; const sandboxName = optionalText(options.sandboxName, 'Sandbox name'); const sandboxIdOption = optionalText(options.sandboxId, 'Sandbox ID'); - // An explicit sandbox identity is a retained/replayable resource. Never - // delete it as collateral when a later verification or dispatch step - // fails; only clean up sandboxes whose identity this invocation minted. - const shouldCleanupSandbox = sandboxIdOption === undefined; const explicitWorkspaceId = optionalText(options.workspaceId, 'Workspace ID'); if (sandboxIdOption !== undefined && !CLOUD_SANDBOX_ID_PATTERN.test(sandboxIdOption)) { throw new Error('--sandbox-id must match lowercase sbx_ using an RFC 4122 UUID.'); @@ -604,14 +597,10 @@ export function registerFleetCommands( ); } - let sandbox: EnsureCloudFleetSandboxResult | undefined; let sandboxRepository: SandboxRepositorySelection | undefined; let liveRepository: CloudRelayfileRepositoryMaterialization | undefined; let sandboxMountPaths: string[] | undefined; let attachProjectRoot: string | undefined; - let workspaceRelay: ReturnType | undefined; - let relaycastClientOptions = clientOptions; - let legacyWorkspaceClientOptions = clientOptions; if (useSandbox) { const coreProjectRoot = deps.core.getProjectPaths().projectRoot; const hasExplicitProjectOverride = Boolean( @@ -673,7 +662,6 @@ export function registerFleetCommands( ...clientOptions, projectRoot: workspaceProjectRoot, }; - relaycastClientOptions = sandboxClientOptions; // Cloud must be the first network authority for a sandbox invocation. // A canonical Relaycast info call would both leak the workspace key and // make it impossible to prove that Cloud's isolated target is the one @@ -681,7 +669,7 @@ export function registerFleetCommands( const workspaceSelection = deps.resolveWorkspaceSelection({ ...sandboxClientOptions, }); - legacyWorkspaceClientOptions = { + const legacyWorkspaceClientOptions = { ...sandboxClientOptions, ...(sandboxProvider === 'agent37' ? {} : { ignorePersistedRelaycastTarget: true }), }; @@ -708,8 +696,9 @@ export function registerFleetCommands( ); } if (!relayWorkspaceId && sandboxProvider !== undefined && sandboxProvider !== 'agent37') { - workspaceRelay = deps.sdk.createWorkspaceRelay(legacyWorkspaceClientOptions); - const workspaceInfo = await workspaceRelay.workspace.info(); + const workspaceInfo = await deps.sdk + .createWorkspaceRelay(legacyWorkspaceClientOptions) + .workspace.info(); relayWorkspaceId = workspaceInfo.id?.trim(); } if (!relayWorkspaceId) { @@ -782,294 +771,127 @@ export function registerFleetCommands( revision: sandboxRepository.revision, }); } - const sandboxId = sandboxIdOption ?? (sandboxName === undefined ? `sbx_${randomUUID()}` : undefined); - const deterministicSandboxName = - sandboxId === undefined ? undefined : `fleet-sandbox-${sandboxId.slice('sbx_'.length)}`; - if ( - sandboxIdOption !== undefined && - sandboxName !== undefined && - sandboxName !== deterministicSandboxName - ) { - throw new Error( - `--sandbox-name must be '${deterministicSandboxName}' when --sandbox-id is supplied; custom names cannot preserve the one-to-one sandbox identity.` - ); - } - const effectiveSandboxName = deterministicSandboxName ?? sandboxName; - // The unpinned/Agent37 path deliberately carries the measured heavy - // 8 CPU / 16 GiB / 20 GiB profile. Daytona and E2B cannot satisfy that - // shape, so an explicit legacy-provider selection must request the - // provider-neutral durable profile instead of becoming unroutable by - // construction. - const workloadProfile = - sandboxProvider === undefined || sandboxProvider === 'agent37' - ? 'long-running-agent' - : 'standard-long-running-agent'; - const useAsyncPreparation = - sandboxId !== undefined && (sandboxProvider === undefined || sandboxProvider === 'agent37'); - if (useAsyncPreparation && sandboxIdOption === undefined) { - // The generated identity is not persisted. Print it before Cloud work - // starts so an interrupted run can be resumed instead of duplicated. - deps.warn( - `Cloud sandbox identity: ${sandboxId}. If this command is interrupted, re-run it with --sandbox-id ${sandboxId} to resume the same sandbox instead of creating another.` - ); - } - try { - const ensureInput: EnsureCloudFleetSandboxInput = { + const handle = await spawnFleetSandbox( + { + cli, + name, + task, workspaceId: relayWorkspaceId, - requiredCapability: `spawn:${cli}`, - ...(useAsyncPreparation ? { preparationMode: 'async-v1' as const } : {}), - maxAgents: 1, + transport: sandboxClientOptions, + ...(sandboxProvider === undefined ? {} : { provider: sandboxProvider }), + ...(sandboxIdOption === undefined ? {} : { sandboxId: sandboxIdOption }), + ...(sandboxName === undefined ? {} : { sandboxName }), mountRelayfile: mountSandboxRelayfile, - ...(sandboxReadonlyPaths === undefined ? {} : { readonlyPaths: sandboxReadonlyPaths }), ...(sandboxMountPaths === undefined ? {} : { relayfilePaths: sandboxMountPaths }), - ...(sandboxId === undefined ? {} : { sandboxId }), - forceProvision: true, - ...(sandboxProvider === undefined ? {} : { providerId: sandboxProvider }), - workloadProfile, - waitTimeoutMs: 90_000, - ...(effectiveSandboxName === undefined ? {} : { name: effectiveSandboxName }), - ...(checkoutRepository && sandboxRepository ? { repos: [sandboxRepository.repository] } : {}), + ...(sandboxReadonlyPaths === undefined ? {} : { readonlyPaths: sandboxReadonlyPaths }), ...(checkoutRepository && sandboxRepository - ? { repoRevisions: { [sandboxRepository.repository]: sandboxRepository.revision } } + ? { + repos: [sandboxRepository.repository], + repoRevisions: { [sandboxRepository.repository]: sandboxRepository.revision }, + } : {}), - }; - sandbox = useAsyncPreparation - ? await deps.ensureCloudFleetSandbox(ensureInput, { - onPreparationProgress: (progress) => { - deps.warn( - `Cloud sandbox preparation: ${progress.phase} (${progress.state}, generation ${progress.generation}).` - ); - }, - }) - : await deps.ensureCloudFleetSandbox(ensureInput); - assertSandboxRepositoryRevision(sandbox, checkoutRepository ? sandboxRepository : undefined); - } catch (error) { - if (error instanceof CloudFleetSandboxProvisionError && error.sandboxAbsent) { - // Cloud's terminal record for this exact identity already proves the - // provider sandbox is gone; a delete here would only race its reaper. - deps.warn( - `${error.message} Cloud confirmed sandbox '${ - error.sandboxId ?? sandboxId ?? 'the requested sandbox' - }' is not running; no sandbox was left running.` - ); - } else if ( - shouldCleanupSandbox && - error instanceof CloudFleetSandboxProvisionError && - error.confirmedProvisioned && - error.cloudWorkspaceId && - error.sandboxId - ) { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: error.cloudWorkspaceId, - sandboxId: error.sandboxId, - ...(error.providerId === undefined ? {} : { providerId: error.providerId }), - }) - .catch((cleanupError) => { - deps.warn( - `Provisioning failed after Cloud confirmed sandbox '${error.sandboxId}', and automatic cleanup failed: ${ - cleanupError instanceof Error ? cleanupError.message : String(cleanupError) - }` - ); - }); - } else if (error instanceof CloudFleetSandboxProvisionError && error.noSandboxCreated) { - deps.warn(error.message); - } else if (error instanceof CloudFleetSandboxProvisionError && error.outcomeUnknown) { - deps.warn( - `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ - error.nodeName ?? effectiveSandboxName ?? 'the requested sandbox' - }'${ - sandboxId === undefined ? '' : ` before retrying with --sandbox-id '${sandboxId}'` - } so a sandbox is not left running.` - ); - } else if ( - shouldCleanupSandbox && - error instanceof CloudFleetSandboxProvisionError && - error.cloudWorkspaceId && - error.sandboxId - ) { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: error.cloudWorkspaceId, - sandboxId: error.sandboxId, - ...(error.providerId === undefined ? {} : { providerId: error.providerId }), - }) - .catch((cleanupError) => { - deps.warn( - `Provisioning failed after Cloud created sandbox '${error.sandboxId}', and automatic cleanup failed: ${ - cleanupError instanceof Error ? cleanupError.message : String(cleanupError) - }` - ); - }); - } - if (shouldCleanupSandbox && sandbox && sandbox.outcome !== 'reused') { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: sandbox.cloudWorkspaceId, - sandboxId: sandbox.sandboxId, - ...(sandbox.providerId === undefined ? {} : { providerId: sandbox.providerId }), - }) - .catch((cleanupError) => { - deps.warn( - `Sandbox repository verification failed and cleanup also failed: ${ - cleanupError instanceof Error ? cleanupError.message : String(cleanupError) - }` + ...(channel ? { channels: [channel] } : {}), + ...(model ? { model } : {}), + spawnMetadata: { ...registrationMetadata, ...(sessionRef ? { session_ref: sessionRef } : {}) }, + confirm: options.confirm !== false, + confirmTimeoutMs, + verifySandbox: (ensured) => + assertSandboxRepositoryRevision(ensured, checkoutRepository ? sandboxRepository : undefined), + resolveWorkerCwd: (ensured) => { + if (workerCwd) return workerCwd; + if ( + liveRepository && + sandboxRepository && + ensured.outcome === 'provisioned' && + ensured.relayfileMounted + ) { + return liveRelayfileWorkerCwd( + ensured.relayfileMountPath ?? '/workspace', + liveRepository, + sandboxRepository.repositoryRelativeCwd ); - }); - } - throw error; - } - if (sandbox.outcome !== 'provisioning_timeout' && sandbox.relaycastTarget) { - // When Cloud returns a closed, server-owned target, apply it for any - // provider and outcome before registration, spawn, or launcher release. - // Rebuild both credentials and origin before any registration, spawn, - // or launcher release, then prove the authenticated client sees the - // exact workspace Cloud returned. - try { - const target = sandbox.relaycastTarget; - const returnedRelayWorkspaceId = - 'relayWorkspaceId' in sandbox ? sandbox.relayWorkspaceId?.trim() : undefined; - if ( - (returnedRelayWorkspaceId !== undefined && - target.workspaceId.trim() !== returnedRelayWorkspaceId) || - (sandbox.outcome === 'provisioned' && !returnedRelayWorkspaceId) - ) { - throw new Error('Cloud returned a Relaycast target for a different workspace.'); - } - relaycastClientOptions = { - ...relaycastClientOptions, - workspaceKey: target.relaycastApiKey, - baseUrl: target.baseUrl, - }; - workspaceRelay = deps.sdk.createWorkspaceRelay(relaycastClientOptions); - const postEnsureWorkspace = await workspaceRelay.workspace.info(); - const postEnsureWorkspaceId = postEnsureWorkspace.id?.trim(); - if ( - !postEnsureWorkspaceId || - (sandbox.outcome === 'provisioned' && postEnsureWorkspaceId !== returnedRelayWorkspaceId) || - postEnsureWorkspaceId !== target.workspaceId.trim() - ) { - throw new Error( - 'Cloud returned a Relaycast workspace that could not be verified on the selected gateway.' - ); - } - const persistedTarget = sandbox.relaycastCloudApiUrl - ? deps.persistWorkspaceRelaycastTarget(workspaceSelection, target, sandbox.relaycastCloudApiUrl) - : deps.persistWorkspaceRelaycastTarget(workspaceSelection, target); - if (!persistedTarget) { + } + return undefined; + }, + resolveTask: (ensured, sandboxWorkerCwd) => { if ( - target.route === 'canonical' && - target.baseUrl === DEV_RELAYCAST_ORIGIN && - sandbox.relaycastCloudApiUrl !== DEV_CLOUD_API_URL + checkoutRepository && + sandboxRepository && + mountSandboxRelayfile && + ensured.outcome === 'provisioned' ) { + return `${task}\n\nAgent Relay sandbox context: Relayfile records are available at ${ensured.relayfileMountPath ?? '/workspace'}. The source checkout is separate; use ${sandboxWorkerCwd ?? 'the worker checkout'} for repository files and the mount for Relayfile records.`; + } + const liveSandboxContext = + ensured.outcome === 'provisioned' && liveRepository && sandboxRepository + ? [ + `Agent Relay sandbox context: ${liveRepository.repository} is mounted as a live Relayfile working tree at ${liveRelayfileWorkerCwd( + ensured.relayfileMountPath ?? '/workspace', + liveRepository, + '' + )}. Its exact source revision is ${liveRepository.revision}.`, + ...(sandboxMountPaths?.includes(`${path.posix.dirname(liveRepository.sentinelPath)}/**`) + ? [ + `The same attestation is recorded at ${mountedRelayfilePath( + ensured.relayfileMountPath ?? '/workspace', + liveRepository.sentinelPath + )}.`, + ] + : []), + ...(sandboxMountPaths?.includes('/.skills/**') + ? [ + `Workspace skills are under ${mountedRelayfilePath( + ensured.relayfileMountPath ?? '/workspace', + '/.skills' + )}.`, + ] + : []), + 'The Relayfile daemon synchronizes this tree; it intentionally has no .git directory.', + ].join(' ') + : undefined; + return liveSandboxContext ? `${task}\n\n${liveSandboxContext}` : task; + }, + }, + { + ensureCloudFleetSandbox: deps.ensureCloudFleetSandbox, + deleteCloudFleetSandbox: deps.deleteCloudFleetSandbox, + resolveWorkspaceByKey: deps.resolveWorkspaceByKey, + createWorkspaceRelay: (relayOptions) => deps.sdk.createWorkspaceRelay(relayOptions), + createAgentRelay: (relayOptions) => deps.sdk.createAgentRelay(relayOptions), + persistRelaycastTarget: (target, { sandbox }) => { + const persistedTarget = sandbox.relaycastCloudApiUrl + ? deps.persistWorkspaceRelaycastTarget( + workspaceSelection, + target, + sandbox.relaycastCloudApiUrl + ) + : deps.persistWorkspaceRelaycastTarget(workspaceSelection, target); + if (!persistedTarget) { throw new Error( - `Cloud returned the DEV canonical Relaycast target, but relaycastCloudApiUrl was not exactly ${DEV_CLOUD_API_URL}; refusing to persist an untrusted route.` + 'Cloud returned a Relaycast target, but no durable project session is available for follow-up attach.' ); } - throw new Error( - 'Cloud returned a Relaycast target, but no durable project session is available for follow-up attach.' - ); - } - } catch (error) { - if (shouldCleanupSandbox && sandbox.outcome === 'provisioned') { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: sandbox.cloudWorkspaceId, - sandboxId: sandbox.sandboxId, - ...(sandbox.providerId === undefined ? {} : { providerId: sandbox.providerId }), - }) - .catch((cleanupError) => { - deps.warn( - `Relaycast workspace verification failed and sandbox cleanup also failed: ${ - cleanupError instanceof Error ? cleanupError.message : String(cleanupError) - }` - ); - }); - } - throw error; - } - } else if (sandbox.outcome !== 'provisioning_timeout') { - // Older non-Agent37 Cloud responses can omit a target. In that - // compatibility case, keep every subsequent client on the canonical - // workspace selection; a stale persisted Agent37 target must not - // leak into registration, dispatch, or launcher release. - relaycastClientOptions = legacyWorkspaceClientOptions; - } - if (sandbox.outcome === 'provisioning_timeout') { - if (shouldCleanupSandbox) { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: sandbox.cloudWorkspaceId, - sandboxId: sandbox.sandboxId, - ...(sandbox.providerId === undefined ? {} : { providerId: sandbox.providerId }), - }) - .catch((error) => { - deps.warn( - `The timed-out sandbox could not be cleaned up automatically: ${ - error instanceof Error ? error.message : String(error) - }` - ); - }); - } - throw new Error( - `Sandbox node '${sandbox.nodeName}' did not become ready within ${sandbox.waitedMs}ms.` - ); - } - if ( - mountSandboxRelayfile && - (sandbox.outcome !== 'provisioned' || sandbox.relayfileMounted !== true) - ) { - if (shouldCleanupSandbox && sandbox.outcome === 'provisioned') { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: sandbox.cloudWorkspaceId, - sandboxId: sandbox.sandboxId, - ...(sandbox.providerId === undefined ? {} : { providerId: sandbox.providerId }), - }) - .catch((error) => { - deps.warn( - `The unmounted sandbox could not be cleaned up automatically and may still be running: ${ - error instanceof Error ? error.message : String(error) - }` - ); - }); + }, + warn: deps.warn, } - throw new Error('Cloud returned a sandbox node without the required Relayfile mount.'); - } - targetNode = sandbox.nodeName; - if ( - liveRepository && - sandboxRepository && - !workerCwd && - sandbox.outcome === 'provisioned' && - sandbox.relayfileMounted - ) { - workerCwd = liveRelayfileWorkerCwd( - sandbox.relayfileMountPath ?? '/workspace', - liveRepository, - sandboxRepository.repositoryRelativeCwd - ); - } - if (!workerCwd && sandbox.outcome === 'provisioned' && sandbox.relayfileMounted) { - workerCwd = sandbox.relayfileMountPath ?? '/workspace'; - } + ); + printJson(deps.sdk, { + sandbox: handle.sandbox, + attachCommand: sandboxAttachCommand(name, attachProjectRoot), + invocation: spawnInvocationWithMergedPlacement( + handle.invocation as unknown as Record + ), + }); + return; } if (targetNode) { let launcherName: string | undefined; + let workspaceRelay: ReturnType | undefined; try { - // Agent tokens are scoped to a Relaycast deployment. Never replay a - // canonical token after Cloud has selected the isolated shard; mint - // a temporary launcher on the validated target instead. - // A sandbox dispatch always mints a launcher on the transport Cloud - // selected (or the canonical compatibility transport when an older - // non-Agent37 response omitted the target). Ambient agent tokens do - // not carry enough provenance to prove they belong to that transport. - let agentToken = sandbox ? undefined : resolveAgentToken(clientOptions); + let agentToken = resolveAgentToken(clientOptions); if (!agentToken) { - workspaceRelay ??= deps.sdk.createWorkspaceRelay( - sandbox?.relaycastTarget ? relaycastClientOptions : legacyWorkspaceClientOptions - ); + workspaceRelay = deps.sdk.createWorkspaceRelay(clientOptions); const pendingLauncherName = `fleet-spawn-launcher-${randomUUID().slice(0, 8)}`; const launcher = await workspaceRelay.workspace.register( { @@ -1085,48 +907,12 @@ export function registerFleetCommands( } } - // A sandbox launcher token is already scoped to the exact workspace - // and Relaycast deployment selected by Cloud. Keep the workspace key - // only on `workspaceRelay`, where it mints and releases that token; - // passing both authorities to the agent client is rejected by the - // SDK and would prevent every sandbox placement from dispatching. - const relay = deps.sdk.createAgentRelay( - sandbox - ? { token: agentToken, baseUrl: relaycastClientOptions.baseUrl } - : { ...relaycastClientOptions, token: agentToken } - ); + const relay = deps.sdk.createAgentRelay({ ...clientOptions, token: agentToken }); // Placement alone only proves the node accepted the dispatch. A node // running an obsolete broker advertises `spawn:` capacity, acks // the invocation and launches nothing, which is indistinguishable from // success here — so wait for the node to confirm unless asked not to. const confirm = options.confirm !== false; - const liveSandboxContext = - sandbox?.outcome === 'provisioned' && liveRepository && sandboxRepository - ? [ - `Agent Relay sandbox context: ${liveRepository.repository} is mounted as a live Relayfile working tree at ${liveRelayfileWorkerCwd( - sandbox.relayfileMountPath ?? '/workspace', - liveRepository, - '' - )}. Its exact source revision is ${liveRepository.revision}.`, - ...(sandboxMountPaths?.includes(`${path.posix.dirname(liveRepository.sentinelPath)}/**`) - ? [ - `The same attestation is recorded at ${mountedRelayfilePath( - sandbox.relayfileMountPath ?? '/workspace', - liveRepository.sentinelPath - )}.`, - ] - : []), - ...(sandboxMountPaths?.includes('/.skills/**') - ? [ - `Workspace skills are under ${mountedRelayfilePath( - sandbox.relayfileMountPath ?? '/workspace', - '/.skills' - )}.`, - ] - : []), - 'The Relayfile daemon synchronizes this tree; it intentionally has no .git directory.', - ].join(' ') - : undefined; const invocation = await relay.messaging.placement.spawn({ capability: `spawn:${cli}`, node: targetNode, @@ -1136,16 +922,7 @@ export function registerFleetCommands( input: { name, cli, - task: - sandbox && - checkoutRepository && - sandboxRepository && - mountSandboxRelayfile && - sandbox.outcome === 'provisioned' - ? `${task}\n\nAgent Relay sandbox context: Relayfile records are available at ${sandbox.relayfileMountPath ?? '/workspace'}. The source checkout is separate; use ${workerCwd ?? 'the worker checkout'} for repository files and the mount for Relayfile records.` - : liveSandboxContext - ? `${task}\n\n${liveSandboxContext}` - : task, + task, ...(channel ? { channels: [channel] } : {}), ...(model ? { model } : {}), ...(workerCwd ? { worker_cwd: workerCwd } : {}), @@ -1153,47 +930,9 @@ export function registerFleetCommands( ...(sessionRef ? { session_ref: sessionRef } : {}), }, }); - const printableSandbox = - (sandbox?.outcome === 'provisioned' || sandbox?.outcome === 'reused') && sandbox.relaycastTarget - ? { - ...sandbox, - relaycastTarget: { - route: sandbox.relaycastTarget.route, - baseUrl: sandbox.relaycastTarget.baseUrl, - workspaceId: sandbox.relaycastTarget.workspaceId, - }, - } - : sandbox; printJson(deps.sdk, { - ...(sandbox - ? { - sandbox: printableSandbox, - attachCommand: sandboxAttachCommand(name, attachProjectRoot), - } - : {}), invocation: spawnInvocationWithMergedPlacement(invocation as unknown as Record), }); - } catch (error) { - if ( - shouldCleanupSandbox && - sandbox?.outcome === 'provisioned' && - !(error instanceof RelayPlacementError && error.state === 'unconfirmed_may_be_running') - ) { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: sandbox.cloudWorkspaceId, - sandboxId: sandbox.sandboxId, - ...(sandbox.providerId === undefined ? {} : { providerId: sandbox.providerId }), - }) - .catch((cleanupError) => { - deps.warn( - `Spawn failed and the sandbox could not be cleaned up automatically: ${ - cleanupError instanceof Error ? cleanupError.message : String(cleanupError) - }` - ); - }); - } - throw error; } finally { if (launcherName && workspaceRelay) { await workspaceRelay.workspace diff --git a/packages/cli/src/cli/lib/sdk-client.ts b/packages/cli/src/cli/lib/sdk-client.ts index 0bc6a1aee9..0aac3a7a4c 100644 --- a/packages/cli/src/cli/lib/sdk-client.ts +++ b/packages/cli/src/cli/lib/sdk-client.ts @@ -1,23 +1,17 @@ -import path from 'node:path'; - import { AgentRelay, type AgentRelayAgent } from '@agent-relay/sdk'; -import { DEV_CLOUD_API_URL, DEV_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; import { resolveBaseUrl, resolveWorkspaceSelection, resolveWorkspaceTransport, } from '@agent-relay/cloud/workspace-transport'; export { + persistWorkspaceRelaycastTarget, resolveBaseUrl, resolveWorkspaceSelection, resolveWorkspaceTransport, type WorkspaceTransport, } from '@agent-relay/cloud/workspace-transport'; -import { - writeProjectWorkspaceTargetIfSelectionCurrent, - type WorkspaceSelection, - type WorkspaceKeySource, -} from '@agent-relay/cloud/workspace-key'; +import { type WorkspaceSelection, type WorkspaceKeySource } from '@agent-relay/cloud/workspace-key'; /** Options shared by the SDK-backed (Relaycast) CLI command groups. */ export interface SdkClientOptions { @@ -65,39 +59,6 @@ export function resolveWorkspaceKey(options: SdkClientOptions = {}): string { return resolveWorkspaceKeyWithSource(options).key; } -/** Persist a server-selected target only while the captured project selection is still current. */ -export function persistWorkspaceRelaycastTarget( - selection: WorkspaceSelection | undefined, - target: { - route: 'canonical' | 'agent37-isolated'; - baseUrl: string; - workspaceId: string; - relaycastApiKey: string; - }, - relaycastCloudApiUrl?: string -): boolean { - if (!selection) return false; - if ( - target.route === 'canonical' && - target.baseUrl === DEV_RELAYCAST_ORIGIN && - relaycastCloudApiUrl !== DEV_CLOUD_API_URL - ) { - return false; - } - const selectionWithProjectDir = selection as WorkspaceSelection & { projectDataDir?: string }; - const dataDir = - selectionWithProjectDir?.projectDataDir ?? - (selection?.source === 'project' && selection.origin ? path.dirname(selection.origin) : undefined); - if (!dataDir) return false; - return writeProjectWorkspaceTargetIfSelectionCurrent(dataDir, selection, { - workspaceId: target.workspaceId, - relaycastRoute: target.route, - relaycastBaseUrl: target.baseUrl, - ...(relaycastCloudApiUrl ? { relaycastCloudApiUrl } : {}), - relaycastApiKey: target.relaycastApiKey, - }); -} - export function resolveAgentToken(options: SdkClientOptions = {}): string | undefined { return trimOrUndefined(options.token) ?? trimOrUndefined(env(options).RELAY_AGENT_TOKEN); } diff --git a/packages/cloud/src/attach.test.ts b/packages/cloud/src/attach.test.ts index 53c58bb4bf..4b77108a1b 100644 --- a/packages/cloud/src/attach.test.ts +++ b/packages/cloud/src/attach.test.ts @@ -49,7 +49,7 @@ it('discovers one agent and pipes raw bytes through a private socket, then remov expect((await stat(proxy!.socketPath)).mode & 0o777).toBe(0o600); local = connect(proxy!.socketPath); await once(local, 'connect'); - send(remote, 'terminal.ready', { screen: 'hello' }); + send(remote, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); expect((await once(local, 'data'))[0].toString()).toBe('hello'); const frame = once(remote, 'message'); const bytes = Buffer.from([0x00, 0x1b, 0xff, 0xc3, 0xa9]); @@ -84,7 +84,7 @@ it('view sockets cannot send terminal input and reject additional clients', asyn const { remote } = await setup('view'); local = connect(proxy!.socketPath); await once(local, 'connect'); - send(remote, 'terminal.ready', { screen: 'ready' }); + send(remote, 'terminal.ready', { screen: Buffer.from('ready').toString('base64') }); await once(local, 'data'); const received = vi.fn(); remote.on('message', received); diff --git a/packages/cloud/src/fleet.ts b/packages/cloud/src/fleet.ts index f34cc48366..84e1ac86fc 100644 --- a/packages/cloud/src/fleet.ts +++ b/packages/cloud/src/fleet.ts @@ -388,7 +388,13 @@ export { ensureCloudFleetSandbox, deleteCloudFleetSandbox, CloudFleetSandboxProvisionError, + DEV_CLOUD_API_URL, + DEV_RELAYCAST_ORIGIN, + type CloudFleetRelaycastTarget, + type CloudFleetSandboxProviderId, + type DeleteCloudFleetSandboxInput, type EnsureCloudFleetSandboxInput, type EnsureCloudFleetSandboxResult, type CloudFleetSandboxRequestOptions, } from './fleet-sandbox.js'; +export { resolveWorkspaceByKey } from './workspaces.js'; diff --git a/packages/cloud/src/workspace-transport.ts b/packages/cloud/src/workspace-transport.ts index 14c90ebe24..8f4c06a6a0 100644 --- a/packages/cloud/src/workspace-transport.ts +++ b/packages/cloud/src/workspace-transport.ts @@ -7,6 +7,7 @@ import { } from './fleet-sandbox.js'; import { resolveWorkspaceSelection as resolveCloudWorkspaceSelection, + writeProjectWorkspaceTargetIfSelectionCurrent, type WorkspaceSelection, type WorkspaceKeySource, } from './workspace-key.js'; @@ -175,3 +176,36 @@ function validatePersistedRelaycastBaseUrl(selection: WorkspaceSelection | undef } return parsed.origin; } + +/** Persist a server-selected target only while the captured project selection is still current. */ +export function persistWorkspaceRelaycastTarget( + selection: WorkspaceSelection | undefined, + target: { + route: 'canonical' | 'agent37-isolated'; + baseUrl: string; + workspaceId: string; + relaycastApiKey: string; + }, + relaycastCloudApiUrl?: string +): boolean { + if (!selection) return false; + if ( + target.route === 'canonical' && + target.baseUrl === DEV_RELAYCAST_ORIGIN && + relaycastCloudApiUrl !== DEV_CLOUD_API_URL + ) { + return false; + } + const selectionWithProjectDir = selection as WorkspaceSelection & { projectDataDir?: string }; + const dataDir = + selectionWithProjectDir?.projectDataDir ?? + (selection?.source === 'project' && selection.origin ? path.dirname(selection.origin) : undefined); + if (!dataDir) return false; + return writeProjectWorkspaceTargetIfSelectionCurrent(dataDir, selection, { + workspaceId: target.workspaceId, + relaycastRoute: target.route, + relaycastBaseUrl: target.baseUrl, + ...(relaycastCloudApiUrl ? { relaycastCloudApiUrl } : {}), + relaycastApiKey: target.relaycastApiKey, + }); +} diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 404786591e..f8683eb032 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -150,6 +150,37 @@ the result when the engine acknowledgment can be matched to the current roster. A successful dispatch remains successful if that optional metadata is missing or the roster read lags; the raw acknowledgment IDs remain available. +## Fleet sandboxes + +`@agent-relay/sdk/fleet` exports `spawnFleetSandbox`, the same path +`agent-relay fleet spawn --sandbox` uses. It provisions a Cloud sandbox, starts +the agent harness on it, waits for the node to confirm the launch, checks that +the agent landed on that sandbox's node, and returns a handle to the live agent: + +```ts +import { spawnFleetSandbox } from '@agent-relay/sdk/fleet'; + +const agent = await spawnFleetSandbox({ + cli: 'claude', + name: 'reviewer', + task: 'Review the draft under /workspace/docs', + provider: 'e2b', + relayfilePaths: ['/docs/**', '/work/**'], + readonlyPaths: ['/docs/**'], // Cloud mounts these subtrees read-only +}); + +const terminal = await agent.attach({ mode: 'drive' }); // { socketPath, finished, close } +// ... pipe stdio to terminal.socketPath ... +await terminal.close(); +await agent.destroy(); // releases the agent; deletes the sandbox when this call provisioned it +``` + +The handle carries `sandboxId`, `nodeId`, `nodeName`, and `agentName`. A +caller-declared `sandboxId` is retained: failure cleanup and `destroy()` never +delete it. A non-empty `env` is rejected, because fleet spawn cannot deliver a +per-agent environment yet. `@agent-relay/sdk/attach` exports +`startFleetNodeAttachProxy` for attaching to any fleet node agent. + ## Events `relay.addListener(selector, handler)` accepts a dotted event name, a `*`/prefix wildcard, or a fluent predicate, and always hands the handler one discriminated event object. It returns an unsubscribe function. diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 39f02143f7..53f529219d 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -64,7 +64,7 @@ "build": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\" && npx tsc -p tsconfig.build.json", "build:full": "npm run build", "check": "tsc -p tsconfig.json --noEmit", - "test": "vitest run src/__tests__/activity-reducer.test.ts src/__tests__/agent-relay.test.ts src/__tests__/facade.test.ts src/__tests__/listeners.test.ts src/__tests__/integrations.test.ts src/__tests__/messaging.test.ts src/__tests__/delivery-actions.test.ts src/__tests__/relaycast-errors.test.ts src/__tests__/register-action-relay.test.ts src/__tests__/thin-client.test.ts src/__tests__/typed-action-handle.test.ts src/__tests__/webhooks.test.ts src/__tests__/event-fanin.test.ts src/__tests__/observer-source.test.ts", + "test": "vitest run src/__tests__/activity-reducer.test.ts src/__tests__/agent-relay.test.ts src/__tests__/facade.test.ts src/__tests__/listeners.test.ts src/__tests__/integrations.test.ts src/__tests__/messaging.test.ts src/__tests__/delivery-actions.test.ts src/__tests__/relaycast-errors.test.ts src/__tests__/register-action-relay.test.ts src/__tests__/thin-client.test.ts src/__tests__/typed-action-handle.test.ts src/__tests__/webhooks.test.ts src/__tests__/event-fanin.test.ts src/__tests__/observer-source.test.ts src/__tests__/fleet-spawn.test.ts", "test:types": "vitest run --typecheck.only --typecheck.tsconfig tsconfig.typetest.json", "prepack": "npm run build" }, diff --git a/packages/sdk/src/__tests__/fleet-spawn.test.ts b/packages/sdk/src/__tests__/fleet-spawn.test.ts new file mode 100644 index 0000000000..34a1526dfb --- /dev/null +++ b/packages/sdk/src/__tests__/fleet-spawn.test.ts @@ -0,0 +1,241 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { spawnFleetSandbox, FleetSandboxSpawnError, type SpawnFleetSandboxDependencies } from '../fleet.js'; + +const SANDBOX_ID = 'sbx_3f2b8c4e-1d2a-4b5c-8d9e-0a1b2c3d4e5f'; + +function provisioned(overrides: Record = {}) { + return { + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cw_1', + nodeId: 'node_1', + nodeName: 'fleet-sandbox-node', + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_1', + relayfileMounted: true, + relayfileMountPath: '/workspace', + providerId: 'e2b' as const, + ...overrides, + }; +} + +function harness( + options: { + ensure?: () => Promise; + spawn?: (input: Record) => Promise; + } = {} +) { + const calls: string[] = []; + const ensureCloudFleetSandbox = vi.fn(async (input: Record) => { + calls.push('ensure'); + void input; + return (options.ensure ?? (async () => provisioned()))(); + }); + const deleteCloudFleetSandbox = vi.fn(async (_input: Record) => { + calls.push('delete-sandbox'); + }); + const release = vi.fn(async (input: { name: string }) => { + calls.push(`release:${input.name}`); + }); + const register = vi.fn(async (input: { name: string }) => { + calls.push(`register:${input.name}`); + return { token: 'launcher-token' }; + }); + const workspaceRelay = { + workspace: { register, release, info: vi.fn(async () => ({ id: 'rw_1' })) }, + }; + const spawn = vi.fn(async (input: Record) => { + calls.push('spawn'); + if (options.spawn) return options.spawn(input); + return { + id: 'inv_1', + status: 'completed', + node: { id: 'node_1', name: 'fleet-sandbox-node', status: 'online', capabilities: [] }, + placement: { + capability: 'spawn:claude', + node: 'fleet-sandbox-node', + attempts: 1, + queued: false, + state: 'ready', + confirmed: true, + }, + }; + }); + const createWorkspaceRelay = vi.fn((_options: Record) => workspaceRelay); + const createAgentRelay = vi.fn((_options: Record) => ({ + messaging: { placement: { spawn } }, + })); + const startFleetNodeAttachProxy = vi.fn(async (_options: Record) => ({ + socketPath: '/tmp/attach.sock', + finished: Promise.resolve(0), + close: async () => undefined, + })); + const deps = { + ensureCloudFleetSandbox, + deleteCloudFleetSandbox, + createWorkspaceRelay, + createAgentRelay, + startFleetNodeAttachProxy, + warn: vi.fn(), + } as unknown as SpawnFleetSandboxDependencies; + return { + deps, + calls, + ensureCloudFleetSandbox, + deleteCloudFleetSandbox, + release, + spawn, + createAgentRelay, + startFleetNodeAttachProxy, + }; +} + +const base = { + cli: 'claude', + name: 'sandbox-worker', + task: 'Review the repository', + workspaceId: 'rw_1', + workspaceKey: 'rk_live_test', + provider: 'e2b' as const, +}; + +describe('spawnFleetSandbox', () => { + it('provisions a sandbox, starts the harness on it, and returns a live attachable handle', async () => { + const h = harness(); + const handle = await spawnFleetSandbox( + { ...base, readonlyPaths: ['/docs/**'], relayfilePaths: ['/docs/**', '/work/**'] }, + h.deps + ); + + const ensureInput = h.ensureCloudFleetSandbox.mock.calls[0][0]; + expect(ensureInput).toMatchObject({ + workspaceId: 'rw_1', + requiredCapability: 'spawn:claude', + maxAgents: 1, + mountRelayfile: true, + readonlyPaths: ['/docs/**'], + relayfilePaths: ['/docs/**', '/work/**'], + providerId: 'e2b', + forceProvision: true, + }); + expect(ensureInput.sandboxId).toMatch(/^sbx_[0-9a-f-]{36}$/); + + const spawnInput = h.spawn.mock.calls[0][0] as Record; + expect(spawnInput).toMatchObject({ + capability: 'spawn:claude', + node: 'fleet-sandbox-node', + confirm: true, + }); + expect(spawnInput.input).toMatchObject({ + name: 'sandbox-worker', + cli: 'claude', + task: 'Review the repository', + worker_cwd: '/workspace', + }); + + expect(handle).toMatchObject({ + sandboxId: SANDBOX_ID, + nodeId: 'node_1', + nodeName: 'fleet-sandbox-node', + agentName: 'sandbox-worker', + ownsSandbox: true, + }); + // The temporary launcher identity never outlives the spawn. + expect(h.calls.filter((call) => call.startsWith('release:fleet-spawn-launcher-'))).toHaveLength(1); + + const proxy = await handle.attach({ mode: 'drive' }); + expect(proxy.socketPath).toBe('/tmp/attach.sock'); + expect(h.startFleetNodeAttachProxy).toHaveBeenCalledWith( + expect.objectContaining({ + node: 'fleet-sandbox-node', + agent: 'sandbox-worker', + mode: 'drive', + workspaceKey: 'rk_live_test', + }) + ); + }); + + it('tears down idempotently: releases the agent, then deletes the sandbox it provisioned', async () => { + const h = harness(); + const handle = await spawnFleetSandbox(base, h.deps); + h.calls.length = 0; + await Promise.all([handle.destroy(), handle.destroy()]); + await handle.destroy(); + expect(h.calls).toEqual(['release:sandbox-worker', 'delete-sandbox']); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledWith({ + cloudWorkspaceId: 'cw_1', + sandboxId: SANDBOX_ID, + providerId: 'e2b', + }); + }); + + it('retains a caller-declared sandbox on destroy', async () => { + const h = harness(); + const handle = await spawnFleetSandbox({ ...base, sandboxId: SANDBOX_ID }, h.deps); + expect(handle.ownsSandbox).toBe(false); + await handle.destroy(); + expect(h.deleteCloudFleetSandbox).not.toHaveBeenCalled(); + }); + + it('fails closed and cleans up when the agent lands on a different node', async () => { + const h = harness({ + spawn: async () => ({ + id: 'inv_1', + status: 'completed', + placement: { + capability: 'spawn:claude', + node: 'some-other-node', + attempts: 1, + queued: false, + state: 'ready', + confirmed: true, + }, + }), + }); + await expect(spawnFleetSandbox(base, h.deps)).rejects.toMatchObject({ + name: 'FleetSandboxSpawnError', + code: 'placement_mismatch', + }); + expect(h.calls).toContain('release:sandbox-worker'); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + }); + + it('deletes the provisioned sandbox when the harness fails to start', async () => { + const h = harness({ + spawn: async () => { + throw new Error('spawn failed on node'); + }, + }); + await expect(spawnFleetSandbox(base, h.deps)).rejects.toThrow('spawn failed on node'); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + }); + + it('deletes a timed-out sandbox and reports the timeout', async () => { + const h = harness({ + ensure: async () => ({ + outcome: 'provisioning_timeout', + cloudWorkspaceId: 'cw_1', + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_1', + nodeName: 'fleet-sandbox-node', + waitedMs: 90_000, + providerId: 'e2b', + }), + }); + await expect(spawnFleetSandbox(base, h.deps)).rejects.toThrow(/did not become ready within 90000ms/); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + expect(h.spawn).not.toHaveBeenCalled(); + }); + + it('omits an empty readonlyPaths list and rejects env it cannot deliver', async () => { + const h = harness(); + await spawnFleetSandbox({ ...base, readonlyPaths: [], env: {} }, h.deps); + expect(h.ensureCloudFleetSandbox.mock.calls[0][0]).not.toHaveProperty('readonlyPaths'); + + const rejected = harness(); + await expect(spawnFleetSandbox({ ...base, env: { SECRET: 'x' } }, rejected.deps)).rejects.toBeInstanceOf( + FleetSandboxSpawnError + ); + expect(rejected.ensureCloudFleetSandbox).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/sdk/src/fleet-spawn.ts b/packages/sdk/src/fleet-spawn.ts new file mode 100644 index 0000000000..1bb1529a1e --- /dev/null +++ b/packages/sdk/src/fleet-spawn.ts @@ -0,0 +1,647 @@ +/** + * `spawnFleetSandbox` — the typed SDK contract for `agent-relay fleet spawn + * --sandbox`. It provisions a Cloud fleet sandbox, starts the agent harness on + * it, confirms the node the agent landed on, and returns a handle to the live + * agent that can be attached to immediately and torn down idempotently. + * + * The CLI command drives this same function, so the two cannot drift. + */ +import { randomUUID } from 'node:crypto'; + +import { + CloudFleetSandboxProvisionError, + DEV_CLOUD_API_URL, + DEV_RELAYCAST_ORIGIN, + deleteCloudFleetSandbox, + ensureCloudFleetSandbox, + resolveWorkspaceByKey, + type CloudFleetRelaycastTarget, + type EnsureCloudFleetSandboxInput, + type EnsureCloudFleetSandboxResult, +} from '@agent-relay/cloud/fleet'; +import { + startFleetNodeAttachProxy, + type FleetNodeAttachOptions, + type FleetNodeAttachProxy, +} from '@agent-relay/cloud/attach'; +import { + persistWorkspaceRelaycastTarget, + resolveBaseUrl, + resolveWorkspaceSelection, + resolveWorkspaceTransport, + type WorkspaceTransportOptions, +} from '@agent-relay/cloud/workspace-transport'; + +import { AgentRelay, type AgentRelayAgent } from './agent-relay.js'; +import { RelayPlacementError } from './messaging/relaycast-placement.js'; +import type { RelaySpawnPlacementAck } from './messaging/types.js'; + +export type FleetSandboxProvider = 'daytona' | 'e2b' | 'agent37'; + +const CLOUD_SANDBOX_ID_PATTERN = + /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const DEFAULT_CONFIRM_TIMEOUT_MS = 120_000; + +/** A sandbox that is ready to receive the agent (never a provisioning timeout). */ +export type FleetSandboxReadyResult = Exclude< + EnsureCloudFleetSandboxResult, + { outcome: 'provisioning_timeout' } +>; + +export interface SpawnFleetSandboxInput { + /** Agent harness to start, e.g. `claude` or `codex`. */ + cli: string; + /** Worker agent name. Defaults to `-sandbox-`. */ + name?: string; + /** Initial task. */ + task?: string; + /** Relay workspace ID used for Cloud provisioning. Resolved from the workspace selection when omitted. */ + workspaceId?: string; + /** Alias for `workspaceId`. */ + workspace?: string; + /** Workspace credential/transport selection; ambient sources are used when omitted. */ + workspaceKey?: string; + baseUrl?: string; + projectRoot?: string; + /** Advanced: full transport selection (overrides workspaceKey/baseUrl/projectRoot). */ + transport?: WorkspaceTransportOptions; + /** + * Harness environment. Fleet spawn cannot deliver per-agent environment yet, + * so a non-empty map is rejected rather than silently dropped. + */ + env?: Record; + provider?: FleetSandboxProvider; + /** + * Caller-declared `sbx_` identity. A declared sandbox is retained: it + * is never deleted by failure cleanup or by `destroy()`. + */ + sandboxId?: string; + sandboxName?: string; + /** Mount Relayfile in the sandbox (default true). */ + mountRelayfile?: boolean; + /** Relayfile subtrees (`/path/**`) to mount. */ + relayfilePaths?: string[]; + /** Relayfile subtrees (`/path/**`) Cloud must mount read-only. An empty list is ignored. */ + readonlyPaths?: string[]; + /** Static repository checkouts for the sandbox. */ + repos?: string[]; + repoRevisions?: Record; + /** Worker working directory. Defaults to the Relayfile mount root. */ + workerCwd?: string; + channels?: string[]; + model?: string; + /** Extra spawn input fields (for example declared workforce metadata or `session_ref`). */ + spawnMetadata?: Record; + /** Wait for the node to confirm the harness launched (default true). */ + confirm?: boolean; + confirmTimeoutMs?: number; + /** Validate the ensured sandbox before any agent is started; throwing aborts and cleans up. */ + verifySandbox?: (sandbox: EnsureCloudFleetSandboxResult) => void; + /** Compute the worker cwd once the sandbox mount is known. */ + resolveWorkerCwd?: (sandbox: FleetSandboxReadyResult) => string | undefined; + /** Compute the task once the sandbox and worker cwd are known. */ + resolveTask?: (sandbox: FleetSandboxReadyResult, workerCwd: string | undefined) => string; +} + +type WorkspaceRelayLike = Pick; +type AgentRelayLike = { messaging: Pick }; + +export interface SpawnFleetSandboxDependencies { + ensureCloudFleetSandbox: typeof ensureCloudFleetSandbox; + deleteCloudFleetSandbox: typeof deleteCloudFleetSandbox; + resolveWorkspaceByKey: (workspaceKey: string) => Promise<{ cloudWorkspaceId: string }>; + createWorkspaceRelay: (options: WorkspaceTransportOptions) => WorkspaceRelayLike; + createAgentRelay: (options: { token: string; baseUrl?: string }) => AgentRelayLike; + /** + * Persist Cloud's server-selected Relaycast target for follow-up commands. + * Throwing aborts the spawn (and cleans up an owned sandbox). The default + * persists to the project session when one exists and otherwise does nothing; + * the handle's `attach()` carries the verified target either way. + */ + persistRelaycastTarget: ( + target: CloudFleetRelaycastTarget, + context: { sandbox: EnsureCloudFleetSandboxResult; transport: WorkspaceTransportOptions } + ) => void; + startFleetNodeAttachProxy: (options: FleetNodeAttachOptions) => Promise; + warn: (message: string) => void; +} + +export interface FleetSandboxHandle { + sandboxId: string; + /** Cloud node ID the agent landed on. */ + nodeId: string; + nodeName: string; + agentName: string; + cloudWorkspaceId: string; + providerId?: string; + workerCwd?: string; + relayfileMountPath?: string; + /** True when this call provisioned the sandbox and `destroy()` deletes it. */ + ownsSandbox: boolean; + /** The ensured sandbox, with the Relaycast credential removed. */ + sandbox: EnsureCloudFleetSandboxResult; + invocation: RelaySpawnPlacementAck; + /** Attach to the live agent's terminal through a private local socket. */ + attach(options?: { mode?: FleetNodeAttachOptions['mode'] }): Promise; + /** Release the agent and delete the sandbox when owned. Idempotent. */ + destroy(): Promise; +} + +export type FleetSandboxSpawnErrorCode = + | 'invalid_input' + | 'unsupported_env' + | 'workspace_unresolved' + | 'provisioning_timeout' + | 'relayfile_unmounted' + | 'relaycast_target_unverified' + | 'placement_mismatch'; + +export class FleetSandboxSpawnError extends Error { + readonly code: FleetSandboxSpawnErrorCode; + constructor(code: FleetSandboxSpawnErrorCode, message: string) { + super(message); + this.name = 'FleetSandboxSpawnError'; + this.code = code; + } +} + +function defaultDependencies(): SpawnFleetSandboxDependencies { + return { + ensureCloudFleetSandbox, + deleteCloudFleetSandbox, + resolveWorkspaceByKey: (workspaceKey) => resolveWorkspaceByKey(workspaceKey), + createWorkspaceRelay: (options) => { + const { workspaceKey, baseUrl } = resolveWorkspaceTransport(options); + return new AgentRelay({ workspaceKey, baseUrl }); + }, + createAgentRelay: ({ token, baseUrl }) => + new AgentRelay({ + agentToken: token, + // A launcher token is scoped to one deployment; never let a persisted + // project route pick a different gateway for it. + baseUrl: baseUrl ?? resolveBaseUrl({ ignorePersistedRelaycastTarget: true }), + }), + persistRelaycastTarget: (target, { sandbox, transport }) => { + persistWorkspaceRelaycastTarget( + resolveWorkspaceSelection(transport), + target, + sandbox.relaycastCloudApiUrl + ); + }, + startFleetNodeAttachProxy, + warn: (message) => process.stderr.write(`${message}\n`), + }; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function nonEmpty(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +/** Remove the Relaycast credential so the result can be logged or returned. */ +export function redactFleetSandbox(sandbox: EnsureCloudFleetSandboxResult): EnsureCloudFleetSandboxResult { + if (!sandbox.relaycastTarget) return sandbox; + const { relaycastApiKey: _secret, ...target } = sandbox.relaycastTarget; + return { ...sandbox, relaycastTarget: target } as EnsureCloudFleetSandboxResult; +} + +export async function spawnFleetSandbox( + input: SpawnFleetSandboxInput, + dependencies: Partial = {} +): Promise { + const deps: SpawnFleetSandboxDependencies = { ...defaultDependencies(), ...dependencies }; + const cli = nonEmpty(input.cli); + if (!cli) throw new FleetSandboxSpawnError('invalid_input', 'A harness cli is required.'); + if (input.env && Object.keys(input.env).length > 0) { + throw new FleetSandboxSpawnError( + 'unsupported_env', + 'Fleet sandbox spawn cannot deliver a per-agent environment yet; remove env or provide it through the workspace.' + ); + } + const mountRelayfile = input.mountRelayfile !== false; + const readonlyPaths = input.readonlyPaths?.length ? [...input.readonlyPaths] : undefined; + if (readonlyPaths && !mountRelayfile) { + throw new FleetSandboxSpawnError('invalid_input', 'readonlyPaths requires mounting Relayfile.'); + } + if (input.sandboxId !== undefined && !CLOUD_SANDBOX_ID_PATTERN.test(input.sandboxId)) { + throw new FleetSandboxSpawnError( + 'invalid_input', + 'sandboxId must match lowercase sbx_ using an RFC 4122 UUID.' + ); + } + const name = nonEmpty(input.name) ?? `${cli}-sandbox-${randomUUID().slice(0, 8)}`; + const provider = input.provider; + // An explicit sandbox identity is a retained/replayable resource. Never + // delete it as collateral when a later verification or dispatch step fails; + // only clean up sandboxes whose identity this invocation minted. + const shouldCleanupSandbox = input.sandboxId === undefined; + const transport: WorkspaceTransportOptions = input.transport ?? { + ...(input.workspaceKey === undefined ? {} : { workspaceKey: input.workspaceKey }), + ...(input.baseUrl === undefined ? {} : { baseUrl: input.baseUrl }), + ...(input.projectRoot === undefined ? {} : { projectRoot: input.projectRoot }), + }; + const legacyTransport: WorkspaceTransportOptions = { + ...transport, + ...(provider === 'agent37' ? {} : { ignorePersistedRelaycastTarget: true }), + }; + let workspaceRelay: WorkspaceRelayLike | undefined; + + let relayWorkspaceId = nonEmpty(input.workspaceId) ?? nonEmpty(input.workspace); + if (!relayWorkspaceId) { + const selection = resolveWorkspaceSelection(transport); + relayWorkspaceId = nonEmpty(selection?.workspaceId); + if (!relayWorkspaceId && selection?.key && (provider === undefined || provider === 'agent37')) { + relayWorkspaceId = (await deps.resolveWorkspaceByKey(selection.key)).cloudWorkspaceId; + } + if (!relayWorkspaceId && provider !== undefined && provider !== 'agent37') { + workspaceRelay = deps.createWorkspaceRelay(legacyTransport); + relayWorkspaceId = nonEmpty((await workspaceRelay.workspace.info()).id); + } + } + if (!relayWorkspaceId) { + throw new FleetSandboxSpawnError( + 'workspace_unresolved', + 'Sandbox provisioning requires a Relay workspace identity; pass workspaceId or bind the workspace.' + ); + } + + const sandboxId = input.sandboxId ?? (input.sandboxName === undefined ? `sbx_${randomUUID()}` : undefined); + const deterministicSandboxName = + sandboxId === undefined ? undefined : `fleet-sandbox-${sandboxId.slice('sbx_'.length)}`; + if ( + input.sandboxId !== undefined && + input.sandboxName !== undefined && + input.sandboxName !== deterministicSandboxName + ) { + throw new FleetSandboxSpawnError( + 'invalid_input', + `--sandbox-name must be '${deterministicSandboxName}' when --sandbox-id is supplied; custom names cannot preserve the one-to-one sandbox identity.` + ); + } + const effectiveSandboxName = deterministicSandboxName ?? input.sandboxName; + // The unpinned/Agent37 path deliberately carries the measured heavy + // 8 CPU / 16 GiB / 20 GiB profile. Daytona and E2B cannot satisfy that + // shape, so an explicit legacy-provider selection must request the + // provider-neutral durable profile instead of becoming unroutable by + // construction. + const workloadProfile = + provider === undefined || provider === 'agent37' ? 'long-running-agent' : 'standard-long-running-agent'; + const useAsyncPreparation = sandboxId !== undefined && (provider === undefined || provider === 'agent37'); + if (useAsyncPreparation && input.sandboxId === undefined) { + // The generated identity is not persisted. Print it before Cloud work + // starts so an interrupted run can be resumed instead of duplicated. + deps.warn( + `Cloud sandbox identity: ${sandboxId}. If this command is interrupted, re-run it with --sandbox-id ${sandboxId} to resume the same sandbox instead of creating another.` + ); + } + + const deleteSandbox = (sandbox: { cloudWorkspaceId: string; sandboxId: string; providerId?: string }) => + deps.deleteCloudFleetSandbox({ + cloudWorkspaceId: sandbox.cloudWorkspaceId, + sandboxId: sandbox.sandboxId, + ...(sandbox.providerId === undefined + ? {} + : { providerId: sandbox.providerId as EnsureCloudFleetSandboxInput['providerId'] & string }), + }); + + let sandbox: EnsureCloudFleetSandboxResult | undefined; + try { + const ensureInput: EnsureCloudFleetSandboxInput = { + workspaceId: relayWorkspaceId, + requiredCapability: `spawn:${cli}`, + ...(useAsyncPreparation ? { preparationMode: 'async-v1' as const } : {}), + maxAgents: 1, + mountRelayfile, + ...(readonlyPaths === undefined ? {} : { readonlyPaths }), + ...(input.relayfilePaths === undefined ? {} : { relayfilePaths: input.relayfilePaths }), + ...(sandboxId === undefined ? {} : { sandboxId }), + forceProvision: true, + ...(provider === undefined ? {} : { providerId: provider }), + workloadProfile, + waitTimeoutMs: 90_000, + ...(effectiveSandboxName === undefined ? {} : { name: effectiveSandboxName }), + ...(input.repos === undefined ? {} : { repos: input.repos }), + ...(input.repoRevisions === undefined ? {} : { repoRevisions: input.repoRevisions }), + }; + sandbox = useAsyncPreparation + ? await deps.ensureCloudFleetSandbox(ensureInput, { + onPreparationProgress: (progress) => { + deps.warn( + `Cloud sandbox preparation: ${progress.phase} (${progress.state}, generation ${progress.generation}).` + ); + }, + }) + : await deps.ensureCloudFleetSandbox(ensureInput); + input.verifySandbox?.(sandbox); + } catch (error) { + if (error instanceof CloudFleetSandboxProvisionError && error.sandboxAbsent) { + // Cloud's terminal record for this exact identity already proves the + // provider sandbox is gone; a delete here would only race its reaper. + deps.warn( + `${error.message} Cloud confirmed sandbox '${ + error.sandboxId ?? sandboxId ?? 'the requested sandbox' + }' is not running; no sandbox was left running.` + ); + } else if ( + shouldCleanupSandbox && + error instanceof CloudFleetSandboxProvisionError && + error.confirmedProvisioned && + error.cloudWorkspaceId && + error.sandboxId + ) { + const failedSandboxId = error.sandboxId; + await deleteSandbox({ + cloudWorkspaceId: error.cloudWorkspaceId, + sandboxId: failedSandboxId, + ...(error.providerId === undefined ? {} : { providerId: error.providerId }), + }).catch((cleanupError) => { + deps.warn( + `Provisioning failed after Cloud confirmed sandbox '${failedSandboxId}', and automatic cleanup failed: ${errorMessage( + cleanupError + )}` + ); + }); + } else if (error instanceof CloudFleetSandboxProvisionError && error.noSandboxCreated) { + deps.warn(error.message); + } else if (error instanceof CloudFleetSandboxProvisionError && error.outcomeUnknown) { + deps.warn( + `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ + error.nodeName ?? effectiveSandboxName ?? 'the requested sandbox' + }'${ + sandboxId === undefined ? '' : ` before retrying with --sandbox-id '${sandboxId}'` + } so a sandbox is not left running.` + ); + } else if ( + shouldCleanupSandbox && + error instanceof CloudFleetSandboxProvisionError && + error.cloudWorkspaceId && + error.sandboxId + ) { + const failedSandboxId = error.sandboxId; + await deleteSandbox({ + cloudWorkspaceId: error.cloudWorkspaceId, + sandboxId: failedSandboxId, + ...(error.providerId === undefined ? {} : { providerId: error.providerId }), + }).catch((cleanupError) => { + deps.warn( + `Provisioning failed after Cloud created sandbox '${failedSandboxId}', and automatic cleanup failed: ${errorMessage( + cleanupError + )}` + ); + }); + } + if (shouldCleanupSandbox && sandbox && sandbox.outcome !== 'reused') { + await deleteSandbox(sandbox).catch((cleanupError) => { + deps.warn( + `Sandbox repository verification failed and cleanup also failed: ${errorMessage(cleanupError)}` + ); + }); + } + throw error; + } + + let relaycastTransport: WorkspaceTransportOptions = transport; + let verifiedTarget: CloudFleetRelaycastTarget | undefined; + if (sandbox.outcome !== 'provisioning_timeout' && sandbox.relaycastTarget) { + // When Cloud returns a closed, server-owned target, apply it for any + // provider and outcome before registration, spawn, or launcher release, + // then prove the authenticated client sees the exact workspace Cloud + // returned. + try { + const target = sandbox.relaycastTarget; + const returnedRelayWorkspaceId = + 'relayWorkspaceId' in sandbox ? sandbox.relayWorkspaceId?.trim() : undefined; + if ( + (returnedRelayWorkspaceId !== undefined && target.workspaceId.trim() !== returnedRelayWorkspaceId) || + (sandbox.outcome === 'provisioned' && !returnedRelayWorkspaceId) + ) { + throw new FleetSandboxSpawnError( + 'relaycast_target_unverified', + 'Cloud returned a Relaycast target for a different workspace.' + ); + } + relaycastTransport = { + ...relaycastTransport, + workspaceKey: target.relaycastApiKey, + baseUrl: target.baseUrl, + }; + workspaceRelay = deps.createWorkspaceRelay(relaycastTransport); + const postEnsureWorkspace = await workspaceRelay.workspace.info(); + const postEnsureWorkspaceId = postEnsureWorkspace.id?.trim(); + if ( + !postEnsureWorkspaceId || + (sandbox.outcome === 'provisioned' && postEnsureWorkspaceId !== returnedRelayWorkspaceId) || + postEnsureWorkspaceId !== target.workspaceId.trim() + ) { + throw new FleetSandboxSpawnError( + 'relaycast_target_unverified', + 'Cloud returned a Relaycast workspace that could not be verified on the selected gateway.' + ); + } + if ( + target.route === 'canonical' && + target.baseUrl === DEV_RELAYCAST_ORIGIN && + sandbox.relaycastCloudApiUrl !== DEV_CLOUD_API_URL + ) { + throw new FleetSandboxSpawnError( + 'relaycast_target_unverified', + `Cloud returned the DEV canonical Relaycast target, but relaycastCloudApiUrl was not exactly ${DEV_CLOUD_API_URL}; refusing to persist an untrusted route.` + ); + } + deps.persistRelaycastTarget(target, { sandbox, transport }); + verifiedTarget = target; + } catch (error) { + if (shouldCleanupSandbox && sandbox.outcome === 'provisioned') { + await deleteSandbox(sandbox).catch((cleanupError) => { + deps.warn( + `Relaycast workspace verification failed and sandbox cleanup also failed: ${errorMessage( + cleanupError + )}` + ); + }); + } + throw error; + } + } else if (sandbox.outcome !== 'provisioning_timeout') { + // Older non-Agent37 Cloud responses can omit a target. In that + // compatibility case, keep every subsequent client on the canonical + // workspace selection; a stale persisted Agent37 target must not leak + // into registration, dispatch, or launcher release. + relaycastTransport = legacyTransport; + } + if (sandbox.outcome === 'provisioning_timeout') { + if (shouldCleanupSandbox) { + await deleteSandbox(sandbox).catch((error) => { + deps.warn(`The timed-out sandbox could not be cleaned up automatically: ${errorMessage(error)}`); + }); + } + throw new FleetSandboxSpawnError( + 'provisioning_timeout', + `Sandbox node '${sandbox.nodeName}' did not become ready within ${sandbox.waitedMs}ms.` + ); + } + const ready: FleetSandboxReadyResult = sandbox; + if (mountRelayfile && (ready.outcome !== 'provisioned' || ready.relayfileMounted !== true)) { + if (shouldCleanupSandbox && ready.outcome === 'provisioned') { + await deleteSandbox(ready).catch((error) => { + deps.warn( + `The unmounted sandbox could not be cleaned up automatically and may still be running: ${errorMessage( + error + )}` + ); + }); + } + throw new FleetSandboxSpawnError( + 'relayfile_unmounted', + 'Cloud returned a sandbox node without the required Relayfile mount.' + ); + } + + const mountPath = + ready.outcome === 'provisioned' && ready.relayfileMounted + ? (ready.relayfileMountPath ?? '/workspace') + : undefined; + const workerCwd = input.resolveWorkerCwd?.(ready) ?? input.workerCwd ?? mountPath; + const task = input.resolveTask?.(ready, workerCwd) ?? input.task ?? ''; + const sandboxIdentity = 'sandboxId' in ready ? ready.sandboxId : (sandboxId ?? ''); + const ownsSandbox = shouldCleanupSandbox && ready.outcome === 'provisioned'; + const confirm = input.confirm !== false; + const workspaceRelayFor = () => + (workspaceRelay ??= deps.createWorkspaceRelay(verifiedTarget ? relaycastTransport : legacyTransport)); + const releaseAgent = (reason: string) => + workspaceRelayFor().workspace.release({ name, reason, deleteAgent: true }); + + let launcherName: string | undefined; + let invocation: RelaySpawnPlacementAck; + try { + // Agent tokens are scoped to a Relaycast deployment. Mint a temporary + // launcher on the transport Cloud selected (or the canonical + // compatibility transport when an older non-Agent37 response omitted the + // target); ambient agent tokens cannot prove they belong to it. + const pendingLauncherName = `fleet-spawn-launcher-${randomUUID().slice(0, 8)}`; + const launcher = await workspaceRelayFor().workspace.register( + { name: pendingLauncherName, metadata: { purpose: 'fleet-spawn-launcher' } }, + { strict: true } + ); + launcherName = pendingLauncherName; + const agentToken = launcher.token; + if (!agentToken) { + throw new Error('The temporary fleet spawn launcher did not receive an agent token.'); + } + // The launcher token is already scoped to the exact workspace and + // deployment Cloud selected; keep the workspace key only on + // `workspaceRelay`, where it mints and releases that token. + const relay = deps.createAgentRelay({ + token: agentToken, + ...(relaycastTransport.baseUrl === undefined ? {} : { baseUrl: relaycastTransport.baseUrl }), + }); + // Placement alone only proves the node accepted the dispatch. A node + // running an obsolete broker advertises `spawn:` capacity, acks the + // invocation and launches nothing — so wait for confirmation unless asked + // not to. + invocation = await relay.messaging.placement.spawn({ + capability: `spawn:${cli}`, + node: ready.nodeName, + failFast: true, + confirm, + ...(confirm ? { confirmTimeoutMs: input.confirmTimeoutMs ?? DEFAULT_CONFIRM_TIMEOUT_MS } : {}), + input: { + name, + cli, + task, + ...(input.channels?.length ? { channels: input.channels } : {}), + ...(input.model ? { model: input.model } : {}), + ...(workerCwd ? { worker_cwd: workerCwd } : {}), + ...(input.spawnMetadata ?? {}), + }, + }); + // Confirm the agent landed on the sandbox this call ensured, not merely + // on some node advertising the same capability. + const landedNodeName = invocation.placement?.node ?? invocation.node?.name; + const landedNodeId = invocation.node?.id ?? invocation.node?.nodeId; + if ( + (landedNodeName !== undefined && landedNodeName !== ready.nodeName) || + (landedNodeId !== undefined && landedNodeId !== ready.nodeId) + ) { + await releaseAgent('Fleet sandbox spawn landed on an unexpected node').catch((error) => { + deps.warn(`Releasing the misplaced agent '${name}' failed: ${errorMessage(error)}`); + }); + throw new FleetSandboxSpawnError( + 'placement_mismatch', + `Agent '${name}' landed on node '${landedNodeName ?? landedNodeId}', not sandbox node '${ready.nodeName}'.` + ); + } + } catch (error) { + if ( + shouldCleanupSandbox && + ready.outcome === 'provisioned' && + !(error instanceof RelayPlacementError && error.state === 'unconfirmed_may_be_running') + ) { + await deleteSandbox(ready).catch((cleanupError) => { + deps.warn( + `Spawn failed and the sandbox could not be cleaned up automatically: ${errorMessage(cleanupError)}` + ); + }); + } + throw error; + } finally { + if (launcherName && workspaceRelay) { + await workspaceRelay.workspace + .release({ + name: launcherName, + reason: 'Temporary fleet spawn launcher completed', + deleteAgent: true, + }) + .catch((error) => { + deps.warn(`Temporary launcher cleanup failed: ${errorMessage(error)}`); + }); + } + } + + let destroyed: Promise | undefined; + return { + sandboxId: sandboxIdentity, + nodeId: ready.nodeId, + nodeName: ready.nodeName, + agentName: name, + cloudWorkspaceId: ready.cloudWorkspaceId, + ...(ready.providerId === undefined ? {} : { providerId: ready.providerId }), + ...(workerCwd === undefined ? {} : { workerCwd }), + ...(mountPath === undefined ? {} : { relayfileMountPath: mountPath }), + ownsSandbox, + sandbox: redactFleetSandbox(ready), + invocation, + attach: async (options = {}) => { + const attachTransport = verifiedTarget + ? { workspaceKey: verifiedTarget.relaycastApiKey, baseUrl: verifiedTarget.baseUrl } + : resolveWorkspaceTransport(legacyTransport); + return deps.startFleetNodeAttachProxy({ + node: ready.nodeName, + agent: name, + mode: options.mode ?? 'drive', + workspaceKey: attachTransport.workspaceKey, + ...(attachTransport.baseUrl === undefined ? {} : { baseUrl: attachTransport.baseUrl }), + }); + }, + destroy: () => + (destroyed ??= (async () => { + const failures: string[] = []; + await releaseAgent('Fleet sandbox handle destroyed').catch((error) => { + failures.push(`agent release failed: ${errorMessage(error)}`); + }); + if (ownsSandbox) { + await deleteSandbox(ready).catch((error) => { + failures.push(`sandbox deletion failed: ${errorMessage(error)}`); + }); + } + if (failures.length > 0) { + destroyed = undefined; + throw new Error(`Fleet sandbox teardown incomplete: ${failures.join('; ')}`); + } + })()), + }; +} diff --git a/packages/sdk/src/fleet.ts b/packages/sdk/src/fleet.ts index caf0e4fd8d..07ddf2a8e7 100644 --- a/packages/sdk/src/fleet.ts +++ b/packages/sdk/src/fleet.ts @@ -1 +1,2 @@ export * from '@agent-relay/cloud/fleet'; +export * from './fleet-spawn.js'; diff --git a/packages/sdk/tsconfig.build.json b/packages/sdk/tsconfig.build.json index 975f698853..bd08700afb 100644 --- a/packages/sdk/tsconfig.build.json +++ b/packages/sdk/tsconfig.build.json @@ -30,7 +30,8 @@ "src/actions/**/*.ts", "src/session/**/*.ts", "src/attach.ts", - "src/fleet.ts" + "src/fleet.ts", + "src/fleet-spawn.ts" ], "exclude": ["src/__tests__/**"] } diff --git a/summary.md b/summary.md deleted file mode 100644 index e43f0d704f..0000000000 --- a/summary.md +++ /dev/null @@ -1,35 +0,0 @@ -## Summary - -Draft implementation of the attach and read-only-request portions of the workforce#338 upstream work. **This does not yet unblock workforce#338.** The reviewed plan's D4 decision is pending: `spawnFleetSandbox` must either orchestrate a running agent or expose a narrower ensure-only contract. No placeholder `spawnFleetSandbox` is exported. - -- Move the attach adapter and workspace transport resolution into Cloud, retaining CLI re-exports and existing broker transport fields. Add Cloud and SDK `/attach` entries, nodeId/optional-agent discovery, a private single-client raw stdio socket, idempotent cleanup, and a completion promise. -- Expose existing provisioning primitives through Cloud and SDK `/fleet`; route the CLI's ensure import through the SDK entry. Preserve existing Cloud enrollment APIs. The running-agent spawn wrapper remains pending D4. -- Validate and forward `readonlyPaths` as explicit `/path/**` subtrees; expose `fleet spawn --sandbox-readonly-path` with mount guards. -- Add socket, request-validation, CLI, and built-entry regression coverage. Add an independent read-only-request RelayFlow proof; leave historical case 1630 unchanged. - -The proxy retains broker transport fields alongside socketPath/finished/close, as the reviewed plan recommends: workforce duck-types the three consumer fields and ignores extras. Completion is an **inferred** status (0 for normal terminal closure/detach, 1 for transport failure), not a remote harness exit code. SDK re-exports add a dependency on Cloud and its transitive dependencies. No package versions changed. - -## Remaining requirements - -- Resolve reviewed-plan.md D4 and implement/test `spawnFleetSandbox` registration, placement confirmation, input mapping, and teardown. -- Cloud's server-side ensure handler/mount builder is absent from this repository. Forwarding does not enforce chmod-444; server implementation and a live Daytona write-denial proof are still required. -- Exports follow the reviewed plan's existing ESM conventions. No CommonJS build or `require` condition was added. -- No live CLI sandbox-spawn or node-attach parity run was performed. - -## Validation - -- `npm run typecheck`: passed. -- `npm run build`: passed (Rust build skipped automatically because Cargo is unavailable). -- SDK suite: 182 passed; fleet suite: 43 passed. -- Cloud suite: 450 passed, 4 skipped after removing inherited Cloud authentication variables. -- Existing and new attach tests: 42 passed, using actual local WebSocket and UNIX socket servers. -- CLI suite via `npx vitest run packages/cli/ --maxWorkers=2`: 1,652 passed, 29 skipped. `npm --prefix packages/cli test` is unavailable because that package has no test script. Tests ran without inherited Cloud/Relay credentials and with local Git file transport enabled for the temporary-repository fixture. -- Built Cloud/SDK entry-resolution regression: passed. -- New proof runner executed locally against base `c978302` and implementation `b6c3bb1`: base `absent`, head `fixed`. Both generated probes and compiled CLI-help checks passed. This was not a hosted Cloud or live Daytona run. - -## RelayFlow Proof - -- Change type: `feature` -- RelayFlow case: `sandbox-readonly-paths` - -This generated base/head probe verifies exact request forwarding and compiled CLI help. It does not provision a sandbox or prove chmod enforcement. From 8c980d44a636969a964bf7a5cd8c724fb5ebe644 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 11:38:43 -0700 Subject: [PATCH 05/19] ci(evals): build the SDK's cloud dependency before the SDK @agent-relay/sdk now imports @agent-relay/cloud/{attach,fleet,workspace-transport}, so the offline-evals job and eval:build must build session, config and cloud first. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- .github/workflows/relay-evals.yml | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/relay-evals.yml b/.github/workflows/relay-evals.yml index dc3ea0f592..ef80c7691d 100644 --- a/.github/workflows/relay-evals.yml +++ b/.github/workflows/relay-evals.yml @@ -56,7 +56,7 @@ jobs: run: npm ci - name: Build SDK - run: npm run build:sdk + run: npm run build:session && npm run build:config && npm run build:cloud && npm run build:sdk - name: Run offline evals run: npm run evals:offline diff --git a/package.json b/package.json index cbdb15133f..d1ef92211d 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "test:integration:broker": "npx tsc -p tests/integration/broker/tsconfig.json && cd tests/integration/broker && node --test dist/*.test.js", "test:integration:broker:build": "npx tsc -p tests/integration/broker/tsconfig.json", "test:integration:broker:run": "cd tests/integration/broker && node --test dist/*.test.js", - "eval:build": "npm run build:sdk && npm run build:harness-driver && npx tsc -p tests/integration/broker/evals/tsconfig.json", + "eval:build": "npm run build:session && npm run build:config && npm run build:cloud && npm run build:sdk && npm run build:harness-driver && npx tsc -p tests/integration/broker/evals/tsconfig.json", "eval:unit": "vitest run tests/integration/broker/evals", "eval:selftest": "npm run eval:build && node tests/integration/broker/dist/evals/selftest.js", "eval:toolcheck": "npm run eval:build && node tests/integration/broker/dist/evals/toolcheck-cli.js", From 31968ec0d8015e4be737d22148d7ce50e8560ce9 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 14:31:09 -0700 Subject: [PATCH 06/19] fix(sdk,cloud): address review on spawnFleetSandbox and attach - spawnFleetSandbox: run resolveWorkerCwd/resolveTask inside the cleanup boundary so a throwing hook cannot strand an owned sandbox. - Landing check uses the dispatch receipt (dispatchedNodeId/handlerNodeId); invocation.node/placement.node only echo the requested target. - spawnMetadata is spread before name/cli/task/worker_cwd so it can never replace the fields the handle attaches to and releases. - A reused node without a sandbox identity is refused instead of returning an empty sandboxId. - attach: buffer output for a not-yet-ready raw socket separately from the event-listener replay buffer (reset at each terminal.ready snapshot). - attach: trust the DEV Relaycast origin so a DEV spawn's handle.attach() works. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- packages/cli/src/cli/commands/fleet.test.ts | 10 +++- packages/cloud/src/attach.test.ts | 25 ++++++++ packages/cloud/src/attach.ts | 35 ++++++++++-- .../sdk/src/__tests__/fleet-spawn.test.ts | 57 ++++++++++++++++++- packages/sdk/src/fleet-spawn.ts | 45 ++++++++++----- 5 files changed, 149 insertions(+), 23 deletions(-) diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index ab5e78a6cb..04a97ba962 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1256,10 +1256,14 @@ describe('fleet command support', () => { }); }); - it('fleet spawn --sandbox rejects an agent that lands on another node and tears the sandbox down', async () => { + it('fleet spawn --sandbox rejects an agent dispatched to another node and tears the sandbox down', async () => { vi.stubEnv('RELAY_AGENT_TOKEN', undefined); const placement = { - spawn: vi.fn(async () => ({ invocationId: 'inv_sandbox', node: { name: 'some-other-node' } })), + spawn: vi.fn(async () => ({ + invocationId: 'inv_sandbox', + node: { name: 'e2b-codex' }, + dispatchedNodeId: 'node-2', + })), }; const release = vi.fn(async () => ({ released: true, deleted: true })); const createWorkspaceRelay = vi.fn(() => ({ @@ -1330,7 +1334,7 @@ describe('fleet command support', () => { { from: 'user' } ) ).rejects.toThrow('__exit__'); - expect(errors.join('\n')).toContain("landed on node 'some-other-node', not sandbox node 'e2b-codex'"); + expect(errors.join('\n')).toContain("landed on node 'node-2', not sandbox node 'e2b-codex'"); expect(release).toHaveBeenCalledWith( expect.objectContaining({ name: 'sandbox-worker', deleteAgent: true }) ); diff --git a/packages/cloud/src/attach.test.ts b/packages/cloud/src/attach.test.ts index 4b77108a1b..84cc781dc6 100644 --- a/packages/cloud/src/attach.test.ts +++ b/packages/cloud/src/attach.test.ts @@ -68,6 +68,23 @@ it('discovers one agent and pipes raw bytes through a private socket, then remov await expect(stat(socketPath)).rejects.toMatchObject({ code: 'ENOENT' }); }); +it('replays output to a late raw client even when an event listener already consumed it', async () => { + const { remote } = await setup(); + const events = new (await import('ws')).default(`${proxy!.brokerUrl.replace('http', 'ws')}/ws`, { + headers: { authorization: `Bearer ${proxy!.apiKey}` }, + }); + await once(events, 'open'); + send(remote, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + const consumed = once(events, 'message'); + send(remote, 'terminal.output', { chunk: ' early' }); + await consumed; + local = connect(proxy!.socketPath); + let received = ''; + local.on('data', (data: Buffer) => (received += data.toString())); + await vi.waitFor(() => expect(received).toBe('hello early')); + events.close(); +}); + it('settles finished on explicit close before terminal.ready', async () => { await setup(); await proxy!.close(); @@ -108,3 +125,11 @@ it.each([[], [{ agentName: 'a' }, { agentName: 'b' }]])( ).rejects.toMatchObject({ code: 'ambiguous_agent' }); } ); + +it('trusts the DEV Relaycast origin for attach, alongside canonical and Agent37', async () => { + const { validateFleetAttachBaseUrl } = await import('./attach.js'); + expect(validateFleetAttachBaseUrl('https://dev-cast.agentrelay.com/')).toBe( + 'https://dev-cast.agentrelay.com' + ); + expect(() => validateFleetAttachBaseUrl('https://evil.example.com')).toThrow('trusted Relaycast origin'); +}); diff --git a/packages/cloud/src/attach.ts b/packages/cloud/src/attach.ts index bb38a4f514..00a6bafac6 100644 --- a/packages/cloud/src/attach.ts +++ b/packages/cloud/src/attach.ts @@ -19,7 +19,11 @@ import { join } from 'node:path'; import WebSocket, { WebSocketServer } from 'ws'; -import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from './fleet-sandbox.js'; +import { + AGENT37_RELAYCAST_ORIGIN, + CANONICAL_RELAYCAST_ORIGIN, + DEV_RELAYCAST_ORIGIN, +} from './fleet-sandbox.js'; import type { AttachMode } from './attach-mode.js'; import { collectWithRetry } from './collect-with-retry.js'; import { resolveWorkspaceTransport } from './workspace-transport.js'; @@ -177,7 +181,11 @@ export class FleetNodeAttachError extends Error { } } -const TRUSTED_RELAYCAST_ORIGINS = new Set([CANONICAL_RELAYCAST_ORIGIN, AGENT37_RELAYCAST_ORIGIN]); +const TRUSTED_RELAYCAST_ORIGINS = new Set([ + CANONICAL_RELAYCAST_ORIGIN, + AGENT37_RELAYCAST_ORIGIN, + DEV_RELAYCAST_ORIGIN, +]); export function validateFleetAttachBaseUrl(value: string): string { let parsed: URL; @@ -688,6 +696,18 @@ export async function startFleetNodeAttachProxy( const inputSockets = new Set(); const outputHistory: Array<{ chunk: string; offset?: number }> = []; let outputHistoryBytes = 0; + // Output since the latest terminal.ready snapshot, held for a raw socket + // client that has not yet become ready. Kept apart from `outputHistory`, + // which event listeners drain, so neither consumer can starve the other. + const rawPending: string[] = []; + let rawPendingBytes = 0; + const retainRawOutput = (chunk: string) => { + rawPending.push(chunk); + rawPendingBytes += Buffer.byteLength(chunk, 'utf8'); + while (rawPendingBytes > MAX_BUFFERED_BYTES && rawPending.length > 0) { + rawPendingBytes -= Buffer.byteLength(rawPending.shift()!, 'utf8'); + } + }; let remote: WebSocket | undefined; let rawSocket: Socket | undefined; let rawConnected = false; @@ -1155,9 +1175,9 @@ export async function startFleetNodeAttachProxy( () => { if (socket.destroyed || stopped || terminalEnded) return; if (snapshot.screenAnsi) socket.write(snapshot.screenAnsi); - for (const entry of outputHistory) socket.write(entry.chunk); - outputHistory.length = 0; - outputHistoryBytes = 0; + for (const chunk of rawPending) socket.write(chunk); + rawPending.length = 0; + rawPendingBytes = 0; rawReady = true; socket.resume(); }, @@ -1405,6 +1425,9 @@ export async function startFleetNodeAttachProxy( // so no later consumer can render it; the HTTP snapshot still serves // the bytes verbatim and lets its own decoder report the problem. snapshot.screenAnsi = decodeAnsiScreenPayload(snapshot.screenBase64) ?? ''; + // The fresh snapshot already contains everything retained so far. + rawPending.length = 0; + rawPendingBytes = 0; snapshot.rows = typeof frame.rows === 'number' ? frame.rows : 24; snapshot.cols = typeof frame.cols === 'number' ? frame.cols : 80; snapshot.offset = typeof frame.offset === 'number' ? frame.offset : 0; @@ -1444,6 +1467,8 @@ export async function startFleetNodeAttachProxy( return; } rawSocket.write(frame.chunk); + } else { + retainRawOutput(frame.chunk); } if (!broadcast(eventSockets, workerStreamEvent(frame.chunk, offset)) && !rawReady) { if (!retainOutput(frame.chunk, offset)) { diff --git a/packages/sdk/src/__tests__/fleet-spawn.test.ts b/packages/sdk/src/__tests__/fleet-spawn.test.ts index 34a1526dfb..a3e24c85d1 100644 --- a/packages/sdk/src/__tests__/fleet-spawn.test.ts +++ b/packages/sdk/src/__tests__/fleet-spawn.test.ts @@ -177,14 +177,17 @@ describe('spawnFleetSandbox', () => { expect(h.deleteCloudFleetSandbox).not.toHaveBeenCalled(); }); - it('fails closed and cleans up when the agent lands on a different node', async () => { + it('fails closed and cleans up when the dispatch receipt names a different node', async () => { const h = harness({ spawn: async () => ({ id: 'inv_1', status: 'completed', + // A targeted placement echoes the requested node; only the receipt is evidence. + dispatchedNodeId: 'node_other', + node: { id: 'node_1', name: 'fleet-sandbox-node', status: 'online', capabilities: [] }, placement: { capability: 'spawn:claude', - node: 'some-other-node', + node: 'fleet-sandbox-node', attempts: 1, queued: false, state: 'ready', @@ -200,6 +203,56 @@ describe('spawnFleetSandbox', () => { expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); }); + it('cleans up the sandbox when a caller hook throws', async () => { + const h = harness(); + await expect( + spawnFleetSandbox( + { + ...base, + resolveTask: () => { + throw new Error('bad context'); + }, + }, + h.deps + ) + ).rejects.toThrow('bad context'); + expect(h.spawn).not.toHaveBeenCalled(); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + }); + + it('never lets spawnMetadata replace the lifecycle fields', async () => { + const h = harness(); + const handle = await spawnFleetSandbox( + { ...base, spawnMetadata: { name: 'other-agent', worker_cwd: '/tmp', session_ref: 'ref-1' } }, + h.deps + ); + expect((h.spawn.mock.calls[0][0] as Record).input).toMatchObject({ + name: 'sandbox-worker', + worker_cwd: '/workspace', + session_ref: 'ref-1', + }); + expect(handle.agentName).toBe('sandbox-worker'); + }); + + it('refuses a reused node without a sandbox identity', async () => { + const h = harness({ + ensure: async () => ({ + outcome: 'reused', + cloudWorkspaceId: 'cw_1', + nodeId: 'node_1', + nodeName: 'existing-node', + status: 'online', + activeAgents: 0, + maxAgents: 1, + }), + }); + await expect( + spawnFleetSandbox({ ...base, sandboxName: 'existing-node', mountRelayfile: false }, h.deps) + ).rejects.toMatchObject({ code: 'sandbox_identity_unknown' }); + expect(h.spawn).not.toHaveBeenCalled(); + expect(h.deleteCloudFleetSandbox).not.toHaveBeenCalled(); + }); + it('deletes the provisioned sandbox when the harness fails to start', async () => { const h = harness({ spawn: async () => { diff --git a/packages/sdk/src/fleet-spawn.ts b/packages/sdk/src/fleet-spawn.ts index 1bb1529a1e..154105d07c 100644 --- a/packages/sdk/src/fleet-spawn.ts +++ b/packages/sdk/src/fleet-spawn.ts @@ -154,6 +154,7 @@ export type FleetSandboxSpawnErrorCode = | 'provisioning_timeout' | 'relayfile_unmounted' | 'relaycast_target_unverified' + | 'sandbox_identity_unknown' | 'placement_mismatch'; export class FleetSandboxSpawnError extends Error { @@ -505,9 +506,14 @@ export async function spawnFleetSandbox( ready.outcome === 'provisioned' && ready.relayfileMounted ? (ready.relayfileMountPath ?? '/workspace') : undefined; - const workerCwd = input.resolveWorkerCwd?.(ready) ?? input.workerCwd ?? mountPath; - const task = input.resolveTask?.(ready, workerCwd) ?? input.task ?? ''; - const sandboxIdentity = 'sandboxId' in ready ? ready.sandboxId : (sandboxId ?? ''); + const sandboxIdentity = 'sandboxId' in ready ? ready.sandboxId : sandboxId; + if (sandboxIdentity === undefined) { + // A reused node is not owned by this call, so there is nothing to clean up. + throw new FleetSandboxSpawnError( + 'sandbox_identity_unknown', + `Cloud reused node '${ready.nodeName}' without a sandbox identity; pass sandboxId to resume a known sandbox.` + ); + } const ownsSandbox = shouldCleanupSandbox && ready.outcome === 'provisioned'; const confirm = input.confirm !== false; const workspaceRelayFor = () => @@ -517,7 +523,12 @@ export async function spawnFleetSandbox( let launcherName: string | undefined; let invocation: RelaySpawnPlacementAck; + let workerCwd: string | undefined; try { + // Caller hooks run inside the cleanup boundary: a throwing hook must not + // strand the sandbox this call just provisioned. + workerCwd = input.resolveWorkerCwd?.(ready) ?? input.workerCwd ?? mountPath; + const task = input.resolveTask?.(ready, workerCwd) ?? input.task ?? ''; // Agent tokens are scoped to a Relaycast deployment. Mint a temporary // launcher on the transport Cloud selected (or the canonical // compatibility transport when an older non-Agent37 response omitted the @@ -550,29 +561,37 @@ export async function spawnFleetSandbox( confirm, ...(confirm ? { confirmTimeoutMs: input.confirmTimeoutMs ?? DEFAULT_CONFIRM_TIMEOUT_MS } : {}), input: { + // Metadata first: it must never replace the lifecycle fields the + // handle attaches to and releases. + ...(input.spawnMetadata ?? {}), name, cli, task, ...(input.channels?.length ? { channels: input.channels } : {}), ...(input.model ? { model: input.model } : {}), ...(workerCwd ? { worker_cwd: workerCwd } : {}), - ...(input.spawnMetadata ?? {}), }, }); - // Confirm the agent landed on the sandbox this call ensured, not merely - // on some node advertising the same capability. - const landedNodeName = invocation.placement?.node ?? invocation.node?.name; - const landedNodeId = invocation.node?.id ?? invocation.node?.nodeId; - if ( - (landedNodeName !== undefined && landedNodeName !== ready.nodeName) || - (landedNodeId !== undefined && landedNodeId !== ready.nodeId) - ) { + // Confirm the agent landed on the sandbox this call ensured. For a + // targeted spawn, `invocation.node` and `placement.node` echo the + // requested roster node, so only the engine's dispatch receipt is + // evidence of where the action actually ran. + const sandboxNodeIds = new Set( + [ + ready.nodeId, + ...(invocation.node?.name === ready.nodeName ? [invocation.node.id, invocation.node.nodeId] : []), + ].filter((id): id is string => typeof id === 'string' && id.length > 0) + ); + const landedNodeId = [invocation.dispatchedNodeId, invocation.handlerNodeId].find( + (id): id is string => typeof id === 'string' && id.length > 0 && !sandboxNodeIds.has(id) + ); + if (landedNodeId !== undefined) { await releaseAgent('Fleet sandbox spawn landed on an unexpected node').catch((error) => { deps.warn(`Releasing the misplaced agent '${name}' failed: ${errorMessage(error)}`); }); throw new FleetSandboxSpawnError( 'placement_mismatch', - `Agent '${name}' landed on node '${landedNodeName ?? landedNodeId}', not sandbox node '${ready.nodeName}'.` + `Agent '${name}' landed on node '${landedNodeId}', not sandbox node '${ready.nodeName}' (${ready.nodeId}).` ); } } catch (error) { From 0cdd92b35d102a0908179b654475498029929192 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 14:39:44 -0700 Subject: [PATCH 07/19] fix(sdk): await async spawnFleetSandbox hooks inside the cleanup boundary verifySandbox, resolveWorkerCwd and resolveTask may return promises; awaiting them keeps a late rejection inside the cleanup that deletes an owned sandbox. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- .../sdk/src/__tests__/fleet-spawn.test.ts | 26 +++++++++++++++++++ packages/sdk/src/fleet-spawn.ts | 12 ++++----- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/packages/sdk/src/__tests__/fleet-spawn.test.ts b/packages/sdk/src/__tests__/fleet-spawn.test.ts index a3e24c85d1..0d3e639686 100644 --- a/packages/sdk/src/__tests__/fleet-spawn.test.ts +++ b/packages/sdk/src/__tests__/fleet-spawn.test.ts @@ -220,6 +220,32 @@ describe('spawnFleetSandbox', () => { expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); }); + it('awaits async hooks and cleans up when async verification rejects', async () => { + const h = harness(); + await expect( + spawnFleetSandbox( + { ...base, verifySandbox: async () => Promise.reject(new Error('stale revision')) }, + h.deps + ) + ).rejects.toThrow('stale revision'); + expect(h.spawn).not.toHaveBeenCalled(); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + + const ok = harness(); + await spawnFleetSandbox( + { + ...base, + resolveWorkerCwd: async () => '/workspace/repo', + resolveTask: async (_s, cwd) => `work in ${cwd}`, + }, + ok.deps + ); + expect((ok.spawn.mock.calls[0][0] as Record).input).toMatchObject({ + worker_cwd: '/workspace/repo', + task: 'work in /workspace/repo', + }); + }); + it('never lets spawnMetadata replace the lifecycle fields', async () => { const h = harness(); const handle = await spawnFleetSandbox( diff --git a/packages/sdk/src/fleet-spawn.ts b/packages/sdk/src/fleet-spawn.ts index 154105d07c..92cf282dad 100644 --- a/packages/sdk/src/fleet-spawn.ts +++ b/packages/sdk/src/fleet-spawn.ts @@ -96,11 +96,11 @@ export interface SpawnFleetSandboxInput { confirm?: boolean; confirmTimeoutMs?: number; /** Validate the ensured sandbox before any agent is started; throwing aborts and cleans up. */ - verifySandbox?: (sandbox: EnsureCloudFleetSandboxResult) => void; + verifySandbox?: (sandbox: EnsureCloudFleetSandboxResult) => void | Promise; /** Compute the worker cwd once the sandbox mount is known. */ - resolveWorkerCwd?: (sandbox: FleetSandboxReadyResult) => string | undefined; + resolveWorkerCwd?: (sandbox: FleetSandboxReadyResult) => string | undefined | Promise; /** Compute the task once the sandbox and worker cwd are known. */ - resolveTask?: (sandbox: FleetSandboxReadyResult, workerCwd: string | undefined) => string; + resolveTask?: (sandbox: FleetSandboxReadyResult, workerCwd: string | undefined) => string | Promise; } type WorkspaceRelayLike = Pick; @@ -337,7 +337,7 @@ export async function spawnFleetSandbox( }, }) : await deps.ensureCloudFleetSandbox(ensureInput); - input.verifySandbox?.(sandbox); + await input.verifySandbox?.(sandbox); } catch (error) { if (error instanceof CloudFleetSandboxProvisionError && error.sandboxAbsent) { // Cloud's terminal record for this exact identity already proves the @@ -527,8 +527,8 @@ export async function spawnFleetSandbox( try { // Caller hooks run inside the cleanup boundary: a throwing hook must not // strand the sandbox this call just provisioned. - workerCwd = input.resolveWorkerCwd?.(ready) ?? input.workerCwd ?? mountPath; - const task = input.resolveTask?.(ready, workerCwd) ?? input.task ?? ''; + workerCwd = (await input.resolveWorkerCwd?.(ready)) ?? input.workerCwd ?? mountPath; + const task = (await input.resolveTask?.(ready, workerCwd)) ?? input.task ?? ''; // Agent tokens are scoped to a Relaycast deployment. Mint a temporary // launcher on the transport Cloud selected (or the canonical // compatibility transport when an older non-Agent37 response omitted the From 813a003617a997601591499ff01d95ffdeb45dd1 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 14:51:19 -0700 Subject: [PATCH 08/19] fix(sdk): require a dispatch receipt proving the sandbox node; redacted sandbox type - spawnFleetSandbox fails closed unless at least one dispatch receipt id (dispatchedNodeId/handlerNodeId) proves the ensured sandbox node, and none names another node. Verified live: production receipts carry both ids and they equal Cloud's sandbox nodeId. - handle.sandbox is typed RedactedFleetSandboxResult, which omits relaycastTarget.relaycastApiKey, instead of the credential-bearing type. - Test harness stubs persistRelaycastTarget so tests never write a project session or credential store; CLI placement doubles return receipts like the engine does. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- .../fleet-lifecycle-integration.test.ts | 6 ++- .../commands/fleet-sandbox-regression.test.ts | 15 +++++- packages/cli/src/cli/commands/fleet.test.ts | 7 ++- .../sdk/src/__tests__/fleet-spawn.test.ts | 47 +++++++++++++++++++ packages/sdk/src/fleet-spawn.ts | 35 ++++++++++---- 5 files changed, 96 insertions(+), 14 deletions(-) diff --git a/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts b/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts index a7e4ffe660..fc6ac1ed19 100644 --- a/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts +++ b/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts @@ -101,7 +101,11 @@ describe('fleet CLI lifecycle routing', () => { } as never; }); const placement = { - spawn: vi.fn(async () => ({ invocationId: 'inv_lifecycle', node: { name: nodeName } })), + spawn: vi.fn(async () => ({ + invocationId: 'inv_lifecycle', + dispatchedNodeId: 'node-1', + node: { name: nodeName }, + })), }; const ensureCloudFleetSandbox = vi.fn(async () => ({ outcome: 'provisioned' as const, diff --git a/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts b/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts index b6f8f1ae4d..3b582e32ef 100644 --- a/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts +++ b/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts @@ -42,6 +42,17 @@ function registerSandboxCommand(overrides: { placement?: ReturnType; }): Command { const placement = overrides.placement ?? vi.fn(async () => ({ invocationId: 'inv_test' })); + // Like the engine, the dispatch receipt names the node the spawn ran on. + let ensuredNodeId: string | undefined; + const ensureCloudFleetSandbox = async (...args: unknown[]) => { + const result = await overrides.ensureCloudFleetSandbox(...args); + ensuredNodeId = (result as { nodeId?: string } | undefined)?.nodeId; + return result; + }; + const spawn = async (input: unknown) => ({ + dispatchedNodeId: ensuredNodeId, + ...((await placement(input)) as Record), + }); const register = vi.fn(async () => ({ token: 'at_live_launcher' })); const release = vi.fn(async () => ({ released: true, deleted: true })); const createWorkspaceRelay = vi.fn(() => ({ @@ -69,7 +80,7 @@ function registerSandboxCommand(overrides: { ...(overrides.materializeCloudRelayfileRepository ? { materializeCloudRelayfileRepository: overrides.materializeCloudRelayfileRepository as never } : {}), - ensureCloudFleetSandbox: overrides.ensureCloudFleetSandbox as never, + ensureCloudFleetSandbox: ensureCloudFleetSandbox as never, resolveWorkspaceSelection: () => ({ key: 'rk_live_workspace', source: 'project', @@ -79,7 +90,7 @@ function registerSandboxCommand(overrides: { persistWorkspaceRelaycastTarget: () => true, deleteCloudFleetSandbox: vi.fn(async () => undefined), sdk: { - createAgentRelay: vi.fn(() => ({ messaging: { placement: { spawn: placement } } })) as never, + createAgentRelay: vi.fn(() => ({ messaging: { placement: { spawn } } })) as never, createWorkspaceRelay: createWorkspaceRelay as never, createWorkspace: vi.fn() as never, log: vi.fn(), diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 04a97ba962..82c088b0bc 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1103,6 +1103,7 @@ describe('fleet command support', () => { const placement = { spawn: vi.fn(async () => ({ invocationId: 'inv_sandbox', + dispatchedNodeId: 'node-1', node: { name: 'e2b-codex' }, })), }; @@ -2223,7 +2224,11 @@ describe('fleet command support', () => { const selectionOptions: Record[] = []; const persistWorkspaceRelaycastTarget = vi.fn(() => true); const placement = { - spawn: vi.fn(async () => ({ invocationId: 'inv_inferred', node: { name: 'cloud-node' } })), + spawn: vi.fn(async () => ({ + invocationId: 'inv_inferred', + dispatchedNodeId: 'node-1', + node: { name: 'cloud-node' }, + })), }; const register = vi.fn(async () => ({ token: 'at_live_launcher' })); const release = vi.fn(async () => ({ released: true, deleted: true })); diff --git a/packages/sdk/src/__tests__/fleet-spawn.test.ts b/packages/sdk/src/__tests__/fleet-spawn.test.ts index 0d3e639686..0ad4402cd1 100644 --- a/packages/sdk/src/__tests__/fleet-spawn.test.ts +++ b/packages/sdk/src/__tests__/fleet-spawn.test.ts @@ -50,6 +50,8 @@ function harness( return { id: 'inv_1', status: 'completed', + dispatchedNodeId: 'node_1', + handlerNodeId: 'node_1', node: { id: 'node_1', name: 'fleet-sandbox-node', status: 'online', capabilities: [] }, placement: { capability: 'spawn:claude', @@ -76,6 +78,8 @@ function harness( createWorkspaceRelay, createAgentRelay, startFleetNodeAttachProxy, + // Never let a test persist a Relaycast target into a real project session. + persistRelaycastTarget: vi.fn(), warn: vi.fn(), } as unknown as SpawnFleetSandboxDependencies; return { @@ -155,6 +159,28 @@ describe('spawnFleetSandbox', () => { ); }); + it('returns the sandbox without its Relaycast credential', async () => { + const h = harness({ + ensure: async () => + provisioned({ + relaycastTarget: { + route: 'canonical', + baseUrl: 'https://cast.agentrelay.com', + workspaceId: 'rw_1', + relaycastApiKey: 'rk_live_secret', + }, + }), + }); + const handle = await spawnFleetSandbox(base, h.deps); + expect(handle.sandbox.relaycastTarget).toEqual({ + route: 'canonical', + baseUrl: 'https://cast.agentrelay.com', + workspaceId: 'rw_1', + }); + // @ts-expect-error the redacted type has no credential field + expect(handle.sandbox.relaycastTarget?.relaycastApiKey).toBeUndefined(); + }); + it('tears down idempotently: releases the agent, then deletes the sandbox it provisioned', async () => { const h = harness(); const handle = await spawnFleetSandbox(base, h.deps); @@ -203,6 +229,27 @@ describe('spawnFleetSandbox', () => { expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); }); + it('fails closed when no dispatch receipt proves the sandbox node', async () => { + const h = harness({ + spawn: async () => ({ + id: 'inv_1', + status: 'completed', + node: { id: 'node_1', name: 'fleet-sandbox-node', status: 'online', capabilities: [] }, + placement: { + capability: 'spawn:claude', + node: 'fleet-sandbox-node', + attempts: 1, + queued: false, + state: 'ready', + confirmed: true, + }, + }), + }); + await expect(spawnFleetSandbox(base, h.deps)).rejects.toMatchObject({ code: 'placement_mismatch' }); + expect(h.calls).toContain('release:sandbox-worker'); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + }); + it('cleans up the sandbox when a caller hook throws', async () => { const h = harness(); await expect( diff --git a/packages/sdk/src/fleet-spawn.ts b/packages/sdk/src/fleet-spawn.ts index 92cf282dad..c20f452bc7 100644 --- a/packages/sdk/src/fleet-spawn.ts +++ b/packages/sdk/src/fleet-spawn.ts @@ -48,6 +48,16 @@ export type FleetSandboxReadyResult = Exclude< { outcome: 'provisioning_timeout' } >; +/** A Relaycast target without its credential. */ +export type RedactedCloudFleetRelaycastTarget = Omit; + +/** An ensured sandbox with the Relaycast credential removed. */ +export type RedactedFleetSandboxResult = EnsureCloudFleetSandboxResult extends infer Result + ? Result extends { relaycastTarget?: CloudFleetRelaycastTarget } + ? Omit & { relaycastTarget?: RedactedCloudFleetRelaycastTarget } + : Result + : never; + export interface SpawnFleetSandboxInput { /** Agent harness to start, e.g. `claude` or `codex`. */ cli: string; @@ -139,7 +149,7 @@ export interface FleetSandboxHandle { /** True when this call provisioned the sandbox and `destroy()` deletes it. */ ownsSandbox: boolean; /** The ensured sandbox, with the Relaycast credential removed. */ - sandbox: EnsureCloudFleetSandboxResult; + sandbox: RedactedFleetSandboxResult; invocation: RelaySpawnPlacementAck; /** Attach to the live agent's terminal through a private local socket. */ attach(options?: { mode?: FleetNodeAttachOptions['mode'] }): Promise; @@ -204,10 +214,10 @@ function nonEmpty(value: string | undefined): string | undefined { } /** Remove the Relaycast credential so the result can be logged or returned. */ -export function redactFleetSandbox(sandbox: EnsureCloudFleetSandboxResult): EnsureCloudFleetSandboxResult { - if (!sandbox.relaycastTarget) return sandbox; +export function redactFleetSandbox(sandbox: EnsureCloudFleetSandboxResult): RedactedFleetSandboxResult { + if (!sandbox.relaycastTarget) return sandbox as RedactedFleetSandboxResult; const { relaycastApiKey: _secret, ...target } = sandbox.relaycastTarget; - return { ...sandbox, relaycastTarget: target } as EnsureCloudFleetSandboxResult; + return { ...sandbox, relaycastTarget: target } as RedactedFleetSandboxResult; } export async function spawnFleetSandbox( @@ -582,16 +592,21 @@ export async function spawnFleetSandbox( ...(invocation.node?.name === ready.nodeName ? [invocation.node.id, invocation.node.nodeId] : []), ].filter((id): id is string => typeof id === 'string' && id.length > 0) ); - const landedNodeId = [invocation.dispatchedNodeId, invocation.handlerNodeId].find( - (id): id is string => typeof id === 'string' && id.length > 0 && !sandboxNodeIds.has(id) + const receiptNodeIds = [invocation.dispatchedNodeId, invocation.handlerNodeId].filter( + (id): id is string => typeof id === 'string' && id.length > 0 ); - if (landedNodeId !== undefined) { - await releaseAgent('Fleet sandbox spawn landed on an unexpected node').catch((error) => { - deps.warn(`Releasing the misplaced agent '${name}' failed: ${errorMessage(error)}`); + const landedNodeId = receiptNodeIds.find((id) => !sandboxNodeIds.has(id)); + if (receiptNodeIds.length === 0 || landedNodeId !== undefined) { + // No receipt proves the sandbox node, or one names another node: fail + // closed so an agent routed elsewhere never escapes the sandbox. + await releaseAgent('Fleet sandbox spawn landed on an unverified node').catch((error) => { + deps.warn(`Releasing the unverified agent '${name}' failed: ${errorMessage(error)}`); }); throw new FleetSandboxSpawnError( 'placement_mismatch', - `Agent '${name}' landed on node '${landedNodeId}', not sandbox node '${ready.nodeName}' (${ready.nodeId}).` + landedNodeId === undefined + ? `Agent '${name}' was dispatched without a node receipt proving sandbox node '${ready.nodeName}' (${ready.nodeId}).` + : `Agent '${name}' landed on node '${landedNodeId}', not sandbox node '${ready.nodeName}' (${ready.nodeId}).` ); } } catch (error) { From c36c3c14760a851c843cc17e08701f5ee687a734 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 14:58:10 -0700 Subject: [PATCH 09/19] fix(cloud): keep /ws output history while a raw attach client is active With no /ws listener and a ready raw client, output is now retained as a bounded newest-first ring for a later listener instead of being dropped; the raw client draining the stream is not backpressure, so the session is never ended for it. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- packages/cloud/src/attach.test.ts | 17 +++++++++++++++++ packages/cloud/src/attach.ts | 15 +++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/cloud/src/attach.test.ts b/packages/cloud/src/attach.test.ts index 84cc781dc6..11794e0e57 100644 --- a/packages/cloud/src/attach.test.ts +++ b/packages/cloud/src/attach.test.ts @@ -85,6 +85,23 @@ it('replays output to a late raw client even when an event listener already cons events.close(); }); +it('replays output to a late event listener even while a raw client consumed it', async () => { + const { remote } = await setup(); + local = connect(proxy!.socketPath); + await once(local, 'connect'); + send(remote, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + await once(local, 'data'); + const rawOutput = once(local, 'data'); + send(remote, 'terminal.output', { chunk: ' during-raw' }); + await rawOutput; + const events = new (await import('ws')).default(`${proxy!.brokerUrl.replace('http', 'ws')}/ws`, { + headers: { authorization: `Bearer ${proxy!.apiKey}` }, + }); + const [message] = await once(events, 'message'); + expect(JSON.stringify(JSON.parse(String(message)))).toContain('during-raw'); + events.close(); +}); + it('settles finished on explicit close before terminal.ready', async () => { await setup(); await proxy!.close(); diff --git a/packages/cloud/src/attach.ts b/packages/cloud/src/attach.ts index 00a6bafac6..396daef719 100644 --- a/packages/cloud/src/attach.ts +++ b/packages/cloud/src/attach.ts @@ -1037,6 +1037,13 @@ export async function startFleetNodeAttachProxy( chunk, ...(offset === undefined ? {} : { offset }), }); + const retainOutputDroppingOldest = (chunk: string, offset: number | undefined) => { + outputHistory.push({ chunk, ...(offset === undefined ? {} : { offset }) }); + outputHistoryBytes += Buffer.byteLength(chunk, 'utf8'); + while (outputHistoryBytes > MAX_BUFFERED_BYTES && outputHistory.length > 0) { + outputHistoryBytes -= Buffer.byteLength(outputHistory.shift()!.chunk, 'utf8'); + } + }; const retainOutput = (chunk: string, offset: number | undefined): boolean => { const bytes = Buffer.byteLength(chunk, 'utf8'); if (outputHistoryBytes + bytes > MAX_BUFFERED_BYTES) return false; @@ -1470,8 +1477,12 @@ export async function startFleetNodeAttachProxy( } else { retainRawOutput(frame.chunk); } - if (!broadcast(eventSockets, workerStreamEvent(frame.chunk, offset)) && !rawReady) { - if (!retainOutput(frame.chunk, offset)) { + if (!broadcast(eventSockets, workerStreamEvent(frame.chunk, offset))) { + if (rawReady) { + // A raw client is draining the stream, so a missing /ws listener is + // not backpressure: keep the newest history for a later listener. + retainOutputDroppingOldest(frame.chunk, offset); + } else if (!retainOutput(frame.chunk, offset)) { endTerminal( new FleetNodeAttachError( 'terminal output exceeded the bounded loopback buffer', From b8ca716df3d9954d152ba0668d801a0d114ad733 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 15:04:56 -0700 Subject: [PATCH 10/19] fix(sdk): await async persistRelaycastTarget inside the cleanup boundary Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- .../sdk/src/__tests__/fleet-spawn.test.ts | 22 +++++++++++++++++++ packages/sdk/src/fleet-spawn.ts | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/sdk/src/__tests__/fleet-spawn.test.ts b/packages/sdk/src/__tests__/fleet-spawn.test.ts index 0ad4402cd1..7debbb98a2 100644 --- a/packages/sdk/src/__tests__/fleet-spawn.test.ts +++ b/packages/sdk/src/__tests__/fleet-spawn.test.ts @@ -181,6 +181,28 @@ describe('spawnFleetSandbox', () => { expect(handle.sandbox.relaycastTarget?.relaycastApiKey).toBeUndefined(); }); + it('awaits async target persistence and cleans up when it rejects', async () => { + const h = harness({ + ensure: async () => + provisioned({ + relaycastTarget: { + route: 'canonical', + baseUrl: 'https://cast.agentrelay.com', + workspaceId: 'rw_1', + relaycastApiKey: 'rk_live_secret', + }, + }), + }); + await expect( + spawnFleetSandbox(base, { + ...h.deps, + persistRelaycastTarget: async () => Promise.reject(new Error('session write failed')), + }) + ).rejects.toThrow('session write failed'); + expect(h.spawn).not.toHaveBeenCalled(); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + }); + it('tears down idempotently: releases the agent, then deletes the sandbox it provisioned', async () => { const h = harness(); const handle = await spawnFleetSandbox(base, h.deps); diff --git a/packages/sdk/src/fleet-spawn.ts b/packages/sdk/src/fleet-spawn.ts index c20f452bc7..b85e61a609 100644 --- a/packages/sdk/src/fleet-spawn.ts +++ b/packages/sdk/src/fleet-spawn.ts @@ -131,7 +131,7 @@ export interface SpawnFleetSandboxDependencies { persistRelaycastTarget: ( target: CloudFleetRelaycastTarget, context: { sandbox: EnsureCloudFleetSandboxResult; transport: WorkspaceTransportOptions } - ) => void; + ) => void | Promise; startFleetNodeAttachProxy: (options: FleetNodeAttachOptions) => Promise; warn: (message: string) => void; } @@ -463,7 +463,7 @@ export async function spawnFleetSandbox( `Cloud returned the DEV canonical Relaycast target, but relaycastCloudApiUrl was not exactly ${DEV_CLOUD_API_URL}; refusing to persist an untrusted route.` ); } - deps.persistRelaycastTarget(target, { sandbox, transport }); + await deps.persistRelaycastTarget(target, { sandbox, transport }); verifiedTarget = target; } catch (error) { if (shouldCleanupSandbox && sandbox.outcome === 'provisioned') { From 475c8c662111509f1b4b9455f4f11bc7062a7f74 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 15:12:12 -0700 Subject: [PATCH 11/19] fix(sdk): destroy() retries only the teardown step that failed Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- .../sdk/src/__tests__/fleet-spawn.test.ts | 11 +++++++ packages/sdk/src/fleet-spawn.ts | 30 ++++++++++++++----- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/packages/sdk/src/__tests__/fleet-spawn.test.ts b/packages/sdk/src/__tests__/fleet-spawn.test.ts index 7debbb98a2..8024b1b1db 100644 --- a/packages/sdk/src/__tests__/fleet-spawn.test.ts +++ b/packages/sdk/src/__tests__/fleet-spawn.test.ts @@ -217,6 +217,17 @@ describe('spawnFleetSandbox', () => { }); }); + it('retries only the teardown step that failed', async () => { + const h = harness(); + const handle = await spawnFleetSandbox(base, h.deps); + h.release.mockRejectedValueOnce(new Error('relaycast busy')); + h.calls.length = 0; + await expect(handle.destroy()).rejects.toThrow('agent release failed: relaycast busy'); + await handle.destroy(); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + expect(h.release.mock.calls.filter(([input]) => input.name === 'sandbox-worker')).toHaveLength(2); + }); + it('retains a caller-declared sandbox on destroy', async () => { const h = harness(); const handle = await spawnFleetSandbox({ ...base, sandboxId: SANDBOX_ID }, h.deps); diff --git a/packages/sdk/src/fleet-spawn.ts b/packages/sdk/src/fleet-spawn.ts index b85e61a609..69840b625b 100644 --- a/packages/sdk/src/fleet-spawn.ts +++ b/packages/sdk/src/fleet-spawn.ts @@ -637,6 +637,8 @@ export async function spawnFleetSandbox( } let destroyed: Promise | undefined; + let agentReleased = false; + let sandboxDeleted = false; return { sandboxId: sandboxIdentity, nodeId: ready.nodeId, @@ -663,14 +665,28 @@ export async function spawnFleetSandbox( }, destroy: () => (destroyed ??= (async () => { + // Each step is recorded once it succeeds, so a retry only replays + // the step that failed (a deleted sandbox would otherwise 404). const failures: string[] = []; - await releaseAgent('Fleet sandbox handle destroyed').catch((error) => { - failures.push(`agent release failed: ${errorMessage(error)}`); - }); - if (ownsSandbox) { - await deleteSandbox(ready).catch((error) => { - failures.push(`sandbox deletion failed: ${errorMessage(error)}`); - }); + if (!agentReleased) { + await releaseAgent('Fleet sandbox handle destroyed').then( + () => { + agentReleased = true; + }, + (error) => { + failures.push(`agent release failed: ${errorMessage(error)}`); + } + ); + } + if (ownsSandbox && !sandboxDeleted) { + await deleteSandbox(ready).then( + () => { + sandboxDeleted = true; + }, + (error) => { + failures.push(`sandbox deletion failed: ${errorMessage(error)}`); + } + ); } if (failures.length > 0) { destroyed = undefined; From a20c0663d62faebf1c26fdb5ae54189ad6cabdcf Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 15:20:31 -0700 Subject: [PATCH 12/19] =?UTF-8?q?chore:=20align=20with=20v13.2.0=20?= =?UTF-8?q?=E2=80=94=20pin=20@agent-relay/cloud=2013.2.0,=20keep=20new=20e?= =?UTF-8?q?ntries=20Unreleased?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Main released v13.2.0 and bumped every internal package; the SDK's exact @agent-relay/cloud pin must follow or npm ci rejects the lockfile. The auto-merge had also filed this PR's changelog entries under the released 13.2.0 section; they belong under Unreleased. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- CHANGELOG.md | 7 +++++-- package-lock.json | 2 +- packages/sdk/package.json | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 022f5809fe..eb879142e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,14 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [13.2.0] - 2026-10-08 - ### Added - `@agent-relay/sdk/fleet` exports `spawnFleetSandbox`, which provisions a Cloud fleet sandbox, starts an agent harness on it, confirms the node the agent landed on, and returns a handle to the live agent that can be attached to immediately and torn down idempotently. `agent-relay fleet spawn --sandbox` uses the same path. - `@agent-relay/cloud/attach` and `@agent-relay/sdk/attach` expose fleet terminal attachment with a private local stdio socket and completion promise. - `@agent-relay/cloud/fleet` and `@agent-relay/sdk/fleet` expose the sandbox ensure and deletion primitives. - Fleet sandbox ensure and `fleet spawn --sandbox-readonly-path` forward explicit read-only Relayfile subtree requests to Cloud for server-side enforcement. + +## [13.2.0] - 2026-10-08 + +### Added + - `agent-relay fleet spawn --sandbox` (unpinned or `--sandbox-provider agent37`) now uses Cloud's durable `async-v1` preparation and prints each preparation phase, so a slow Relayfile initial sync no longer ends as an unknown outcome. Older Cloud deployments keep working through the synchronous response. - `@agent-relay/cloud` `ensureCloudFleetSandbox` accepts `preparationMode: 'async-v1'` and an `onPreparationProgress` callback; `CloudFleetSandboxProvisionError` exposes `sandboxAbsent` and a typed `preparationFailure` (`code`, `phase`, `causeStage`). diff --git a/package-lock.json b/package-lock.json index c530c64572..a7384efc2b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14311,7 +14311,7 @@ "@relaycast/sdk": "^8.0.7", "@relaycast/types": "^8.0.7", "zod": "^4.4.3", - "@agent-relay/cloud": "13.1.5" + "@agent-relay/cloud": "13.2.0" }, "devDependencies": { "@types/node": "^22.13.10" diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 99f1ba937f..dce09f907a 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -75,6 +75,6 @@ "@relaycast/sdk": "^8.0.7", "@relaycast/types": "^8.0.7", "zod": "^4.4.3", - "@agent-relay/cloud": "13.1.5" + "@agent-relay/cloud": "13.2.0" } } From 734d30f69c6605c900854344f0b4720cad2c782d Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 15:24:22 -0700 Subject: [PATCH 13/19] fix(cloud): end raw attachments on connection-fatal session errors A request-less connection-fatal terminal.error after ready now closes the raw stdio socket and settles finished with 1, as broker input sockets are already closed, so socketPath consumers stop writing to a dead session. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- packages/cloud/src/attach.test.ts | 12 ++++++++++++ packages/cloud/src/attach.ts | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/packages/cloud/src/attach.test.ts b/packages/cloud/src/attach.test.ts index 11794e0e57..6ee987bd0c 100644 --- a/packages/cloud/src/attach.test.ts +++ b/packages/cloud/src/attach.test.ts @@ -102,6 +102,18 @@ it('replays output to a late event listener even while a raw client consumed it' events.close(); }); +it('ends the raw attachment on a connection-fatal session error after ready', async () => { + const { remote } = await setup(); + local = connect(proxy!.socketPath); + await once(local, 'connect'); + send(remote, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + await once(local, 'data'); + const closed = once(local, 'close'); + send(remote, 'terminal.error', { code: 'node_unreachable', message: 'node went away' }); + await expect(proxy!.finished).resolves.toBe(1); + await closed; +}); + it('settles finished on explicit close before terminal.ready', async () => { await setup(); await proxy!.close(); diff --git a/packages/cloud/src/attach.ts b/packages/cloud/src/attach.ts index 396daef719..aa96761b63 100644 --- a/packages/cloud/src/attach.ts +++ b/packages/cloud/src/attach.ts @@ -1556,6 +1556,12 @@ export async function startFleetNodeAttachProxy( endTerminal(new FleetNodeAttachError(message, code)); } else if (!frameRid) { failAllInputSockets(code, message); + if (inputErrorIsConnectionFatal(code)) { + // The raw stdio client has no error frame to read: close it and + // report the failure so callers stop writing to a dead session. + finish(1); + rawSocket?.end(); + } } } else if (frame.type === 'terminal.closed') { endTerminal(new FleetNodeAttachError('remote terminal session closed', 'terminal_closed')); From e3c17c78bf24ad3f3e541294181fd54c716880bf Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 15:25:26 -0700 Subject: [PATCH 14/19] docs(changelog): mark the pending release Minor (new SDK subpaths and API) Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb879142e4..f2f63c6803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Minor] ### Added From 1ce7c7c12131bd323f35303f3d4c14924e73ee3f Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 15:35:28 -0700 Subject: [PATCH 15/19] fix(cloud,sdk): hold raw input across reconnects; pin attach transport at spawn - attach: while the terminal transport reconnects, raw input is paused and unshifted, then resumed at the next terminal.ready, instead of ending a recoverable session. Input overflow on an open transport still ends it. - spawnFleetSandbox resolves the compatibility (no relaycastTarget) transport once at spawn and pins it for attach(), so a later rebind or env change cannot redirect attach to another workspace. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- packages/cloud/src/attach.test.ts | 45 +++++++++++++++++++ packages/cloud/src/attach.ts | 17 ++++--- .../sdk/src/__tests__/fleet-spawn.test.ts | 12 +++++ packages/sdk/src/fleet-spawn.ts | 18 ++++++-- 4 files changed, 84 insertions(+), 8 deletions(-) diff --git a/packages/cloud/src/attach.test.ts b/packages/cloud/src/attach.test.ts index 6ee987bd0c..8766234725 100644 --- a/packages/cloud/src/attach.test.ts +++ b/packages/cloud/src/attach.test.ts @@ -114,6 +114,51 @@ it('ends the raw attachment on a connection-fatal session error after ready', as await closed; }); +it('holds raw input during a transient reconnect and delivers it once the terminal is ready again', async () => { + server = new WebSocketServer({ host: '127.0.0.1', port: 0 }); + await once(server, 'listening'); + const connections: WebSocket[] = []; + server.on('connection', (socket) => connections.push(socket as WebSocket)); + const fetch = vi.fn(async (url: string | URL | Request) => + Response.json({ + ok: true, + data: String(url).endsWith('/agents') + ? [{ agentName: 'worker' }] + : { + session_id: 'session', + resume_token: 'resume', + terminal_url: `ws://127.0.0.1:${(server!.address() as AddressInfo).port}/terminal`, + }, + }) + ); + proxy = await startFleetNodeAttachProxy({ + nodeId: 'node', + mode: 'drive', + workspaceKey: 'rk_test', + env: {}, + fetch, + reconnectDelay: { initialMs: 300, maxMs: 300 }, + }); + await vi.waitFor(() => expect(connections).toHaveLength(1)); + local = connect(proxy!.socketPath); + await once(local, 'connect'); + send(connections[0]!, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + await once(local, 'data'); + connections[0]!.terminate(); + await new Promise((resolve) => setTimeout(resolve, 50)); + local.write('typed-during-reconnect'); + await vi.waitFor(() => expect(connections).toHaveLength(2)); + const input = once(connections[1]!, 'message'); + send(connections[1]!, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + const frame = JSON.parse(String((await input)[0])); + expect(frame).toMatchObject({ type: 'terminal.input' }); + expect(Buffer.from(frame.data_base64, 'base64').toString()).toBe('typed-during-reconnect'); + let settled = false; + void proxy!.finished.then(() => (settled = true)); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(settled).toBe(false); +}); + it('settles finished on explicit close before terminal.ready', async () => { await setup(); await proxy!.close(); diff --git a/packages/cloud/src/attach.ts b/packages/cloud/src/attach.ts index aa96761b63..2468fa0b33 100644 --- a/packages/cloud/src/attach.ts +++ b/packages/cloud/src/attach.ts @@ -1162,11 +1162,16 @@ export async function startFleetNodeAttachProxy( }); socket.on('data', (data: Buffer) => { if (options.mode === 'view') return; - if ( - !remote || - remote.readyState !== WebSocket.OPEN || - remote.bufferedAmount + data.length > MAX_BUFFERED_BYTES - ) { + if (stopped || terminalEnded) return; + if (!remote || remote.readyState !== WebSocket.OPEN || !activeReadiness.settled) { + // A transient reconnect is in progress: hold the keystrokes on the + // socket and resume at the next terminal.ready rather than ending a + // recoverable session. + socket.pause(); + socket.unshift(data); + return; + } + if (remote.bufferedAmount + data.length > MAX_BUFFERED_BYTES) { endTerminal(new FleetNodeAttachError('terminal input transport unavailable', 'node_unreachable')); return; } @@ -1456,6 +1461,8 @@ export async function startFleetNodeAttachProxy( // DECODED grid: a `worker_stream` chunk is raw PTY bytes that the // attach client writes straight to the terminal, so the base64 form // renders as a wall of text instead of a repaint (relay#1829). + // Release any input held while the transport was reconnecting. + if (rawReady && rawSocket && !rawSocket.destroyed) rawSocket.resume(); if (readiness.generation > 1 && snapshot.screenAnsi) { broadcast(eventSockets, workerStreamEvent(snapshot.screenAnsi)); rawSocket?.write(snapshot.screenAnsi); diff --git a/packages/sdk/src/__tests__/fleet-spawn.test.ts b/packages/sdk/src/__tests__/fleet-spawn.test.ts index 8024b1b1db..93a77dc288 100644 --- a/packages/sdk/src/__tests__/fleet-spawn.test.ts +++ b/packages/sdk/src/__tests__/fleet-spawn.test.ts @@ -203,6 +203,18 @@ describe('spawnFleetSandbox', () => { expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); }); + it('pins the compatibility transport at spawn time for attach', async () => { + const h = harness(); + const env: NodeJS.ProcessEnv = { RELAY_WORKSPACE_KEY: 'rk_live_spawned' }; + const { workspaceKey: _key, ...rest } = base; + const handle = await spawnFleetSandbox({ ...rest, transport: { env } }, h.deps); + env.RELAY_WORKSPACE_KEY = 'rk_live_rebound'; + await handle.attach(); + expect(h.startFleetNodeAttachProxy).toHaveBeenCalledWith( + expect.objectContaining({ workspaceKey: 'rk_live_spawned' }) + ); + }); + it('tears down idempotently: releases the agent, then deletes the sandbox it provisioned', async () => { const h = harness(); const handle = await spawnFleetSandbox(base, h.deps); diff --git a/packages/sdk/src/fleet-spawn.ts b/packages/sdk/src/fleet-spawn.ts index 69840b625b..7ce19cfc68 100644 --- a/packages/sdk/src/fleet-spawn.ts +++ b/packages/sdk/src/fleet-spawn.ts @@ -636,6 +636,19 @@ export async function spawnFleetSandbox( } } + // Pin the credential/origin pair the agent was launched with: a later + // project rebind or environment change must not redirect attach. + let spawnTransport: { workspaceKey: string; baseUrl?: string } | Error; + if (verifiedTarget) { + spawnTransport = { workspaceKey: verifiedTarget.relaycastApiKey, baseUrl: verifiedTarget.baseUrl }; + } else { + try { + const { workspaceKey, baseUrl } = resolveWorkspaceTransport(legacyTransport); + spawnTransport = { workspaceKey, ...(baseUrl === undefined ? {} : { baseUrl }) }; + } catch (error) { + spawnTransport = error instanceof Error ? error : new Error(String(error)); + } + } let destroyed: Promise | undefined; let agentReleased = false; let sandboxDeleted = false; @@ -652,9 +665,8 @@ export async function spawnFleetSandbox( sandbox: redactFleetSandbox(ready), invocation, attach: async (options = {}) => { - const attachTransport = verifiedTarget - ? { workspaceKey: verifiedTarget.relaycastApiKey, baseUrl: verifiedTarget.baseUrl } - : resolveWorkspaceTransport(legacyTransport); + if (spawnTransport instanceof Error) throw spawnTransport; + const attachTransport = spawnTransport; return deps.startFleetNodeAttachProxy({ node: ready.nodeName, agent: name, From 5c155d3c4119d51df6eda8f0e72a89f0120e3df8 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 15:42:28 -0700 Subject: [PATCH 16/19] fix(sdk,cloud): pin the launcher gateway; refuse raw clients after a fatal error - spawnFleetSandbox creates the launcher client on the base URL of the transport it resolved at spawn (verified target or compatibility transport), not the ambient process environment. - attach refuses later raw socketPath clients once a connection-fatal session error has been reported through finished, even when no client was attached. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- packages/cloud/src/attach.test.ts | 14 +++++++++ packages/cloud/src/attach.ts | 5 ++- .../sdk/src/__tests__/fleet-spawn.test.ts | 18 +++++++++++ packages/sdk/src/fleet-spawn.ts | 31 ++++++++++--------- 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/packages/cloud/src/attach.test.ts b/packages/cloud/src/attach.test.ts index 8766234725..258447113f 100644 --- a/packages/cloud/src/attach.test.ts +++ b/packages/cloud/src/attach.test.ts @@ -159,6 +159,20 @@ it('holds raw input during a transient reconnect and delivers it once the termin expect(settled).toBe(false); }); +it('refuses raw clients after a connection-fatal session error', async () => { + const { remote } = await setup(); + send(remote, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + await new Promise((resolve) => setTimeout(resolve, 20)); + send(remote, 'terminal.error', { code: 'node_unreachable', message: 'node went away' }); + await expect(proxy!.finished).resolves.toBe(1); + local = connect(proxy!.socketPath); + let received = ''; + local.on('data', (data: Buffer) => (received += data.toString())); + local.on('error', () => undefined); + await once(local, 'close'); + expect(received).toBe(''); +}); + it('settles finished on explicit close before terminal.ready', async () => { await setup(); await proxy!.close(); diff --git a/packages/cloud/src/attach.ts b/packages/cloud/src/attach.ts index 2468fa0b33..5ae75f0636 100644 --- a/packages/cloud/src/attach.ts +++ b/packages/cloud/src/attach.ts @@ -711,6 +711,8 @@ export async function startFleetNodeAttachProxy( let remote: WebSocket | undefined; let rawSocket: Socket | undefined; let rawConnected = false; + // Set once a connection-fatal session error was reported through `finished`. + let rawRefused = false; let rawReady = false; let finish!: (code: number) => void; const finished = new Promise((resolve) => { @@ -1149,7 +1151,7 @@ export async function startFleetNodeAttachProxy( let socketDirectory: string | undefined; let socketPath: string; const rawServer = createSocketServer((socket) => { - if (rawConnected || stopped || terminalEnded) { + if (rawConnected || rawRefused || stopped || terminalEnded) { socket.destroy(); return; } @@ -1567,6 +1569,7 @@ export async function startFleetNodeAttachProxy( // The raw stdio client has no error frame to read: close it and // report the failure so callers stop writing to a dead session. finish(1); + rawRefused = true; rawSocket?.end(); } } diff --git a/packages/sdk/src/__tests__/fleet-spawn.test.ts b/packages/sdk/src/__tests__/fleet-spawn.test.ts index 93a77dc288..e3e13361fe 100644 --- a/packages/sdk/src/__tests__/fleet-spawn.test.ts +++ b/packages/sdk/src/__tests__/fleet-spawn.test.ts @@ -215,6 +215,24 @@ describe('spawnFleetSandbox', () => { ); }); + it('creates the launcher client on the transport the spawn resolved', async () => { + const h = harness(); + const { workspaceKey: _key, ...rest } = base; + await spawnFleetSandbox( + { + ...rest, + transport: { + env: { RELAY_WORKSPACE_KEY: 'rk_live_spawned', RELAY_BASE_URL: 'https://cast.agentrelay.com' }, + }, + }, + h.deps + ); + expect(h.createAgentRelay).toHaveBeenCalledWith({ + token: 'launcher-token', + baseUrl: 'https://cast.agentrelay.com', + }); + }); + it('tears down idempotently: releases the agent, then deletes the sandbox it provisioned', async () => { const h = harness(); const handle = await spawnFleetSandbox(base, h.deps); diff --git a/packages/sdk/src/fleet-spawn.ts b/packages/sdk/src/fleet-spawn.ts index 7ce19cfc68..b67858420c 100644 --- a/packages/sdk/src/fleet-spawn.ts +++ b/packages/sdk/src/fleet-spawn.ts @@ -531,6 +531,19 @@ export async function spawnFleetSandbox( const releaseAgent = (reason: string) => workspaceRelayFor().workspace.release({ name, reason, deleteAgent: true }); + // Pin the credential/origin pair the agent was launched with: a later + // project rebind or environment change must not redirect attach. + let spawnTransport: { workspaceKey: string; baseUrl?: string } | Error; + if (verifiedTarget) { + spawnTransport = { workspaceKey: verifiedTarget.relaycastApiKey, baseUrl: verifiedTarget.baseUrl }; + } else { + try { + const { workspaceKey, baseUrl } = resolveWorkspaceTransport(legacyTransport); + spawnTransport = { workspaceKey, ...(baseUrl === undefined ? {} : { baseUrl }) }; + } catch (error) { + spawnTransport = error instanceof Error ? error : new Error(String(error)); + } + } let launcherName: string | undefined; let invocation: RelaySpawnPlacementAck; let workerCwd: string | undefined; @@ -556,9 +569,12 @@ export async function spawnFleetSandbox( // The launcher token is already scoped to the exact workspace and // deployment Cloud selected; keep the workspace key only on // `workspaceRelay`, where it mints and releases that token. + // The launcher token is valid only on the gateway that minted it. + const launcherBaseUrl = + spawnTransport instanceof Error ? relaycastTransport.baseUrl : spawnTransport.baseUrl; const relay = deps.createAgentRelay({ token: agentToken, - ...(relaycastTransport.baseUrl === undefined ? {} : { baseUrl: relaycastTransport.baseUrl }), + ...(launcherBaseUrl === undefined ? {} : { baseUrl: launcherBaseUrl }), }); // Placement alone only proves the node accepted the dispatch. A node // running an obsolete broker advertises `spawn:` capacity, acks the @@ -636,19 +652,6 @@ export async function spawnFleetSandbox( } } - // Pin the credential/origin pair the agent was launched with: a later - // project rebind or environment change must not redirect attach. - let spawnTransport: { workspaceKey: string; baseUrl?: string } | Error; - if (verifiedTarget) { - spawnTransport = { workspaceKey: verifiedTarget.relaycastApiKey, baseUrl: verifiedTarget.baseUrl }; - } else { - try { - const { workspaceKey, baseUrl } = resolveWorkspaceTransport(legacyTransport); - spawnTransport = { workspaceKey, ...(baseUrl === undefined ? {} : { baseUrl }) }; - } catch (error) { - spawnTransport = error instanceof Error ? error : new Error(String(error)); - } - } let destroyed: Promise | undefined; let agentReleased = false; let sandboxDeleted = false; From 5e05ea5ddf8759a376e8f28b797db262b8ca4518 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 15:56:03 -0700 Subject: [PATCH 17/19] fix(cloud): settle finished with 0 when the raw attach client detaches Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- packages/cloud/src/attach.test.ts | 10 ++++++++++ packages/cloud/src/attach.ts | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/packages/cloud/src/attach.test.ts b/packages/cloud/src/attach.test.ts index 258447113f..c04eb6800b 100644 --- a/packages/cloud/src/attach.test.ts +++ b/packages/cloud/src/attach.test.ts @@ -173,6 +173,16 @@ it('refuses raw clients after a connection-fatal session error', async () => { expect(received).toBe(''); }); +it('settles finished with 0 when the raw client detaches', async () => { + const { remote } = await setup(); + local = connect(proxy!.socketPath); + await once(local, 'connect'); + send(remote, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + await once(local, 'data'); + local.end(); + await expect(proxy!.finished).resolves.toBe(0); +}); + it('settles finished on explicit close before terminal.ready', async () => { await setup(); await proxy!.close(); diff --git a/packages/cloud/src/attach.ts b/packages/cloud/src/attach.ts index 5ae75f0636..2c91cff76f 100644 --- a/packages/cloud/src/attach.ts +++ b/packages/cloud/src/attach.ts @@ -1161,6 +1161,10 @@ export async function startFleetNodeAttachProxy( socket.on('error', () => socket.destroy()); socket.on('close', () => { if (rawSocket === socket) rawSocket = undefined; + // The single raw client leaving is a detach: report it and refuse a + // replacement. The caller still owns close() for the remote session. + rawRefused = true; + finish(0); }); socket.on('data', (data: Buffer) => { if (options.mode === 'view') return; From 5ec40ec9aff75cf957d894d8a5a01ceac76c6e1f Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 16:45:39 -0700 Subject: [PATCH 18/19] fix(cloud,sdk): keep handle.attach() on the spawn-pinned transport startFleetNodeAttachProxy gains pinnedTransport: the given workspaceKey and baseUrl are used exactly instead of re-resolving the selection, where a matching persisted project route could replace the credential or reject the origin. handle.attach() always pins (Relaycast's canonical origin when the spawn transport had none). The CLI's selector semantics are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 5fe0438a-3323-4877-8199-91318ed6457a --- packages/cloud/src/attach.test.ts | 37 +++++++++++++++++++ packages/cloud/src/attach.ts | 23 +++++++++--- packages/cloud/src/fleet.ts | 1 + .../sdk/src/__tests__/fleet-spawn.test.ts | 2 + packages/sdk/src/fleet-spawn.ts | 6 ++- 5 files changed, 63 insertions(+), 6 deletions(-) diff --git a/packages/cloud/src/attach.test.ts b/packages/cloud/src/attach.test.ts index c04eb6800b..cc2be0fd5d 100644 --- a/packages/cloud/src/attach.test.ts +++ b/packages/cloud/src/attach.test.ts @@ -183,6 +183,43 @@ it('settles finished with 0 when the raw client detaches', async () => { await expect(proxy!.finished).resolves.toBe(0); }); +it('uses a pinned workspaceKey and baseUrl as given, ignoring a persisted project route', async () => { + const { mkdtemp, mkdir, writeFile } = await import('node:fs/promises'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const project = await mkdtemp(join(tmpdir(), 'attach-explicit-')); + await mkdir(join(project, '.agentworkforce', 'relay'), { recursive: true }); + await writeFile( + join(project, '.agentworkforce', 'relay', 'workspace-key.json'), + JSON.stringify({ + workspaceKey: 'rk_live_project', + workspaceId: 'rw_project', + relaycastRoute: 'canonical', + relaycastBaseUrl: 'https://cast.agentrelay.com', + }) + ); + const fetch = vi.fn(async () => + Response.json({ ok: false, error: { code: 'stop', message: 'stop' } }, { status: 400 }) + ); + await expect( + startFleetNodeAttachProxy({ + node: 'node', + agent: 'worker', + mode: 'drive', + workspaceKey: 'rk_live_project', + baseUrl: 'https://agent37-cast.agentrelay.com', + pinnedTransport: true, + env: { AGENT_RELAY_PROJECT: project }, + fetch, + sessionRequest: { timeoutMs: 50, totalTimeoutMs: 50 }, + }) + ).rejects.toBeInstanceOf(Error); + expect(fetch).toHaveBeenCalled(); + const [url, init] = fetch.mock.calls[0] as unknown as [string, RequestInit]; + expect(url.startsWith('https://agent37-cast.agentrelay.com/')).toBe(true); + expect((init.headers as Record).Authorization).toBe('Bearer rk_live_project'); +}); + it('settles finished on explicit close before terminal.ready', async () => { await setup(); await proxy!.close(); diff --git a/packages/cloud/src/attach.ts b/packages/cloud/src/attach.ts index 2c91cff76f..0adc13c1c5 100644 --- a/packages/cloud/src/attach.ts +++ b/packages/cloud/src/attach.ts @@ -138,6 +138,8 @@ export interface FleetNodeAttachOptions { env?: NodeJS.ProcessEnv; baseUrl?: string; workspaceKey?: string; + /** Use workspaceKey/baseUrl exactly as given instead of resolving the selection. */ + pinnedTransport?: boolean; fetch?: typeof globalThis.fetch; /** Deterministic test seam for the bounded session-request retry delay. */ sessionRequest?: { @@ -438,11 +440,22 @@ export async function startFleetNodeAttachProxy( } const env = options.env ?? process.env; const fetchFn = options.fetch ?? globalThis.fetch; - const { workspaceKey, baseUrl: requestedBaseUrl } = resolveWorkspaceTransport({ - workspaceKey: options.workspaceKey, - baseUrl: options.baseUrl, - env, - }); + // A pinned credential/origin pair (e.g. the one a spawn verified) is used + // exactly as given. Otherwise an explicit key is a selector that may bind to + // the project's persisted route and credential. + const pinnedKey = options.pinnedTransport ? options.workspaceKey?.trim() : undefined; + const pinnedBaseUrl = options.pinnedTransport ? options.baseUrl?.trim() : undefined; + if (options.pinnedTransport && (!pinnedKey || !pinnedBaseUrl)) { + throw new FleetNodeAttachError('pinnedTransport requires both workspaceKey and baseUrl.'); + } + const { workspaceKey, baseUrl: requestedBaseUrl } = + pinnedKey && pinnedBaseUrl + ? { workspaceKey: pinnedKey, baseUrl: pinnedBaseUrl } + : resolveWorkspaceTransport({ + workspaceKey: options.workspaceKey, + baseUrl: options.baseUrl, + env, + }); const baseUrl = validateFleetAttachBaseUrl(requestedBaseUrl ?? CANONICAL_RELAYCAST_ORIGIN); const nodePath = safeNodePath(options.node); if (!options.agent) { diff --git a/packages/cloud/src/fleet.ts b/packages/cloud/src/fleet.ts index 84e1ac86fc..c7025a11b7 100644 --- a/packages/cloud/src/fleet.ts +++ b/packages/cloud/src/fleet.ts @@ -388,6 +388,7 @@ export { ensureCloudFleetSandbox, deleteCloudFleetSandbox, CloudFleetSandboxProvisionError, + CANONICAL_RELAYCAST_ORIGIN, DEV_CLOUD_API_URL, DEV_RELAYCAST_ORIGIN, type CloudFleetRelaycastTarget, diff --git a/packages/sdk/src/__tests__/fleet-spawn.test.ts b/packages/sdk/src/__tests__/fleet-spawn.test.ts index e3e13361fe..2f632592e4 100644 --- a/packages/sdk/src/__tests__/fleet-spawn.test.ts +++ b/packages/sdk/src/__tests__/fleet-spawn.test.ts @@ -155,6 +155,8 @@ describe('spawnFleetSandbox', () => { agent: 'sandbox-worker', mode: 'drive', workspaceKey: 'rk_live_test', + pinnedTransport: true, + baseUrl: 'https://cast.agentrelay.com', }) ); }); diff --git a/packages/sdk/src/fleet-spawn.ts b/packages/sdk/src/fleet-spawn.ts index b67858420c..9e5c79b217 100644 --- a/packages/sdk/src/fleet-spawn.ts +++ b/packages/sdk/src/fleet-spawn.ts @@ -9,6 +9,7 @@ import { randomUUID } from 'node:crypto'; import { + CANONICAL_RELAYCAST_ORIGIN, CloudFleetSandboxProvisionError, DEV_CLOUD_API_URL, DEV_RELAYCAST_ORIGIN, @@ -674,8 +675,11 @@ export async function spawnFleetSandbox( node: ready.nodeName, agent: name, mode: options.mode ?? 'drive', + // Keep attach on the exact pair the spawn resolved and verified; a + // transport without an origin used Relaycast's canonical default. + pinnedTransport: true, workspaceKey: attachTransport.workspaceKey, - ...(attachTransport.baseUrl === undefined ? {} : { baseUrl: attachTransport.baseUrl }), + baseUrl: attachTransport.baseUrl ?? CANONICAL_RELAYCAST_ORIGIN, }); }, destroy: () => From 8a7e5683e7fd6c1427e9579bf6ce531e780e6bc2 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 10 Oct 2026 01:06:04 -0700 Subject: [PATCH 19/19] test(relayflows): give sandbox spawn probe mocks a dispatch receipt spawnFleetSandbox fails closed (placement_mismatch) unless the engine's dispatch receipt names the ensured sandbox node. The 1656, 1744 and 1746 probes mocked a spawn with no receipt, so 1656 stopped before reaching the output-sink failure it relies on. Their subject is sandbox identity and launcher authority, not placement, so the mocks now report the receipt a real dispatch to that node returns. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../relayflows/cases/1656-long-running-agent37-sandbox/run.mjs | 3 +++ .../cases/1744-explicit-provider-workload-profile/run.mjs | 2 ++ .../cases/1746-sandbox-launcher-token-authority/run.mjs | 2 ++ 3 files changed, 7 insertions(+) diff --git a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs index 916cd2edb4..8065facf04 100644 --- a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs +++ b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs @@ -270,6 +270,9 @@ test('fleet spawn --sandbox replays and retains an exact provider-attributed ide spawn: vi.fn(async () => ({ invocationId: 'inv_relayflow', node: { name: REPLAY_SANDBOX_NAME }, + // The engine's dispatch receipt: sandbox spawns fail closed + // (placement_mismatch) unless it names the ensured sandbox node. + dispatchedNodeId: 'node-relayflow', })), }, }, diff --git a/tests/relayflows/cases/1744-explicit-provider-workload-profile/run.mjs b/tests/relayflows/cases/1744-explicit-provider-workload-profile/run.mjs index 468b88e766..2a43fa6b52 100644 --- a/tests/relayflows/cases/1744-explicit-provider-workload-profile/run.mjs +++ b/tests/relayflows/cases/1744-explicit-provider-workload-profile/run.mjs @@ -194,6 +194,8 @@ test('explicit provider selection emits the provider-compatible Cloud workload p spawn: vi.fn(async () => ({ invocationId: 'inv_' + provider, node: { name: provider + '-worker' }, + // Sandbox spawns fail closed unless the dispatch receipt names the sandbox node. + dispatchedNodeId: 'node-' + provider, })), }, }, diff --git a/tests/relayflows/cases/1746-sandbox-launcher-token-authority/run.mjs b/tests/relayflows/cases/1746-sandbox-launcher-token-authority/run.mjs index 300507cde2..fff4acc93f 100644 --- a/tests/relayflows/cases/1746-sandbox-launcher-token-authority/run.mjs +++ b/tests/relayflows/cases/1746-sandbox-launcher-token-authority/run.mjs @@ -120,6 +120,8 @@ test('sandbox dispatch constructs an agent client with exactly one authority', a spawn: vi.fn(async () => ({ invocationId: 'inv_relayflow_1746', node: { name: 'daytona-worker' }, + // Sandbox spawns fail closed unless the dispatch receipt names the sandbox node. + dispatchedNodeId: 'node-daytona', })), }; let constructorOptions: Record | undefined;