diff --git a/.agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/summary.md b/.agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/summary.md new file mode 100644 index 0000000000..b04d0f9c86 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/summary.md @@ -0,0 +1,23 @@ +# Trajectory: Publish fleet and attach subpaths and forward sandbox readonlyPaths + +> **Status:** ❌ Abandoned +> **Started:** September 18, 2026 at 03:11 PM +> **Completed:** September 18, 2026 at 03:21 PM + +--- + +## Key Decisions + +### Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics +- **Chose:** Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics +- **Reasoning:** Reviewed plan identifies Cloud as the consumer import target, no remote exit-code protocol, and no server mount handler in this repository. D4 running-agent orchestration remains pending user scope clarification. + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics: Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics +- Abandoned: Attach extraction and readonlyPaths client implementation ready; full spawnFleetSandbox orchestration awaits reviewed-plan.md D4 scope decision. Server chmod enforcement is outside this repository. diff --git a/.agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/trajectory.json b/.agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/trajectory.json new file mode 100644 index 0000000000..7b06f12566 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-09/traj_qdtkdvki0xoh/trajectory.json @@ -0,0 +1,54 @@ +{ + "id": "traj_qdtkdvki0xoh", + "version": 1, + "task": { + "title": "Publish fleet and attach subpaths and forward sandbox readonlyPaths" + }, + "status": "abandoned", + "startedAt": "2026-09-18T15:11:05.520Z", + "completedAt": "2026-09-18T15:21:12.959Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-09-18T15:20:29.731Z" + } + ], + "chapters": [ + { + "id": "chap_jnv5kvm9t42d", + "title": "Work", + "agentName": "default", + "startedAt": "2026-09-18T15:20:29.731Z", + "endedAt": "2026-09-18T15:21:12.959Z", + "events": [ + { + "ts": 1789744829731, + "type": "decision", + "content": "Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics: Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics", + "raw": { + "question": "Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics", + "chosen": "Implement shared attach transport in Cloud with SDK re-exports; preserve ESM exports and inferred completion semantics", + "alternatives": [], + "reasoning": "Reviewed plan identifies Cloud as the consumer import target, no remote exit-code protocol, and no server mount handler in this repository. D4 running-agent orchestration remains pending user scope clarification." + }, + "significance": "high" + }, + { + "ts": 1789744872960, + "type": "note", + "content": "Abandoned: Attach extraction and readonlyPaths client implementation ready; full spawnFleetSandbox orchestration awaits reviewed-plan.md D4 scope decision. Server chmod enforcement is outside this repository.", + "significance": "high" + } + ] + } + ], + "commits": [], + "filesChanged": [], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "c97830275c0849f8ae1ed7fc1f347a5198124b11", + "endRef": "c97830275c0849f8ae1ed7fc1f347a5198124b11" + } +} \ No newline at end of file diff --git a/.github/workflows/relay-evals.yml b/.github/workflows/relay-evals.yml index 46ccb49868..9e5e77147d 100644 --- a/.github/workflows/relay-evals.yml +++ b/.github/workflows/relay-evals.yml @@ -57,7 +57,7 @@ jobs: run: npm ci - name: Build SDK - run: npm run build:sdk + run: npm run build:session && npm run build:config && npm run build:cloud && npm run build:sdk - name: Run offline evals run: npm run evals:offline diff --git a/CHANGELOG.md b/CHANGELOG.md index 44c90c85ba..a5a90350f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The `agent-relay mcp` server adds `upload_file` (local path or base64) and `download_file`, and `send_group_dm` accepts `attachments`. - Agents spawned by the broker receive message attachments (up to 25 MiB each) as local files: the injected message ends with an `Attachments:` list giving each file's path under `.agent-relay/attachments/` in the agent's working directory, or an `agent-relay message file download ` command when a file could not be fetched. Messages that carry only attachments are now delivered too. - `@agent-relay/sdk` adds `files.upload` / `files.get` / `files.download` on `AgentRelay` and `RelaycastMessagingClient`, plus `uploadRelayFile` / `downloadRelayFile` helpers. +- `@agent-relay/sdk/fleet` exports `spawnFleetSandbox`, which provisions a Cloud fleet sandbox, starts an agent harness on it, confirms the node the agent landed on, and returns a handle to the live agent that can be attached to immediately and torn down idempotently. `agent-relay fleet spawn --sandbox` uses the same path. +- `@agent-relay/cloud/attach` and `@agent-relay/sdk/attach` expose fleet terminal attachment with a private local stdio socket and completion promise. +- `@agent-relay/cloud/fleet` and `@agent-relay/sdk/fleet` expose the sandbox ensure and deletion primitives. +- Fleet sandbox ensure and `fleet spawn --sandbox-readonly-path` forward explicit read-only Relayfile subtree requests to Cloud for server-side enforcement. ### Fixed diff --git a/package-lock.json b/package-lock.json index 6736bed65e..a7384efc2b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14170,12 +14170,14 @@ "@agent-relay/config": "12.4.1", "@aws-sdk/client-s3": "3.1020.0", "ignore": "^7.0.5", - "tar": "^7.5.21" + "tar": "^7.5.21", + "ws": "^8.18.3" }, "devDependencies": { "@types/node": "^22.19.3", "@types/ssh2": "^1.15.5", - "vitest": "^4.1.0" + "vitest": "^4.1.0", + "@types/ws": "^8.18.1" }, "engines": { "node": ">=22.0.0" @@ -14308,7 +14310,8 @@ "dependencies": { "@relaycast/sdk": "^8.0.7", "@relaycast/types": "^8.0.7", - "zod": "^4.4.3" + "zod": "^4.4.3", + "@agent-relay/cloud": "13.2.0" }, "devDependencies": { "@types/node": "^22.13.10" diff --git a/package.json b/package.json index 8de28a98c6..d68a617de4 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "test:integration:broker": "npx tsc -p tests/integration/broker/tsconfig.json && cd tests/integration/broker && node --test dist/*.test.js", "test:integration:broker:build": "npx tsc -p tests/integration/broker/tsconfig.json", "test:integration:broker:run": "cd tests/integration/broker && node --test dist/*.test.js", - "eval:build": "npm run build:sdk && npm run build:harness-driver && npx tsc -p tests/integration/broker/evals/tsconfig.json", + "eval:build": "npm run build:session && npm run build:config && npm run build:cloud && npm run build:sdk && npm run build:harness-driver && npx tsc -p tests/integration/broker/evals/tsconfig.json", "eval:unit": "vitest run tests/integration/broker/evals", "eval:selftest": "npm run eval:build && node tests/integration/broker/dist/evals/selftest.js", "eval:toolcheck": "npm run eval:build && node tests/integration/broker/dist/evals/toolcheck-cli.js", diff --git a/packages/cli/README.md b/packages/cli/README.md index 55dc8763c3..ce0858be0c 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -476,6 +476,19 @@ agent-relay fleet spawn claude \ --task 'Review the live draft under /workspace/live-review/run-123' ``` +Use `--sandbox-readonly-path '/reference/**'` to request a read-only Relayfile +subtree. It requires `--sandbox` with Relayfile enabled and accepts explicit +`/path/**` subtrees (no traversal or other wildcards). This client forwards the +request; the deployed Cloud ensure handler must implement chmod enforcement. + +The ESM `@agent-relay/cloud/attach` and `@agent-relay/sdk/attach` entries expose +`startFleetNodeAttachProxy({ nodeId, mode })`. An omitted agent is discovered +only when the node has exactly one agent. `socketPath` accepts one raw stdio +connection; `close()` removes it. `finished` reports an inferred status (0 for +terminal closure or detach, 1 for transport failure), because Relaycast does +not currently transmit the remote harness exit code. Existing CLI broker +transport fields remain available for compatibility. + `--session-ref` is a real CLI resume, not a logical collaboration label. Pass the actual Claude session ID or Codex thread ID and target its origin node. `--cwd` must name an absolute directory that exists on the selected node. The diff --git a/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts b/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts index a7e4ffe660..fc6ac1ed19 100644 --- a/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts +++ b/packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts @@ -101,7 +101,11 @@ describe('fleet CLI lifecycle routing', () => { } as never; }); const placement = { - spawn: vi.fn(async () => ({ invocationId: 'inv_lifecycle', node: { name: nodeName } })), + spawn: vi.fn(async () => ({ + invocationId: 'inv_lifecycle', + dispatchedNodeId: 'node-1', + node: { name: nodeName }, + })), }; const ensureCloudFleetSandbox = vi.fn(async () => ({ outcome: 'provisioned' as const, diff --git a/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts b/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts index b6f8f1ae4d..3b582e32ef 100644 --- a/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts +++ b/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts @@ -42,6 +42,17 @@ function registerSandboxCommand(overrides: { placement?: ReturnType; }): Command { const placement = overrides.placement ?? vi.fn(async () => ({ invocationId: 'inv_test' })); + // Like the engine, the dispatch receipt names the node the spawn ran on. + let ensuredNodeId: string | undefined; + const ensureCloudFleetSandbox = async (...args: unknown[]) => { + const result = await overrides.ensureCloudFleetSandbox(...args); + ensuredNodeId = (result as { nodeId?: string } | undefined)?.nodeId; + return result; + }; + const spawn = async (input: unknown) => ({ + dispatchedNodeId: ensuredNodeId, + ...((await placement(input)) as Record), + }); const register = vi.fn(async () => ({ token: 'at_live_launcher' })); const release = vi.fn(async () => ({ released: true, deleted: true })); const createWorkspaceRelay = vi.fn(() => ({ @@ -69,7 +80,7 @@ function registerSandboxCommand(overrides: { ...(overrides.materializeCloudRelayfileRepository ? { materializeCloudRelayfileRepository: overrides.materializeCloudRelayfileRepository as never } : {}), - ensureCloudFleetSandbox: overrides.ensureCloudFleetSandbox as never, + ensureCloudFleetSandbox: ensureCloudFleetSandbox as never, resolveWorkspaceSelection: () => ({ key: 'rk_live_workspace', source: 'project', @@ -79,7 +90,7 @@ function registerSandboxCommand(overrides: { persistWorkspaceRelaycastTarget: () => true, deleteCloudFleetSandbox: vi.fn(async () => undefined), sdk: { - createAgentRelay: vi.fn(() => ({ messaging: { placement: { spawn: placement } } })) as never, + createAgentRelay: vi.fn(() => ({ messaging: { placement: { spawn } } })) as never, createWorkspaceRelay: createWorkspaceRelay as never, createWorkspace: vi.fn() as never, log: vi.fn(), diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index e12fc0df12..82c088b0bc 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1103,6 +1103,7 @@ describe('fleet command support', () => { const placement = { spawn: vi.fn(async () => ({ invocationId: 'inv_sandbox', + dispatchedNodeId: 'node-1', node: { name: 'e2b-codex' }, })), }; @@ -1178,6 +1179,8 @@ describe('fleet command support', () => { REPLAY_SANDBOX_NAME, '--sandbox-relayfile-path', '/live-review/run-123/**', + '--sandbox-readonly-path', + '/live-review/run-123/reference/**', '--name', 'sandbox-worker', '--task', @@ -1195,6 +1198,7 @@ describe('fleet command support', () => { maxAgents: 1, mountRelayfile: true, relayfilePaths: ['/live-review/run-123/**'], + readonlyPaths: ['/live-review/run-123/reference/**'], sandboxId: REPLAY_SANDBOX_ID, forceProvision: true, providerId: 'agent37', @@ -1253,6 +1257,95 @@ describe('fleet command support', () => { }); }); + it('fleet spawn --sandbox rejects an agent dispatched to another node and tears the sandbox down', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', undefined); + const placement = { + spawn: vi.fn(async () => ({ + invocationId: 'inv_sandbox', + node: { name: 'e2b-codex' }, + dispatchedNodeId: 'node-2', + })), + }; + const release = vi.fn(async () => ({ released: true, deleted: true })); + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { + info: vi.fn(async () => ({ id: 'rw_abc' })), + register: vi.fn(async () => ({ token: 'at_live_launcher' })), + release, + }, + })); + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const errors: string[] = []; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + resolveSandboxRepository: vi.fn(() => undefined), + sdk: { + createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, + createWorkspaceRelay: createWorkspaceRelay as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: (message: unknown) => errors.push(String(message)), + exit: vi.fn(() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => ({ + outcome: 'provisioned' as const, + providerId: 'e2b' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-1', + nodeName: 'e2b-codex', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + relayfileMountPath: '/workspace', + })), + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/tmp/agent-relay-test/workspace-key.json', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-provider', + 'e2b', + '--no-sandbox-relayfile', + '--name', + 'sandbox-worker', + '--task', + 'Wait for VERIFY', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + expect(errors.join('\n')).toContain("landed on node 'node-2', not sandbox node 'e2b-codex'"); + expect(release).toHaveBeenCalledWith( + expect.objectContaining({ name: 'sandbox-worker', deleteAgent: true }) + ); + expect(deleteCloudFleetSandbox).toHaveBeenCalledWith({ + cloudWorkspaceId: 'cloud-workspace', + sandboxId: 'sandbox-1', + providerId: 'e2b', + }); + }); + it('plain --sandbox materializes the inferred repository through Relayfile and starts in its live relative cwd', async () => { vi.stubEnv('RELAY_AGENT_TOKEN', undefined); const revision = '0123456789abcdef0123456789abcdef01234567'; @@ -2131,7 +2224,11 @@ describe('fleet command support', () => { const selectionOptions: Record[] = []; const persistWorkspaceRelaycastTarget = vi.fn(() => true); const placement = { - spawn: vi.fn(async () => ({ invocationId: 'inv_inferred', node: { name: 'cloud-node' } })), + spawn: vi.fn(async () => ({ + invocationId: 'inv_inferred', + dispatchedNodeId: 'node-1', + node: { name: 'cloud-node' }, + })), }; const register = vi.fn(async () => ({ token: 'at_live_launcher' })); const release = vi.fn(async () => ({ released: true, deleted: true })); @@ -5428,3 +5525,38 @@ describe('fleet command support', () => { expect(errors.join('\n')).toMatch(/relay cloud enroll/); }); }); + +describe('sandbox read-only path guards', () => { + it.each([[], ['--sandbox', '--no-sandbox-relayfile']])( + 'rejects unavailable mounts (%j)', + async (...flags) => { + const ensureCloudFleetSandbox = vi.fn(); + const error = vi.fn(); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + ensureCloudFleetSandbox, + sdk: { error, exit: vi.fn() as never }, + }); + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'claude', + ...flags, + '--sandbox-readonly-path', + '/reference/**', + '--name', + 'worker', + '--task', + 'review', + ], + { from: 'user' } + ); + expect(error).toHaveBeenCalledWith( + expect.stringContaining('--sandbox-readonly-path requires --sandbox') + ); + expect(ensureCloudFleetSandbox).not.toHaveBeenCalled(); + } + ); +}); diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index fd1ff04647..c63c090257 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -4,18 +4,14 @@ import path from 'node:path'; import { InvalidArgumentError, type Command } from 'commander'; import { findProjectRoot } from '@agent-relay/config'; import { - CloudFleetSandboxProvisionError, - DEV_CLOUD_API_URL, - DEV_RELAYCAST_ORIGIN, deleteCloudFleetSandbox, - ensureCloudFleetSandbox, materializeCloudRelayfileRepository, resolveWorkspaceByKey, type CloudFleetSandboxProviderId, type CloudRelayfileRepositoryMaterialization, - type EnsureCloudFleetSandboxInput, type EnsureCloudFleetSandboxResult, } from '@agent-relay/cloud'; +import { ensureCloudFleetSandbox, spawnFleetSandbox } from '@agent-relay/sdk/fleet'; import { HarnessDriverClient } from '@agent-relay/harness-driver'; import { createWorkspaceClient, @@ -468,6 +464,10 @@ export function registerFleetCommands( '--sandbox-relayfile-path ', 'Mount only these Relayfile subtrees (each path must end in /**)' ) + .option( + '--sandbox-readonly-path ', + 'Read-only Relayfile subtrees (/path/**); requires Cloud enforcement' + ) .option('--no-sandbox-relayfile', 'Provision the sandbox without mounting Relayfile') .option('--channel ', 'Channel for the worker to join') .option('--persona ', 'Worker persona (automatic placement)') @@ -496,7 +496,8 @@ export function registerFleetCommands( const clientOptions = sdkOptionsFromOpts(options); const name = requiredText(options.name, 'Worker name'); const task = requiredText(options.task, 'Task'); - let targetNode = optionalText(options.targetNode, 'Target node') ?? optionalText(options.node, 'Node'); + const targetNode = + optionalText(options.targetNode, 'Target node') ?? optionalText(options.node, 'Node'); const useSandbox = options.sandbox === true; // Explicit hosted credentials/transport and personas retain their legacy // automatic-placement contract. Ambient credentials and a persisted @@ -513,10 +514,6 @@ export function registerFleetCommands( const checkoutRepository = options.checkout === true; const sandboxName = optionalText(options.sandboxName, 'Sandbox name'); const sandboxIdOption = optionalText(options.sandboxId, 'Sandbox ID'); - // An explicit sandbox identity is a retained/replayable resource. Never - // delete it as collateral when a later verification or dispatch step - // fails; only clean up sandboxes whose identity this invocation minted. - const shouldCleanupSandbox = sandboxIdOption === undefined; const explicitWorkspaceId = optionalText(options.workspaceId, 'Workspace ID'); if (sandboxIdOption !== undefined && !CLOUD_SANDBOX_ID_PATTERN.test(sandboxIdOption)) { throw new Error('--sandbox-id must match lowercase sbx_ using an RFC 4122 UUID.'); @@ -534,6 +531,12 @@ export function registerFleetCommands( throw new Error('--sandbox-provider must be daytona, e2b, or agent37.'); } const mountSandboxRelayfile = options.sandboxRelayfile !== false; + const sandboxReadonlyPaths = optionalTextList(options.sandboxReadonlyPath, 'Sandbox read-only path'); + if (sandboxReadonlyPaths && (!useSandbox || !mountSandboxRelayfile)) { + throw new Error( + '--sandbox-readonly-path requires --sandbox and cannot be combined with --no-sandbox-relayfile.' + ); + } const sandboxRelayfilePaths = optionalTextList(options.sandboxRelayfilePath, 'Sandbox Relayfile path'); if (useSandbox && targetNode) { throw new Error('--sandbox cannot be combined with --node or --target-node.'); @@ -594,14 +597,10 @@ export function registerFleetCommands( ); } - let sandbox: EnsureCloudFleetSandboxResult | undefined; let sandboxRepository: SandboxRepositorySelection | undefined; let liveRepository: CloudRelayfileRepositoryMaterialization | undefined; let sandboxMountPaths: string[] | undefined; let attachProjectRoot: string | undefined; - let workspaceRelay: ReturnType | undefined; - let relaycastClientOptions = clientOptions; - let legacyWorkspaceClientOptions = clientOptions; if (useSandbox) { const coreProjectRoot = deps.core.getProjectPaths().projectRoot; const hasExplicitProjectOverride = Boolean( @@ -663,7 +662,6 @@ export function registerFleetCommands( ...clientOptions, projectRoot: workspaceProjectRoot, }; - relaycastClientOptions = sandboxClientOptions; // Cloud must be the first network authority for a sandbox invocation. // A canonical Relaycast info call would both leak the workspace key and // make it impossible to prove that Cloud's isolated target is the one @@ -671,7 +669,7 @@ export function registerFleetCommands( const workspaceSelection = deps.resolveWorkspaceSelection({ ...sandboxClientOptions, }); - legacyWorkspaceClientOptions = { + const legacyWorkspaceClientOptions = { ...sandboxClientOptions, ...(sandboxProvider === 'agent37' ? {} : { ignorePersistedRelaycastTarget: true }), }; @@ -698,8 +696,9 @@ export function registerFleetCommands( ); } if (!relayWorkspaceId && sandboxProvider !== undefined && sandboxProvider !== 'agent37') { - workspaceRelay = deps.sdk.createWorkspaceRelay(legacyWorkspaceClientOptions); - const workspaceInfo = await workspaceRelay.workspace.info(); + const workspaceInfo = await deps.sdk + .createWorkspaceRelay(legacyWorkspaceClientOptions) + .workspace.info(); relayWorkspaceId = workspaceInfo.id?.trim(); } if (!relayWorkspaceId) { @@ -772,293 +771,127 @@ export function registerFleetCommands( revision: sandboxRepository.revision, }); } - const sandboxId = sandboxIdOption ?? (sandboxName === undefined ? `sbx_${randomUUID()}` : undefined); - const deterministicSandboxName = - sandboxId === undefined ? undefined : `fleet-sandbox-${sandboxId.slice('sbx_'.length)}`; - if ( - sandboxIdOption !== undefined && - sandboxName !== undefined && - sandboxName !== deterministicSandboxName - ) { - throw new Error( - `--sandbox-name must be '${deterministicSandboxName}' when --sandbox-id is supplied; custom names cannot preserve the one-to-one sandbox identity.` - ); - } - const effectiveSandboxName = deterministicSandboxName ?? sandboxName; - // The unpinned/Agent37 path deliberately carries the measured heavy - // 8 CPU / 16 GiB / 20 GiB profile. Daytona and E2B cannot satisfy that - // shape, so an explicit legacy-provider selection must request the - // provider-neutral durable profile instead of becoming unroutable by - // construction. - const workloadProfile = - sandboxProvider === undefined || sandboxProvider === 'agent37' - ? 'long-running-agent' - : 'standard-long-running-agent'; - const useAsyncPreparation = - sandboxId !== undefined && (sandboxProvider === undefined || sandboxProvider === 'agent37'); - if (useAsyncPreparation && sandboxIdOption === undefined) { - // The generated identity is not persisted. Print it before Cloud work - // starts so an interrupted run can be resumed instead of duplicated. - deps.warn( - `Cloud sandbox identity: ${sandboxId}. If this command is interrupted, re-run it with --sandbox-id ${sandboxId} to resume the same sandbox instead of creating another.` - ); - } - try { - const ensureInput: EnsureCloudFleetSandboxInput = { + const handle = await spawnFleetSandbox( + { + cli, + name, + task, workspaceId: relayWorkspaceId, - requiredCapability: `spawn:${cli}`, - ...(useAsyncPreparation ? { preparationMode: 'async-v1' as const } : {}), - maxAgents: 1, + transport: sandboxClientOptions, + ...(sandboxProvider === undefined ? {} : { provider: sandboxProvider }), + ...(sandboxIdOption === undefined ? {} : { sandboxId: sandboxIdOption }), + ...(sandboxName === undefined ? {} : { sandboxName }), mountRelayfile: mountSandboxRelayfile, ...(sandboxMountPaths === undefined ? {} : { relayfilePaths: sandboxMountPaths }), - ...(sandboxId === undefined ? {} : { sandboxId }), - forceProvision: true, - ...(sandboxProvider === undefined ? {} : { providerId: sandboxProvider }), - workloadProfile, - waitTimeoutMs: 90_000, - ...(effectiveSandboxName === undefined ? {} : { name: effectiveSandboxName }), - ...(checkoutRepository && sandboxRepository ? { repos: [sandboxRepository.repository] } : {}), + ...(sandboxReadonlyPaths === undefined ? {} : { readonlyPaths: sandboxReadonlyPaths }), ...(checkoutRepository && sandboxRepository - ? { repoRevisions: { [sandboxRepository.repository]: sandboxRepository.revision } } + ? { + repos: [sandboxRepository.repository], + repoRevisions: { [sandboxRepository.repository]: sandboxRepository.revision }, + } : {}), - }; - sandbox = useAsyncPreparation - ? await deps.ensureCloudFleetSandbox(ensureInput, { - onPreparationProgress: (progress) => { - deps.warn( - `Cloud sandbox preparation: ${progress.phase} (${progress.state}, generation ${progress.generation}).` - ); - }, - }) - : await deps.ensureCloudFleetSandbox(ensureInput); - assertSandboxRepositoryRevision(sandbox, checkoutRepository ? sandboxRepository : undefined); - } catch (error) { - if (error instanceof CloudFleetSandboxProvisionError && error.sandboxAbsent) { - // Cloud's terminal record for this exact identity already proves the - // provider sandbox is gone; a delete here would only race its reaper. - deps.warn( - `${error.message} Cloud confirmed sandbox '${ - error.sandboxId ?? sandboxId ?? 'the requested sandbox' - }' is not running; no sandbox was left running.` - ); - } else if ( - shouldCleanupSandbox && - error instanceof CloudFleetSandboxProvisionError && - error.confirmedProvisioned && - error.cloudWorkspaceId && - error.sandboxId - ) { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: error.cloudWorkspaceId, - sandboxId: error.sandboxId, - ...(error.providerId === undefined ? {} : { providerId: error.providerId }), - }) - .catch((cleanupError) => { - deps.warn( - `Provisioning failed after Cloud confirmed sandbox '${error.sandboxId}', and automatic cleanup failed: ${ - cleanupError instanceof Error ? cleanupError.message : String(cleanupError) - }` - ); - }); - } else if (error instanceof CloudFleetSandboxProvisionError && error.noSandboxCreated) { - deps.warn(error.message); - } else if (error instanceof CloudFleetSandboxProvisionError && error.outcomeUnknown) { - deps.warn( - `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ - error.nodeName ?? effectiveSandboxName ?? 'the requested sandbox' - }'${ - sandboxId === undefined ? '' : ` before retrying with --sandbox-id '${sandboxId}'` - } so a sandbox is not left running.` - ); - } else if ( - shouldCleanupSandbox && - error instanceof CloudFleetSandboxProvisionError && - error.cloudWorkspaceId && - error.sandboxId - ) { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: error.cloudWorkspaceId, - sandboxId: error.sandboxId, - ...(error.providerId === undefined ? {} : { providerId: error.providerId }), - }) - .catch((cleanupError) => { - deps.warn( - `Provisioning failed after Cloud created sandbox '${error.sandboxId}', and automatic cleanup failed: ${ - cleanupError instanceof Error ? cleanupError.message : String(cleanupError) - }` - ); - }); - } - if (shouldCleanupSandbox && sandbox && sandbox.outcome !== 'reused') { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: sandbox.cloudWorkspaceId, - sandboxId: sandbox.sandboxId, - ...(sandbox.providerId === undefined ? {} : { providerId: sandbox.providerId }), - }) - .catch((cleanupError) => { - deps.warn( - `Sandbox repository verification failed and cleanup also failed: ${ - cleanupError instanceof Error ? cleanupError.message : String(cleanupError) - }` + ...(channel ? { channels: [channel] } : {}), + ...(model ? { model } : {}), + spawnMetadata: { ...registrationMetadata, ...(sessionRef ? { session_ref: sessionRef } : {}) }, + confirm: options.confirm !== false, + confirmTimeoutMs, + verifySandbox: (ensured) => + assertSandboxRepositoryRevision(ensured, checkoutRepository ? sandboxRepository : undefined), + resolveWorkerCwd: (ensured) => { + if (workerCwd) return workerCwd; + if ( + liveRepository && + sandboxRepository && + ensured.outcome === 'provisioned' && + ensured.relayfileMounted + ) { + return liveRelayfileWorkerCwd( + ensured.relayfileMountPath ?? '/workspace', + liveRepository, + sandboxRepository.repositoryRelativeCwd ); - }); - } - throw error; - } - if (sandbox.outcome !== 'provisioning_timeout' && sandbox.relaycastTarget) { - // When Cloud returns a closed, server-owned target, apply it for any - // provider and outcome before registration, spawn, or launcher release. - // Rebuild both credentials and origin before any registration, spawn, - // or launcher release, then prove the authenticated client sees the - // exact workspace Cloud returned. - try { - const target = sandbox.relaycastTarget; - const returnedRelayWorkspaceId = - 'relayWorkspaceId' in sandbox ? sandbox.relayWorkspaceId?.trim() : undefined; - if ( - (returnedRelayWorkspaceId !== undefined && - target.workspaceId.trim() !== returnedRelayWorkspaceId) || - (sandbox.outcome === 'provisioned' && !returnedRelayWorkspaceId) - ) { - throw new Error('Cloud returned a Relaycast target for a different workspace.'); - } - relaycastClientOptions = { - ...relaycastClientOptions, - workspaceKey: target.relaycastApiKey, - baseUrl: target.baseUrl, - }; - workspaceRelay = deps.sdk.createWorkspaceRelay(relaycastClientOptions); - const postEnsureWorkspace = await workspaceRelay.workspace.info(); - const postEnsureWorkspaceId = postEnsureWorkspace.id?.trim(); - if ( - !postEnsureWorkspaceId || - (sandbox.outcome === 'provisioned' && postEnsureWorkspaceId !== returnedRelayWorkspaceId) || - postEnsureWorkspaceId !== target.workspaceId.trim() - ) { - throw new Error( - 'Cloud returned a Relaycast workspace that could not be verified on the selected gateway.' - ); - } - const persistedTarget = sandbox.relaycastCloudApiUrl - ? deps.persistWorkspaceRelaycastTarget(workspaceSelection, target, sandbox.relaycastCloudApiUrl) - : deps.persistWorkspaceRelaycastTarget(workspaceSelection, target); - if (!persistedTarget) { + } + return undefined; + }, + resolveTask: (ensured, sandboxWorkerCwd) => { if ( - target.route === 'canonical' && - target.baseUrl === DEV_RELAYCAST_ORIGIN && - sandbox.relaycastCloudApiUrl !== DEV_CLOUD_API_URL + checkoutRepository && + sandboxRepository && + mountSandboxRelayfile && + ensured.outcome === 'provisioned' ) { + return `${task}\n\nAgent Relay sandbox context: Relayfile records are available at ${ensured.relayfileMountPath ?? '/workspace'}. The source checkout is separate; use ${sandboxWorkerCwd ?? 'the worker checkout'} for repository files and the mount for Relayfile records.`; + } + const liveSandboxContext = + ensured.outcome === 'provisioned' && liveRepository && sandboxRepository + ? [ + `Agent Relay sandbox context: ${liveRepository.repository} is mounted as a live Relayfile working tree at ${liveRelayfileWorkerCwd( + ensured.relayfileMountPath ?? '/workspace', + liveRepository, + '' + )}. Its exact source revision is ${liveRepository.revision}.`, + ...(sandboxMountPaths?.includes(`${path.posix.dirname(liveRepository.sentinelPath)}/**`) + ? [ + `The same attestation is recorded at ${mountedRelayfilePath( + ensured.relayfileMountPath ?? '/workspace', + liveRepository.sentinelPath + )}.`, + ] + : []), + ...(sandboxMountPaths?.includes('/.skills/**') + ? [ + `Workspace skills are under ${mountedRelayfilePath( + ensured.relayfileMountPath ?? '/workspace', + '/.skills' + )}.`, + ] + : []), + 'The Relayfile daemon synchronizes this tree; it intentionally has no .git directory.', + ].join(' ') + : undefined; + return liveSandboxContext ? `${task}\n\n${liveSandboxContext}` : task; + }, + }, + { + ensureCloudFleetSandbox: deps.ensureCloudFleetSandbox, + deleteCloudFleetSandbox: deps.deleteCloudFleetSandbox, + resolveWorkspaceByKey: deps.resolveWorkspaceByKey, + createWorkspaceRelay: (relayOptions) => deps.sdk.createWorkspaceRelay(relayOptions), + createAgentRelay: (relayOptions) => deps.sdk.createAgentRelay(relayOptions), + persistRelaycastTarget: (target, { sandbox }) => { + const persistedTarget = sandbox.relaycastCloudApiUrl + ? deps.persistWorkspaceRelaycastTarget( + workspaceSelection, + target, + sandbox.relaycastCloudApiUrl + ) + : deps.persistWorkspaceRelaycastTarget(workspaceSelection, target); + if (!persistedTarget) { throw new Error( - `Cloud returned the DEV canonical Relaycast target, but relaycastCloudApiUrl was not exactly ${DEV_CLOUD_API_URL}; refusing to persist an untrusted route.` + 'Cloud returned a Relaycast target, but no durable project session is available for follow-up attach.' ); } - throw new Error( - 'Cloud returned a Relaycast target, but no durable project session is available for follow-up attach.' - ); - } - } catch (error) { - if (shouldCleanupSandbox && sandbox.outcome === 'provisioned') { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: sandbox.cloudWorkspaceId, - sandboxId: sandbox.sandboxId, - ...(sandbox.providerId === undefined ? {} : { providerId: sandbox.providerId }), - }) - .catch((cleanupError) => { - deps.warn( - `Relaycast workspace verification failed and sandbox cleanup also failed: ${ - cleanupError instanceof Error ? cleanupError.message : String(cleanupError) - }` - ); - }); - } - throw error; - } - } else if (sandbox.outcome !== 'provisioning_timeout') { - // Older non-Agent37 Cloud responses can omit a target. In that - // compatibility case, keep every subsequent client on the canonical - // workspace selection; a stale persisted Agent37 target must not - // leak into registration, dispatch, or launcher release. - relaycastClientOptions = legacyWorkspaceClientOptions; - } - if (sandbox.outcome === 'provisioning_timeout') { - if (shouldCleanupSandbox) { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: sandbox.cloudWorkspaceId, - sandboxId: sandbox.sandboxId, - ...(sandbox.providerId === undefined ? {} : { providerId: sandbox.providerId }), - }) - .catch((error) => { - deps.warn( - `The timed-out sandbox could not be cleaned up automatically: ${ - error instanceof Error ? error.message : String(error) - }` - ); - }); - } - throw new Error( - `Sandbox node '${sandbox.nodeName}' did not become ready within ${sandbox.waitedMs}ms.` - ); - } - if ( - mountSandboxRelayfile && - (sandbox.outcome !== 'provisioned' || sandbox.relayfileMounted !== true) - ) { - if (shouldCleanupSandbox && sandbox.outcome === 'provisioned') { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: sandbox.cloudWorkspaceId, - sandboxId: sandbox.sandboxId, - ...(sandbox.providerId === undefined ? {} : { providerId: sandbox.providerId }), - }) - .catch((error) => { - deps.warn( - `The unmounted sandbox could not be cleaned up automatically and may still be running: ${ - error instanceof Error ? error.message : String(error) - }` - ); - }); + }, + warn: deps.warn, } - throw new Error('Cloud returned a sandbox node without the required Relayfile mount.'); - } - targetNode = sandbox.nodeName; - if ( - liveRepository && - sandboxRepository && - !workerCwd && - sandbox.outcome === 'provisioned' && - sandbox.relayfileMounted - ) { - workerCwd = liveRelayfileWorkerCwd( - sandbox.relayfileMountPath ?? '/workspace', - liveRepository, - sandboxRepository.repositoryRelativeCwd - ); - } - if (!workerCwd && sandbox.outcome === 'provisioned' && sandbox.relayfileMounted) { - workerCwd = sandbox.relayfileMountPath ?? '/workspace'; - } + ); + printJson(deps.sdk, { + sandbox: handle.sandbox, + attachCommand: sandboxAttachCommand(name, attachProjectRoot), + invocation: spawnInvocationWithMergedPlacement( + handle.invocation as unknown as Record + ), + }); + return; } if (targetNode) { let launcherName: string | undefined; + let workspaceRelay: ReturnType | undefined; try { - // Agent tokens are scoped to a Relaycast deployment. Never replay a - // canonical token after Cloud has selected the isolated shard; mint - // a temporary launcher on the validated target instead. - // A sandbox dispatch always mints a launcher on the transport Cloud - // selected (or the canonical compatibility transport when an older - // non-Agent37 response omitted the target). Ambient agent tokens do - // not carry enough provenance to prove they belong to that transport. - let agentToken = sandbox ? undefined : resolveAgentToken(clientOptions); + let agentToken = resolveAgentToken(clientOptions); if (!agentToken) { - workspaceRelay ??= deps.sdk.createWorkspaceRelay( - sandbox?.relaycastTarget ? relaycastClientOptions : legacyWorkspaceClientOptions - ); + workspaceRelay = deps.sdk.createWorkspaceRelay(clientOptions); const pendingLauncherName = `fleet-spawn-launcher-${randomUUID().slice(0, 8)}`; const launcher = await workspaceRelay.workspace.register( { @@ -1074,48 +907,12 @@ export function registerFleetCommands( } } - // A sandbox launcher token is already scoped to the exact workspace - // and Relaycast deployment selected by Cloud. Keep the workspace key - // only on `workspaceRelay`, where it mints and releases that token; - // passing both authorities to the agent client is rejected by the - // SDK and would prevent every sandbox placement from dispatching. - const relay = deps.sdk.createAgentRelay( - sandbox - ? { token: agentToken, baseUrl: relaycastClientOptions.baseUrl } - : { ...relaycastClientOptions, token: agentToken } - ); + const relay = deps.sdk.createAgentRelay({ ...clientOptions, token: agentToken }); // Placement alone only proves the node accepted the dispatch. A node // running an obsolete broker advertises `spawn:` capacity, acks // the invocation and launches nothing, which is indistinguishable from // success here — so wait for the node to confirm unless asked not to. const confirm = options.confirm !== false; - const liveSandboxContext = - sandbox?.outcome === 'provisioned' && liveRepository && sandboxRepository - ? [ - `Agent Relay sandbox context: ${liveRepository.repository} is mounted as a live Relayfile working tree at ${liveRelayfileWorkerCwd( - sandbox.relayfileMountPath ?? '/workspace', - liveRepository, - '' - )}. Its exact source revision is ${liveRepository.revision}.`, - ...(sandboxMountPaths?.includes(`${path.posix.dirname(liveRepository.sentinelPath)}/**`) - ? [ - `The same attestation is recorded at ${mountedRelayfilePath( - sandbox.relayfileMountPath ?? '/workspace', - liveRepository.sentinelPath - )}.`, - ] - : []), - ...(sandboxMountPaths?.includes('/.skills/**') - ? [ - `Workspace skills are under ${mountedRelayfilePath( - sandbox.relayfileMountPath ?? '/workspace', - '/.skills' - )}.`, - ] - : []), - 'The Relayfile daemon synchronizes this tree; it intentionally has no .git directory.', - ].join(' ') - : undefined; const invocation = await relay.messaging.placement.spawn({ capability: `spawn:${cli}`, node: targetNode, @@ -1125,16 +922,7 @@ export function registerFleetCommands( input: { name, cli, - task: - sandbox && - checkoutRepository && - sandboxRepository && - mountSandboxRelayfile && - sandbox.outcome === 'provisioned' - ? `${task}\n\nAgent Relay sandbox context: Relayfile records are available at ${sandbox.relayfileMountPath ?? '/workspace'}. The source checkout is separate; use ${workerCwd ?? 'the worker checkout'} for repository files and the mount for Relayfile records.` - : liveSandboxContext - ? `${task}\n\n${liveSandboxContext}` - : task, + task, ...(channel ? { channels: [channel] } : {}), ...(model ? { model } : {}), ...(workerCwd ? { worker_cwd: workerCwd } : {}), @@ -1142,47 +930,9 @@ export function registerFleetCommands( ...(sessionRef ? { session_ref: sessionRef } : {}), }, }); - const printableSandbox = - (sandbox?.outcome === 'provisioned' || sandbox?.outcome === 'reused') && sandbox.relaycastTarget - ? { - ...sandbox, - relaycastTarget: { - route: sandbox.relaycastTarget.route, - baseUrl: sandbox.relaycastTarget.baseUrl, - workspaceId: sandbox.relaycastTarget.workspaceId, - }, - } - : sandbox; printJson(deps.sdk, { - ...(sandbox - ? { - sandbox: printableSandbox, - attachCommand: sandboxAttachCommand(name, attachProjectRoot), - } - : {}), invocation: spawnInvocationWithMergedPlacement(invocation as unknown as Record), }); - } catch (error) { - if ( - shouldCleanupSandbox && - sandbox?.outcome === 'provisioned' && - !(error instanceof RelayPlacementError && error.state === 'unconfirmed_may_be_running') - ) { - await deps - .deleteCloudFleetSandbox({ - cloudWorkspaceId: sandbox.cloudWorkspaceId, - sandboxId: sandbox.sandboxId, - ...(sandbox.providerId === undefined ? {} : { providerId: sandbox.providerId }), - }) - .catch((cleanupError) => { - deps.warn( - `Spawn failed and the sandbox could not be cleaned up automatically: ${ - cleanupError instanceof Error ? cleanupError.message : String(cleanupError) - }` - ); - }); - } - throw error; } finally { if (launcherName && workspaceRelay) { await workspaceRelay.workspace diff --git a/packages/cli/src/cli/lib/attach-fleet-node.ts b/packages/cli/src/cli/lib/attach-fleet-node.ts index ddfa812c73..71c417be17 100644 --- a/packages/cli/src/cli/lib/attach-fleet-node.ts +++ b/packages/cli/src/cli/lib/attach-fleet-node.ts @@ -1,1551 +1 @@ -/** - * Ticketed fleet-node attach adapter. - * - * The established attach clients intentionally continue to speak the local - * broker HTTP/WebSocket contract. This short-lived loopback adapter maps that - * contract onto Relaycast's authenticated terminal session, so view/drive and - * passthrough retain their behaviour without exposing a remote broker listener - * or copying a broker API key off a physical or Daytona node. - */ - -import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; -import { once } from 'node:events'; -import { Buffer } from 'node:buffer'; -import { randomBytes } from 'node:crypto'; - -import WebSocket, { WebSocketServer } from 'ws'; - -import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; -import type { AttachMode } from './attach-mode.js'; -import { collectWithRetry } from './collect-with-retry.js'; -import { resolveWorkspaceTransport } from './sdk-client.js'; - -const MAX_BUFFERED_BYTES = 1024 * 1024; -const MAX_WEBSOCKET_CLOSE_REASON_BYTES = 123; -// Use the same finite 30s request window as the broker's Relaycast HTTP calls. -// Terminal-session creation can exceed the shorter startup-handshake latency -// under load, and this POST cannot be replayed safely after an ambiguous client -// timeout because allocation may have completed server-side. -const SESSION_REQUEST_TIMEOUT_MS = 30_000; -// Immediate structured reachability failures still receive the complete -// five-attempt/31.2s retry schedule, while slow responses cannot multiply the -// per-attempt timeout into a roughly three-minute CLI hang. -const SESSION_REQUEST_TOTAL_TIMEOUT_MS = 90_000; -const SESSION_REQUEST_RETRIES = 4; -// The fleet node publishes liveness every 12s. With four retries, the helper's -// deterministic delays are 6s, 7.2s, 8.4s, and 9.6s: 31.2s total. That spans -// more than two heartbeat intervals and matches the established terminal -// transport's bounded recovery window instead of exhausting every retry inside -// the same stale control-plane read. -const SESSION_REQUEST_RETRY_DELAY_MS = 6_000; -const TERMINAL_CONNECT_TIMEOUT_MS = 10_000; -const INITIAL_RECONNECT_DELAY_MS = 500; -const MAX_RECONNECT_DELAY_MS = 30_000; -// Six delays (0.5s + 1s + 2s + 4s + 8s + 16s) cover the node terminal -// transport's independent 30s maximum reconnect backoff without leaving this -// client unbounded. Five attempts previously stopped after only 15.5s. -const MAX_RECONNECT_ATTEMPTS = 6; -// Bounds the acknowledgement for a delivery-mode command after readiness. -// The command is not replayable across reconnects, so this is intentionally a -// separate post-readiness phase rather than another full reconnect window. -const DELIVERY_MODE_TIMEOUT_MS = 10_000; -// The caller starts its HTTP deadline before the loopback handler starts its -// readiness timer. Leave a small response-delivery margin so the proxy's -// actionable readiness error wins that race. -const LOOPBACK_REQUEST_TIMEOUT_MARGIN_MS = 1_000; - -type FleetSessionResponse = { - ok?: boolean; - data?: { - session_id?: string; - terminal_url?: string; - resume_token?: string; - expires_at?: string; - }; - error?: { code?: string; message?: string }; -}; - -type TerminalFrame = Record & { - type?: string; - session_id?: string; - blocked_reason_code?: unknown; - head_sequence?: unknown; - acked_up_to_sequence?: unknown; - received_up_to_sequence?: unknown; - next_ackable_sequence?: unknown; - reconciliation_action?: unknown; -}; - -type GapDiagnosticsResult = { - blocked_reason_code?: string; - head_sequence?: number; - acked_up_to_sequence?: number; - received_up_to_sequence?: number; - next_ackable_sequence?: number; - reconciliation_action?: string; -}; - -function gapDiagnosticsFromFrame(frame: { - blocked_reason_code?: unknown; - head_sequence?: unknown; - acked_up_to_sequence?: unknown; - received_up_to_sequence?: unknown; - next_ackable_sequence?: unknown; - reconciliation_action?: unknown; -}): GapDiagnosticsResult { - return { - ...(typeof frame.blocked_reason_code === 'string' - ? { blocked_reason_code: frame.blocked_reason_code } - : {}), - ...(typeof frame.head_sequence === 'number' ? { head_sequence: frame.head_sequence } : {}), - ...(typeof frame.acked_up_to_sequence === 'number' - ? { acked_up_to_sequence: frame.acked_up_to_sequence } - : {}), - ...(typeof frame.received_up_to_sequence === 'number' - ? { received_up_to_sequence: frame.received_up_to_sequence } - : {}), - ...(typeof frame.next_ackable_sequence === 'number' - ? { next_ackable_sequence: frame.next_ackable_sequence } - : {}), - ...(typeof frame.reconciliation_action === 'string' - ? { reconciliation_action: frame.reconciliation_action } - : {}), - }; -} - -type TerminalReadiness = { - generation: number; - settled: boolean; - promise: Promise; - resolve: () => void; - reject: (error: Error) => void; -}; - -export interface FleetNodeAttachOptions { - agent: string; - node: string; - mode: AttachMode; - env?: NodeJS.ProcessEnv; - baseUrl?: string; - workspaceKey?: string; - fetch?: typeof globalThis.fetch; - /** Deterministic test seam for the bounded session-request retry delay. */ - sessionRequest?: { - timeoutMs?: number; - totalTimeoutMs?: number; - sleep?: (ms: number) => Promise; - }; - /** Deterministic test seam for established-session reconnect timing. */ - reconnectDelay?: { - initialMs?: number; - maxMs?: number; - handshakeTimeoutMs?: number; - readyTimeoutMs?: number; - beforeReadyTimeoutTerminate?: (socket: WebSocket) => void; - }; -} - -export interface FleetNodeAttachProxy { - brokerUrl: string; - apiKey: string; - requestTimeoutMs: number; - close(): Promise; -} - -export class FleetNodeAttachError extends Error { - constructor( - message: string, - readonly code?: string - ) { - super(message); - this.name = 'FleetNodeAttachError'; - } -} - -const TRUSTED_RELAYCAST_ORIGINS = new Set([CANONICAL_RELAYCAST_ORIGIN, AGENT37_RELAYCAST_ORIGIN]); - -export function validateFleetAttachBaseUrl(value: string): string { - let parsed: URL; - try { - parsed = new URL(value); - } catch { - throw new FleetNodeAttachError('Fleet node attach requires a trusted Relaycast origin.'); - } - if ( - parsed.protocol !== 'https:' || - parsed.username || - parsed.password || - parsed.port || - parsed.search || - parsed.hash || - (parsed.pathname !== '' && parsed.pathname !== '/') || - !TRUSTED_RELAYCAST_ORIGINS.has(parsed.origin) - ) { - throw new FleetNodeAttachError('Fleet node attach requires a trusted Relaycast origin.'); - } - return parsed.origin; -} - -class TerminalSessionAttemptError extends FleetNodeAttachError { - constructor( - message: string, - code: string | undefined, - readonly status: number | undefined, - readonly retryable: boolean, - readonly completionUnknown: boolean - ) { - super(message, code); - } -} - -/** - * Canonical HTTP status for a terminal failure code. - * - * `agent_not_found` must stay 404: {@link switchInboundDeliveryModeOrAbort} - * only emits the "no agent named X" message and the cross-node placement hint - * on a 404, so collapsing it into 503 replaces actionable guidance with an - * opaque unreachable-node error. `unsupported_runtime` stays 409. Everything - * else is a transport-level failure and reports 503. - */ -function terminalErrorStatus(code: string | undefined): number { - if (code === 'agent_not_found') return 404; - if (code === 'unsupported_runtime') return 409; - return 503; -} - -function json(response: ServerResponse, status: number, payload: unknown): void { - response.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - response.end(JSON.stringify(payload)); -} - -function readBody(request: IncomingMessage): Promise> { - return new Promise((resolve) => { - let settled = false; - const finish = (body: Record) => { - if (settled) return; - settled = true; - resolve(body); - }; - let body = ''; - request.setEncoding('utf8'); - request.on('data', (chunk: string) => { - body += chunk; - }); - request.on('end', () => { - try { - const parsed = JSON.parse(body) as unknown; - finish( - parsed && typeof parsed === 'object' && !Array.isArray(parsed) - ? (parsed as Record) - : {} - ); - } catch { - finish({}); - } - }); - request.on('error', () => finish({})); - request.on('aborted', () => finish({})); - }); -} - -function asWsUrl(value: string): string { - const lower = value.toLowerCase(); - if (lower.startsWith('https://')) return 'wss://' + value.slice(8); - if (lower.startsWith('http://')) return 'ws://' + value.slice(7); - return value; -} - -function safeNodePath(node: string): string { - const trimmed = node.trim().replace(/^#/, ''); - if (!trimmed) throw new FleetNodeAttachError('Error: --node requires a node name or id.', 'invalid_node'); - return encodeURIComponent(trimmed); -} - -function parseFrame(data: WebSocket.RawData): TerminalFrame | null { - try { - const parsed = JSON.parse(rawDataToString(data)) as unknown; - return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as TerminalFrame) : null; - } catch { - return null; - } -} - -/** - * Decode a `terminal.ready` / `terminal.snapshot` `screen` field into the raw - * ANSI bytes a terminal can render. - * - * The two payloads this adapter bridges are encoded differently and the - * difference is invisible at the type level — both are `string`: - * - * - `screen` is the visible grid rendered by `Snapshot::to_ansi()` and then - * **base64-encoded**, because the broker asks the worker for - * `format: "ansi"` (`fleet.rs`) and that arm encodes (`pty_worker.rs`). - * Broker HTTP snapshot consumers decode it (`attach.ts` `captureAndRender…`). - * - `worker_stream.chunk` (and `terminal.output.chunk`, which is the same - * value forwarded) is **raw PTY bytes**. Attach clients write it to stdout - * verbatim (`applyServerOutput` in `attach-drive.ts`). - * - * Handing an undecoded `screen` to {@link workerStreamEvent} therefore prints - * the base64 text itself into the operator's terminal — the reconnect flood in - * relay#1829. Returns `null` when the payload is not canonical base64, so a - * malformed or protocol-violating frame is dropped rather than rendered: a - * stale-but-coherent screen beats writing unintelligible bytes to a live TTY. - */ -export function decodeAnsiScreenPayload(screen: string): string | null { - if (screen === '') return ''; - if (screen.length % 4 !== 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(screen)) return null; - const decoded = Buffer.from(screen, 'base64'); - // `Buffer.from(…, 'base64')` silently skips characters outside the alphabet, - // so the round-trip is what actually rejects a non-base64 payload. - if (decoded.toString('base64') !== screen) return null; - return decoded.toString('utf8'); -} - -/** - * Mirror of the broker's `pty_input_error_is_connection_fatal` - * (`crates/broker/src/listen_api.rs`). The two lists must agree, because the - * SDK's `PtyInputStream` latches `closed` **only** when its socket closes - * (`harness-driver/src/transport.ts`): a fatal error delivered on a socket - * that stays open leaves `isUsable()` true forever. - * - * That matters most for a fatal error that arrives with **no write in - * flight** — the session-scoped `terminal.error` path below. The client's - * `failAll()` then has nothing to reject, so nothing marks the stream dead: - * the drive session never enters recovery and keeps writing into a stream the - * node has already declared unusable. Closing the socket, as the broker does, - * is what turns that into a reported outage the session can recover from. - */ -export function inputErrorIsConnectionFatal(code: string): boolean { - return code !== 'worker_timeout' && code !== 'pty_write_queue_full'; -} - -function rawDataToString(data: WebSocket.RawData): string { - if (Buffer.isBuffer(data)) return data.toString('utf8'); - if (data instanceof ArrayBuffer) return Buffer.from(data).toString('utf8'); - if (Array.isArray(data)) return Buffer.concat(data).toString('utf8'); - return String(data); -} - -function diagnosticEndpoint(value: string): string { - try { - const url = new URL(value); - url.username = ''; - url.password = ''; - url.search = ''; - url.hash = ''; - return url.toString(); - } catch { - return '(invalid endpoint)'; - } -} - -function diagnosticEndpointOrigin(value: string): string { - try { - return new URL(value).origin; - } catch { - return '(invalid endpoint)'; - } -} - -function boundedWebSocketCloseReason(reason: string): string { - const encoded = Buffer.from(reason, 'utf8'); - if (encoded.length <= MAX_WEBSOCKET_CLOSE_REASON_BYTES) return reason; - - const suffix = Buffer.from('…', 'utf8'); - let end = MAX_WEBSOCKET_CLOSE_REASON_BYTES - suffix.length; - // Do not cut through a UTF-8 continuation sequence. Excluding the leading - // byte at this boundary also excludes the incomplete code point. - while (end > 0 && (encoded[end] & 0xc0) === 0x80) end -= 1; - return Buffer.concat([encoded.subarray(0, end), suffix]).toString('utf8'); -} - -function diagnosticValue(value: string): string { - // JSON quoting keeps node names and upstream text from injecting terminal - // control characters into the operator-facing error. - return JSON.stringify(value); -} - -function resolvedNodeIdFromTerminalUrl(value: string): string | undefined { - try { - const match = /^\/v1\/nodes\/([^/]+)\/terminal\/connect$/.exec(new URL(value).pathname); - return match?.[1] ? decodeURIComponent(match[1]) : undefined; - } catch { - return undefined; - } -} - -function retryDelayBudgetMs(attempts: number, initialMs: number, maxMs: number): number { - let total = 0; - for (let attempt = 0; attempt < attempts; attempt += 1) { - total += Math.min(initialMs * 2 ** attempt, maxMs); - } - return total; -} - -function terminalSessionFailureSummary(error: TerminalSessionAttemptError): string { - if (error.code === 'node_not_found') return 'Control-plane node lookup found no matching record'; - if (error.code === 'node_unreachable') { - return /no terminal transport/i.test(error.message) - ? 'The node record was found, but its terminal transport was unavailable' - : 'The control plane classified the node as unreachable'; - } - if (error.code === 'terminal_session_unavailable') { - return 'The control plane could not allocate a terminal session'; - } - if (error.code === 'control_plane_timeout') return 'Control-plane terminal-session lookup timed out'; - if (error.code === 'control_plane_unavailable') return 'Control-plane terminal-session lookup failed'; - return 'The terminal-session request was rejected'; -} - -function isRetryableTerminalSessionFailure(code: string | undefined): boolean { - // These structured responses are emitted before a session is returned, so - // retrying cannot duplicate a successful allocation. A fetch timeout, - // network failure, or unclassified 5xx is different: this POST may already - // have completed server-side, and retrying it could create a second session. - return code === 'node_unreachable' || code === 'terminal_session_unavailable'; -} - -/** Start a broker-compatible loopback proxy for one remote terminal session. */ -export async function startFleetNodeAttachProxy( - options: FleetNodeAttachOptions -): Promise { - const env = options.env ?? process.env; - const fetchFn = options.fetch ?? globalThis.fetch; - const { workspaceKey, baseUrl: requestedBaseUrl } = resolveWorkspaceTransport({ - workspaceKey: options.workspaceKey, - baseUrl: options.baseUrl, - env, - }); - const baseUrl = validateFleetAttachBaseUrl(requestedBaseUrl ?? CANONICAL_RELAYCAST_ORIGIN); - const nodePath = safeNodePath(options.node); - const sessionEndpoint = `${baseUrl}/v1/nodes/${nodePath}/terminal/sessions`; - const sessionRequestTimeoutMs = options.sessionRequest?.timeoutMs ?? SESSION_REQUEST_TIMEOUT_MS; - const sessionRequestTotalTimeoutMs = - options.sessionRequest?.totalTimeoutMs ?? SESSION_REQUEST_TOTAL_TIMEOUT_MS; - const sessionRequestDeadline = Date.now() + sessionRequestTotalTimeoutMs; - const sessionRequestSleep = - options.sessionRequest?.sleep ?? - ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); - let sessionRequestAttempts = 0; - let sessionRequestBudgetExhaustedBetweenAttempts = false; - let lastSessionError: TerminalSessionAttemptError | undefined; - const sessionResult = await collectWithRetry( - 'terminal session request', - async () => { - const remainingRequestBudgetMs = sessionRequestDeadline - Date.now(); - if (remainingRequestBudgetMs <= 0) { - sessionRequestBudgetExhaustedBetweenAttempts = true; - const exhausted = - lastSessionError === undefined - ? new TerminalSessionAttemptError( - 'overall terminal-session request deadline exhausted', - 'control_plane_timeout', - undefined, - false, - false - ) - : new TerminalSessionAttemptError( - lastSessionError.message, - lastSessionError.code, - lastSessionError.status, - false, - lastSessionError.completionUnknown - ); - lastSessionError = exhausted; - throw exhausted; - } - sessionRequestAttempts += 1; - const controller = new AbortController(); - let timedOut = false; - const attemptTimeoutMs = Math.min(sessionRequestTimeoutMs, remainingRequestBudgetMs); - const timeout = setTimeout(() => { - timedOut = true; - controller.abort(); - }, attemptTimeoutMs); - let ticketResponse: Response; - let ticketPayload: FleetSessionResponse; - try { - ticketResponse = await fetchFn(sessionEndpoint, { - method: 'POST', - headers: { Authorization: `Bearer ${workspaceKey}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ agent: options.agent, mode: options.mode }), - signal: controller.signal, - }); - const parsedPayload = (await ticketResponse.json()) as unknown; - ticketPayload = - parsedPayload && typeof parsedPayload === 'object' && !Array.isArray(parsedPayload) - ? (parsedPayload as FleetSessionResponse) - : {}; - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - lastSessionError = new TerminalSessionAttemptError( - timedOut ? 'request exceeded its deadline' : detail, - timedOut ? 'control_plane_timeout' : 'control_plane_unavailable', - undefined, - false, - true - ); - throw lastSessionError; - } finally { - clearTimeout(timeout); - } - - const terminalUrl = ticketPayload.data?.terminal_url; - const sessionId = ticketPayload.data?.session_id; - const resumeToken = ticketPayload.data?.resume_token; - if (!ticketResponse.ok || !terminalUrl || !sessionId || !resumeToken) { - const code = ticketPayload.error?.code; - const message = - ticketPayload.error?.message ?? `terminal session request failed (HTTP ${ticketResponse.status})`; - lastSessionError = new TerminalSessionAttemptError( - message, - code, - ticketResponse.status, - isRetryableTerminalSessionFailure(code), - false - ); - throw lastSessionError; - } - return { terminalUrl, sessionId, resumeToken, expiresAt: ticketPayload.data?.expires_at }; - }, - { - retries: SESSION_REQUEST_RETRIES, - baseDelayMs: SESSION_REQUEST_RETRY_DELAY_MS, - sleep: async (delayMs) => { - const remainingRequestBudgetMs = sessionRequestDeadline - Date.now(); - if (remainingRequestBudgetMs <= 0) return; - await sessionRequestSleep(Math.min(delayMs, remainingRequestBudgetMs)); - }, - shouldRetry: (error) => error instanceof TerminalSessionAttemptError && error.retryable, - } - ); - if (!sessionResult.ok) { - const failure = lastSessionError; - const status = failure?.status === undefined ? '' : ` HTTP ${failure.status};`; - const code = failure?.code === undefined ? '' : ` code ${failure.code};`; - const retryNote = sessionRequestBudgetExhaustedBetweenAttempts - ? sessionRequestAttempts > 1 - ? `retried ${sessionRequestAttempts - 1} time${sessionRequestAttempts === 2 ? '' : 's'};` + - ' overall budget exhausted before the next attempt' - : 'not retried because the overall budget was exhausted before the next attempt' - : sessionRequestAttempts > 1 - ? `retried ${sessionRequestAttempts - 1} time${sessionRequestAttempts === 2 ? '' : 's'}` + - (failure?.completionUnknown - ? '; final POST not retried because it may have completed server-side' - : '') - : failure?.completionUnknown - ? 'not retried because the POST may have completed server-side' - : 'not retried because the failure was terminal'; - const upstream = failure?.message ? ` Upstream message ${diagnosticValue(failure.message)}.` : ''; - throw new FleetNodeAttachError( - `Error: ${failure ? terminalSessionFailureSummary(failure) : 'Terminal-session request failed'}.` + - `${upstream} Node ref ${diagnosticValue(options.node.trim())}, resolved node id unavailable (session creation did not complete);` + - ` endpoint ${diagnosticValue(diagnosticEndpoint(sessionEndpoint))};${status}${code}` + - ` timeout ${sessionRequestTimeoutMs}ms per attempt; overall budget ${sessionRequestTotalTimeoutMs}ms;` + - ` attempts ${sessionRequestAttempts} (${retryNote}).`, - failure?.code - ); - } - let { terminalUrl, sessionId, resumeToken, expiresAt } = sessionResult.value; - const resolvedNodeId = resolvedNodeIdFromTerminalUrl(terminalUrl); - const remoteEndpoint = () => diagnosticEndpoint(terminalUrl); - const reconnectInitialDelayMs = options.reconnectDelay?.initialMs ?? INITIAL_RECONNECT_DELAY_MS; - const reconnectMaxDelayMs = options.reconnectDelay?.maxMs ?? MAX_RECONNECT_DELAY_MS; - const terminalHandshakeTimeoutMs = - options.reconnectDelay?.handshakeTimeoutMs ?? TERMINAL_CONNECT_TIMEOUT_MS; - const terminalReadyTimeoutMs = options.reconnectDelay?.readyTimeoutMs ?? TERMINAL_CONNECT_TIMEOUT_MS; - // A readiness-gated local request follows activeReadiness across reconnect - // generations. Its own deadline therefore has to cover the same complete, - // finite recovery path: every backoff plus every handshake/readiness pair. - const terminalWaitTimeoutMs = - retryDelayBudgetMs(MAX_RECONNECT_ATTEMPTS, reconnectInitialDelayMs, reconnectMaxDelayMs) + - MAX_RECONNECT_ATTEMPTS * (terminalHandshakeTimeoutMs + terminalReadyTimeoutMs) + - sessionRequestTotalTimeoutMs; - - let connectionGeneration = 0; - const createReadiness = (): TerminalReadiness => { - let resolve!: () => void; - let reject!: (error: Error) => void; - const promise = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise; - reject = rejectPromise; - }); - // A failure can land before a snapshot request attaches its waiter. Keep - // the rejection observable while avoiding an unhandled-rejection warning. - void promise.catch(() => undefined); - return { generation: ++connectionGeneration, settled: false, promise, resolve, reject }; - }; - let activeReadiness = createReadiness(); - const resolveReadiness = (readiness: TerminalReadiness) => { - if (readiness.settled) return; - readiness.settled = true; - readiness.resolve(); - }; - const rejectReadiness = (readiness: TerminalReadiness, error: Error) => { - if (readiness.settled) return; - readiness.settled = true; - readiness.reject(error); - }; - const waitForCurrentReadiness = async (): Promise => { - for (;;) { - const readiness = activeReadiness; - await readiness.promise; - if (readiness === activeReadiness) return; - } - }; - /** - * Await the live readiness generation through both the WebSocket handshake - * and the post-open terminal.ready allowance. - * Every handler that must not run before `terminal.ready` — snapshot, - * delivery-mode PUT, resize — goes through this one helper so the - * timer/clearTimeout/settle logic cannot diverge between copies. - * - * Rejects with the underlying {@link FleetNodeAttachError} when the terminal - * failed (preserving its `code`), or a plain `Error` carrying - * `timeoutMessage` when the wait expired. - */ - const waitForTerminalReady = (timeoutMessage: string): Promise => - new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error(timeoutMessage)), terminalWaitTimeoutMs); - void waitForCurrentReadiness().then( - () => { - clearTimeout(timer); - resolve(); - }, - (error: Error) => { - clearTimeout(timer); - reject(error); - } - ); - }); - /** - * The two screen representations are kept as separate, differently named - * fields on purpose. They are both strings and only one of them may reach a - * terminal; a single `screen` field spread into both consumers is what let - * the encoded form escape to stdout (relay#1829). - * - * `screenBase64` is the HTTP snapshot wire format (callers decode it). - * `screenAnsi` is the renderable form, and the only one that may be emitted - * as a `worker_stream` chunk. - */ - const snapshot: { - screenBase64: string; - screenAnsi: string; - rows: number; - cols: number; - offset: number; - } = { - screenBase64: '', - screenAnsi: '', - rows: 24, - cols: 80, - offset: 0, - }; - const eventSockets = new Set(); - const inputSockets = new Set(); - const outputHistory: Array<{ chunk: string; offset?: number }> = []; - let outputHistoryBytes = 0; - let remote: WebSocket | undefined; - let stopped = false; - let terminalEnded = false; - let terminalEverReady = false; - let reconnecting = false; - let reconnectAttempts = 0; - let reconnectTimer: ReturnType | undefined; - const terminalReadinessTimers = new Set>(); - /** Locally-tracked delivery mode, kept in sync with each broker reply. */ - let loopbackDeliveryMode: 'manual_flush' | 'auto_inject' = - options.mode === 'drive' ? 'manual_flush' : 'auto_inject'; - type DeliveryModeResult = GapDiagnosticsResult & { - mode: string; - flushed: number; - dead_lettered?: number; - matched: boolean; - revision: string; - blocked_reason?: string; - }; - type FlushResult = GapDiagnosticsResult & { - flushed: number; - dead_lettered: number; - held: number; - blocked_reason: string | null; - }; - let pendingFlush: { - requestId: string; - resolve: (result: FlushResult) => void; - reject: (error: Error) => void; - timer: ReturnType; - } | null = null; - /** At most one in-flight delivery-mode PUT at a time. */ - let pendingDeliveryMode: { - requestId: string; - resolve: (result: DeliveryModeResult) => void; - reject: (err: Error) => void; - timer: ReturnType; - } | null = null; - const loopbackApiKey = randomBytes(32).toString('base64url'); - const loopbackAuthorized = (headers: IncomingMessage['headers']) => - headers.authorization === `Bearer ${loopbackApiKey}` || headers['x-api-key'] === loopbackApiKey; - - const server = createServer(async (request, response) => { - if (!loopbackAuthorized(request.headers)) { - json(response, 401, { - error: { code: 'unauthorized', message: 'loopback terminal token is required' }, - }); - return; - } - const path = new URL(request.url ?? '/', 'http://127.0.0.1').pathname; - if (request.method === 'GET' && path === `/api/spawned/${encodeURIComponent(options.agent)}/snapshot`) { - try { - await waitForTerminalReady('terminal snapshot timed out'); - } catch (error) { - const terminalError = error instanceof FleetNodeAttachError ? error : undefined; - json(response, terminalErrorStatus(terminalError?.code), { - error: { - code: terminalError?.code ?? 'snapshot_unavailable', - message: - terminalError?.message ?? (error instanceof Error ? error.message : 'snapshot unavailable'), - }, - }); - return; - } - // Explicit field mapping, not a spread: this endpoint's `screen` is - // contractually base64 (the caller decodes it), and only the encoded - // form may appear here. - json(response, 200, { - format: 'ansi', - screen: snapshot.screenBase64, - rows: snapshot.rows, - cols: snapshot.cols, - offset: snapshot.offset, - }); - return; - } - const name = encodeURIComponent(options.agent); - if (path === `/api/spawned/${name}/flush` && request.method === 'POST') { - // Forward the flush to the remote broker over the terminal websocket. - // Without this the `--node` form of `node agent message flush` reached - // only the LOCAL broker's worker registry and returned agent_not_found - // for a name that `node agent list` and `attach` both resolve. - try { - await waitForTerminalReady('terminal connection timed out'); - } catch (error) { - const terminalError = error instanceof FleetNodeAttachError ? error : undefined; - json(response, terminalErrorStatus(terminalError?.code), { - error: { - code: terminalError?.code ?? 'node_unreachable', - message: - terminalError?.message ?? (error instanceof Error ? error.message : 'terminal unavailable'), - }, - }); - return; - } - if (!remote || remote.readyState !== WebSocket.OPEN) { - json(response, 503, { - error: { code: 'node_unreachable', message: 'terminal transport is not connected' }, - }); - return; - } - if (pendingFlush) { - json(response, 503, { - error: { code: 'flush_conflict', message: 'a flush request is already in flight' }, - }); - return; - } - const requestId = randomBytes(8).toString('hex'); - const result = await new Promise((resolve) => { - const timer = setTimeout(() => { - pendingFlush = null; - resolve(new FleetNodeAttachError('flush request timed out', 'flush_timeout')); - }, DELIVERY_MODE_TIMEOUT_MS); - pendingFlush = { - requestId, - resolve: (r) => resolve(r), - reject: (e) => resolve(e), - timer, - }; - remote!.send( - JSON.stringify({ - type: 'terminal.flush_pending', - session_id: sessionId, - request_id: requestId, - }) - ); - }); - if (result instanceof Error) { - const errCode = - result instanceof FleetNodeAttachError ? (result.code ?? 'flush_failed') : 'flush_failed'; - json(response, terminalErrorStatus(errCode), { - error: { code: errCode, message: result.message }, - }); - return; - } - json(response, 200, result); - return; - } - if (path === `/api/spawned/${name}/delivery-mode`) { - if (request.method === 'GET') { - json(response, 200, { mode: loopbackDeliveryMode }); - return; - } - // PUT — forward the request to the remote broker via the terminal WS and - // await the broker's real reply. This is the path that was previously a - // static stub returning manual_flush, causing drive attach to fail with - // "broker remained in manual_flush mode". - const body = await readBody(request); - const requestedMode = - body.mode === 'auto_inject' ? 'auto_inject' : body.mode === 'manual_flush' ? 'manual_flush' : null; - if (requestedMode === null) { - json(response, 400, { - error: { code: 'invalid_mode', message: `unsupported delivery mode '${String(body.mode)}'` }, - }); - return; - } - // Drive attach changes delivery mode before it requests the initial - // snapshot. Gate the PUT on terminal.ready so a fast local caller does - // not lose a race with the remote websocket handshake and receive the - // misleading "terminal transport is not connected" failure. - try { - await waitForTerminalReady('terminal connection timed out'); - } catch (error) { - // Preserve the canonical status mapping here too: a readiness failure - // carrying `agent_not_found` has to reach the preflight as a 404 or - // the operator loses the cross-node placement hint that tells them - // which machine to run the attach on. - const terminalError = error instanceof FleetNodeAttachError ? error : undefined; - json(response, terminalErrorStatus(terminalError?.code), { - error: { - code: terminalError?.code ?? 'node_unreachable', - message: - terminalError?.message ?? (error instanceof Error ? error.message : 'terminal unavailable'), - }, - }); - return; - } - if (!remote || remote.readyState !== WebSocket.OPEN) { - json(response, 503, { - error: { code: 'node_unreachable', message: 'terminal transport is not connected' }, - }); - return; - } - if (pendingDeliveryMode) { - json(response, 503, { - error: { code: 'delivery_mode_conflict', message: 'a delivery mode request is already in flight' }, - }); - return; - } - const requestId = randomBytes(8).toString('hex'); - const result = await new Promise((resolve) => { - const timer = setTimeout(() => { - pendingDeliveryMode = null; - resolve(new FleetNodeAttachError('delivery mode request timed out', 'delivery_mode_timeout')); - }, DELIVERY_MODE_TIMEOUT_MS); - pendingDeliveryMode = { - requestId, - resolve: (r) => resolve(r), - reject: (e) => resolve(e), - timer, - }; - const frame: Record = { - type: 'terminal.set_delivery_mode', - session_id: sessionId, - mode: requestedMode, - request_id: requestId, - }; - if (typeof body.expected_mode === 'string') frame.expected_mode = body.expected_mode; - if (typeof body.expected_revision === 'string') frame.expected_revision = body.expected_revision; - remote!.send(JSON.stringify(frame)); - }); - if (result instanceof Error) { - const errCode = - result instanceof FleetNodeAttachError - ? (result.code ?? 'delivery_mode_failed') - : 'delivery_mode_failed'; - // Same canonical mapping as the readiness gate: 404 for - // agent_not_found so the attach preflight can produce the no-agent or - // cross-node placement error, 409 for unsupported_runtime, 503 for - // everything else. - json(response, terminalErrorStatus(errCode), { - error: { code: errCode, message: result.message }, - }); - return; - } - loopbackDeliveryMode = result.mode === 'manual_flush' ? 'manual_flush' : 'auto_inject'; - json(response, 200, { - mode: result.mode, - flushed: result.flushed, - ...(result.dead_lettered !== undefined ? { dead_lettered: result.dead_lettered } : {}), - matched: result.matched, - revision: result.revision, - ...(result.blocked_reason !== undefined ? { blocked_reason: result.blocked_reason } : {}), - ...gapDiagnosticsFromFrame(result), - }); - return; - } - if (request.method === 'GET' && path === `/api/spawned/${name}/pending`) { - json(response, 200, { pending: [] }); - return; - } - if (request.method === 'POST' && path === `/api/spawned/${name}/flush`) { - json(response, 200, { flushed: 0 }); - return; - } - if (request.method === 'GET' && path === '/api/spawned') { - json(response, 200, { agents: [{ name: options.agent, workerPid: 1 }] }); - return; - } - if (request.method === 'POST' && path === `/api/resize/${name}`) { - const body = await readBody(request); - if (body.release === true) { - json(response, 200, { name: options.agent, released: true }); - return; - } - const rows = typeof body.rows === 'number' ? body.rows : 0; - const cols = typeof body.cols === 'number' ? body.cols : 0; - if (!Number.isInteger(rows) || !Number.isInteger(cols) || rows < 1 || cols < 1) { - json(response, 400, { - error: { code: 'invalid_dimensions', message: 'rows and cols must be positive integers' }, - }); - return; - } - try { - await waitForTerminalReady('terminal resize timed out'); - } catch (error) { - json(response, 503, { - error: { - code: 'session_not_ready', - message: error instanceof Error ? error.message : 'terminal session is not ready', - }, - }); - return; - } - if (!remote || remote.readyState !== WebSocket.OPEN || remote.bufferedAmount > MAX_BUFFERED_BYTES) { - json(response, 503, { - error: { code: 'node_unreachable', message: 'terminal transport is unavailable' }, - }); - return; - } - remote.send(JSON.stringify({ type: 'terminal.resize', session_id: sessionId, rows, cols })); - json(response, 200, { name: options.agent, rows, cols, applied: true }); - return; - } - json(response, 404, { error: { code: 'not_found', message: 'loopback terminal endpoint not found' } }); - }); - const websocketServer = new WebSocketServer({ noServer: true }); - - const closeSocket = (socket: WebSocket, code: number, reason: string) => { - try { - socket.close(code, boundedWebSocketCloseReason(reason)); - } catch { - /* connection already gone */ - } - }; - const broadcast = (sockets: Set, payload: unknown): boolean => { - const encoded = JSON.stringify(payload); - let accepted = false; - for (const socket of sockets) { - if (socket.readyState !== WebSocket.OPEN) continue; - if (socket.bufferedAmount > MAX_BUFFERED_BYTES) { - closeSocket(socket, 1013, 'loopback client backpressure exceeded'); - sockets.delete(socket); - continue; - } - try { - socket.send(encoded); - accepted = true; - } catch { - sockets.delete(socket); - } - } - return accepted; - }; - const workerStreamEvent = (chunk: string, offset?: number) => ({ - kind: 'worker_stream', - name: options.agent, - stream: 'stdout', - chunk, - ...(offset === undefined ? {} : { offset }), - }); - const retainOutput = (chunk: string, offset: number | undefined): boolean => { - const bytes = Buffer.byteLength(chunk, 'utf8'); - if (outputHistoryBytes + bytes > MAX_BUFFERED_BYTES) return false; - outputHistory.push({ chunk, ...(offset === undefined ? {} : { offset }) }); - outputHistoryBytes += bytes; - return true; - }; - - /** - * Report a PTY-input failure to one input socket the way the broker does: - * with the `retryable` flag the SDK reads, and — for a connection-fatal - * code — by closing the socket so the client's `PtyInputStream` latches - * `closed` and its recovery can start buffering. See - * {@link inputErrorIsConnectionFatal}. - */ - const failInputSocket = (socket: WebSocket, code: string, message: string): void => { - const fatal = inputErrorIsConnectionFatal(code); - if (socket.readyState === WebSocket.OPEN) { - try { - socket.send(JSON.stringify({ type: 'pty_input_error', code, message, retryable: !fatal })); - } catch { - // The socket is already gone; the close below is still correct. - } - } - if (!fatal) return; - inputSockets.delete(socket); - closeSocket(socket, 1011, message); - }; - - /** Same, for every attached input socket (session-scoped failures). */ - const failAllInputSockets = (code: string, message: string): void => { - for (const socket of [...inputSockets]) failInputSocket(socket, code, message); - }; - - websocketServer.on('connection', (socket, request) => { - const path = new URL(request.url ?? '/', 'http://127.0.0.1').pathname; - if (path === '/ws') { - eventSockets.add(socket); - socket.on('close', () => eventSockets.delete(socket)); - let replayed = 0; - for (const event of outputHistory) { - if (socket.readyState !== WebSocket.OPEN || socket.bufferedAmount > MAX_BUFFERED_BYTES) break; - try { - socket.send(JSON.stringify(workerStreamEvent(event.chunk, event.offset))); - } catch { - break; - } - replayed += 1; - } - if (replayed > 0) { - const sentBytes = outputHistory - .slice(0, replayed) - .reduce((total, event) => total + Buffer.byteLength(event.chunk, 'utf8'), 0); - outputHistory.splice(0, replayed); - outputHistoryBytes -= sentBytes; - } - return; - } - if (path === `/api/input/${encodeURIComponent(options.agent)}/stream`) { - inputSockets.add(socket); - socket.on('close', () => inputSockets.delete(socket)); - socket.send(JSON.stringify({ type: 'pty_input_ready', name: options.agent })); - socket.on('message', (data) => { - if (!remote || remote.readyState !== WebSocket.OPEN || remote.bufferedAmount > MAX_BUFFERED_BYTES) { - // Only THIS socket failed to write; a sibling input stream must not - // be torn down for it. `node_unreachable` is connection-fatal, so - // this also closes the socket — see failInputSocket. - failInputSocket(socket, 'node_unreachable', 'terminal transport is unavailable'); - return; - } - const raw = rawDataToString(data); - remote.send( - JSON.stringify({ - type: 'terminal.input', - session_id: sessionId, - data_base64: Buffer.from(raw, 'utf8').toString('base64'), - }) - ); - }); - return; - } - closeSocket(socket, 1008, 'unknown loopback endpoint'); - }); - server.on('upgrade', (request, socket, head) => { - if (!loopbackAuthorized(request.headers)) { - socket.write('HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n'); - socket.destroy(); - return; - } - websocketServer.handleUpgrade(request, socket, head, (client) => - websocketServer.emit('connection', client, request) - ); - }); - server.listen(0, '127.0.0.1'); - await once(server, 'listening'); - const address = server.address(); - if (!address || typeof address === 'string') - throw new FleetNodeAttachError( - 'Error: could not allocate loopback terminal listener.', - 'loopback_unavailable' - ); - - const resumeUrlForCurrentSession = () => { - const resumeUrl = new URL(terminalUrl); - resumeUrl.searchParams.delete('ticket'); - resumeUrl.searchParams.set('session_id', sessionId); - resumeUrl.searchParams.set('resume', resumeToken); - return resumeUrl.toString(); - }; - const terminalSessionExpired = () => { - if (!expiresAt) return false; - const expiresAtMs = Date.parse(expiresAt); - return Number.isFinite(expiresAtMs) && expiresAtMs <= Date.now(); - }; - // A disconnected terminal can need one replacement session when its resume - // credential has expired. Keep this scoped to the whole reconnect incident: - // a replacement that itself drops still follows the normal bounded resume - // budget instead of repeatedly allocating sessions. - let replacementAllocatedForReconnect = false; - const allocateReplacementTerminalSession = async (): Promise<{ - terminalUrl: string; - sessionId: string; - resumeToken: string; - expiresAt: string | undefined; - }> => { - const replacementDeadline = Date.now() + sessionRequestTotalTimeoutMs; - let lastReplacementError: TerminalSessionAttemptError | undefined; - const result = await collectWithRetry( - 'replacement terminal session request', - async () => { - const remainingMs = replacementDeadline - Date.now(); - if (remainingMs <= 0) { - throw new TerminalSessionAttemptError( - lastReplacementError?.message ?? - 'overall replacement terminal-session request deadline exhausted', - lastReplacementError?.code ?? 'control_plane_timeout', - lastReplacementError?.status, - false, - lastReplacementError?.completionUnknown ?? false - ); - } - const controller = new AbortController(); - let timedOut = false; - const timeout = setTimeout( - () => { - timedOut = true; - controller.abort(); - }, - Math.min(sessionRequestTimeoutMs, remainingMs) - ); - try { - const response = await fetchFn(sessionEndpoint, { - method: 'POST', - headers: { Authorization: `Bearer ${workspaceKey}`, 'Content-Type': 'application/json' }, - // Keep the replacement bound to precisely the session this proxy was - // already serving; never infer a different agent or delivery mode. - body: JSON.stringify({ agent: options.agent, mode: options.mode }), - signal: controller.signal, - }); - const parsed = (await response.json()) as unknown; - const payload = - parsed && typeof parsed === 'object' && !Array.isArray(parsed) - ? (parsed as FleetSessionResponse) - : {}; - const replacementUrl = payload.data?.terminal_url; - const replacementSessionId = payload.data?.session_id; - const replacementResumeToken = payload.data?.resume_token; - if (!response.ok || !replacementUrl || !replacementSessionId || !replacementResumeToken) { - const error = new TerminalSessionAttemptError( - payload.error?.message ?? - 'terminal session could not be replaced after its resume credential expired', - payload.error?.code, - response.status, - isRetryableTerminalSessionFailure(payload.error?.code), - false - ); - lastReplacementError = error; - throw error; - } - return { - terminalUrl: replacementUrl, - sessionId: replacementSessionId, - resumeToken: replacementResumeToken, - expiresAt: payload.data?.expires_at, - }; - } catch (error) { - if (error instanceof TerminalSessionAttemptError) throw error; - const replacementError = new TerminalSessionAttemptError( - timedOut - ? 'replacement request exceeded its deadline' - : 'replacement terminal session request failed', - timedOut ? 'control_plane_timeout' : 'control_plane_unavailable', - undefined, - false, - true - ); - lastReplacementError = replacementError; - throw replacementError; - } finally { - clearTimeout(timeout); - } - }, - { - retries: SESSION_REQUEST_RETRIES, - baseDelayMs: SESSION_REQUEST_RETRY_DELAY_MS, - sleep: async (delayMs) => { - const remainingMs = replacementDeadline - Date.now(); - if (remainingMs > 0) await sessionRequestSleep(Math.min(delayMs, remainingMs)); - }, - shouldRetry: (error) => error instanceof TerminalSessionAttemptError && error.retryable, - } - ); - if (result.ok) return result.value; - if (lastReplacementError) throw lastReplacementError; - throw new FleetNodeAttachError( - 'terminal session could not be replaced after its resume credential expired', - 'terminal_session_unavailable' - ); - }; - /** Reject and clear any in-flight delivery-mode PUT, if one is pending. */ - const rejectPendingDeliveryMode = (error: FleetNodeAttachError) => { - if (pendingFlush) { - const flush = pendingFlush; - pendingFlush = null; - clearTimeout(flush.timer); - // A flush never changes delivery mode, so it must not inherit the - // delivery-mode disconnect wording. An operator debugging a failed flush - // would otherwise be told the "delivery-mode change" was interrupted — - // pointing at an operation their command never performed. - const flushError = - error.code === 'delivery_mode_disconnected' - ? new FleetNodeAttachError( - 'terminal transport disconnected while the flush was in flight', - 'flush_disconnected' - ) - : error; - flush.reject(flushError); - } - if (!pendingDeliveryMode) return; - const pending = pendingDeliveryMode; - pendingDeliveryMode = null; - clearTimeout(pending.timer); - pending.reject(error); - }; - const endTerminal = (error: FleetNodeAttachError, eventCloseReason = error.message) => { - if (terminalEnded) return; - terminalEnded = true; - if (reconnectTimer) { - clearTimeout(reconnectTimer); - reconnectTimer = undefined; - } - for (const timer of terminalReadinessTimers) clearTimeout(timer); - terminalReadinessTimers.clear(); - rejectPendingDeliveryMode(error); - const activeRemote = remote; - remote = undefined; - rejectReadiness(activeReadiness, error); - failAllInputSockets(error.code ?? 'terminal_error', error.message); - for (const socket of eventSockets) closeSocket(socket, 1011, eventCloseReason); - if (activeRemote && activeRemote.readyState !== WebSocket.CLOSED) { - try { - activeRemote.terminate(); - } catch { - // The socket may have closed between the state check and terminate. - } - } - }; - const failRemote = (message: string, eventCloseReason?: string) => { - endTerminal(new FleetNodeAttachError(message, 'node_unreachable'), eventCloseReason); - }; - const connect = (url: string, readiness: TerminalReadiness) => { - if (stopped || terminalEnded) return; - const socket = new WebSocket(asWsUrl(url), { handshakeTimeout: terminalHandshakeTimeoutMs }); - remote = socket; - let readinessTimer: ReturnType | undefined; - let readinessExpired = false; - let resumeRejectedAsExpired = false; - const clearReadinessTimer = () => { - if (!readinessTimer) return; - clearTimeout(readinessTimer); - terminalReadinessTimers.delete(readinessTimer); - readinessTimer = undefined; - }; - socket.on('open', () => { - // `handshakeTimeout` independently bounds the HTTP upgrade. Start the - // terminal.ready allowance only after that upgrade succeeds so a slow - // but valid handshake cannot consume the readiness window. - readinessTimer = setTimeout(() => { - const expiredTimer = readinessTimer; - readinessTimer = undefined; - if (expiredTimer) terminalReadinessTimers.delete(expiredTimer); - if (remote !== socket || stopped || terminalEnded || readiness.settled) return; - // Mark this generation stale before terminating. The ws receiver may - // still deliver data already buffered on the socket while close is - // propagating; none of it may restore readiness or reset retry state. - readinessExpired = true; - if (!terminalEverReady) { - failRemote( - `terminal transport connected but did not become ready (node ref ${diagnosticValue(options.node.trim())},` + - ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint())},` + - ` readiness timeout ${terminalReadyTimeoutMs}ms, attempts 1; not retried because no terminal session became ready)` - ); - return; - } - // A successful WebSocket upgrade is not sufficient: Relaycast may - // accept a resume lane that never produces terminal.ready. Terminating - // it drives the same bounded close/retry path as a transport failure. - options.reconnectDelay?.beforeReadyTimeoutTerminate?.(socket); - socket.terminate(); - }, terminalReadyTimeoutMs); - terminalReadinessTimers.add(readinessTimer); - }); - socket.on('message', (data) => { - // A late frame from a transport superseded during reconnect must never - // overwrite the fresh snapshot or end the replacement session. - if (remote !== socket || stopped || terminalEnded || readinessExpired) return; - const frame = parseFrame(data); - if (!frame || frame.session_id !== sessionId) return; - if (frame.type === 'terminal.ready') { - clearReadinessTimer(); - snapshot.screenBase64 = typeof frame.screen === 'string' ? frame.screen : ''; - // A payload that will not decode is kept out of `screenAnsi` entirely - // so no later consumer can render it; the HTTP snapshot still serves - // the bytes verbatim and lets its own decoder report the problem. - snapshot.screenAnsi = decodeAnsiScreenPayload(snapshot.screenBase64) ?? ''; - snapshot.rows = typeof frame.rows === 'number' ? frame.rows : 24; - snapshot.cols = typeof frame.cols === 'number' ? frame.cols : 80; - snapshot.offset = typeof frame.offset === 'number' ? frame.offset : 0; - // Seed loopbackDeliveryMode from the broker's actual state so that - // detach restores the correct mode even when the worker started in a - // different mode than our local inference at line 214. - if (frame.delivery_mode === 'manual_flush' || frame.delivery_mode === 'auto_inject') { - loopbackDeliveryMode = frame.delivery_mode; - } - terminalEverReady = true; - reconnectAttempts = 0; - replacementAllocatedForReconnect = false; - if (readiness === activeReadiness) { - resolveReadiness(readiness); - // A reconnect gets a fresh ANSI grid but existing local `/ws` - // consumers have already performed their initial HTTP snapshot. - // Re-emit this screen without an offset so they repaint instead of - // retaining a stale pre-reconnect terminal image. It must be the - // DECODED grid: a `worker_stream` chunk is raw PTY bytes that the - // attach client writes straight to the terminal, so the base64 form - // renders as a wall of text instead of a repaint (relay#1829). - if (readiness.generation > 1 && snapshot.screenAnsi) { - broadcast(eventSockets, workerStreamEvent(snapshot.screenAnsi)); - } - } - } else if (frame.type === 'terminal.output' && typeof frame.chunk === 'string') { - const offset = typeof frame.offset === 'number' ? frame.offset : undefined; - if (!broadcast(eventSockets, workerStreamEvent(frame.chunk, offset))) { - if (!retainOutput(frame.chunk, offset)) { - endTerminal( - new FleetNodeAttachError( - 'terminal output exceeded the bounded loopback buffer', - 'output_backpressure' - ) - ); - } - } - } else if (frame.type === 'terminal.input_ack') { - broadcast(inputSockets, { - type: 'pty_input_ack', - name: options.agent, - bytes_written: typeof frame.bytes_written === 'number' ? frame.bytes_written : 0, - }); - } else if (frame.type === 'terminal.delivery_mode') { - const frameRid = typeof frame.request_id === 'string' ? frame.request_id : undefined; - if (pendingDeliveryMode && (frameRid === undefined || frameRid === pendingDeliveryMode.requestId)) { - const pending = pendingDeliveryMode; - pendingDeliveryMode = null; - clearTimeout(pending.timer); - pending.resolve({ - mode: typeof frame.mode === 'string' ? frame.mode : 'auto_inject', - flushed: typeof frame.flushed === 'number' ? frame.flushed : 0, - ...(typeof frame.dead_lettered === 'number' ? { dead_lettered: frame.dead_lettered } : {}), - matched: typeof frame.matched === 'boolean' ? frame.matched : true, - revision: typeof frame.revision === 'string' ? frame.revision : '1', - ...(typeof frame.blocked_reason === 'string' ? { blocked_reason: frame.blocked_reason } : {}), - ...gapDiagnosticsFromFrame(frame), - }); - } - } else if (frame.type === 'terminal.flush_pending') { - const frameRid = typeof frame.request_id === 'string' ? frame.request_id : undefined; - // Exact match only. The proxy always sends a request_id, and unlike - // delivery-mode there is no older-broker reply shape to stay - // compatible with, so a reply without one is not ours — accepting it - // would resolve the caller's flush with an unrelated result. - if (pendingFlush && frameRid === pendingFlush.requestId) { - const pending = pendingFlush; - pendingFlush = null; - clearTimeout(pending.timer); - pending.resolve({ - flushed: typeof frame.flushed === 'number' ? frame.flushed : 0, - dead_lettered: typeof frame.dead_lettered === 'number' ? frame.dead_lettered : 0, - held: typeof frame.held === 'number' ? frame.held : 0, - blocked_reason: typeof frame.blocked_reason === 'string' ? frame.blocked_reason : null, - ...gapDiagnosticsFromFrame(frame), - }); - } - } else if (frame.type === 'terminal.error') { - const message = typeof frame.message === 'string' ? frame.message : 'remote terminal failed'; - const code = typeof frame.code === 'string' ? frame.code : 'terminal_error'; - const frameRid = typeof frame.request_id === 'string' ? frame.request_id : undefined; - // Route the error to the pending delivery-mode request only when the - // request_id matches (or the broker sent no request_id at all — older - // broker compat). An unrelated session-level error must not cancel a - // live delivery-mode PUT and vice-versa. - if (pendingFlush && frameRid !== undefined && frameRid === pendingFlush.requestId) { - const pending = pendingFlush; - pendingFlush = null; - clearTimeout(pending.timer); - pending.reject(new FleetNodeAttachError(message, code)); - } else if ( - pendingDeliveryMode && - (frameRid === undefined || frameRid === pendingDeliveryMode.requestId) - ) { - const pending = pendingDeliveryMode; - pendingDeliveryMode = null; - clearTimeout(pending.timer); - pending.reject(new FleetNodeAttachError(message, code)); - } else if (!frameRid && readiness === activeReadiness && !readiness.settled) { - endTerminal(new FleetNodeAttachError(message, code)); - } else if (!frameRid) { - failAllInputSockets(code, message); - } - } else if (frame.type === 'terminal.closed') { - endTerminal(new FleetNodeAttachError('remote terminal session closed', 'terminal_closed')); - } - }); - socket.on('error', (error) => { - readinessExpired = true; - clearReadinessTimer(); - // `ws` exposes a refused HTTP upgrade as this error before `close`. - // Only an expired resume credential may mint a replacement terminal; - // notably a transient or 5xx failure stays on the resume path. - if ( - terminalEverReady && - error instanceof Error && - /Unexpected server response: (401|410)\b/.test(error.message) - ) { - resumeRejectedAsExpired = true; - } - // Initial connection failure has no terminal state worth preserving. - // Fail promptly with the canonical unavailable-node error instead of - // letting the HTTP snapshot timeout mask it. Once Ready has been seen, - // the close handler retains the bounded resume/backoff behaviour. - if (remote === socket && readiness === activeReadiness && !readiness.settled && !terminalEverReady) { - failRemote( - `terminal transport could not connect to the fleet node (node ref ${diagnosticValue(options.node.trim())},` + - ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint())},` + - ` handshake budget ${terminalHandshakeTimeoutMs}ms, attempts 1; not retried because no terminal session became ready)` - ); - } - }); - socket.on('close', () => { - readinessExpired = true; - clearReadinessTimer(); - if (remote !== socket || stopped || terminalEnded || reconnecting) return; - // Relaycast drops the old lane's terminal session state on disconnect, - // so a set_delivery_mode frame already sent on this dying socket is - // lost and will never get a reply on the replacement socket — even - // once reconnect succeeds. Fail the pending PUT fast with a retryable - // error instead of leaving it to hang out the full timeout. - rejectPendingDeliveryMode( - new FleetNodeAttachError( - 'terminal transport disconnected while the delivery-mode change was in flight', - 'delivery_mode_disconnected' - ) - ); - // Any waiter that observed the prior connection must retry against the - // fresh generation instead of receiving its stale resolved snapshot. - resolveReadiness(readiness); - const nextReadiness = createReadiness(); - activeReadiness = nextReadiness; - if (reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) { - const backoffBudgetMs = retryDelayBudgetMs( - MAX_RECONNECT_ATTEMPTS, - reconnectInitialDelayMs, - reconnectMaxDelayMs - ); - const message = - `terminal transport could not reconnect to the fleet node (node ref ${diagnosticValue(options.node.trim())},` + - ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint())},` + - ` handshake timeout ${terminalHandshakeTimeoutMs}ms, readiness timeout ${terminalReadyTimeoutMs}ms,` + - ` attempts ${reconnectAttempts},` + - ` backoff budget ${backoffBudgetMs}ms)`; - failRemote( - message, - `terminal reconnect failed; attempts=${reconnectAttempts}; budget=${backoffBudgetMs}ms;` + - ` endpoint=${diagnosticValue(diagnosticEndpointOrigin(terminalUrl))};` + - ` node=${diagnosticValue(resolvedNodeId ?? options.node.trim())}` - ); - return; - } - reconnecting = true; - const delay = Math.min(reconnectInitialDelayMs * 2 ** reconnectAttempts, reconnectMaxDelayMs); - reconnectAttempts += 1; - reconnectTimer = setTimeout(() => { - reconnectTimer = undefined; - reconnecting = false; - const needsReplacement = - !replacementAllocatedForReconnect && (resumeRejectedAsExpired || terminalSessionExpired()); - if (!needsReplacement) { - connect(resumeUrlForCurrentSession(), nextReadiness); - return; - } - replacementAllocatedForReconnect = true; - void allocateReplacementTerminalSession().then( - (replacement) => { - if (stopped || terminalEnded || activeReadiness !== nextReadiness) return; - // This has no await between assignments, so every loopback - // handler observes one coherent replacement session. - terminalUrl = replacement.terminalUrl; - sessionId = replacement.sessionId; - resumeToken = replacement.resumeToken; - expiresAt = replacement.expiresAt; - connect(terminalUrl, nextReadiness); - }, - (error) => { - if (stopped || terminalEnded || activeReadiness !== nextReadiness) return; - const replacementError = - error instanceof FleetNodeAttachError - ? error - : new FleetNodeAttachError( - 'terminal session could not be replaced after its resume credential expired', - 'terminal_session_unavailable' - ); - endTerminal( - new FleetNodeAttachError( - `terminal transport could not replace an expired terminal session (node ref ${diagnosticValue(options.node.trim())},` + - ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint())})`, - replacementError.code - ) - ); - } - ); - }, delay); - }); - }; - connect(terminalUrl, activeReadiness); - - return { - brokerUrl: `http://127.0.0.1:${address.port}`, - apiKey: loopbackApiKey, - requestTimeoutMs: terminalWaitTimeoutMs + DELIVERY_MODE_TIMEOUT_MS + LOOPBACK_REQUEST_TIMEOUT_MARGIN_MS, - async close() { - if (stopped) return; - stopped = true; - terminalEnded = true; - if (reconnectTimer) { - clearTimeout(reconnectTimer); - reconnectTimer = undefined; - } - for (const timer of terminalReadinessTimers) clearTimeout(timer); - terminalReadinessTimers.clear(); - rejectReadiness(activeReadiness, new FleetNodeAttachError('terminal attach closed', 'closed')); - rejectPendingDeliveryMode(new FleetNodeAttachError('terminal attach closed', 'closed')); - const activeRemote = remote; - remote = undefined; - if (activeRemote && activeRemote.readyState === WebSocket.OPEN) { - try { - activeRemote.send(JSON.stringify({ type: 'terminal.close', session_id: sessionId })); - } catch { - // Best effort; terminate below still prevents a late reconnect. - } - } - if (activeRemote && activeRemote.readyState !== WebSocket.CLOSED) { - try { - activeRemote.terminate(); - } catch { - // Socket is already gone. - } - } - for (const socket of [...eventSockets, ...inputSockets]) - closeSocket(socket, 1000, 'terminal attach closed'); - websocketServer.close(); - await new Promise((resolve) => server.close(() => resolve())); - }, - }; -} +export * from '@agent-relay/cloud/attach'; diff --git a/packages/cli/src/cli/lib/attach-mode.ts b/packages/cli/src/cli/lib/attach-mode.ts index b1107eecec..b10726bba6 100644 --- a/packages/cli/src/cli/lib/attach-mode.ts +++ b/packages/cli/src/cli/lib/attach-mode.ts @@ -1 +1 @@ -export type AttachMode = 'drive' | 'view' | 'passthrough'; +export type { AttachMode } from '@agent-relay/cloud/attach'; diff --git a/packages/cli/src/cli/lib/collect-with-retry.ts b/packages/cli/src/cli/lib/collect-with-retry.ts index fb726a2315..71c417be17 100644 --- a/packages/cli/src/cli/lib/collect-with-retry.ts +++ b/packages/cli/src/cli/lib/collect-with-retry.ts @@ -1,42 +1 @@ -export interface CollectWithRetryOptions { - retries?: number; - baseDelayMs?: number; - sleep?: (ms: number) => Promise; - /** Return false for terminal failures that must be surfaced immediately. */ - shouldRetry?: (error: unknown, attempt: number) => boolean; -} - -/** - * Run a bounded operation with the fleet CLI's small retry delay. - * - * The result records whether another attempt actually ran. Callers that need - * richer failure metadata can retain the thrown value in their operation; - * this helper deliberately keeps the existing fleet-list rendering contract. - */ -export async function collectWithRetry( - label: string, - fn: () => Promise, - options?: CollectWithRetryOptions -): Promise<{ ok: true; value: T; retried: boolean } | { ok: false; error: string; retried: boolean }> { - const retries = options?.retries ?? 1; - const baseDelay = options?.baseDelayMs ?? 500; - const sleep = options?.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); - let lastError: unknown; - let attempts = 0; - for (let attempt = 0; attempt <= retries; attempt += 1) { - attempts += 1; - try { - const value = await fn(); - return { ok: true, value, retried: attempt > 0 }; - } catch (error) { - lastError = error; - if (attempt >= retries || options?.shouldRetry?.(error, attempt + 1) === false) break; - // Keep the established fleet-list delay shape while making later - // attempts wait slightly longer, without introducing test randomness. - const delay = baseDelay + Math.floor(baseDelay * (attempt / (retries + 1))); - await sleep(delay); - } - } - const message = lastError instanceof Error ? lastError.message : String(lastError); - return { ok: false, error: `${label}: ${message}`, retried: attempts > 1 }; -} +export * from '@agent-relay/cloud/attach'; diff --git a/packages/cli/src/cli/lib/sdk-client.ts b/packages/cli/src/cli/lib/sdk-client.ts index 235e3b13ae..0aac3a7a4c 100644 --- a/packages/cli/src/cli/lib/sdk-client.ts +++ b/packages/cli/src/cli/lib/sdk-client.ts @@ -1,18 +1,17 @@ -import path from 'node:path'; - import { AgentRelay, type AgentRelayAgent } from '@agent-relay/sdk'; import { - AGENT37_RELAYCAST_ORIGIN, - CANONICAL_RELAYCAST_ORIGIN, - DEV_CLOUD_API_URL, - DEV_RELAYCAST_ORIGIN, -} from '@agent-relay/cloud'; -import { - resolveWorkspaceSelection as resolveCloudWorkspaceSelection, - writeProjectWorkspaceTargetIfSelectionCurrent, - type WorkspaceSelection, - type WorkspaceKeySource, -} from '@agent-relay/cloud/workspace-key'; + resolveBaseUrl, + resolveWorkspaceSelection, + resolveWorkspaceTransport, +} from '@agent-relay/cloud/workspace-transport'; +export { + persistWorkspaceRelaycastTarget, + resolveBaseUrl, + resolveWorkspaceSelection, + resolveWorkspaceTransport, + type WorkspaceTransport, +} from '@agent-relay/cloud/workspace-transport'; +import { type WorkspaceSelection, type WorkspaceKeySource } from '@agent-relay/cloud/workspace-key'; /** Options shared by the SDK-backed (Relaycast) CLI command groups. */ export interface SdkClientOptions { @@ -41,23 +40,6 @@ function trimOrUndefined(value: string | undefined): string | undefined { export type { WorkspaceKeySource }; export type { WorkspaceSelection }; -export type WorkspaceTransport = { - workspaceKey: string; - baseUrl?: string; - source: WorkspaceKeySource; -}; - -/** Resolve the selected key and any previously persisted Relay workspace identity. */ -export function resolveWorkspaceSelection(options: SdkClientOptions = {}): WorkspaceSelection | undefined { - const explicitProject = trimOrUndefined(env(options).AGENT_RELAY_PROJECT); - const projectRoot = explicitProject ? path.resolve(explicitProject) : options.projectRoot; - return resolveCloudWorkspaceSelection({ - workspaceKey: options.workspaceKey, - env: env(options), - ...(projectRoot ? { projectRoot } : {}), - }); -} - /** * Resolve the workspace key and report which source it came from. Precedence: * explicit flag → `RELAY_WORKSPACE_KEY`/`RELAY_API_KEY` env → the key the local @@ -77,162 +59,6 @@ export function resolveWorkspaceKey(options: SdkClientOptions = {}): string { return resolveWorkspaceKeyWithSource(options).key; } -export function resolveBaseUrl(options: SdkClientOptions = {}): string | undefined { - const selection = selectionForTransport(options); - return resolveBaseUrlForSelection(selection, options); -} - -function selectionForTransport(options: SdkClientOptions): WorkspaceSelection | undefined { - const selection = resolveWorkspaceSelection(options); - if (!selection || !options.ignorePersistedRelaycastTarget) return selection; - const { - relaycastRoute: _relaycastRoute, - relaycastBaseUrl: _relaycastBaseUrl, - relaycastApiKey: _relaycastApiKey, - relaycastApiKeyRef: _relaycastApiKeyRef, - ...canonicalSelection - } = selection; - return canonicalSelection; -} - -/** Resolve an origin while keeping a persisted route paired with its credential. */ -function resolveBaseUrlForSelection( - selection: WorkspaceSelection | undefined, - options: SdkClientOptions -): string | undefined { - const persisted = validatePersistedRelaycastBaseUrl(selection); - const explicit = trimOrUndefined(options.baseUrl); - const fallback = trimOrUndefined(options.fallbackBaseUrl) ?? trimOrUndefined(env(options).RELAY_BASE_URL); - const requested = explicit ?? fallback; - // Precedence is explicit --base-url (validated against a persisted route), - // then the persisted route, fallbackBaseUrl, and finally RELAY_BASE_URL. - // This keeps a server-selected route paired with its credential. - if (persisted && explicit) { - let parsed: URL; - try { - parsed = new URL(explicit); - } catch { - throw new Error('The requested Relaycast base URL is invalid.'); - } - const authority = /^https:\/\/([^/?#]+)/i.exec(explicit)?.[1] ?? ''; - if ( - !/^https:\/\/[^/?#]+\/?$/i.test(explicit) || - parsed.protocol !== 'https:' || - parsed.username || - parsed.password || - parsed.port || - /:\d+$/.test(authority) || - parsed.search || - parsed.hash || - (parsed.pathname !== '' && parsed.pathname !== '/') - ) { - throw new Error('The requested Relaycast base URL is not a trusted origin.'); - } - if (parsed.origin !== persisted) { - throw new Error('The requested Relaycast base URL does not match the persisted workspace route.'); - } - } - return persisted ?? requested; -} - -/** Resolve one credential/origin pair from one workspace selection. */ -export function resolveWorkspaceTransport(options: SdkClientOptions = {}): WorkspaceTransport { - const selection = selectionForTransport(options); - if (!selection) { - throw new Error( - 'No workspace key found. Pass --workspace-key, set RELAY_WORKSPACE_KEY, or run `relay workspace set_key `.' - ); - } - const baseUrl = resolveBaseUrlForSelection(selection, options); - // Project-session loading already validates the reference against the - // project/workspace/route/base tuple. Never re-read a ref here: doing so - // would let a tampered ref bypass that binding and pair an unrelated key - // with this route. - const routeCredential = trimOrUndefined(selection.relaycastApiKey); - if (selection.relaycastRoute === 'agent37-isolated' && !routeCredential) { - throw new Error( - 'The persisted isolated Relaycast credential is unavailable or mismatched; rerun the sandbox command to mint a fresh route.' - ); - } - return { - workspaceKey: routeCredential ?? selection.key, - ...(baseUrl ? { baseUrl } : {}), - source: selection.source, - }; -} - -function validatePersistedRelaycastBaseUrl(selection: WorkspaceSelection | undefined): string | undefined { - const baseUrl = trimOrUndefined(selection?.relaycastBaseUrl); - const route = selection?.relaycastRoute; - const relaycastApiKey = trimOrUndefined(selection?.relaycastApiKey); - const relaycastCloudApiUrl = trimOrUndefined(selection?.relaycastCloudApiUrl); - if (!baseUrl && !route && !relaycastApiKey) return undefined; - if (!baseUrl || !route) { - throw new Error('The persisted Relaycast workspace route is incomplete.'); - } - let parsed: URL; - try { - parsed = new URL(baseUrl); - } catch { - throw new Error('The persisted Relaycast workspace route is invalid.'); - } - const expectedOrigin = - route === 'canonical' - ? relaycastCloudApiUrl === DEV_CLOUD_API_URL - ? DEV_RELAYCAST_ORIGIN - : CANONICAL_RELAYCAST_ORIGIN - : route === 'agent37-isolated' - ? AGENT37_RELAYCAST_ORIGIN - : undefined; - if ( - !expectedOrigin || - parsed.origin !== expectedOrigin || - parsed.protocol !== 'https:' || - parsed.username || - parsed.password || - parsed.port || - parsed.search || - parsed.hash || - (parsed.pathname !== '' && parsed.pathname !== '/') - ) { - throw new Error('The persisted Relaycast workspace route is not trusted.'); - } - return parsed.origin; -} - -/** Persist a server-selected target only while the captured project selection is still current. */ -export function persistWorkspaceRelaycastTarget( - selection: WorkspaceSelection | undefined, - target: { - route: 'canonical' | 'agent37-isolated'; - baseUrl: string; - workspaceId: string; - relaycastApiKey: string; - }, - relaycastCloudApiUrl?: string -): boolean { - if (!selection) return false; - if ( - target.route === 'canonical' && - target.baseUrl === DEV_RELAYCAST_ORIGIN && - relaycastCloudApiUrl !== DEV_CLOUD_API_URL - ) { - return false; - } - const selectionWithProjectDir = selection as WorkspaceSelection & { projectDataDir?: string }; - const dataDir = - selectionWithProjectDir?.projectDataDir ?? - (selection?.source === 'project' && selection.origin ? path.dirname(selection.origin) : undefined); - if (!dataDir) return false; - return writeProjectWorkspaceTargetIfSelectionCurrent(dataDir, selection, { - workspaceId: target.workspaceId, - relaycastRoute: target.route, - relaycastBaseUrl: target.baseUrl, - ...(relaycastCloudApiUrl ? { relaycastCloudApiUrl } : {}), - relaycastApiKey: target.relaycastApiKey, - }); -} - export function resolveAgentToken(options: SdkClientOptions = {}): string | undefined { return trimOrUndefined(options.token) ?? trimOrUndefined(env(options).RELAY_AGENT_TOKEN); } diff --git a/packages/cloud/package.json b/packages/cloud/package.json index c4bcdd156f..4b874a5f0d 100644 --- a/packages/cloud/package.json +++ b/packages/cloud/package.json @@ -49,7 +49,19 @@ "types": "./dist/workspace-key.d.ts", "import": "./dist/workspace-key.js" }, - "./package.json": "./package.json" + "./package.json": "./package.json", + "./attach": { + "types": "./dist/attach.d.ts", + "import": "./dist/attach.js" + }, + "./workspace-transport": { + "types": "./dist/workspace-transport.d.ts", + "import": "./dist/workspace-transport.js" + }, + "./fleet": { + "types": "./dist/fleet.d.ts", + "import": "./dist/fleet.js" + } }, "files": [ "dist", @@ -65,7 +77,8 @@ "@agent-relay/config": "13.2.0", "@aws-sdk/client-s3": "3.1020.0", "ignore": "^7.0.5", - "tar": "^7.5.21" + "tar": "^7.5.21", + "ws": "^8.18.3" }, "optionalDependencies": { "ssh2": "^1.17.0" @@ -73,7 +86,8 @@ "devDependencies": { "@types/node": "^22.19.3", "@types/ssh2": "^1.15.5", - "vitest": "^4.1.0" + "vitest": "^4.1.0", + "@types/ws": "^8.18.1" }, "publishConfig": { "access": "public" diff --git a/packages/cloud/src/attach-mode.ts b/packages/cloud/src/attach-mode.ts new file mode 100644 index 0000000000..b1107eecec --- /dev/null +++ b/packages/cloud/src/attach-mode.ts @@ -0,0 +1 @@ +export type AttachMode = 'drive' | 'view' | 'passthrough'; diff --git a/packages/cloud/src/attach.test.ts b/packages/cloud/src/attach.test.ts new file mode 100644 index 0000000000..cc2be0fd5d --- /dev/null +++ b/packages/cloud/src/attach.test.ts @@ -0,0 +1,270 @@ +import { once } from 'node:events'; +import { connect, type AddressInfo, type Socket } from 'node:net'; +import { stat } from 'node:fs/promises'; +import { WebSocketServer, type WebSocket } from 'ws'; +import { afterEach, expect, it, vi } from 'vitest'; +import { startFleetNodeAttachProxy, type FleetNodeAttachProxy } from './attach.js'; + +let proxy: FleetNodeAttachProxy | undefined; +let server: WebSocketServer | undefined; +let local: Socket | undefined; +afterEach(async () => { + local?.destroy(); + await proxy?.close(); + proxy = undefined; + if (server) { + for (const client of server.clients) client.terminate(); + await new Promise((resolve) => server!.close(() => resolve())); + } + server = undefined; +}); + +async function setup(mode: 'drive' | 'view' = 'drive') { + server = new WebSocketServer({ host: '127.0.0.1', port: 0 }); + await once(server, 'listening'); + const remoteConnection = once(server, 'connection'); + const fetch = vi.fn(async (url: string | URL | Request, _init?: RequestInit) => + Response.json({ + ok: true, + data: String(url).endsWith('/agents') + ? [{ agentName: 'worker' }] + : { + session_id: 'session', + resume_token: 'resume', + terminal_url: `ws://127.0.0.1:${(server!.address() as AddressInfo).port}/terminal`, + }, + }) + ); + proxy = await startFleetNodeAttachProxy({ nodeId: 'node', mode, workspaceKey: 'rk_test', env: {}, fetch }); + const [remote] = (await remoteConnection) as [WebSocket]; + return { remote, fetch }; +} +function send(remote: WebSocket, type: string, data = {}) { + remote.send(JSON.stringify({ session_id: 'session', type, ...data })); +} + +it('discovers one agent and pipes raw bytes through a private socket, then removes it', async () => { + const { remote, fetch } = await setup(); + expect(JSON.parse(String(fetch.mock.calls[1]?.[1]?.body)).agent).toBe('worker'); + expect((await stat(proxy!.socketPath)).mode & 0o777).toBe(0o600); + local = connect(proxy!.socketPath); + await once(local, 'connect'); + send(remote, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + expect((await once(local, 'data'))[0].toString()).toBe('hello'); + const frame = once(remote, 'message'); + const bytes = Buffer.from([0x00, 0x1b, 0xff, 0xc3, 0xa9]); + local.write(bytes); + expect(JSON.parse((await frame)[0].toString())).toMatchObject({ + type: 'terminal.input', + data_base64: bytes.toString('base64'), + }); + const output = once(local, 'data'); + send(remote, 'terminal.output', { chunk: 'world' }); + expect((await output)[0].toString()).toBe('world'); + const socketPath = proxy!.socketPath; + send(remote, 'terminal.closed'); + await expect(proxy!.finished).resolves.toBe(0); + await Promise.all([proxy!.close(), proxy!.close()]); + await expect(stat(socketPath)).rejects.toMatchObject({ code: 'ENOENT' }); +}); + +it('replays output to a late raw client even when an event listener already consumed it', async () => { + const { remote } = await setup(); + const events = new (await import('ws')).default(`${proxy!.brokerUrl.replace('http', 'ws')}/ws`, { + headers: { authorization: `Bearer ${proxy!.apiKey}` }, + }); + await once(events, 'open'); + send(remote, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + const consumed = once(events, 'message'); + send(remote, 'terminal.output', { chunk: ' early' }); + await consumed; + local = connect(proxy!.socketPath); + let received = ''; + local.on('data', (data: Buffer) => (received += data.toString())); + await vi.waitFor(() => expect(received).toBe('hello early')); + events.close(); +}); + +it('replays output to a late event listener even while a raw client consumed it', async () => { + const { remote } = await setup(); + local = connect(proxy!.socketPath); + await once(local, 'connect'); + send(remote, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + await once(local, 'data'); + const rawOutput = once(local, 'data'); + send(remote, 'terminal.output', { chunk: ' during-raw' }); + await rawOutput; + const events = new (await import('ws')).default(`${proxy!.brokerUrl.replace('http', 'ws')}/ws`, { + headers: { authorization: `Bearer ${proxy!.apiKey}` }, + }); + const [message] = await once(events, 'message'); + expect(JSON.stringify(JSON.parse(String(message)))).toContain('during-raw'); + events.close(); +}); + +it('ends the raw attachment on a connection-fatal session error after ready', async () => { + const { remote } = await setup(); + local = connect(proxy!.socketPath); + await once(local, 'connect'); + send(remote, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + await once(local, 'data'); + const closed = once(local, 'close'); + send(remote, 'terminal.error', { code: 'node_unreachable', message: 'node went away' }); + await expect(proxy!.finished).resolves.toBe(1); + await closed; +}); + +it('holds raw input during a transient reconnect and delivers it once the terminal is ready again', async () => { + server = new WebSocketServer({ host: '127.0.0.1', port: 0 }); + await once(server, 'listening'); + const connections: WebSocket[] = []; + server.on('connection', (socket) => connections.push(socket as WebSocket)); + const fetch = vi.fn(async (url: string | URL | Request) => + Response.json({ + ok: true, + data: String(url).endsWith('/agents') + ? [{ agentName: 'worker' }] + : { + session_id: 'session', + resume_token: 'resume', + terminal_url: `ws://127.0.0.1:${(server!.address() as AddressInfo).port}/terminal`, + }, + }) + ); + proxy = await startFleetNodeAttachProxy({ + nodeId: 'node', + mode: 'drive', + workspaceKey: 'rk_test', + env: {}, + fetch, + reconnectDelay: { initialMs: 300, maxMs: 300 }, + }); + await vi.waitFor(() => expect(connections).toHaveLength(1)); + local = connect(proxy!.socketPath); + await once(local, 'connect'); + send(connections[0]!, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + await once(local, 'data'); + connections[0]!.terminate(); + await new Promise((resolve) => setTimeout(resolve, 50)); + local.write('typed-during-reconnect'); + await vi.waitFor(() => expect(connections).toHaveLength(2)); + const input = once(connections[1]!, 'message'); + send(connections[1]!, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + const frame = JSON.parse(String((await input)[0])); + expect(frame).toMatchObject({ type: 'terminal.input' }); + expect(Buffer.from(frame.data_base64, 'base64').toString()).toBe('typed-during-reconnect'); + let settled = false; + void proxy!.finished.then(() => (settled = true)); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(settled).toBe(false); +}); + +it('refuses raw clients after a connection-fatal session error', async () => { + const { remote } = await setup(); + send(remote, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + await new Promise((resolve) => setTimeout(resolve, 20)); + send(remote, 'terminal.error', { code: 'node_unreachable', message: 'node went away' }); + await expect(proxy!.finished).resolves.toBe(1); + local = connect(proxy!.socketPath); + let received = ''; + local.on('data', (data: Buffer) => (received += data.toString())); + local.on('error', () => undefined); + await once(local, 'close'); + expect(received).toBe(''); +}); + +it('settles finished with 0 when the raw client detaches', async () => { + const { remote } = await setup(); + local = connect(proxy!.socketPath); + await once(local, 'connect'); + send(remote, 'terminal.ready', { screen: Buffer.from('hello').toString('base64') }); + await once(local, 'data'); + local.end(); + await expect(proxy!.finished).resolves.toBe(0); +}); + +it('uses a pinned workspaceKey and baseUrl as given, ignoring a persisted project route', async () => { + const { mkdtemp, mkdir, writeFile } = await import('node:fs/promises'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const project = await mkdtemp(join(tmpdir(), 'attach-explicit-')); + await mkdir(join(project, '.agentworkforce', 'relay'), { recursive: true }); + await writeFile( + join(project, '.agentworkforce', 'relay', 'workspace-key.json'), + JSON.stringify({ + workspaceKey: 'rk_live_project', + workspaceId: 'rw_project', + relaycastRoute: 'canonical', + relaycastBaseUrl: 'https://cast.agentrelay.com', + }) + ); + const fetch = vi.fn(async () => + Response.json({ ok: false, error: { code: 'stop', message: 'stop' } }, { status: 400 }) + ); + await expect( + startFleetNodeAttachProxy({ + node: 'node', + agent: 'worker', + mode: 'drive', + workspaceKey: 'rk_live_project', + baseUrl: 'https://agent37-cast.agentrelay.com', + pinnedTransport: true, + env: { AGENT_RELAY_PROJECT: project }, + fetch, + sessionRequest: { timeoutMs: 50, totalTimeoutMs: 50 }, + }) + ).rejects.toBeInstanceOf(Error); + expect(fetch).toHaveBeenCalled(); + const [url, init] = fetch.mock.calls[0] as unknown as [string, RequestInit]; + expect(url.startsWith('https://agent37-cast.agentrelay.com/')).toBe(true); + expect((init.headers as Record).Authorization).toBe('Bearer rk_live_project'); +}); + +it('settles finished on explicit close before terminal.ready', async () => { + await setup(); + await proxy!.close(); + await expect(proxy!.finished).resolves.toBe(0); +}); + +it('resolves failure without an unhandled rejection', async () => { + const { remote } = await setup(); + send(remote, 'terminal.error', { code: 'node_unreachable', message: 'offline' }); + await expect(proxy!.finished).resolves.toBe(1); +}); + +it('view sockets cannot send terminal input and reject additional clients', async () => { + const { remote } = await setup('view'); + local = connect(proxy!.socketPath); + await once(local, 'connect'); + send(remote, 'terminal.ready', { screen: Buffer.from('ready').toString('base64') }); + await once(local, 'data'); + const received = vi.fn(); + remote.on('message', received); + local.write('ignored'); + const extra = connect(proxy!.socketPath); + await once(extra, 'close'); + expect(received).not.toHaveBeenCalled(); +}); + +it.each([[], [{ agentName: 'a' }, { agentName: 'b' }]])( + 'rejects missing or ambiguous agents', + async (...agents) => { + await expect( + startFleetNodeAttachProxy({ + nodeId: 'node', + mode: 'drive', + workspaceKey: 'rk_test', + env: {}, + fetch: vi.fn(async () => Response.json({ data: agents.length === 0 ? [] : agents })), + }) + ).rejects.toMatchObject({ code: 'ambiguous_agent' }); + } +); + +it('trusts the DEV Relaycast origin for attach, alongside canonical and Agent37', async () => { + const { validateFleetAttachBaseUrl } = await import('./attach.js'); + expect(validateFleetAttachBaseUrl('https://dev-cast.agentrelay.com/')).toBe( + 'https://dev-cast.agentrelay.com' + ); + expect(() => validateFleetAttachBaseUrl('https://evil.example.com')).toThrow('trusted Relaycast origin'); +}); diff --git a/packages/cloud/src/attach.ts b/packages/cloud/src/attach.ts new file mode 100644 index 0000000000..0adc13c1c5 --- /dev/null +++ b/packages/cloud/src/attach.ts @@ -0,0 +1,1760 @@ +/** + * Ticketed fleet-node attach adapter. + * + * The established attach clients intentionally continue to speak the local + * broker HTTP/WebSocket contract. This short-lived loopback adapter maps that + * contract onto Relaycast's authenticated terminal session, so view/drive and + * passthrough retain their behaviour without exposing a remote broker listener + * or copying a broker API key off a physical or Daytona node. + */ + +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { once } from 'node:events'; +import { Buffer } from 'node:buffer'; +import { randomBytes } from 'node:crypto'; +import { createServer as createSocketServer, type Socket } from 'node:net'; +import { mkdtemp, chmod, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import WebSocket, { WebSocketServer } from 'ws'; + +import { + AGENT37_RELAYCAST_ORIGIN, + CANONICAL_RELAYCAST_ORIGIN, + DEV_RELAYCAST_ORIGIN, +} from './fleet-sandbox.js'; +import type { AttachMode } from './attach-mode.js'; +import { collectWithRetry } from './collect-with-retry.js'; +import { resolveWorkspaceTransport } from './workspace-transport.js'; + +const MAX_BUFFERED_BYTES = 1024 * 1024; +const MAX_WEBSOCKET_CLOSE_REASON_BYTES = 123; +// Use the same finite 30s request window as the broker's Relaycast HTTP calls. +// Terminal-session creation can exceed the shorter startup-handshake latency +// under load, and this POST cannot be replayed safely after an ambiguous client +// timeout because allocation may have completed server-side. +const SESSION_REQUEST_TIMEOUT_MS = 30_000; +// Immediate structured reachability failures still receive the complete +// five-attempt/31.2s retry schedule, while slow responses cannot multiply the +// per-attempt timeout into a roughly three-minute CLI hang. +const SESSION_REQUEST_TOTAL_TIMEOUT_MS = 90_000; +const SESSION_REQUEST_RETRIES = 4; +// The fleet node publishes liveness every 12s. With four retries, the helper's +// deterministic delays are 6s, 7.2s, 8.4s, and 9.6s: 31.2s total. That spans +// more than two heartbeat intervals and matches the established terminal +// transport's bounded recovery window instead of exhausting every retry inside +// the same stale control-plane read. +const SESSION_REQUEST_RETRY_DELAY_MS = 6_000; +const TERMINAL_CONNECT_TIMEOUT_MS = 10_000; +const INITIAL_RECONNECT_DELAY_MS = 500; +const MAX_RECONNECT_DELAY_MS = 30_000; +// Six delays (0.5s + 1s + 2s + 4s + 8s + 16s) cover the node terminal +// transport's independent 30s maximum reconnect backoff without leaving this +// client unbounded. Five attempts previously stopped after only 15.5s. +const MAX_RECONNECT_ATTEMPTS = 6; +// Bounds the acknowledgement for a delivery-mode command after readiness. +// The command is not replayable across reconnects, so this is intentionally a +// separate post-readiness phase rather than another full reconnect window. +const DELIVERY_MODE_TIMEOUT_MS = 10_000; +// The caller starts its HTTP deadline before the loopback handler starts its +// readiness timer. Leave a small response-delivery margin so the proxy's +// actionable readiness error wins that race. +const LOOPBACK_REQUEST_TIMEOUT_MARGIN_MS = 1_000; + +type FleetSessionResponse = { + ok?: boolean; + data?: { + session_id?: string; + terminal_url?: string; + resume_token?: string; + expires_at?: string; + }; + error?: { code?: string; message?: string }; +}; + +type TerminalFrame = Record & { + type?: string; + session_id?: string; + blocked_reason_code?: unknown; + head_sequence?: unknown; + acked_up_to_sequence?: unknown; + received_up_to_sequence?: unknown; + next_ackable_sequence?: unknown; + reconciliation_action?: unknown; +}; + +type GapDiagnosticsResult = { + blocked_reason_code?: string; + head_sequence?: number; + acked_up_to_sequence?: number; + received_up_to_sequence?: number; + next_ackable_sequence?: number; + reconciliation_action?: string; +}; + +function gapDiagnosticsFromFrame(frame: { + blocked_reason_code?: unknown; + head_sequence?: unknown; + acked_up_to_sequence?: unknown; + received_up_to_sequence?: unknown; + next_ackable_sequence?: unknown; + reconciliation_action?: unknown; +}): GapDiagnosticsResult { + return { + ...(typeof frame.blocked_reason_code === 'string' + ? { blocked_reason_code: frame.blocked_reason_code } + : {}), + ...(typeof frame.head_sequence === 'number' ? { head_sequence: frame.head_sequence } : {}), + ...(typeof frame.acked_up_to_sequence === 'number' + ? { acked_up_to_sequence: frame.acked_up_to_sequence } + : {}), + ...(typeof frame.received_up_to_sequence === 'number' + ? { received_up_to_sequence: frame.received_up_to_sequence } + : {}), + ...(typeof frame.next_ackable_sequence === 'number' + ? { next_ackable_sequence: frame.next_ackable_sequence } + : {}), + ...(typeof frame.reconciliation_action === 'string' + ? { reconciliation_action: frame.reconciliation_action } + : {}), + }; +} + +type TerminalReadiness = { + generation: number; + settled: boolean; + promise: Promise; + resolve: () => void; + reject: (error: Error) => void; +}; + +export interface FleetNodeAttachOptions { + /** Defaults to the only agent on the node; ambiguous nodes require an explicit agent. */ + agent?: string; + node?: string; + nodeId?: string; + mode: AttachMode; + env?: NodeJS.ProcessEnv; + baseUrl?: string; + workspaceKey?: string; + /** Use workspaceKey/baseUrl exactly as given instead of resolving the selection. */ + pinnedTransport?: boolean; + fetch?: typeof globalThis.fetch; + /** Deterministic test seam for the bounded session-request retry delay. */ + sessionRequest?: { + timeoutMs?: number; + totalTimeoutMs?: number; + sleep?: (ms: number) => Promise; + }; + /** Deterministic test seam for established-session reconnect timing. */ + reconnectDelay?: { + initialMs?: number; + maxMs?: number; + handshakeTimeoutMs?: number; + readyTimeoutMs?: number; + beforeReadyTimeoutTerminate?: (socket: WebSocket) => void; + }; +} + +/** Internal compatibility transport for broker-based CLI attach clients. */ +export interface FleetNodeAttachTransport { + brokerUrl: string; + apiKey: string; + requestTimeoutMs: number; +} + +export interface FleetNodeAttachProxy extends FleetNodeAttachTransport { + /** Private UNIX byte-stream socket (named pipe on Windows). */ + socketPath: string; + /** Inferred status: 0 on terminal.closed or detach, 1 on transport failure. + * Relaycast does not currently transmit the remote process exit code. */ + finished: Promise; + close(): Promise; +} + +export class FleetNodeAttachError extends Error { + constructor( + message: string, + readonly code?: string + ) { + super(message); + this.name = 'FleetNodeAttachError'; + } +} + +const TRUSTED_RELAYCAST_ORIGINS = new Set([ + CANONICAL_RELAYCAST_ORIGIN, + AGENT37_RELAYCAST_ORIGIN, + DEV_RELAYCAST_ORIGIN, +]); + +export function validateFleetAttachBaseUrl(value: string): string { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new FleetNodeAttachError('Fleet node attach requires a trusted Relaycast origin.'); + } + if ( + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') || + !TRUSTED_RELAYCAST_ORIGINS.has(parsed.origin) + ) { + throw new FleetNodeAttachError('Fleet node attach requires a trusted Relaycast origin.'); + } + return parsed.origin; +} + +class TerminalSessionAttemptError extends FleetNodeAttachError { + constructor( + message: string, + code: string | undefined, + readonly status: number | undefined, + readonly retryable: boolean, + readonly completionUnknown: boolean + ) { + super(message, code); + } +} + +/** + * Canonical HTTP status for a terminal failure code. + * + * `agent_not_found` must stay 404: {@link switchInboundDeliveryModeOrAbort} + * only emits the "no agent named X" message and the cross-node placement hint + * on a 404, so collapsing it into 503 replaces actionable guidance with an + * opaque unreachable-node error. `unsupported_runtime` stays 409. Everything + * else is a transport-level failure and reports 503. + */ +function terminalErrorStatus(code: string | undefined): number { + if (code === 'agent_not_found') return 404; + if (code === 'unsupported_runtime') return 409; + return 503; +} + +function json(response: ServerResponse, status: number, payload: unknown): void { + response.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + response.end(JSON.stringify(payload)); +} + +function readBody(request: IncomingMessage): Promise> { + return new Promise((resolve) => { + let settled = false; + const finish = (body: Record) => { + if (settled) return; + settled = true; + resolve(body); + }; + let body = ''; + request.setEncoding('utf8'); + request.on('data', (chunk: string) => { + body += chunk; + }); + request.on('end', () => { + try { + const parsed = JSON.parse(body) as unknown; + finish( + parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : {} + ); + } catch { + finish({}); + } + }); + request.on('error', () => finish({})); + request.on('aborted', () => finish({})); + }); +} + +function asWsUrl(value: string): string { + const lower = value.toLowerCase(); + if (lower.startsWith('https://')) return 'wss://' + value.slice(8); + if (lower.startsWith('http://')) return 'ws://' + value.slice(7); + return value; +} + +function safeNodePath(node: string): string { + const trimmed = node.trim().replace(/^#/, ''); + if (!trimmed) throw new FleetNodeAttachError('Error: --node requires a node name or id.', 'invalid_node'); + return encodeURIComponent(trimmed); +} + +function parseFrame(data: WebSocket.RawData): TerminalFrame | null { + try { + const parsed = JSON.parse(rawDataToString(data)) as unknown; + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as TerminalFrame) : null; + } catch { + return null; + } +} + +/** + * Decode a `terminal.ready` / `terminal.snapshot` `screen` field into the raw + * ANSI bytes a terminal can render. + * + * The two payloads this adapter bridges are encoded differently and the + * difference is invisible at the type level — both are `string`: + * + * - `screen` is the visible grid rendered by `Snapshot::to_ansi()` and then + * **base64-encoded**, because the broker asks the worker for + * `format: "ansi"` (`fleet.rs`) and that arm encodes (`pty_worker.rs`). + * Broker HTTP snapshot consumers decode it (`attach.ts` `captureAndRender…`). + * - `worker_stream.chunk` (and `terminal.output.chunk`, which is the same + * value forwarded) is **raw PTY bytes**. Attach clients write it to stdout + * verbatim (`applyServerOutput` in `attach-drive.ts`). + * + * Handing an undecoded `screen` to {@link workerStreamEvent} therefore prints + * the base64 text itself into the operator's terminal — the reconnect flood in + * relay#1829. Returns `null` when the payload is not canonical base64, so a + * malformed or protocol-violating frame is dropped rather than rendered: a + * stale-but-coherent screen beats writing unintelligible bytes to a live TTY. + */ +export function decodeAnsiScreenPayload(screen: string): string | null { + if (screen === '') return ''; + if (screen.length % 4 !== 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(screen)) return null; + const decoded = Buffer.from(screen, 'base64'); + // `Buffer.from(…, 'base64')` silently skips characters outside the alphabet, + // so the round-trip is what actually rejects a non-base64 payload. + if (decoded.toString('base64') !== screen) return null; + return decoded.toString('utf8'); +} + +/** + * Mirror of the broker's `pty_input_error_is_connection_fatal` + * (`crates/broker/src/listen_api.rs`). The two lists must agree, because the + * SDK's `PtyInputStream` latches `closed` **only** when its socket closes + * (`harness-driver/src/transport.ts`): a fatal error delivered on a socket + * that stays open leaves `isUsable()` true forever. + * + * That matters most for a fatal error that arrives with **no write in + * flight** — the session-scoped `terminal.error` path below. The client's + * `failAll()` then has nothing to reject, so nothing marks the stream dead: + * the drive session never enters recovery and keeps writing into a stream the + * node has already declared unusable. Closing the socket, as the broker does, + * is what turns that into a reported outage the session can recover from. + */ +export function inputErrorIsConnectionFatal(code: string): boolean { + return code !== 'worker_timeout' && code !== 'pty_write_queue_full'; +} + +function rawDataToString(data: WebSocket.RawData): string { + if (Buffer.isBuffer(data)) return data.toString('utf8'); + if (data instanceof ArrayBuffer) return Buffer.from(data).toString('utf8'); + if (Array.isArray(data)) return Buffer.concat(data).toString('utf8'); + return String(data); +} + +function diagnosticEndpoint(value: string): string { + try { + const url = new URL(value); + url.username = ''; + url.password = ''; + url.search = ''; + url.hash = ''; + return url.toString(); + } catch { + return '(invalid endpoint)'; + } +} + +function diagnosticEndpointOrigin(value: string): string { + try { + return new URL(value).origin; + } catch { + return '(invalid endpoint)'; + } +} + +function boundedWebSocketCloseReason(reason: string): string { + const encoded = Buffer.from(reason, 'utf8'); + if (encoded.length <= MAX_WEBSOCKET_CLOSE_REASON_BYTES) return reason; + + const suffix = Buffer.from('…', 'utf8'); + let end = MAX_WEBSOCKET_CLOSE_REASON_BYTES - suffix.length; + // Do not cut through a UTF-8 continuation sequence. Excluding the leading + // byte at this boundary also excludes the incomplete code point. + while (end > 0 && (encoded[end] & 0xc0) === 0x80) end -= 1; + return Buffer.concat([encoded.subarray(0, end), suffix]).toString('utf8'); +} + +function diagnosticValue(value: string): string { + // JSON quoting keeps node names and upstream text from injecting terminal + // control characters into the operator-facing error. + return JSON.stringify(value); +} + +function resolvedNodeIdFromTerminalUrl(value: string): string | undefined { + try { + const match = /^\/v1\/nodes\/([^/]+)\/terminal\/connect$/.exec(new URL(value).pathname); + return match?.[1] ? decodeURIComponent(match[1]) : undefined; + } catch { + return undefined; + } +} + +function retryDelayBudgetMs(attempts: number, initialMs: number, maxMs: number): number { + let total = 0; + for (let attempt = 0; attempt < attempts; attempt += 1) { + total += Math.min(initialMs * 2 ** attempt, maxMs); + } + return total; +} + +function terminalSessionFailureSummary(error: TerminalSessionAttemptError): string { + if (error.code === 'node_not_found') return 'Control-plane node lookup found no matching record'; + if (error.code === 'node_unreachable') { + return /no terminal transport/i.test(error.message) + ? 'The node record was found, but its terminal transport was unavailable' + : 'The control plane classified the node as unreachable'; + } + if (error.code === 'terminal_session_unavailable') { + return 'The control plane could not allocate a terminal session'; + } + if (error.code === 'control_plane_timeout') return 'Control-plane terminal-session lookup timed out'; + if (error.code === 'control_plane_unavailable') return 'Control-plane terminal-session lookup failed'; + return 'The terminal-session request was rejected'; +} + +function isRetryableTerminalSessionFailure(code: string | undefined): boolean { + // These structured responses are emitted before a session is returned, so + // retrying cannot duplicate a successful allocation. A fetch timeout, + // network failure, or unclassified 5xx is different: this POST may already + // have completed server-side, and retrying it could create a second session. + return code === 'node_unreachable' || code === 'terminal_session_unavailable'; +} + +/** Start a broker-compatible loopback proxy for one remote terminal session. */ +export async function startFleetNodeAttachProxy( + input: FleetNodeAttachOptions +): Promise { + const options = { ...input, node: input.node ?? input.nodeId ?? '', agent: input.agent ?? '' }; + if (input.node && input.nodeId && input.node !== input.nodeId) { + throw new FleetNodeAttachError('node and nodeId must identify the same node.', 'invalid_node'); + } + const env = options.env ?? process.env; + const fetchFn = options.fetch ?? globalThis.fetch; + // A pinned credential/origin pair (e.g. the one a spawn verified) is used + // exactly as given. Otherwise an explicit key is a selector that may bind to + // the project's persisted route and credential. + const pinnedKey = options.pinnedTransport ? options.workspaceKey?.trim() : undefined; + const pinnedBaseUrl = options.pinnedTransport ? options.baseUrl?.trim() : undefined; + if (options.pinnedTransport && (!pinnedKey || !pinnedBaseUrl)) { + throw new FleetNodeAttachError('pinnedTransport requires both workspaceKey and baseUrl.'); + } + const { workspaceKey, baseUrl: requestedBaseUrl } = + pinnedKey && pinnedBaseUrl + ? { workspaceKey: pinnedKey, baseUrl: pinnedBaseUrl } + : resolveWorkspaceTransport({ + workspaceKey: options.workspaceKey, + baseUrl: options.baseUrl, + env, + }); + const baseUrl = validateFleetAttachBaseUrl(requestedBaseUrl ?? CANONICAL_RELAYCAST_ORIGIN); + const nodePath = safeNodePath(options.node); + if (!options.agent) { + const response = await fetchFn(`${baseUrl}/v1/nodes/${nodePath}/agents`, { + headers: { Authorization: `Bearer ${workspaceKey}` }, + signal: AbortSignal.timeout(SESSION_REQUEST_TIMEOUT_MS), + }); + const payload = (await response.json()) as { data?: Array<{ agentName?: string }> }; + if (!response.ok || !Array.isArray(payload.data)) { + throw new FleetNodeAttachError('Could not discover the fleet node agent.', 'agent_discovery_failed'); + } + const names = [ + ...new Set( + payload.data + .map((agent) => agent.agentName) + .filter((name): name is string => typeof name === 'string' && !!name.trim()) + ), + ]; + if (names.length !== 1) { + throw new FleetNodeAttachError( + 'Specify agent explicitly unless the node has exactly one agent.', + 'ambiguous_agent' + ); + } + options.agent = names[0]; + } + const sessionEndpoint = `${baseUrl}/v1/nodes/${nodePath}/terminal/sessions`; + const sessionRequestTimeoutMs = options.sessionRequest?.timeoutMs ?? SESSION_REQUEST_TIMEOUT_MS; + const sessionRequestTotalTimeoutMs = + options.sessionRequest?.totalTimeoutMs ?? SESSION_REQUEST_TOTAL_TIMEOUT_MS; + const sessionRequestDeadline = Date.now() + sessionRequestTotalTimeoutMs; + const sessionRequestSleep = + options.sessionRequest?.sleep ?? + ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + let sessionRequestAttempts = 0; + let sessionRequestBudgetExhaustedBetweenAttempts = false; + let lastSessionError: TerminalSessionAttemptError | undefined; + const sessionResult = await collectWithRetry( + 'terminal session request', + async () => { + const remainingRequestBudgetMs = sessionRequestDeadline - Date.now(); + if (remainingRequestBudgetMs <= 0) { + sessionRequestBudgetExhaustedBetweenAttempts = true; + const exhausted = + lastSessionError === undefined + ? new TerminalSessionAttemptError( + 'overall terminal-session request deadline exhausted', + 'control_plane_timeout', + undefined, + false, + false + ) + : new TerminalSessionAttemptError( + lastSessionError.message, + lastSessionError.code, + lastSessionError.status, + false, + lastSessionError.completionUnknown + ); + lastSessionError = exhausted; + throw exhausted; + } + sessionRequestAttempts += 1; + const controller = new AbortController(); + let timedOut = false; + const attemptTimeoutMs = Math.min(sessionRequestTimeoutMs, remainingRequestBudgetMs); + const timeout = setTimeout(() => { + timedOut = true; + controller.abort(); + }, attemptTimeoutMs); + let ticketResponse: Response; + let ticketPayload: FleetSessionResponse; + try { + ticketResponse = await fetchFn(sessionEndpoint, { + method: 'POST', + headers: { Authorization: `Bearer ${workspaceKey}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ agent: options.agent, mode: options.mode }), + signal: controller.signal, + }); + const parsedPayload = (await ticketResponse.json()) as unknown; + ticketPayload = + parsedPayload && typeof parsedPayload === 'object' && !Array.isArray(parsedPayload) + ? (parsedPayload as FleetSessionResponse) + : {}; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + lastSessionError = new TerminalSessionAttemptError( + timedOut ? 'request exceeded its deadline' : detail, + timedOut ? 'control_plane_timeout' : 'control_plane_unavailable', + undefined, + false, + true + ); + throw lastSessionError; + } finally { + clearTimeout(timeout); + } + + const terminalUrl = ticketPayload.data?.terminal_url; + const sessionId = ticketPayload.data?.session_id; + const resumeToken = ticketPayload.data?.resume_token; + if (!ticketResponse.ok || !terminalUrl || !sessionId || !resumeToken) { + const code = ticketPayload.error?.code; + const message = + ticketPayload.error?.message ?? `terminal session request failed (HTTP ${ticketResponse.status})`; + lastSessionError = new TerminalSessionAttemptError( + message, + code, + ticketResponse.status, + isRetryableTerminalSessionFailure(code), + false + ); + throw lastSessionError; + } + return { terminalUrl, sessionId, resumeToken, expiresAt: ticketPayload.data?.expires_at }; + }, + { + retries: SESSION_REQUEST_RETRIES, + baseDelayMs: SESSION_REQUEST_RETRY_DELAY_MS, + sleep: async (delayMs) => { + const remainingRequestBudgetMs = sessionRequestDeadline - Date.now(); + if (remainingRequestBudgetMs <= 0) return; + await sessionRequestSleep(Math.min(delayMs, remainingRequestBudgetMs)); + }, + shouldRetry: (error) => error instanceof TerminalSessionAttemptError && error.retryable, + } + ); + if (!sessionResult.ok) { + const failure = lastSessionError; + const status = failure?.status === undefined ? '' : ` HTTP ${failure.status};`; + const code = failure?.code === undefined ? '' : ` code ${failure.code};`; + const retryNote = sessionRequestBudgetExhaustedBetweenAttempts + ? sessionRequestAttempts > 1 + ? `retried ${sessionRequestAttempts - 1} time${sessionRequestAttempts === 2 ? '' : 's'};` + + ' overall budget exhausted before the next attempt' + : 'not retried because the overall budget was exhausted before the next attempt' + : sessionRequestAttempts > 1 + ? `retried ${sessionRequestAttempts - 1} time${sessionRequestAttempts === 2 ? '' : 's'}` + + (failure?.completionUnknown + ? '; final POST not retried because it may have completed server-side' + : '') + : failure?.completionUnknown + ? 'not retried because the POST may have completed server-side' + : 'not retried because the failure was terminal'; + const upstream = failure?.message ? ` Upstream message ${diagnosticValue(failure.message)}.` : ''; + throw new FleetNodeAttachError( + `Error: ${failure ? terminalSessionFailureSummary(failure) : 'Terminal-session request failed'}.` + + `${upstream} Node ref ${diagnosticValue(options.node.trim())}, resolved node id unavailable (session creation did not complete);` + + ` endpoint ${diagnosticValue(diagnosticEndpoint(sessionEndpoint))};${status}${code}` + + ` timeout ${sessionRequestTimeoutMs}ms per attempt; overall budget ${sessionRequestTotalTimeoutMs}ms;` + + ` attempts ${sessionRequestAttempts} (${retryNote}).`, + failure?.code + ); + } + let { terminalUrl, sessionId, resumeToken, expiresAt } = sessionResult.value; + const resolvedNodeId = resolvedNodeIdFromTerminalUrl(terminalUrl); + const remoteEndpoint = () => diagnosticEndpoint(terminalUrl); + const reconnectInitialDelayMs = options.reconnectDelay?.initialMs ?? INITIAL_RECONNECT_DELAY_MS; + const reconnectMaxDelayMs = options.reconnectDelay?.maxMs ?? MAX_RECONNECT_DELAY_MS; + const terminalHandshakeTimeoutMs = + options.reconnectDelay?.handshakeTimeoutMs ?? TERMINAL_CONNECT_TIMEOUT_MS; + const terminalReadyTimeoutMs = options.reconnectDelay?.readyTimeoutMs ?? TERMINAL_CONNECT_TIMEOUT_MS; + // A readiness-gated local request follows activeReadiness across reconnect + // generations. Its own deadline therefore has to cover the same complete, + // finite recovery path: every backoff plus every handshake/readiness pair. + const terminalWaitTimeoutMs = + retryDelayBudgetMs(MAX_RECONNECT_ATTEMPTS, reconnectInitialDelayMs, reconnectMaxDelayMs) + + MAX_RECONNECT_ATTEMPTS * (terminalHandshakeTimeoutMs + terminalReadyTimeoutMs) + + sessionRequestTotalTimeoutMs; + + let connectionGeneration = 0; + const createReadiness = (): TerminalReadiness => { + let resolve!: () => void; + let reject!: (error: Error) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + // A failure can land before a snapshot request attaches its waiter. Keep + // the rejection observable while avoiding an unhandled-rejection warning. + void promise.catch(() => undefined); + return { generation: ++connectionGeneration, settled: false, promise, resolve, reject }; + }; + let activeReadiness = createReadiness(); + const resolveReadiness = (readiness: TerminalReadiness) => { + if (readiness.settled) return; + readiness.settled = true; + readiness.resolve(); + }; + const rejectReadiness = (readiness: TerminalReadiness, error: Error) => { + if (readiness.settled) return; + readiness.settled = true; + readiness.reject(error); + }; + const waitForCurrentReadiness = async (): Promise => { + for (;;) { + const readiness = activeReadiness; + await readiness.promise; + if (readiness === activeReadiness) return; + } + }; + /** + * Await the live readiness generation through both the WebSocket handshake + * and the post-open terminal.ready allowance. + * Every handler that must not run before `terminal.ready` — snapshot, + * delivery-mode PUT, resize — goes through this one helper so the + * timer/clearTimeout/settle logic cannot diverge between copies. + * + * Rejects with the underlying {@link FleetNodeAttachError} when the terminal + * failed (preserving its `code`), or a plain `Error` carrying + * `timeoutMessage` when the wait expired. + */ + const waitForTerminalReady = (timeoutMessage: string): Promise => + new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(timeoutMessage)), terminalWaitTimeoutMs); + void waitForCurrentReadiness().then( + () => { + clearTimeout(timer); + resolve(); + }, + (error: Error) => { + clearTimeout(timer); + reject(error); + } + ); + }); + /** + * The two screen representations are kept as separate, differently named + * fields on purpose. They are both strings and only one of them may reach a + * terminal; a single `screen` field spread into both consumers is what let + * the encoded form escape to stdout (relay#1829). + * + * `screenBase64` is the HTTP snapshot wire format (callers decode it). + * `screenAnsi` is the renderable form, and the only one that may be emitted + * as a `worker_stream` chunk. + */ + const snapshot: { + screenBase64: string; + screenAnsi: string; + rows: number; + cols: number; + offset: number; + } = { + screenBase64: '', + screenAnsi: '', + rows: 24, + cols: 80, + offset: 0, + }; + const eventSockets = new Set(); + const inputSockets = new Set(); + const outputHistory: Array<{ chunk: string; offset?: number }> = []; + let outputHistoryBytes = 0; + // Output since the latest terminal.ready snapshot, held for a raw socket + // client that has not yet become ready. Kept apart from `outputHistory`, + // which event listeners drain, so neither consumer can starve the other. + const rawPending: string[] = []; + let rawPendingBytes = 0; + const retainRawOutput = (chunk: string) => { + rawPending.push(chunk); + rawPendingBytes += Buffer.byteLength(chunk, 'utf8'); + while (rawPendingBytes > MAX_BUFFERED_BYTES && rawPending.length > 0) { + rawPendingBytes -= Buffer.byteLength(rawPending.shift()!, 'utf8'); + } + }; + let remote: WebSocket | undefined; + let rawSocket: Socket | undefined; + let rawConnected = false; + // Set once a connection-fatal session error was reported through `finished`. + let rawRefused = false; + let rawReady = false; + let finish!: (code: number) => void; + const finished = new Promise((resolve) => { + finish = resolve; + }); + let closePromise: Promise | undefined; + let stopped = false; + let terminalEnded = false; + let terminalEverReady = false; + let reconnecting = false; + let reconnectAttempts = 0; + let reconnectTimer: ReturnType | undefined; + const terminalReadinessTimers = new Set>(); + /** Locally-tracked delivery mode, kept in sync with each broker reply. */ + let loopbackDeliveryMode: 'manual_flush' | 'auto_inject' = + options.mode === 'drive' ? 'manual_flush' : 'auto_inject'; + type DeliveryModeResult = GapDiagnosticsResult & { + mode: string; + flushed: number; + dead_lettered?: number; + matched: boolean; + revision: string; + blocked_reason?: string; + }; + type FlushResult = GapDiagnosticsResult & { + flushed: number; + dead_lettered: number; + held: number; + blocked_reason: string | null; + }; + let pendingFlush: { + requestId: string; + resolve: (result: FlushResult) => void; + reject: (error: Error) => void; + timer: ReturnType; + } | null = null; + /** At most one in-flight delivery-mode PUT at a time. */ + let pendingDeliveryMode: { + requestId: string; + resolve: (result: DeliveryModeResult) => void; + reject: (err: Error) => void; + timer: ReturnType; + } | null = null; + const loopbackApiKey = randomBytes(32).toString('base64url'); + const loopbackAuthorized = (headers: IncomingMessage['headers']) => + headers.authorization === `Bearer ${loopbackApiKey}` || headers['x-api-key'] === loopbackApiKey; + + const server = createServer(async (request, response) => { + if (!loopbackAuthorized(request.headers)) { + json(response, 401, { + error: { code: 'unauthorized', message: 'loopback terminal token is required' }, + }); + return; + } + const path = new URL(request.url ?? '/', 'http://127.0.0.1').pathname; + if (request.method === 'GET' && path === `/api/spawned/${encodeURIComponent(options.agent)}/snapshot`) { + try { + await waitForTerminalReady('terminal snapshot timed out'); + } catch (error) { + const terminalError = error instanceof FleetNodeAttachError ? error : undefined; + json(response, terminalErrorStatus(terminalError?.code), { + error: { + code: terminalError?.code ?? 'snapshot_unavailable', + message: + terminalError?.message ?? (error instanceof Error ? error.message : 'snapshot unavailable'), + }, + }); + return; + } + // Explicit field mapping, not a spread: this endpoint's `screen` is + // contractually base64 (the caller decodes it), and only the encoded + // form may appear here. + json(response, 200, { + format: 'ansi', + screen: snapshot.screenBase64, + rows: snapshot.rows, + cols: snapshot.cols, + offset: snapshot.offset, + }); + return; + } + const name = encodeURIComponent(options.agent); + if (path === `/api/spawned/${name}/flush` && request.method === 'POST') { + // Forward the flush to the remote broker over the terminal websocket. + // Without this the `--node` form of `node agent message flush` reached + // only the LOCAL broker's worker registry and returned agent_not_found + // for a name that `node agent list` and `attach` both resolve. + try { + await waitForTerminalReady('terminal connection timed out'); + } catch (error) { + const terminalError = error instanceof FleetNodeAttachError ? error : undefined; + json(response, terminalErrorStatus(terminalError?.code), { + error: { + code: terminalError?.code ?? 'node_unreachable', + message: + terminalError?.message ?? (error instanceof Error ? error.message : 'terminal unavailable'), + }, + }); + return; + } + if (!remote || remote.readyState !== WebSocket.OPEN) { + json(response, 503, { + error: { code: 'node_unreachable', message: 'terminal transport is not connected' }, + }); + return; + } + if (pendingFlush) { + json(response, 503, { + error: { code: 'flush_conflict', message: 'a flush request is already in flight' }, + }); + return; + } + const requestId = randomBytes(8).toString('hex'); + const result = await new Promise((resolve) => { + const timer = setTimeout(() => { + pendingFlush = null; + resolve(new FleetNodeAttachError('flush request timed out', 'flush_timeout')); + }, DELIVERY_MODE_TIMEOUT_MS); + pendingFlush = { + requestId, + resolve: (r) => resolve(r), + reject: (e) => resolve(e), + timer, + }; + remote!.send( + JSON.stringify({ + type: 'terminal.flush_pending', + session_id: sessionId, + request_id: requestId, + }) + ); + }); + if (result instanceof Error) { + const errCode = + result instanceof FleetNodeAttachError ? (result.code ?? 'flush_failed') : 'flush_failed'; + json(response, terminalErrorStatus(errCode), { + error: { code: errCode, message: result.message }, + }); + return; + } + json(response, 200, result); + return; + } + if (path === `/api/spawned/${name}/delivery-mode`) { + if (request.method === 'GET') { + json(response, 200, { mode: loopbackDeliveryMode }); + return; + } + // PUT — forward the request to the remote broker via the terminal WS and + // await the broker's real reply. This is the path that was previously a + // static stub returning manual_flush, causing drive attach to fail with + // "broker remained in manual_flush mode". + const body = await readBody(request); + const requestedMode = + body.mode === 'auto_inject' ? 'auto_inject' : body.mode === 'manual_flush' ? 'manual_flush' : null; + if (requestedMode === null) { + json(response, 400, { + error: { code: 'invalid_mode', message: `unsupported delivery mode '${String(body.mode)}'` }, + }); + return; + } + // Drive attach changes delivery mode before it requests the initial + // snapshot. Gate the PUT on terminal.ready so a fast local caller does + // not lose a race with the remote websocket handshake and receive the + // misleading "terminal transport is not connected" failure. + try { + await waitForTerminalReady('terminal connection timed out'); + } catch (error) { + // Preserve the canonical status mapping here too: a readiness failure + // carrying `agent_not_found` has to reach the preflight as a 404 or + // the operator loses the cross-node placement hint that tells them + // which machine to run the attach on. + const terminalError = error instanceof FleetNodeAttachError ? error : undefined; + json(response, terminalErrorStatus(terminalError?.code), { + error: { + code: terminalError?.code ?? 'node_unreachable', + message: + terminalError?.message ?? (error instanceof Error ? error.message : 'terminal unavailable'), + }, + }); + return; + } + if (!remote || remote.readyState !== WebSocket.OPEN) { + json(response, 503, { + error: { code: 'node_unreachable', message: 'terminal transport is not connected' }, + }); + return; + } + if (pendingDeliveryMode) { + json(response, 503, { + error: { code: 'delivery_mode_conflict', message: 'a delivery mode request is already in flight' }, + }); + return; + } + const requestId = randomBytes(8).toString('hex'); + const result = await new Promise((resolve) => { + const timer = setTimeout(() => { + pendingDeliveryMode = null; + resolve(new FleetNodeAttachError('delivery mode request timed out', 'delivery_mode_timeout')); + }, DELIVERY_MODE_TIMEOUT_MS); + pendingDeliveryMode = { + requestId, + resolve: (r) => resolve(r), + reject: (e) => resolve(e), + timer, + }; + const frame: Record = { + type: 'terminal.set_delivery_mode', + session_id: sessionId, + mode: requestedMode, + request_id: requestId, + }; + if (typeof body.expected_mode === 'string') frame.expected_mode = body.expected_mode; + if (typeof body.expected_revision === 'string') frame.expected_revision = body.expected_revision; + remote!.send(JSON.stringify(frame)); + }); + if (result instanceof Error) { + const errCode = + result instanceof FleetNodeAttachError + ? (result.code ?? 'delivery_mode_failed') + : 'delivery_mode_failed'; + // Same canonical mapping as the readiness gate: 404 for + // agent_not_found so the attach preflight can produce the no-agent or + // cross-node placement error, 409 for unsupported_runtime, 503 for + // everything else. + json(response, terminalErrorStatus(errCode), { + error: { code: errCode, message: result.message }, + }); + return; + } + loopbackDeliveryMode = result.mode === 'manual_flush' ? 'manual_flush' : 'auto_inject'; + json(response, 200, { + mode: result.mode, + flushed: result.flushed, + ...(result.dead_lettered !== undefined ? { dead_lettered: result.dead_lettered } : {}), + matched: result.matched, + revision: result.revision, + ...(result.blocked_reason !== undefined ? { blocked_reason: result.blocked_reason } : {}), + ...gapDiagnosticsFromFrame(result), + }); + return; + } + if (request.method === 'GET' && path === `/api/spawned/${name}/pending`) { + json(response, 200, { pending: [] }); + return; + } + if (request.method === 'POST' && path === `/api/spawned/${name}/flush`) { + json(response, 200, { flushed: 0 }); + return; + } + if (request.method === 'GET' && path === '/api/spawned') { + json(response, 200, { agents: [{ name: options.agent, workerPid: 1 }] }); + return; + } + if (request.method === 'POST' && path === `/api/resize/${name}`) { + const body = await readBody(request); + if (body.release === true) { + json(response, 200, { name: options.agent, released: true }); + return; + } + const rows = typeof body.rows === 'number' ? body.rows : 0; + const cols = typeof body.cols === 'number' ? body.cols : 0; + if (!Number.isInteger(rows) || !Number.isInteger(cols) || rows < 1 || cols < 1) { + json(response, 400, { + error: { code: 'invalid_dimensions', message: 'rows and cols must be positive integers' }, + }); + return; + } + try { + await waitForTerminalReady('terminal resize timed out'); + } catch (error) { + json(response, 503, { + error: { + code: 'session_not_ready', + message: error instanceof Error ? error.message : 'terminal session is not ready', + }, + }); + return; + } + if (!remote || remote.readyState !== WebSocket.OPEN || remote.bufferedAmount > MAX_BUFFERED_BYTES) { + json(response, 503, { + error: { code: 'node_unreachable', message: 'terminal transport is unavailable' }, + }); + return; + } + remote.send(JSON.stringify({ type: 'terminal.resize', session_id: sessionId, rows, cols })); + json(response, 200, { name: options.agent, rows, cols, applied: true }); + return; + } + json(response, 404, { error: { code: 'not_found', message: 'loopback terminal endpoint not found' } }); + }); + const websocketServer = new WebSocketServer({ noServer: true }); + + const closeSocket = (socket: WebSocket, code: number, reason: string) => { + try { + socket.close(code, boundedWebSocketCloseReason(reason)); + } catch { + /* connection already gone */ + } + }; + const broadcast = (sockets: Set, payload: unknown): boolean => { + const encoded = JSON.stringify(payload); + let accepted = false; + for (const socket of sockets) { + if (socket.readyState !== WebSocket.OPEN) continue; + if (socket.bufferedAmount > MAX_BUFFERED_BYTES) { + closeSocket(socket, 1013, 'loopback client backpressure exceeded'); + sockets.delete(socket); + continue; + } + try { + socket.send(encoded); + accepted = true; + } catch { + sockets.delete(socket); + } + } + return accepted; + }; + const workerStreamEvent = (chunk: string, offset?: number) => ({ + kind: 'worker_stream', + name: options.agent, + stream: 'stdout', + chunk, + ...(offset === undefined ? {} : { offset }), + }); + const retainOutputDroppingOldest = (chunk: string, offset: number | undefined) => { + outputHistory.push({ chunk, ...(offset === undefined ? {} : { offset }) }); + outputHistoryBytes += Buffer.byteLength(chunk, 'utf8'); + while (outputHistoryBytes > MAX_BUFFERED_BYTES && outputHistory.length > 0) { + outputHistoryBytes -= Buffer.byteLength(outputHistory.shift()!.chunk, 'utf8'); + } + }; + const retainOutput = (chunk: string, offset: number | undefined): boolean => { + const bytes = Buffer.byteLength(chunk, 'utf8'); + if (outputHistoryBytes + bytes > MAX_BUFFERED_BYTES) return false; + outputHistory.push({ chunk, ...(offset === undefined ? {} : { offset }) }); + outputHistoryBytes += bytes; + return true; + }; + + /** + * Report a PTY-input failure to one input socket the way the broker does: + * with the `retryable` flag the SDK reads, and — for a connection-fatal + * code — by closing the socket so the client's `PtyInputStream` latches + * `closed` and its recovery can start buffering. See + * {@link inputErrorIsConnectionFatal}. + */ + const failInputSocket = (socket: WebSocket, code: string, message: string): void => { + const fatal = inputErrorIsConnectionFatal(code); + if (socket.readyState === WebSocket.OPEN) { + try { + socket.send(JSON.stringify({ type: 'pty_input_error', code, message, retryable: !fatal })); + } catch { + // The socket is already gone; the close below is still correct. + } + } + if (!fatal) return; + inputSockets.delete(socket); + closeSocket(socket, 1011, message); + }; + + /** Same, for every attached input socket (session-scoped failures). */ + const failAllInputSockets = (code: string, message: string): void => { + for (const socket of [...inputSockets]) failInputSocket(socket, code, message); + }; + + websocketServer.on('connection', (socket, request) => { + const path = new URL(request.url ?? '/', 'http://127.0.0.1').pathname; + if (path === '/ws') { + eventSockets.add(socket); + socket.on('close', () => eventSockets.delete(socket)); + let replayed = 0; + for (const event of outputHistory) { + if (socket.readyState !== WebSocket.OPEN || socket.bufferedAmount > MAX_BUFFERED_BYTES) break; + try { + socket.send(JSON.stringify(workerStreamEvent(event.chunk, event.offset))); + } catch { + break; + } + replayed += 1; + } + if (replayed > 0) { + const sentBytes = outputHistory + .slice(0, replayed) + .reduce((total, event) => total + Buffer.byteLength(event.chunk, 'utf8'), 0); + outputHistory.splice(0, replayed); + outputHistoryBytes -= sentBytes; + } + return; + } + if (path === `/api/input/${encodeURIComponent(options.agent)}/stream`) { + inputSockets.add(socket); + socket.on('close', () => inputSockets.delete(socket)); + socket.send(JSON.stringify({ type: 'pty_input_ready', name: options.agent })); + socket.on('message', (data) => { + if (!remote || remote.readyState !== WebSocket.OPEN || remote.bufferedAmount > MAX_BUFFERED_BYTES) { + // Only THIS socket failed to write; a sibling input stream must not + // be torn down for it. `node_unreachable` is connection-fatal, so + // this also closes the socket — see failInputSocket. + failInputSocket(socket, 'node_unreachable', 'terminal transport is unavailable'); + return; + } + const raw = rawDataToString(data); + remote.send( + JSON.stringify({ + type: 'terminal.input', + session_id: sessionId, + data_base64: Buffer.from(raw, 'utf8').toString('base64'), + }) + ); + }); + return; + } + closeSocket(socket, 1008, 'unknown loopback endpoint'); + }); + server.on('upgrade', (request, socket, head) => { + if (!loopbackAuthorized(request.headers)) { + socket.write('HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n'); + socket.destroy(); + return; + } + websocketServer.handleUpgrade(request, socket, head, (client) => + websocketServer.emit('connection', client, request) + ); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const address = server.address(); + if (!address || typeof address === 'string') + throw new FleetNodeAttachError( + 'Error: could not allocate loopback terminal listener.', + 'loopback_unavailable' + ); + + let socketDirectory: string | undefined; + let socketPath: string; + const rawServer = createSocketServer((socket) => { + if (rawConnected || rawRefused || stopped || terminalEnded) { + socket.destroy(); + return; + } + rawConnected = true; + rawSocket = socket; + socket.pause(); + socket.on('error', () => socket.destroy()); + socket.on('close', () => { + if (rawSocket === socket) rawSocket = undefined; + // The single raw client leaving is a detach: report it and refuse a + // replacement. The caller still owns close() for the remote session. + rawRefused = true; + finish(0); + }); + socket.on('data', (data: Buffer) => { + if (options.mode === 'view') return; + if (stopped || terminalEnded) return; + if (!remote || remote.readyState !== WebSocket.OPEN || !activeReadiness.settled) { + // A transient reconnect is in progress: hold the keystrokes on the + // socket and resume at the next terminal.ready rather than ending a + // recoverable session. + socket.pause(); + socket.unshift(data); + return; + } + if (remote.bufferedAmount + data.length > MAX_BUFFERED_BYTES) { + endTerminal(new FleetNodeAttachError('terminal input transport unavailable', 'node_unreachable')); + return; + } + remote.send( + JSON.stringify({ + type: 'terminal.input', + session_id: sessionId, + data_base64: data.toString('base64'), + }) + ); + }); + void waitForTerminalReady('terminal socket readiness timed out').then( + () => { + if (socket.destroyed || stopped || terminalEnded) return; + if (snapshot.screenAnsi) socket.write(snapshot.screenAnsi); + for (const chunk of rawPending) socket.write(chunk); + rawPending.length = 0; + rawPendingBytes = 0; + rawReady = true; + socket.resume(); + }, + () => socket.destroy() + ); + }); + try { + socketPath = + process.platform === 'win32' + ? `\\\\.\\pipe\\agent-relay-attach-${randomBytes(16).toString('hex')}` + : join((socketDirectory = await mkdtemp(join(tmpdir(), 'relay-attach-'))), 'stdio.sock'); + rawServer.listen(socketPath); + await once(rawServer, 'listening'); + if (socketDirectory) await chmod(socketPath, 0o600); + } catch (error) { + rawServer.close(); + websocketServer.close(); + await new Promise((resolve) => server.close(() => resolve())); + if (socketDirectory) await rm(socketDirectory, { recursive: true, force: true }); + throw error; + } + + const resumeUrlForCurrentSession = () => { + const resumeUrl = new URL(terminalUrl); + resumeUrl.searchParams.delete('ticket'); + resumeUrl.searchParams.set('session_id', sessionId); + resumeUrl.searchParams.set('resume', resumeToken); + return resumeUrl.toString(); + }; + const terminalSessionExpired = () => { + if (!expiresAt) return false; + const expiresAtMs = Date.parse(expiresAt); + return Number.isFinite(expiresAtMs) && expiresAtMs <= Date.now(); + }; + // A disconnected terminal can need one replacement session when its resume + // credential has expired. Keep this scoped to the whole reconnect incident: + // a replacement that itself drops still follows the normal bounded resume + // budget instead of repeatedly allocating sessions. + let replacementAllocatedForReconnect = false; + const allocateReplacementTerminalSession = async (): Promise<{ + terminalUrl: string; + sessionId: string; + resumeToken: string; + expiresAt: string | undefined; + }> => { + const replacementDeadline = Date.now() + sessionRequestTotalTimeoutMs; + let lastReplacementError: TerminalSessionAttemptError | undefined; + const result = await collectWithRetry( + 'replacement terminal session request', + async () => { + const remainingMs = replacementDeadline - Date.now(); + if (remainingMs <= 0) { + throw new TerminalSessionAttemptError( + lastReplacementError?.message ?? + 'overall replacement terminal-session request deadline exhausted', + lastReplacementError?.code ?? 'control_plane_timeout', + lastReplacementError?.status, + false, + lastReplacementError?.completionUnknown ?? false + ); + } + const controller = new AbortController(); + let timedOut = false; + const timeout = setTimeout( + () => { + timedOut = true; + controller.abort(); + }, + Math.min(sessionRequestTimeoutMs, remainingMs) + ); + try { + const response = await fetchFn(sessionEndpoint, { + method: 'POST', + headers: { Authorization: `Bearer ${workspaceKey}`, 'Content-Type': 'application/json' }, + // Keep the replacement bound to precisely the session this proxy was + // already serving; never infer a different agent or delivery mode. + body: JSON.stringify({ agent: options.agent, mode: options.mode }), + signal: controller.signal, + }); + const parsed = (await response.json()) as unknown; + const payload = + parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as FleetSessionResponse) + : {}; + const replacementUrl = payload.data?.terminal_url; + const replacementSessionId = payload.data?.session_id; + const replacementResumeToken = payload.data?.resume_token; + if (!response.ok || !replacementUrl || !replacementSessionId || !replacementResumeToken) { + const error = new TerminalSessionAttemptError( + payload.error?.message ?? + 'terminal session could not be replaced after its resume credential expired', + payload.error?.code, + response.status, + isRetryableTerminalSessionFailure(payload.error?.code), + false + ); + lastReplacementError = error; + throw error; + } + return { + terminalUrl: replacementUrl, + sessionId: replacementSessionId, + resumeToken: replacementResumeToken, + expiresAt: payload.data?.expires_at, + }; + } catch (error) { + if (error instanceof TerminalSessionAttemptError) throw error; + const replacementError = new TerminalSessionAttemptError( + timedOut + ? 'replacement request exceeded its deadline' + : 'replacement terminal session request failed', + timedOut ? 'control_plane_timeout' : 'control_plane_unavailable', + undefined, + false, + true + ); + lastReplacementError = replacementError; + throw replacementError; + } finally { + clearTimeout(timeout); + } + }, + { + retries: SESSION_REQUEST_RETRIES, + baseDelayMs: SESSION_REQUEST_RETRY_DELAY_MS, + sleep: async (delayMs) => { + const remainingMs = replacementDeadline - Date.now(); + if (remainingMs > 0) await sessionRequestSleep(Math.min(delayMs, remainingMs)); + }, + shouldRetry: (error) => error instanceof TerminalSessionAttemptError && error.retryable, + } + ); + if (result.ok) return result.value; + if (lastReplacementError) throw lastReplacementError; + throw new FleetNodeAttachError( + 'terminal session could not be replaced after its resume credential expired', + 'terminal_session_unavailable' + ); + }; + /** Reject and clear any in-flight delivery-mode PUT, if one is pending. */ + const rejectPendingDeliveryMode = (error: FleetNodeAttachError) => { + if (pendingFlush) { + const flush = pendingFlush; + pendingFlush = null; + clearTimeout(flush.timer); + // A flush never changes delivery mode, so it must not inherit the + // delivery-mode disconnect wording. An operator debugging a failed flush + // would otherwise be told the "delivery-mode change" was interrupted — + // pointing at an operation their command never performed. + const flushError = + error.code === 'delivery_mode_disconnected' + ? new FleetNodeAttachError( + 'terminal transport disconnected while the flush was in flight', + 'flush_disconnected' + ) + : error; + flush.reject(flushError); + } + if (!pendingDeliveryMode) return; + const pending = pendingDeliveryMode; + pendingDeliveryMode = null; + clearTimeout(pending.timer); + pending.reject(error); + }; + const endTerminal = (error: FleetNodeAttachError, eventCloseReason = error.message) => { + if (terminalEnded) return; + terminalEnded = true; + finish(error.code === 'terminal_closed' ? 0 : 1); + rawSocket?.end(); + if (reconnectTimer) { + clearTimeout(reconnectTimer); + reconnectTimer = undefined; + } + for (const timer of terminalReadinessTimers) clearTimeout(timer); + terminalReadinessTimers.clear(); + rejectPendingDeliveryMode(error); + const activeRemote = remote; + remote = undefined; + rejectReadiness(activeReadiness, error); + failAllInputSockets(error.code ?? 'terminal_error', error.message); + for (const socket of eventSockets) closeSocket(socket, 1011, eventCloseReason); + if (activeRemote && activeRemote.readyState !== WebSocket.CLOSED) { + try { + activeRemote.terminate(); + } catch { + // The socket may have closed between the state check and terminate. + } + } + }; + const failRemote = (message: string, eventCloseReason?: string) => { + endTerminal(new FleetNodeAttachError(message, 'node_unreachable'), eventCloseReason); + }; + const connect = (url: string, readiness: TerminalReadiness) => { + if (stopped || terminalEnded) return; + const socket = new WebSocket(asWsUrl(url), { handshakeTimeout: terminalHandshakeTimeoutMs }); + remote = socket; + let readinessTimer: ReturnType | undefined; + let readinessExpired = false; + let resumeRejectedAsExpired = false; + const clearReadinessTimer = () => { + if (!readinessTimer) return; + clearTimeout(readinessTimer); + terminalReadinessTimers.delete(readinessTimer); + readinessTimer = undefined; + }; + socket.on('open', () => { + // `handshakeTimeout` independently bounds the HTTP upgrade. Start the + // terminal.ready allowance only after that upgrade succeeds so a slow + // but valid handshake cannot consume the readiness window. + readinessTimer = setTimeout(() => { + const expiredTimer = readinessTimer; + readinessTimer = undefined; + if (expiredTimer) terminalReadinessTimers.delete(expiredTimer); + if (remote !== socket || stopped || terminalEnded || readiness.settled) return; + // Mark this generation stale before terminating. The ws receiver may + // still deliver data already buffered on the socket while close is + // propagating; none of it may restore readiness or reset retry state. + readinessExpired = true; + if (!terminalEverReady) { + failRemote( + `terminal transport connected but did not become ready (node ref ${diagnosticValue(options.node.trim())},` + + ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint())},` + + ` readiness timeout ${terminalReadyTimeoutMs}ms, attempts 1; not retried because no terminal session became ready)` + ); + return; + } + // A successful WebSocket upgrade is not sufficient: Relaycast may + // accept a resume lane that never produces terminal.ready. Terminating + // it drives the same bounded close/retry path as a transport failure. + options.reconnectDelay?.beforeReadyTimeoutTerminate?.(socket); + socket.terminate(); + }, terminalReadyTimeoutMs); + terminalReadinessTimers.add(readinessTimer); + }); + socket.on('message', (data) => { + // A late frame from a transport superseded during reconnect must never + // overwrite the fresh snapshot or end the replacement session. + if (remote !== socket || stopped || terminalEnded || readinessExpired) return; + const frame = parseFrame(data); + if (!frame || frame.session_id !== sessionId) return; + if (frame.type === 'terminal.ready') { + clearReadinessTimer(); + snapshot.screenBase64 = typeof frame.screen === 'string' ? frame.screen : ''; + // A payload that will not decode is kept out of `screenAnsi` entirely + // so no later consumer can render it; the HTTP snapshot still serves + // the bytes verbatim and lets its own decoder report the problem. + snapshot.screenAnsi = decodeAnsiScreenPayload(snapshot.screenBase64) ?? ''; + // The fresh snapshot already contains everything retained so far. + rawPending.length = 0; + rawPendingBytes = 0; + snapshot.rows = typeof frame.rows === 'number' ? frame.rows : 24; + snapshot.cols = typeof frame.cols === 'number' ? frame.cols : 80; + snapshot.offset = typeof frame.offset === 'number' ? frame.offset : 0; + // Seed loopbackDeliveryMode from the broker's actual state so that + // detach restores the correct mode even when the worker started in a + // different mode than our local inference at line 214. + if (frame.delivery_mode === 'manual_flush' || frame.delivery_mode === 'auto_inject') { + loopbackDeliveryMode = frame.delivery_mode; + } + terminalEverReady = true; + reconnectAttempts = 0; + replacementAllocatedForReconnect = false; + if (readiness === activeReadiness) { + resolveReadiness(readiness); + // A reconnect gets a fresh ANSI grid but existing local `/ws` + // consumers have already performed their initial HTTP snapshot. + // Re-emit this screen without an offset so they repaint instead of + // retaining a stale pre-reconnect terminal image. It must be the + // DECODED grid: a `worker_stream` chunk is raw PTY bytes that the + // attach client writes straight to the terminal, so the base64 form + // renders as a wall of text instead of a repaint (relay#1829). + // Release any input held while the transport was reconnecting. + if (rawReady && rawSocket && !rawSocket.destroyed) rawSocket.resume(); + if (readiness.generation > 1 && snapshot.screenAnsi) { + broadcast(eventSockets, workerStreamEvent(snapshot.screenAnsi)); + rawSocket?.write(snapshot.screenAnsi); + } + } + } else if (frame.type === 'terminal.output' && typeof frame.chunk === 'string') { + const offset = typeof frame.offset === 'number' ? frame.offset : undefined; + if (rawReady && rawSocket && !rawSocket.destroyed) { + if (rawSocket.writableLength + Buffer.byteLength(frame.chunk) > MAX_BUFFERED_BYTES) { + endTerminal( + new FleetNodeAttachError( + 'terminal output exceeded the bounded socket buffer', + 'output_backpressure' + ) + ); + return; + } + rawSocket.write(frame.chunk); + } else { + retainRawOutput(frame.chunk); + } + if (!broadcast(eventSockets, workerStreamEvent(frame.chunk, offset))) { + if (rawReady) { + // A raw client is draining the stream, so a missing /ws listener is + // not backpressure: keep the newest history for a later listener. + retainOutputDroppingOldest(frame.chunk, offset); + } else if (!retainOutput(frame.chunk, offset)) { + endTerminal( + new FleetNodeAttachError( + 'terminal output exceeded the bounded loopback buffer', + 'output_backpressure' + ) + ); + } + } + } else if (frame.type === 'terminal.input_ack') { + broadcast(inputSockets, { + type: 'pty_input_ack', + name: options.agent, + bytes_written: typeof frame.bytes_written === 'number' ? frame.bytes_written : 0, + }); + } else if (frame.type === 'terminal.delivery_mode') { + const frameRid = typeof frame.request_id === 'string' ? frame.request_id : undefined; + if (pendingDeliveryMode && (frameRid === undefined || frameRid === pendingDeliveryMode.requestId)) { + const pending = pendingDeliveryMode; + pendingDeliveryMode = null; + clearTimeout(pending.timer); + pending.resolve({ + mode: typeof frame.mode === 'string' ? frame.mode : 'auto_inject', + flushed: typeof frame.flushed === 'number' ? frame.flushed : 0, + ...(typeof frame.dead_lettered === 'number' ? { dead_lettered: frame.dead_lettered } : {}), + matched: typeof frame.matched === 'boolean' ? frame.matched : true, + revision: typeof frame.revision === 'string' ? frame.revision : '1', + ...(typeof frame.blocked_reason === 'string' ? { blocked_reason: frame.blocked_reason } : {}), + ...gapDiagnosticsFromFrame(frame), + }); + } + } else if (frame.type === 'terminal.flush_pending') { + const frameRid = typeof frame.request_id === 'string' ? frame.request_id : undefined; + // Exact match only. The proxy always sends a request_id, and unlike + // delivery-mode there is no older-broker reply shape to stay + // compatible with, so a reply without one is not ours — accepting it + // would resolve the caller's flush with an unrelated result. + if (pendingFlush && frameRid === pendingFlush.requestId) { + const pending = pendingFlush; + pendingFlush = null; + clearTimeout(pending.timer); + pending.resolve({ + flushed: typeof frame.flushed === 'number' ? frame.flushed : 0, + dead_lettered: typeof frame.dead_lettered === 'number' ? frame.dead_lettered : 0, + held: typeof frame.held === 'number' ? frame.held : 0, + blocked_reason: typeof frame.blocked_reason === 'string' ? frame.blocked_reason : null, + ...gapDiagnosticsFromFrame(frame), + }); + } + } else if (frame.type === 'terminal.error') { + const message = typeof frame.message === 'string' ? frame.message : 'remote terminal failed'; + const code = typeof frame.code === 'string' ? frame.code : 'terminal_error'; + const frameRid = typeof frame.request_id === 'string' ? frame.request_id : undefined; + // Route the error to the pending delivery-mode request only when the + // request_id matches (or the broker sent no request_id at all — older + // broker compat). An unrelated session-level error must not cancel a + // live delivery-mode PUT and vice-versa. + if (pendingFlush && frameRid !== undefined && frameRid === pendingFlush.requestId) { + const pending = pendingFlush; + pendingFlush = null; + clearTimeout(pending.timer); + pending.reject(new FleetNodeAttachError(message, code)); + } else if ( + pendingDeliveryMode && + (frameRid === undefined || frameRid === pendingDeliveryMode.requestId) + ) { + const pending = pendingDeliveryMode; + pendingDeliveryMode = null; + clearTimeout(pending.timer); + pending.reject(new FleetNodeAttachError(message, code)); + } else if (!frameRid && readiness === activeReadiness && !readiness.settled) { + endTerminal(new FleetNodeAttachError(message, code)); + } else if (!frameRid) { + failAllInputSockets(code, message); + if (inputErrorIsConnectionFatal(code)) { + // The raw stdio client has no error frame to read: close it and + // report the failure so callers stop writing to a dead session. + finish(1); + rawRefused = true; + rawSocket?.end(); + } + } + } else if (frame.type === 'terminal.closed') { + endTerminal(new FleetNodeAttachError('remote terminal session closed', 'terminal_closed')); + } + }); + socket.on('error', (error) => { + readinessExpired = true; + clearReadinessTimer(); + // `ws` exposes a refused HTTP upgrade as this error before `close`. + // Only an expired resume credential may mint a replacement terminal; + // notably a transient or 5xx failure stays on the resume path. + if ( + terminalEverReady && + error instanceof Error && + /Unexpected server response: (401|410)\b/.test(error.message) + ) { + resumeRejectedAsExpired = true; + } + // Initial connection failure has no terminal state worth preserving. + // Fail promptly with the canonical unavailable-node error instead of + // letting the HTTP snapshot timeout mask it. Once Ready has been seen, + // the close handler retains the bounded resume/backoff behaviour. + if (remote === socket && readiness === activeReadiness && !readiness.settled && !terminalEverReady) { + failRemote( + `terminal transport could not connect to the fleet node (node ref ${diagnosticValue(options.node.trim())},` + + ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint())},` + + ` handshake budget ${terminalHandshakeTimeoutMs}ms, attempts 1; not retried because no terminal session became ready)` + ); + } + }); + socket.on('close', () => { + readinessExpired = true; + clearReadinessTimer(); + if (remote !== socket || stopped || terminalEnded || reconnecting) return; + // Relaycast drops the old lane's terminal session state on disconnect, + // so a set_delivery_mode frame already sent on this dying socket is + // lost and will never get a reply on the replacement socket — even + // once reconnect succeeds. Fail the pending PUT fast with a retryable + // error instead of leaving it to hang out the full timeout. + rejectPendingDeliveryMode( + new FleetNodeAttachError( + 'terminal transport disconnected while the delivery-mode change was in flight', + 'delivery_mode_disconnected' + ) + ); + // Any waiter that observed the prior connection must retry against the + // fresh generation instead of receiving its stale resolved snapshot. + resolveReadiness(readiness); + const nextReadiness = createReadiness(); + activeReadiness = nextReadiness; + if (reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) { + const backoffBudgetMs = retryDelayBudgetMs( + MAX_RECONNECT_ATTEMPTS, + reconnectInitialDelayMs, + reconnectMaxDelayMs + ); + const message = + `terminal transport could not reconnect to the fleet node (node ref ${diagnosticValue(options.node.trim())},` + + ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint())},` + + ` handshake timeout ${terminalHandshakeTimeoutMs}ms, readiness timeout ${terminalReadyTimeoutMs}ms,` + + ` attempts ${reconnectAttempts},` + + ` backoff budget ${backoffBudgetMs}ms)`; + failRemote( + message, + `terminal reconnect failed; attempts=${reconnectAttempts}; budget=${backoffBudgetMs}ms;` + + ` endpoint=${diagnosticValue(diagnosticEndpointOrigin(terminalUrl))};` + + ` node=${diagnosticValue(resolvedNodeId ?? options.node.trim())}` + ); + return; + } + reconnecting = true; + const delay = Math.min(reconnectInitialDelayMs * 2 ** reconnectAttempts, reconnectMaxDelayMs); + reconnectAttempts += 1; + reconnectTimer = setTimeout(() => { + reconnectTimer = undefined; + reconnecting = false; + const needsReplacement = + !replacementAllocatedForReconnect && (resumeRejectedAsExpired || terminalSessionExpired()); + if (!needsReplacement) { + connect(resumeUrlForCurrentSession(), nextReadiness); + return; + } + replacementAllocatedForReconnect = true; + void allocateReplacementTerminalSession().then( + (replacement) => { + if (stopped || terminalEnded || activeReadiness !== nextReadiness) return; + // This has no await between assignments, so every loopback + // handler observes one coherent replacement session. + terminalUrl = replacement.terminalUrl; + sessionId = replacement.sessionId; + resumeToken = replacement.resumeToken; + expiresAt = replacement.expiresAt; + connect(terminalUrl, nextReadiness); + }, + (error) => { + if (stopped || terminalEnded || activeReadiness !== nextReadiness) return; + const replacementError = + error instanceof FleetNodeAttachError + ? error + : new FleetNodeAttachError( + 'terminal session could not be replaced after its resume credential expired', + 'terminal_session_unavailable' + ); + endTerminal( + new FleetNodeAttachError( + `terminal transport could not replace an expired terminal session (node ref ${diagnosticValue(options.node.trim())},` + + ` resolved node id ${diagnosticValue(resolvedNodeId ?? 'unavailable')}, endpoint ${diagnosticValue(remoteEndpoint())})`, + replacementError.code + ) + ); + } + ); + }, delay); + }); + }; + connect(terminalUrl, activeReadiness); + + return { + socketPath, + finished, + brokerUrl: `http://127.0.0.1:${address.port}`, + apiKey: loopbackApiKey, + requestTimeoutMs: terminalWaitTimeoutMs + DELIVERY_MODE_TIMEOUT_MS + LOOPBACK_REQUEST_TIMEOUT_MARGIN_MS, + close() { + return (closePromise ??= (async () => { + if (stopped) return; + stopped = true; + finish(0); + rawSocket?.destroy(); + terminalEnded = true; + if (reconnectTimer) { + clearTimeout(reconnectTimer); + reconnectTimer = undefined; + } + for (const timer of terminalReadinessTimers) clearTimeout(timer); + terminalReadinessTimers.clear(); + rejectReadiness(activeReadiness, new FleetNodeAttachError('terminal attach closed', 'closed')); + rejectPendingDeliveryMode(new FleetNodeAttachError('terminal attach closed', 'closed')); + const activeRemote = remote; + remote = undefined; + if (activeRemote && activeRemote.readyState === WebSocket.OPEN) { + try { + activeRemote.send(JSON.stringify({ type: 'terminal.close', session_id: sessionId })); + } catch { + // Best effort; terminate below still prevents a late reconnect. + } + } + if (activeRemote && activeRemote.readyState !== WebSocket.CLOSED) { + try { + activeRemote.terminate(); + } catch { + // Socket is already gone. + } + } + for (const socket of [...eventSockets, ...inputSockets]) + closeSocket(socket, 1000, 'terminal attach closed'); + websocketServer.close(); + await Promise.all([ + new Promise((resolve) => server.close(() => resolve())), + new Promise((resolve) => rawServer.close(() => resolve())), + ]); + if (socketDirectory) await rm(socketDirectory, { recursive: true, force: true }); + })()); + }, + }; +} + +export type { AttachMode } from './attach-mode.js'; +export { collectWithRetry } from './collect-with-retry.js'; diff --git a/packages/cloud/src/collect-with-retry.ts b/packages/cloud/src/collect-with-retry.ts new file mode 100644 index 0000000000..fb726a2315 --- /dev/null +++ b/packages/cloud/src/collect-with-retry.ts @@ -0,0 +1,42 @@ +export interface CollectWithRetryOptions { + retries?: number; + baseDelayMs?: number; + sleep?: (ms: number) => Promise; + /** Return false for terminal failures that must be surfaced immediately. */ + shouldRetry?: (error: unknown, attempt: number) => boolean; +} + +/** + * Run a bounded operation with the fleet CLI's small retry delay. + * + * The result records whether another attempt actually ran. Callers that need + * richer failure metadata can retain the thrown value in their operation; + * this helper deliberately keeps the existing fleet-list rendering contract. + */ +export async function collectWithRetry( + label: string, + fn: () => Promise, + options?: CollectWithRetryOptions +): Promise<{ ok: true; value: T; retried: boolean } | { ok: false; error: string; retried: boolean }> { + const retries = options?.retries ?? 1; + const baseDelay = options?.baseDelayMs ?? 500; + const sleep = options?.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); + let lastError: unknown; + let attempts = 0; + for (let attempt = 0; attempt <= retries; attempt += 1) { + attempts += 1; + try { + const value = await fn(); + return { ok: true, value, retried: attempt > 0 }; + } catch (error) { + lastError = error; + if (attempt >= retries || options?.shouldRetry?.(error, attempt + 1) === false) break; + // Keep the established fleet-list delay shape while making later + // attempts wait slightly longer, without introducing test randomness. + const delay = baseDelay + Math.floor(baseDelay * (attempt / (retries + 1))); + await sleep(delay); + } + } + const message = lastError instanceof Error ? lastError.message : String(lastError); + return { ok: false, error: `${label}: ${message}`, retried: attempts > 1 }; +} diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index c32e710ce6..3fc4b535ab 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -2766,6 +2766,7 @@ describe('Cloud fleet sandbox client', () => { requiredCapability: 'spawn:claude', forceProvision: true, relayfilePaths: ['/live-review/run-123/**'], + readonlyPaths: ['/live-review/run-123/reference/**'], }); const ensureCall = mocks.authorizedApiFetch.mock.calls[1]; @@ -2774,9 +2775,36 @@ describe('Cloud fleet sandbox client', () => { requiredCapability: 'spawn:claude', forceProvision: true, relayfilePaths: ['/live-review/run-123/**'], + readonlyPaths: ['/live-review/run-123/reference/**'], }); }); + it.each([[], ['/'], ['relative/**'], ['/foo'], ['/foo/../bar/**'], ['/foo/*/**']])( + 'rejects invalid readonlyPaths before authentication: %j', + async (...paths) => { + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:claude', + readonlyPaths: paths as string[], + }) + ).rejects.toThrow('readonlyPaths'); + expect(mocks.ensureCloudSession).not.toHaveBeenCalled(); + } + ); + + it('rejects readonly paths without a mount', async () => { + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:claude', + mountRelayfile: false, + readonlyPaths: ['/foo/**'], + }) + ).rejects.toThrow('requires mounting'); + expect(mocks.ensureCloudSession).not.toHaveBeenCalled(); + }); + it('rejects incomplete revision maps before Cloud authentication', async () => { await expect( ensureCloudFleetSandbox({ diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index b71bb4810b..1d464cf545 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -178,6 +178,8 @@ export type EnsureCloudFleetSandboxInput = { * must use the explicit `/path/**` subtree form accepted by Cloud. */ relayfilePaths?: readonly string[]; + /** Read-only Relayfile subtrees in explicit `/path/**` form. Cloud owns chmod enforcement. */ + readonlyPaths?: readonly string[]; forceProvision?: boolean; /** Constrain provisioning to a provider that Cloud has enabled for routing. */ providerId?: CloudFleetSandboxProviderId; @@ -1197,6 +1199,24 @@ export async function ensureCloudFleetSandbox( if (input.relayfilePaths !== undefined && input.relayfilePaths.length === 0) { throw new Error('At least one Relayfile subtree path is required when relayfilePaths is provided.'); } + if (input.readonlyPaths !== undefined) { + if ( + input.readonlyPaths.length === 0 || + input.readonlyPaths.some( + (path) => + !/^\/(?:[^/*?\\\x00-\x1f]+\/)+\*\*$/.test(path) || + path + .slice(1, -3) + .split('/') + .some((segment) => segment === '.' || segment === '..') + ) + ) { + throw new Error( + 'readonlyPaths must contain at least one explicit /path/** subtree without traversal or wildcards.' + ); + } + if (input.mountRelayfile === false) throw new Error('readonlyPaths requires mounting Relayfile.'); + } validateRequestedRepos(input.repos); const repoRevisions = validateRepoRevisions(input.repos, input.repoRevisions); const asyncPreparation = input.preparationMode === 'async-v1'; @@ -1235,6 +1255,7 @@ export async function ensureCloudFleetSandbox( ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), ...(input.relayfilePaths === undefined ? {} : { relayfilePaths: [...input.relayfilePaths] }), + ...(input.readonlyPaths === undefined ? {} : { readonlyPaths: [...input.readonlyPaths] }), ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), ...(input.providerId !== undefined ? { providerId: input.providerId } : {}), ...(input.workloadProfile !== undefined ? { workloadProfile: input.workloadProfile } : {}), diff --git a/packages/cloud/src/fleet.ts b/packages/cloud/src/fleet.ts index 36c67c5709..c7025a11b7 100644 --- a/packages/cloud/src/fleet.ts +++ b/packages/cloud/src/fleet.ts @@ -382,3 +382,20 @@ export function resolveActiveFleetNodeEnrollment( `Multiple fleet node enrollments match; pass baseUrl and workspaceId to disambiguate. Candidates: ${candidates}.` ); } + +/** Fleet sandbox provisioning primitives. */ +export { + ensureCloudFleetSandbox, + deleteCloudFleetSandbox, + CloudFleetSandboxProvisionError, + CANONICAL_RELAYCAST_ORIGIN, + DEV_CLOUD_API_URL, + DEV_RELAYCAST_ORIGIN, + type CloudFleetRelaycastTarget, + type CloudFleetSandboxProviderId, + type DeleteCloudFleetSandboxInput, + type EnsureCloudFleetSandboxInput, + type EnsureCloudFleetSandboxResult, + type CloudFleetSandboxRequestOptions, +} from './fleet-sandbox.js'; +export { resolveWorkspaceByKey } from './workspaces.js'; diff --git a/packages/cloud/src/public-entries.test.ts b/packages/cloud/src/public-entries.test.ts new file mode 100644 index 0000000000..17063bf707 --- /dev/null +++ b/packages/cloud/src/public-entries.test.ts @@ -0,0 +1,28 @@ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { execFileSync } from 'node:child_process'; +import { expect, it } from 'vitest'; + +it('resolves the published Cloud and SDK fleet/attach ESM entries after build', () => { + for (const name of ['cloud', 'sdk']) { + const manifest = JSON.parse(readFileSync(new URL(`../../${name}/package.json`, import.meta.url), 'utf8')); + for (const subpath of ['fleet', 'attach']) { + const entry = manifest.exports[`./${subpath}`]; + expect(entry.types).toBe(`./dist/${subpath}.d.ts`); + expect(entry.import).toBe(`./dist/${subpath}.js`); + expect(readFileSync(new URL(`../../${name}/${entry.types}`, import.meta.url), 'utf8')).toBeTruthy(); + const exportedFunction = subpath === 'attach' ? 'startFleetNodeAttachProxy' : 'ensureCloudFleetSandbox'; + expect( + execFileSync( + process.execPath, + [ + '--input-type=module', + '-e', + `import { ${exportedFunction} } from '@agent-relay/${name}/${subpath}'; console.log(typeof ${exportedFunction});`, + ], + { cwd: fileURLToPath(new URL('../../..', import.meta.url)), encoding: 'utf8' } + ).trim() + ).toBe('function'); + } + } +}); diff --git a/packages/cloud/src/workspace-transport.ts b/packages/cloud/src/workspace-transport.ts new file mode 100644 index 0000000000..8f4c06a6a0 --- /dev/null +++ b/packages/cloud/src/workspace-transport.ts @@ -0,0 +1,211 @@ +import path from 'node:path'; +import { + AGENT37_RELAYCAST_ORIGIN, + CANONICAL_RELAYCAST_ORIGIN, + DEV_CLOUD_API_URL, + DEV_RELAYCAST_ORIGIN, +} from './fleet-sandbox.js'; +import { + resolveWorkspaceSelection as resolveCloudWorkspaceSelection, + writeProjectWorkspaceTargetIfSelectionCurrent, + type WorkspaceSelection, + type WorkspaceKeySource, +} from './workspace-key.js'; + +/** Options shared by the SDK-backed (Relaycast) CLI command groups. */ +export interface WorkspaceTransportOptions { + workspaceKey?: string; + token?: string; + /** Non-authoritative session URL used only when no persisted route or explicit URL exists. */ + fallbackBaseUrl?: string; + baseUrl?: string; + env?: NodeJS.ProcessEnv; + /** Explicit project root for nested invocations such as packages/web. */ + projectRoot?: string; + /** Use the canonical gateway instead of a persisted server-selected route. */ + ignorePersistedRelaycastTarget?: boolean; +} + +function env(options: WorkspaceTransportOptions): NodeJS.ProcessEnv { + return options.env ?? process.env; +} + +function trimOrUndefined(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +export type WorkspaceTransport = { + workspaceKey: string; + baseUrl?: string; + source: WorkspaceKeySource; +}; + +/** Resolve the selected key and any previously persisted Relay workspace identity. */ +export function resolveWorkspaceSelection( + options: WorkspaceTransportOptions = {} +): WorkspaceSelection | undefined { + const explicitProject = trimOrUndefined(env(options).AGENT_RELAY_PROJECT); + const projectRoot = explicitProject ? path.resolve(explicitProject) : options.projectRoot; + return resolveCloudWorkspaceSelection({ + workspaceKey: options.workspaceKey, + env: env(options), + ...(projectRoot ? { projectRoot } : {}), + }); +} + +export function resolveBaseUrl(options: WorkspaceTransportOptions = {}): string | undefined { + const selection = selectionForTransport(options); + return resolveBaseUrlForSelection(selection, options); +} + +function selectionForTransport(options: WorkspaceTransportOptions): WorkspaceSelection | undefined { + const selection = resolveWorkspaceSelection(options); + if (!selection || !options.ignorePersistedRelaycastTarget) return selection; + const { + relaycastRoute: _relaycastRoute, + relaycastBaseUrl: _relaycastBaseUrl, + relaycastApiKey: _relaycastApiKey, + relaycastApiKeyRef: _relaycastApiKeyRef, + ...canonicalSelection + } = selection; + return canonicalSelection; +} + +/** Resolve an origin while keeping a persisted route paired with its credential. */ +function resolveBaseUrlForSelection( + selection: WorkspaceSelection | undefined, + options: WorkspaceTransportOptions +): string | undefined { + const persisted = validatePersistedRelaycastBaseUrl(selection); + const explicit = trimOrUndefined(options.baseUrl); + const fallback = trimOrUndefined(options.fallbackBaseUrl) ?? trimOrUndefined(env(options).RELAY_BASE_URL); + const requested = explicit ?? fallback; + // Precedence is explicit --base-url (validated against a persisted route), + // then the persisted route, fallbackBaseUrl, and finally RELAY_BASE_URL. + // This keeps a server-selected route paired with its credential. + if (persisted && explicit) { + let parsed: URL; + try { + parsed = new URL(explicit); + } catch { + throw new Error('The requested Relaycast base URL is invalid.'); + } + const authority = /^https:\/\/([^/?#]+)/i.exec(explicit)?.[1] ?? ''; + if ( + !/^https:\/\/[^/?#]+\/?$/i.test(explicit) || + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + /:\d+$/.test(authority) || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') + ) { + throw new Error('The requested Relaycast base URL is not a trusted origin.'); + } + if (parsed.origin !== persisted) { + throw new Error('The requested Relaycast base URL does not match the persisted workspace route.'); + } + } + return persisted ?? requested; +} + +/** Resolve one credential/origin pair from one workspace selection. */ +export function resolveWorkspaceTransport(options: WorkspaceTransportOptions = {}): WorkspaceTransport { + const selection = selectionForTransport(options); + if (!selection) { + throw new Error( + 'No workspace key found. Pass --workspace-key, set RELAY_WORKSPACE_KEY, or run `relay workspace set_key `.' + ); + } + const baseUrl = resolveBaseUrlForSelection(selection, options); + // Project-session loading already validates the reference against the + // project/workspace/route/base tuple. Never re-read a ref here: doing so + // would let a tampered ref bypass that binding and pair an unrelated key + // with this route. + const routeCredential = trimOrUndefined(selection.relaycastApiKey); + if (selection.relaycastRoute === 'agent37-isolated' && !routeCredential) { + throw new Error( + 'The persisted isolated Relaycast credential is unavailable or mismatched; rerun the sandbox command to mint a fresh route.' + ); + } + return { + workspaceKey: routeCredential ?? selection.key, + ...(baseUrl ? { baseUrl } : {}), + source: selection.source, + }; +} + +function validatePersistedRelaycastBaseUrl(selection: WorkspaceSelection | undefined): string | undefined { + const baseUrl = trimOrUndefined(selection?.relaycastBaseUrl); + const route = selection?.relaycastRoute; + const relaycastApiKey = trimOrUndefined(selection?.relaycastApiKey); + const relaycastCloudApiUrl = trimOrUndefined(selection?.relaycastCloudApiUrl); + if (!baseUrl && !route && !relaycastApiKey) return undefined; + if (!baseUrl || !route) { + throw new Error('The persisted Relaycast workspace route is incomplete.'); + } + let parsed: URL; + try { + parsed = new URL(baseUrl); + } catch { + throw new Error('The persisted Relaycast workspace route is invalid.'); + } + const expectedOrigin = + route === 'canonical' + ? relaycastCloudApiUrl === DEV_CLOUD_API_URL + ? DEV_RELAYCAST_ORIGIN + : CANONICAL_RELAYCAST_ORIGIN + : route === 'agent37-isolated' + ? AGENT37_RELAYCAST_ORIGIN + : undefined; + if ( + !expectedOrigin || + parsed.origin !== expectedOrigin || + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') + ) { + throw new Error('The persisted Relaycast workspace route is not trusted.'); + } + return parsed.origin; +} + +/** Persist a server-selected target only while the captured project selection is still current. */ +export function persistWorkspaceRelaycastTarget( + selection: WorkspaceSelection | undefined, + target: { + route: 'canonical' | 'agent37-isolated'; + baseUrl: string; + workspaceId: string; + relaycastApiKey: string; + }, + relaycastCloudApiUrl?: string +): boolean { + if (!selection) return false; + if ( + target.route === 'canonical' && + target.baseUrl === DEV_RELAYCAST_ORIGIN && + relaycastCloudApiUrl !== DEV_CLOUD_API_URL + ) { + return false; + } + const selectionWithProjectDir = selection as WorkspaceSelection & { projectDataDir?: string }; + const dataDir = + selectionWithProjectDir?.projectDataDir ?? + (selection?.source === 'project' && selection.origin ? path.dirname(selection.origin) : undefined); + if (!dataDir) return false; + return writeProjectWorkspaceTargetIfSelectionCurrent(dataDir, selection, { + workspaceId: target.workspaceId, + relaycastRoute: target.route, + relaycastBaseUrl: target.baseUrl, + ...(relaycastCloudApiUrl ? { relaycastCloudApiUrl } : {}), + relaycastApiKey: target.relaycastApiKey, + }); +} diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 404786591e..f8683eb032 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -150,6 +150,37 @@ the result when the engine acknowledgment can be matched to the current roster. A successful dispatch remains successful if that optional metadata is missing or the roster read lags; the raw acknowledgment IDs remain available. +## Fleet sandboxes + +`@agent-relay/sdk/fleet` exports `spawnFleetSandbox`, the same path +`agent-relay fleet spawn --sandbox` uses. It provisions a Cloud sandbox, starts +the agent harness on it, waits for the node to confirm the launch, checks that +the agent landed on that sandbox's node, and returns a handle to the live agent: + +```ts +import { spawnFleetSandbox } from '@agent-relay/sdk/fleet'; + +const agent = await spawnFleetSandbox({ + cli: 'claude', + name: 'reviewer', + task: 'Review the draft under /workspace/docs', + provider: 'e2b', + relayfilePaths: ['/docs/**', '/work/**'], + readonlyPaths: ['/docs/**'], // Cloud mounts these subtrees read-only +}); + +const terminal = await agent.attach({ mode: 'drive' }); // { socketPath, finished, close } +// ... pipe stdio to terminal.socketPath ... +await terminal.close(); +await agent.destroy(); // releases the agent; deletes the sandbox when this call provisioned it +``` + +The handle carries `sandboxId`, `nodeId`, `nodeName`, and `agentName`. A +caller-declared `sandboxId` is retained: failure cleanup and `destroy()` never +delete it. A non-empty `env` is rejected, because fleet spawn cannot deliver a +per-agent environment yet. `@agent-relay/sdk/attach` exports +`startFleetNodeAttachProxy` for attaching to any fleet node agent. + ## Events `relay.addListener(selector, handler)` accepts a dotted event name, a `*`/prefix wildcard, or a fluent predicate, and always hands the handler one discriminated event object. It returns an unsubscribe function. diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 5cef01e5b4..6fe44d73b5 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -37,6 +37,16 @@ "types": "./dist/capabilities.d.ts", "import": "./dist/capabilities.js", "default": "./dist/capabilities.js" + }, + "./attach": { + "types": "./dist/attach.d.ts", + "import": "./dist/attach.js", + "default": "./dist/attach.js" + }, + "./fleet": { + "types": "./dist/fleet.d.ts", + "import": "./dist/fleet.js", + "default": "./dist/fleet.js" } }, "files": [ @@ -54,7 +64,7 @@ "build": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\" && npx tsc -p tsconfig.build.json", "build:full": "npm run build", "check": "tsc -p tsconfig.json --noEmit", - "test": "vitest run src/__tests__/activity-reducer.test.ts src/__tests__/agent-relay.test.ts src/__tests__/facade.test.ts src/__tests__/files.test.ts src/__tests__/listeners.test.ts src/__tests__/integrations.test.ts src/__tests__/messaging.test.ts src/__tests__/delivery-actions.test.ts src/__tests__/relaycast-errors.test.ts src/__tests__/register-action-relay.test.ts src/__tests__/thin-client.test.ts src/__tests__/typed-action-handle.test.ts src/__tests__/webhooks.test.ts src/__tests__/event-fanin.test.ts src/__tests__/observer-source.test.ts", + "test": "vitest run src/__tests__/activity-reducer.test.ts src/__tests__/agent-relay.test.ts src/__tests__/facade.test.ts src/__tests__/files.test.ts src/__tests__/listeners.test.ts src/__tests__/integrations.test.ts src/__tests__/messaging.test.ts src/__tests__/delivery-actions.test.ts src/__tests__/relaycast-errors.test.ts src/__tests__/register-action-relay.test.ts src/__tests__/thin-client.test.ts src/__tests__/typed-action-handle.test.ts src/__tests__/webhooks.test.ts src/__tests__/event-fanin.test.ts src/__tests__/observer-source.test.ts src/__tests__/fleet-spawn.test.ts", "test:types": "vitest run --typecheck.only --typecheck.tsconfig tsconfig.typetest.json", "prepack": "npm run build" }, @@ -64,6 +74,7 @@ "dependencies": { "@relaycast/sdk": "^8.0.7", "@relaycast/types": "^8.0.7", - "zod": "^4.4.3" + "zod": "^4.4.3", + "@agent-relay/cloud": "13.2.0" } } diff --git a/packages/sdk/src/__tests__/fleet-spawn.test.ts b/packages/sdk/src/__tests__/fleet-spawn.test.ts new file mode 100644 index 0000000000..2f632592e4 --- /dev/null +++ b/packages/sdk/src/__tests__/fleet-spawn.test.ts @@ -0,0 +1,432 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { spawnFleetSandbox, FleetSandboxSpawnError, type SpawnFleetSandboxDependencies } from '../fleet.js'; + +const SANDBOX_ID = 'sbx_3f2b8c4e-1d2a-4b5c-8d9e-0a1b2c3d4e5f'; + +function provisioned(overrides: Record = {}) { + return { + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cw_1', + nodeId: 'node_1', + nodeName: 'fleet-sandbox-node', + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_1', + relayfileMounted: true, + relayfileMountPath: '/workspace', + providerId: 'e2b' as const, + ...overrides, + }; +} + +function harness( + options: { + ensure?: () => Promise; + spawn?: (input: Record) => Promise; + } = {} +) { + const calls: string[] = []; + const ensureCloudFleetSandbox = vi.fn(async (input: Record) => { + calls.push('ensure'); + void input; + return (options.ensure ?? (async () => provisioned()))(); + }); + const deleteCloudFleetSandbox = vi.fn(async (_input: Record) => { + calls.push('delete-sandbox'); + }); + const release = vi.fn(async (input: { name: string }) => { + calls.push(`release:${input.name}`); + }); + const register = vi.fn(async (input: { name: string }) => { + calls.push(`register:${input.name}`); + return { token: 'launcher-token' }; + }); + const workspaceRelay = { + workspace: { register, release, info: vi.fn(async () => ({ id: 'rw_1' })) }, + }; + const spawn = vi.fn(async (input: Record) => { + calls.push('spawn'); + if (options.spawn) return options.spawn(input); + return { + id: 'inv_1', + status: 'completed', + dispatchedNodeId: 'node_1', + handlerNodeId: 'node_1', + node: { id: 'node_1', name: 'fleet-sandbox-node', status: 'online', capabilities: [] }, + placement: { + capability: 'spawn:claude', + node: 'fleet-sandbox-node', + attempts: 1, + queued: false, + state: 'ready', + confirmed: true, + }, + }; + }); + const createWorkspaceRelay = vi.fn((_options: Record) => workspaceRelay); + const createAgentRelay = vi.fn((_options: Record) => ({ + messaging: { placement: { spawn } }, + })); + const startFleetNodeAttachProxy = vi.fn(async (_options: Record) => ({ + socketPath: '/tmp/attach.sock', + finished: Promise.resolve(0), + close: async () => undefined, + })); + const deps = { + ensureCloudFleetSandbox, + deleteCloudFleetSandbox, + createWorkspaceRelay, + createAgentRelay, + startFleetNodeAttachProxy, + // Never let a test persist a Relaycast target into a real project session. + persistRelaycastTarget: vi.fn(), + warn: vi.fn(), + } as unknown as SpawnFleetSandboxDependencies; + return { + deps, + calls, + ensureCloudFleetSandbox, + deleteCloudFleetSandbox, + release, + spawn, + createAgentRelay, + startFleetNodeAttachProxy, + }; +} + +const base = { + cli: 'claude', + name: 'sandbox-worker', + task: 'Review the repository', + workspaceId: 'rw_1', + workspaceKey: 'rk_live_test', + provider: 'e2b' as const, +}; + +describe('spawnFleetSandbox', () => { + it('provisions a sandbox, starts the harness on it, and returns a live attachable handle', async () => { + const h = harness(); + const handle = await spawnFleetSandbox( + { ...base, readonlyPaths: ['/docs/**'], relayfilePaths: ['/docs/**', '/work/**'] }, + h.deps + ); + + const ensureInput = h.ensureCloudFleetSandbox.mock.calls[0][0]; + expect(ensureInput).toMatchObject({ + workspaceId: 'rw_1', + requiredCapability: 'spawn:claude', + maxAgents: 1, + mountRelayfile: true, + readonlyPaths: ['/docs/**'], + relayfilePaths: ['/docs/**', '/work/**'], + providerId: 'e2b', + forceProvision: true, + }); + expect(ensureInput.sandboxId).toMatch(/^sbx_[0-9a-f-]{36}$/); + + const spawnInput = h.spawn.mock.calls[0][0] as Record; + expect(spawnInput).toMatchObject({ + capability: 'spawn:claude', + node: 'fleet-sandbox-node', + confirm: true, + }); + expect(spawnInput.input).toMatchObject({ + name: 'sandbox-worker', + cli: 'claude', + task: 'Review the repository', + worker_cwd: '/workspace', + }); + + expect(handle).toMatchObject({ + sandboxId: SANDBOX_ID, + nodeId: 'node_1', + nodeName: 'fleet-sandbox-node', + agentName: 'sandbox-worker', + ownsSandbox: true, + }); + // The temporary launcher identity never outlives the spawn. + expect(h.calls.filter((call) => call.startsWith('release:fleet-spawn-launcher-'))).toHaveLength(1); + + const proxy = await handle.attach({ mode: 'drive' }); + expect(proxy.socketPath).toBe('/tmp/attach.sock'); + expect(h.startFleetNodeAttachProxy).toHaveBeenCalledWith( + expect.objectContaining({ + node: 'fleet-sandbox-node', + agent: 'sandbox-worker', + mode: 'drive', + workspaceKey: 'rk_live_test', + pinnedTransport: true, + baseUrl: 'https://cast.agentrelay.com', + }) + ); + }); + + it('returns the sandbox without its Relaycast credential', async () => { + const h = harness({ + ensure: async () => + provisioned({ + relaycastTarget: { + route: 'canonical', + baseUrl: 'https://cast.agentrelay.com', + workspaceId: 'rw_1', + relaycastApiKey: 'rk_live_secret', + }, + }), + }); + const handle = await spawnFleetSandbox(base, h.deps); + expect(handle.sandbox.relaycastTarget).toEqual({ + route: 'canonical', + baseUrl: 'https://cast.agentrelay.com', + workspaceId: 'rw_1', + }); + // @ts-expect-error the redacted type has no credential field + expect(handle.sandbox.relaycastTarget?.relaycastApiKey).toBeUndefined(); + }); + + it('awaits async target persistence and cleans up when it rejects', async () => { + const h = harness({ + ensure: async () => + provisioned({ + relaycastTarget: { + route: 'canonical', + baseUrl: 'https://cast.agentrelay.com', + workspaceId: 'rw_1', + relaycastApiKey: 'rk_live_secret', + }, + }), + }); + await expect( + spawnFleetSandbox(base, { + ...h.deps, + persistRelaycastTarget: async () => Promise.reject(new Error('session write failed')), + }) + ).rejects.toThrow('session write failed'); + expect(h.spawn).not.toHaveBeenCalled(); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + }); + + it('pins the compatibility transport at spawn time for attach', async () => { + const h = harness(); + const env: NodeJS.ProcessEnv = { RELAY_WORKSPACE_KEY: 'rk_live_spawned' }; + const { workspaceKey: _key, ...rest } = base; + const handle = await spawnFleetSandbox({ ...rest, transport: { env } }, h.deps); + env.RELAY_WORKSPACE_KEY = 'rk_live_rebound'; + await handle.attach(); + expect(h.startFleetNodeAttachProxy).toHaveBeenCalledWith( + expect.objectContaining({ workspaceKey: 'rk_live_spawned' }) + ); + }); + + it('creates the launcher client on the transport the spawn resolved', async () => { + const h = harness(); + const { workspaceKey: _key, ...rest } = base; + await spawnFleetSandbox( + { + ...rest, + transport: { + env: { RELAY_WORKSPACE_KEY: 'rk_live_spawned', RELAY_BASE_URL: 'https://cast.agentrelay.com' }, + }, + }, + h.deps + ); + expect(h.createAgentRelay).toHaveBeenCalledWith({ + token: 'launcher-token', + baseUrl: 'https://cast.agentrelay.com', + }); + }); + + it('tears down idempotently: releases the agent, then deletes the sandbox it provisioned', async () => { + const h = harness(); + const handle = await spawnFleetSandbox(base, h.deps); + h.calls.length = 0; + await Promise.all([handle.destroy(), handle.destroy()]); + await handle.destroy(); + expect(h.calls).toEqual(['release:sandbox-worker', 'delete-sandbox']); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledWith({ + cloudWorkspaceId: 'cw_1', + sandboxId: SANDBOX_ID, + providerId: 'e2b', + }); + }); + + it('retries only the teardown step that failed', async () => { + const h = harness(); + const handle = await spawnFleetSandbox(base, h.deps); + h.release.mockRejectedValueOnce(new Error('relaycast busy')); + h.calls.length = 0; + await expect(handle.destroy()).rejects.toThrow('agent release failed: relaycast busy'); + await handle.destroy(); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + expect(h.release.mock.calls.filter(([input]) => input.name === 'sandbox-worker')).toHaveLength(2); + }); + + it('retains a caller-declared sandbox on destroy', async () => { + const h = harness(); + const handle = await spawnFleetSandbox({ ...base, sandboxId: SANDBOX_ID }, h.deps); + expect(handle.ownsSandbox).toBe(false); + await handle.destroy(); + expect(h.deleteCloudFleetSandbox).not.toHaveBeenCalled(); + }); + + it('fails closed and cleans up when the dispatch receipt names a different node', async () => { + const h = harness({ + spawn: async () => ({ + id: 'inv_1', + status: 'completed', + // A targeted placement echoes the requested node; only the receipt is evidence. + dispatchedNodeId: 'node_other', + node: { id: 'node_1', name: 'fleet-sandbox-node', status: 'online', capabilities: [] }, + placement: { + capability: 'spawn:claude', + node: 'fleet-sandbox-node', + attempts: 1, + queued: false, + state: 'ready', + confirmed: true, + }, + }), + }); + await expect(spawnFleetSandbox(base, h.deps)).rejects.toMatchObject({ + name: 'FleetSandboxSpawnError', + code: 'placement_mismatch', + }); + expect(h.calls).toContain('release:sandbox-worker'); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + }); + + it('fails closed when no dispatch receipt proves the sandbox node', async () => { + const h = harness({ + spawn: async () => ({ + id: 'inv_1', + status: 'completed', + node: { id: 'node_1', name: 'fleet-sandbox-node', status: 'online', capabilities: [] }, + placement: { + capability: 'spawn:claude', + node: 'fleet-sandbox-node', + attempts: 1, + queued: false, + state: 'ready', + confirmed: true, + }, + }), + }); + await expect(spawnFleetSandbox(base, h.deps)).rejects.toMatchObject({ code: 'placement_mismatch' }); + expect(h.calls).toContain('release:sandbox-worker'); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + }); + + it('cleans up the sandbox when a caller hook throws', async () => { + const h = harness(); + await expect( + spawnFleetSandbox( + { + ...base, + resolveTask: () => { + throw new Error('bad context'); + }, + }, + h.deps + ) + ).rejects.toThrow('bad context'); + expect(h.spawn).not.toHaveBeenCalled(); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + }); + + it('awaits async hooks and cleans up when async verification rejects', async () => { + const h = harness(); + await expect( + spawnFleetSandbox( + { ...base, verifySandbox: async () => Promise.reject(new Error('stale revision')) }, + h.deps + ) + ).rejects.toThrow('stale revision'); + expect(h.spawn).not.toHaveBeenCalled(); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + + const ok = harness(); + await spawnFleetSandbox( + { + ...base, + resolveWorkerCwd: async () => '/workspace/repo', + resolveTask: async (_s, cwd) => `work in ${cwd}`, + }, + ok.deps + ); + expect((ok.spawn.mock.calls[0][0] as Record).input).toMatchObject({ + worker_cwd: '/workspace/repo', + task: 'work in /workspace/repo', + }); + }); + + it('never lets spawnMetadata replace the lifecycle fields', async () => { + const h = harness(); + const handle = await spawnFleetSandbox( + { ...base, spawnMetadata: { name: 'other-agent', worker_cwd: '/tmp', session_ref: 'ref-1' } }, + h.deps + ); + expect((h.spawn.mock.calls[0][0] as Record).input).toMatchObject({ + name: 'sandbox-worker', + worker_cwd: '/workspace', + session_ref: 'ref-1', + }); + expect(handle.agentName).toBe('sandbox-worker'); + }); + + it('refuses a reused node without a sandbox identity', async () => { + const h = harness({ + ensure: async () => ({ + outcome: 'reused', + cloudWorkspaceId: 'cw_1', + nodeId: 'node_1', + nodeName: 'existing-node', + status: 'online', + activeAgents: 0, + maxAgents: 1, + }), + }); + await expect( + spawnFleetSandbox({ ...base, sandboxName: 'existing-node', mountRelayfile: false }, h.deps) + ).rejects.toMatchObject({ code: 'sandbox_identity_unknown' }); + expect(h.spawn).not.toHaveBeenCalled(); + expect(h.deleteCloudFleetSandbox).not.toHaveBeenCalled(); + }); + + it('deletes the provisioned sandbox when the harness fails to start', async () => { + const h = harness({ + spawn: async () => { + throw new Error('spawn failed on node'); + }, + }); + await expect(spawnFleetSandbox(base, h.deps)).rejects.toThrow('spawn failed on node'); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + }); + + it('deletes a timed-out sandbox and reports the timeout', async () => { + const h = harness({ + ensure: async () => ({ + outcome: 'provisioning_timeout', + cloudWorkspaceId: 'cw_1', + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_1', + nodeName: 'fleet-sandbox-node', + waitedMs: 90_000, + providerId: 'e2b', + }), + }); + await expect(spawnFleetSandbox(base, h.deps)).rejects.toThrow(/did not become ready within 90000ms/); + expect(h.deleteCloudFleetSandbox).toHaveBeenCalledTimes(1); + expect(h.spawn).not.toHaveBeenCalled(); + }); + + it('omits an empty readonlyPaths list and rejects env it cannot deliver', async () => { + const h = harness(); + await spawnFleetSandbox({ ...base, readonlyPaths: [], env: {} }, h.deps); + expect(h.ensureCloudFleetSandbox.mock.calls[0][0]).not.toHaveProperty('readonlyPaths'); + + const rejected = harness(); + await expect(spawnFleetSandbox({ ...base, env: { SECRET: 'x' } }, rejected.deps)).rejects.toBeInstanceOf( + FleetSandboxSpawnError + ); + expect(rejected.ensureCloudFleetSandbox).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/sdk/src/attach.ts b/packages/sdk/src/attach.ts new file mode 100644 index 0000000000..71c417be17 --- /dev/null +++ b/packages/sdk/src/attach.ts @@ -0,0 +1 @@ +export * from '@agent-relay/cloud/attach'; diff --git a/packages/sdk/src/fleet-spawn.ts b/packages/sdk/src/fleet-spawn.ts new file mode 100644 index 0000000000..9e5c79b217 --- /dev/null +++ b/packages/sdk/src/fleet-spawn.ts @@ -0,0 +1,716 @@ +/** + * `spawnFleetSandbox` — the typed SDK contract for `agent-relay fleet spawn + * --sandbox`. It provisions a Cloud fleet sandbox, starts the agent harness on + * it, confirms the node the agent landed on, and returns a handle to the live + * agent that can be attached to immediately and torn down idempotently. + * + * The CLI command drives this same function, so the two cannot drift. + */ +import { randomUUID } from 'node:crypto'; + +import { + CANONICAL_RELAYCAST_ORIGIN, + CloudFleetSandboxProvisionError, + DEV_CLOUD_API_URL, + DEV_RELAYCAST_ORIGIN, + deleteCloudFleetSandbox, + ensureCloudFleetSandbox, + resolveWorkspaceByKey, + type CloudFleetRelaycastTarget, + type EnsureCloudFleetSandboxInput, + type EnsureCloudFleetSandboxResult, +} from '@agent-relay/cloud/fleet'; +import { + startFleetNodeAttachProxy, + type FleetNodeAttachOptions, + type FleetNodeAttachProxy, +} from '@agent-relay/cloud/attach'; +import { + persistWorkspaceRelaycastTarget, + resolveBaseUrl, + resolveWorkspaceSelection, + resolveWorkspaceTransport, + type WorkspaceTransportOptions, +} from '@agent-relay/cloud/workspace-transport'; + +import { AgentRelay, type AgentRelayAgent } from './agent-relay.js'; +import { RelayPlacementError } from './messaging/relaycast-placement.js'; +import type { RelaySpawnPlacementAck } from './messaging/types.js'; + +export type FleetSandboxProvider = 'daytona' | 'e2b' | 'agent37'; + +const CLOUD_SANDBOX_ID_PATTERN = + /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const DEFAULT_CONFIRM_TIMEOUT_MS = 120_000; + +/** A sandbox that is ready to receive the agent (never a provisioning timeout). */ +export type FleetSandboxReadyResult = Exclude< + EnsureCloudFleetSandboxResult, + { outcome: 'provisioning_timeout' } +>; + +/** A Relaycast target without its credential. */ +export type RedactedCloudFleetRelaycastTarget = Omit; + +/** An ensured sandbox with the Relaycast credential removed. */ +export type RedactedFleetSandboxResult = EnsureCloudFleetSandboxResult extends infer Result + ? Result extends { relaycastTarget?: CloudFleetRelaycastTarget } + ? Omit & { relaycastTarget?: RedactedCloudFleetRelaycastTarget } + : Result + : never; + +export interface SpawnFleetSandboxInput { + /** Agent harness to start, e.g. `claude` or `codex`. */ + cli: string; + /** Worker agent name. Defaults to `-sandbox-`. */ + name?: string; + /** Initial task. */ + task?: string; + /** Relay workspace ID used for Cloud provisioning. Resolved from the workspace selection when omitted. */ + workspaceId?: string; + /** Alias for `workspaceId`. */ + workspace?: string; + /** Workspace credential/transport selection; ambient sources are used when omitted. */ + workspaceKey?: string; + baseUrl?: string; + projectRoot?: string; + /** Advanced: full transport selection (overrides workspaceKey/baseUrl/projectRoot). */ + transport?: WorkspaceTransportOptions; + /** + * Harness environment. Fleet spawn cannot deliver per-agent environment yet, + * so a non-empty map is rejected rather than silently dropped. + */ + env?: Record; + provider?: FleetSandboxProvider; + /** + * Caller-declared `sbx_` identity. A declared sandbox is retained: it + * is never deleted by failure cleanup or by `destroy()`. + */ + sandboxId?: string; + sandboxName?: string; + /** Mount Relayfile in the sandbox (default true). */ + mountRelayfile?: boolean; + /** Relayfile subtrees (`/path/**`) to mount. */ + relayfilePaths?: string[]; + /** Relayfile subtrees (`/path/**`) Cloud must mount read-only. An empty list is ignored. */ + readonlyPaths?: string[]; + /** Static repository checkouts for the sandbox. */ + repos?: string[]; + repoRevisions?: Record; + /** Worker working directory. Defaults to the Relayfile mount root. */ + workerCwd?: string; + channels?: string[]; + model?: string; + /** Extra spawn input fields (for example declared workforce metadata or `session_ref`). */ + spawnMetadata?: Record; + /** Wait for the node to confirm the harness launched (default true). */ + confirm?: boolean; + confirmTimeoutMs?: number; + /** Validate the ensured sandbox before any agent is started; throwing aborts and cleans up. */ + verifySandbox?: (sandbox: EnsureCloudFleetSandboxResult) => void | Promise; + /** Compute the worker cwd once the sandbox mount is known. */ + resolveWorkerCwd?: (sandbox: FleetSandboxReadyResult) => string | undefined | Promise; + /** Compute the task once the sandbox and worker cwd are known. */ + resolveTask?: (sandbox: FleetSandboxReadyResult, workerCwd: string | undefined) => string | Promise; +} + +type WorkspaceRelayLike = Pick; +type AgentRelayLike = { messaging: Pick }; + +export interface SpawnFleetSandboxDependencies { + ensureCloudFleetSandbox: typeof ensureCloudFleetSandbox; + deleteCloudFleetSandbox: typeof deleteCloudFleetSandbox; + resolveWorkspaceByKey: (workspaceKey: string) => Promise<{ cloudWorkspaceId: string }>; + createWorkspaceRelay: (options: WorkspaceTransportOptions) => WorkspaceRelayLike; + createAgentRelay: (options: { token: string; baseUrl?: string }) => AgentRelayLike; + /** + * Persist Cloud's server-selected Relaycast target for follow-up commands. + * Throwing aborts the spawn (and cleans up an owned sandbox). The default + * persists to the project session when one exists and otherwise does nothing; + * the handle's `attach()` carries the verified target either way. + */ + persistRelaycastTarget: ( + target: CloudFleetRelaycastTarget, + context: { sandbox: EnsureCloudFleetSandboxResult; transport: WorkspaceTransportOptions } + ) => void | Promise; + startFleetNodeAttachProxy: (options: FleetNodeAttachOptions) => Promise; + warn: (message: string) => void; +} + +export interface FleetSandboxHandle { + sandboxId: string; + /** Cloud node ID the agent landed on. */ + nodeId: string; + nodeName: string; + agentName: string; + cloudWorkspaceId: string; + providerId?: string; + workerCwd?: string; + relayfileMountPath?: string; + /** True when this call provisioned the sandbox and `destroy()` deletes it. */ + ownsSandbox: boolean; + /** The ensured sandbox, with the Relaycast credential removed. */ + sandbox: RedactedFleetSandboxResult; + invocation: RelaySpawnPlacementAck; + /** Attach to the live agent's terminal through a private local socket. */ + attach(options?: { mode?: FleetNodeAttachOptions['mode'] }): Promise; + /** Release the agent and delete the sandbox when owned. Idempotent. */ + destroy(): Promise; +} + +export type FleetSandboxSpawnErrorCode = + | 'invalid_input' + | 'unsupported_env' + | 'workspace_unresolved' + | 'provisioning_timeout' + | 'relayfile_unmounted' + | 'relaycast_target_unverified' + | 'sandbox_identity_unknown' + | 'placement_mismatch'; + +export class FleetSandboxSpawnError extends Error { + readonly code: FleetSandboxSpawnErrorCode; + constructor(code: FleetSandboxSpawnErrorCode, message: string) { + super(message); + this.name = 'FleetSandboxSpawnError'; + this.code = code; + } +} + +function defaultDependencies(): SpawnFleetSandboxDependencies { + return { + ensureCloudFleetSandbox, + deleteCloudFleetSandbox, + resolveWorkspaceByKey: (workspaceKey) => resolveWorkspaceByKey(workspaceKey), + createWorkspaceRelay: (options) => { + const { workspaceKey, baseUrl } = resolveWorkspaceTransport(options); + return new AgentRelay({ workspaceKey, baseUrl }); + }, + createAgentRelay: ({ token, baseUrl }) => + new AgentRelay({ + agentToken: token, + // A launcher token is scoped to one deployment; never let a persisted + // project route pick a different gateway for it. + baseUrl: baseUrl ?? resolveBaseUrl({ ignorePersistedRelaycastTarget: true }), + }), + persistRelaycastTarget: (target, { sandbox, transport }) => { + persistWorkspaceRelaycastTarget( + resolveWorkspaceSelection(transport), + target, + sandbox.relaycastCloudApiUrl + ); + }, + startFleetNodeAttachProxy, + warn: (message) => process.stderr.write(`${message}\n`), + }; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function nonEmpty(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +/** Remove the Relaycast credential so the result can be logged or returned. */ +export function redactFleetSandbox(sandbox: EnsureCloudFleetSandboxResult): RedactedFleetSandboxResult { + if (!sandbox.relaycastTarget) return sandbox as RedactedFleetSandboxResult; + const { relaycastApiKey: _secret, ...target } = sandbox.relaycastTarget; + return { ...sandbox, relaycastTarget: target } as RedactedFleetSandboxResult; +} + +export async function spawnFleetSandbox( + input: SpawnFleetSandboxInput, + dependencies: Partial = {} +): Promise { + const deps: SpawnFleetSandboxDependencies = { ...defaultDependencies(), ...dependencies }; + const cli = nonEmpty(input.cli); + if (!cli) throw new FleetSandboxSpawnError('invalid_input', 'A harness cli is required.'); + if (input.env && Object.keys(input.env).length > 0) { + throw new FleetSandboxSpawnError( + 'unsupported_env', + 'Fleet sandbox spawn cannot deliver a per-agent environment yet; remove env or provide it through the workspace.' + ); + } + const mountRelayfile = input.mountRelayfile !== false; + const readonlyPaths = input.readonlyPaths?.length ? [...input.readonlyPaths] : undefined; + if (readonlyPaths && !mountRelayfile) { + throw new FleetSandboxSpawnError('invalid_input', 'readonlyPaths requires mounting Relayfile.'); + } + if (input.sandboxId !== undefined && !CLOUD_SANDBOX_ID_PATTERN.test(input.sandboxId)) { + throw new FleetSandboxSpawnError( + 'invalid_input', + 'sandboxId must match lowercase sbx_ using an RFC 4122 UUID.' + ); + } + const name = nonEmpty(input.name) ?? `${cli}-sandbox-${randomUUID().slice(0, 8)}`; + const provider = input.provider; + // An explicit sandbox identity is a retained/replayable resource. Never + // delete it as collateral when a later verification or dispatch step fails; + // only clean up sandboxes whose identity this invocation minted. + const shouldCleanupSandbox = input.sandboxId === undefined; + const transport: WorkspaceTransportOptions = input.transport ?? { + ...(input.workspaceKey === undefined ? {} : { workspaceKey: input.workspaceKey }), + ...(input.baseUrl === undefined ? {} : { baseUrl: input.baseUrl }), + ...(input.projectRoot === undefined ? {} : { projectRoot: input.projectRoot }), + }; + const legacyTransport: WorkspaceTransportOptions = { + ...transport, + ...(provider === 'agent37' ? {} : { ignorePersistedRelaycastTarget: true }), + }; + let workspaceRelay: WorkspaceRelayLike | undefined; + + let relayWorkspaceId = nonEmpty(input.workspaceId) ?? nonEmpty(input.workspace); + if (!relayWorkspaceId) { + const selection = resolveWorkspaceSelection(transport); + relayWorkspaceId = nonEmpty(selection?.workspaceId); + if (!relayWorkspaceId && selection?.key && (provider === undefined || provider === 'agent37')) { + relayWorkspaceId = (await deps.resolveWorkspaceByKey(selection.key)).cloudWorkspaceId; + } + if (!relayWorkspaceId && provider !== undefined && provider !== 'agent37') { + workspaceRelay = deps.createWorkspaceRelay(legacyTransport); + relayWorkspaceId = nonEmpty((await workspaceRelay.workspace.info()).id); + } + } + if (!relayWorkspaceId) { + throw new FleetSandboxSpawnError( + 'workspace_unresolved', + 'Sandbox provisioning requires a Relay workspace identity; pass workspaceId or bind the workspace.' + ); + } + + const sandboxId = input.sandboxId ?? (input.sandboxName === undefined ? `sbx_${randomUUID()}` : undefined); + const deterministicSandboxName = + sandboxId === undefined ? undefined : `fleet-sandbox-${sandboxId.slice('sbx_'.length)}`; + if ( + input.sandboxId !== undefined && + input.sandboxName !== undefined && + input.sandboxName !== deterministicSandboxName + ) { + throw new FleetSandboxSpawnError( + 'invalid_input', + `--sandbox-name must be '${deterministicSandboxName}' when --sandbox-id is supplied; custom names cannot preserve the one-to-one sandbox identity.` + ); + } + const effectiveSandboxName = deterministicSandboxName ?? input.sandboxName; + // The unpinned/Agent37 path deliberately carries the measured heavy + // 8 CPU / 16 GiB / 20 GiB profile. Daytona and E2B cannot satisfy that + // shape, so an explicit legacy-provider selection must request the + // provider-neutral durable profile instead of becoming unroutable by + // construction. + const workloadProfile = + provider === undefined || provider === 'agent37' ? 'long-running-agent' : 'standard-long-running-agent'; + const useAsyncPreparation = sandboxId !== undefined && (provider === undefined || provider === 'agent37'); + if (useAsyncPreparation && input.sandboxId === undefined) { + // The generated identity is not persisted. Print it before Cloud work + // starts so an interrupted run can be resumed instead of duplicated. + deps.warn( + `Cloud sandbox identity: ${sandboxId}. If this command is interrupted, re-run it with --sandbox-id ${sandboxId} to resume the same sandbox instead of creating another.` + ); + } + + const deleteSandbox = (sandbox: { cloudWorkspaceId: string; sandboxId: string; providerId?: string }) => + deps.deleteCloudFleetSandbox({ + cloudWorkspaceId: sandbox.cloudWorkspaceId, + sandboxId: sandbox.sandboxId, + ...(sandbox.providerId === undefined + ? {} + : { providerId: sandbox.providerId as EnsureCloudFleetSandboxInput['providerId'] & string }), + }); + + let sandbox: EnsureCloudFleetSandboxResult | undefined; + try { + const ensureInput: EnsureCloudFleetSandboxInput = { + workspaceId: relayWorkspaceId, + requiredCapability: `spawn:${cli}`, + ...(useAsyncPreparation ? { preparationMode: 'async-v1' as const } : {}), + maxAgents: 1, + mountRelayfile, + ...(readonlyPaths === undefined ? {} : { readonlyPaths }), + ...(input.relayfilePaths === undefined ? {} : { relayfilePaths: input.relayfilePaths }), + ...(sandboxId === undefined ? {} : { sandboxId }), + forceProvision: true, + ...(provider === undefined ? {} : { providerId: provider }), + workloadProfile, + waitTimeoutMs: 90_000, + ...(effectiveSandboxName === undefined ? {} : { name: effectiveSandboxName }), + ...(input.repos === undefined ? {} : { repos: input.repos }), + ...(input.repoRevisions === undefined ? {} : { repoRevisions: input.repoRevisions }), + }; + sandbox = useAsyncPreparation + ? await deps.ensureCloudFleetSandbox(ensureInput, { + onPreparationProgress: (progress) => { + deps.warn( + `Cloud sandbox preparation: ${progress.phase} (${progress.state}, generation ${progress.generation}).` + ); + }, + }) + : await deps.ensureCloudFleetSandbox(ensureInput); + await input.verifySandbox?.(sandbox); + } catch (error) { + if (error instanceof CloudFleetSandboxProvisionError && error.sandboxAbsent) { + // Cloud's terminal record for this exact identity already proves the + // provider sandbox is gone; a delete here would only race its reaper. + deps.warn( + `${error.message} Cloud confirmed sandbox '${ + error.sandboxId ?? sandboxId ?? 'the requested sandbox' + }' is not running; no sandbox was left running.` + ); + } else if ( + shouldCleanupSandbox && + error instanceof CloudFleetSandboxProvisionError && + error.confirmedProvisioned && + error.cloudWorkspaceId && + error.sandboxId + ) { + const failedSandboxId = error.sandboxId; + await deleteSandbox({ + cloudWorkspaceId: error.cloudWorkspaceId, + sandboxId: failedSandboxId, + ...(error.providerId === undefined ? {} : { providerId: error.providerId }), + }).catch((cleanupError) => { + deps.warn( + `Provisioning failed after Cloud confirmed sandbox '${failedSandboxId}', and automatic cleanup failed: ${errorMessage( + cleanupError + )}` + ); + }); + } else if (error instanceof CloudFleetSandboxProvisionError && error.noSandboxCreated) { + deps.warn(error.message); + } else if (error instanceof CloudFleetSandboxProvisionError && error.outcomeUnknown) { + deps.warn( + `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ + error.nodeName ?? effectiveSandboxName ?? 'the requested sandbox' + }'${ + sandboxId === undefined ? '' : ` before retrying with --sandbox-id '${sandboxId}'` + } so a sandbox is not left running.` + ); + } else if ( + shouldCleanupSandbox && + error instanceof CloudFleetSandboxProvisionError && + error.cloudWorkspaceId && + error.sandboxId + ) { + const failedSandboxId = error.sandboxId; + await deleteSandbox({ + cloudWorkspaceId: error.cloudWorkspaceId, + sandboxId: failedSandboxId, + ...(error.providerId === undefined ? {} : { providerId: error.providerId }), + }).catch((cleanupError) => { + deps.warn( + `Provisioning failed after Cloud created sandbox '${failedSandboxId}', and automatic cleanup failed: ${errorMessage( + cleanupError + )}` + ); + }); + } + if (shouldCleanupSandbox && sandbox && sandbox.outcome !== 'reused') { + await deleteSandbox(sandbox).catch((cleanupError) => { + deps.warn( + `Sandbox repository verification failed and cleanup also failed: ${errorMessage(cleanupError)}` + ); + }); + } + throw error; + } + + let relaycastTransport: WorkspaceTransportOptions = transport; + let verifiedTarget: CloudFleetRelaycastTarget | undefined; + if (sandbox.outcome !== 'provisioning_timeout' && sandbox.relaycastTarget) { + // When Cloud returns a closed, server-owned target, apply it for any + // provider and outcome before registration, spawn, or launcher release, + // then prove the authenticated client sees the exact workspace Cloud + // returned. + try { + const target = sandbox.relaycastTarget; + const returnedRelayWorkspaceId = + 'relayWorkspaceId' in sandbox ? sandbox.relayWorkspaceId?.trim() : undefined; + if ( + (returnedRelayWorkspaceId !== undefined && target.workspaceId.trim() !== returnedRelayWorkspaceId) || + (sandbox.outcome === 'provisioned' && !returnedRelayWorkspaceId) + ) { + throw new FleetSandboxSpawnError( + 'relaycast_target_unverified', + 'Cloud returned a Relaycast target for a different workspace.' + ); + } + relaycastTransport = { + ...relaycastTransport, + workspaceKey: target.relaycastApiKey, + baseUrl: target.baseUrl, + }; + workspaceRelay = deps.createWorkspaceRelay(relaycastTransport); + const postEnsureWorkspace = await workspaceRelay.workspace.info(); + const postEnsureWorkspaceId = postEnsureWorkspace.id?.trim(); + if ( + !postEnsureWorkspaceId || + (sandbox.outcome === 'provisioned' && postEnsureWorkspaceId !== returnedRelayWorkspaceId) || + postEnsureWorkspaceId !== target.workspaceId.trim() + ) { + throw new FleetSandboxSpawnError( + 'relaycast_target_unverified', + 'Cloud returned a Relaycast workspace that could not be verified on the selected gateway.' + ); + } + if ( + target.route === 'canonical' && + target.baseUrl === DEV_RELAYCAST_ORIGIN && + sandbox.relaycastCloudApiUrl !== DEV_CLOUD_API_URL + ) { + throw new FleetSandboxSpawnError( + 'relaycast_target_unverified', + `Cloud returned the DEV canonical Relaycast target, but relaycastCloudApiUrl was not exactly ${DEV_CLOUD_API_URL}; refusing to persist an untrusted route.` + ); + } + await deps.persistRelaycastTarget(target, { sandbox, transport }); + verifiedTarget = target; + } catch (error) { + if (shouldCleanupSandbox && sandbox.outcome === 'provisioned') { + await deleteSandbox(sandbox).catch((cleanupError) => { + deps.warn( + `Relaycast workspace verification failed and sandbox cleanup also failed: ${errorMessage( + cleanupError + )}` + ); + }); + } + throw error; + } + } else if (sandbox.outcome !== 'provisioning_timeout') { + // Older non-Agent37 Cloud responses can omit a target. In that + // compatibility case, keep every subsequent client on the canonical + // workspace selection; a stale persisted Agent37 target must not leak + // into registration, dispatch, or launcher release. + relaycastTransport = legacyTransport; + } + if (sandbox.outcome === 'provisioning_timeout') { + if (shouldCleanupSandbox) { + await deleteSandbox(sandbox).catch((error) => { + deps.warn(`The timed-out sandbox could not be cleaned up automatically: ${errorMessage(error)}`); + }); + } + throw new FleetSandboxSpawnError( + 'provisioning_timeout', + `Sandbox node '${sandbox.nodeName}' did not become ready within ${sandbox.waitedMs}ms.` + ); + } + const ready: FleetSandboxReadyResult = sandbox; + if (mountRelayfile && (ready.outcome !== 'provisioned' || ready.relayfileMounted !== true)) { + if (shouldCleanupSandbox && ready.outcome === 'provisioned') { + await deleteSandbox(ready).catch((error) => { + deps.warn( + `The unmounted sandbox could not be cleaned up automatically and may still be running: ${errorMessage( + error + )}` + ); + }); + } + throw new FleetSandboxSpawnError( + 'relayfile_unmounted', + 'Cloud returned a sandbox node without the required Relayfile mount.' + ); + } + + const mountPath = + ready.outcome === 'provisioned' && ready.relayfileMounted + ? (ready.relayfileMountPath ?? '/workspace') + : undefined; + const sandboxIdentity = 'sandboxId' in ready ? ready.sandboxId : sandboxId; + if (sandboxIdentity === undefined) { + // A reused node is not owned by this call, so there is nothing to clean up. + throw new FleetSandboxSpawnError( + 'sandbox_identity_unknown', + `Cloud reused node '${ready.nodeName}' without a sandbox identity; pass sandboxId to resume a known sandbox.` + ); + } + const ownsSandbox = shouldCleanupSandbox && ready.outcome === 'provisioned'; + const confirm = input.confirm !== false; + const workspaceRelayFor = () => + (workspaceRelay ??= deps.createWorkspaceRelay(verifiedTarget ? relaycastTransport : legacyTransport)); + const releaseAgent = (reason: string) => + workspaceRelayFor().workspace.release({ name, reason, deleteAgent: true }); + + // Pin the credential/origin pair the agent was launched with: a later + // project rebind or environment change must not redirect attach. + let spawnTransport: { workspaceKey: string; baseUrl?: string } | Error; + if (verifiedTarget) { + spawnTransport = { workspaceKey: verifiedTarget.relaycastApiKey, baseUrl: verifiedTarget.baseUrl }; + } else { + try { + const { workspaceKey, baseUrl } = resolveWorkspaceTransport(legacyTransport); + spawnTransport = { workspaceKey, ...(baseUrl === undefined ? {} : { baseUrl }) }; + } catch (error) { + spawnTransport = error instanceof Error ? error : new Error(String(error)); + } + } + let launcherName: string | undefined; + let invocation: RelaySpawnPlacementAck; + let workerCwd: string | undefined; + try { + // Caller hooks run inside the cleanup boundary: a throwing hook must not + // strand the sandbox this call just provisioned. + workerCwd = (await input.resolveWorkerCwd?.(ready)) ?? input.workerCwd ?? mountPath; + const task = (await input.resolveTask?.(ready, workerCwd)) ?? input.task ?? ''; + // Agent tokens are scoped to a Relaycast deployment. Mint a temporary + // launcher on the transport Cloud selected (or the canonical + // compatibility transport when an older non-Agent37 response omitted the + // target); ambient agent tokens cannot prove they belong to it. + const pendingLauncherName = `fleet-spawn-launcher-${randomUUID().slice(0, 8)}`; + const launcher = await workspaceRelayFor().workspace.register( + { name: pendingLauncherName, metadata: { purpose: 'fleet-spawn-launcher' } }, + { strict: true } + ); + launcherName = pendingLauncherName; + const agentToken = launcher.token; + if (!agentToken) { + throw new Error('The temporary fleet spawn launcher did not receive an agent token.'); + } + // The launcher token is already scoped to the exact workspace and + // deployment Cloud selected; keep the workspace key only on + // `workspaceRelay`, where it mints and releases that token. + // The launcher token is valid only on the gateway that minted it. + const launcherBaseUrl = + spawnTransport instanceof Error ? relaycastTransport.baseUrl : spawnTransport.baseUrl; + const relay = deps.createAgentRelay({ + token: agentToken, + ...(launcherBaseUrl === undefined ? {} : { baseUrl: launcherBaseUrl }), + }); + // Placement alone only proves the node accepted the dispatch. A node + // running an obsolete broker advertises `spawn:` capacity, acks the + // invocation and launches nothing — so wait for confirmation unless asked + // not to. + invocation = await relay.messaging.placement.spawn({ + capability: `spawn:${cli}`, + node: ready.nodeName, + failFast: true, + confirm, + ...(confirm ? { confirmTimeoutMs: input.confirmTimeoutMs ?? DEFAULT_CONFIRM_TIMEOUT_MS } : {}), + input: { + // Metadata first: it must never replace the lifecycle fields the + // handle attaches to and releases. + ...(input.spawnMetadata ?? {}), + name, + cli, + task, + ...(input.channels?.length ? { channels: input.channels } : {}), + ...(input.model ? { model: input.model } : {}), + ...(workerCwd ? { worker_cwd: workerCwd } : {}), + }, + }); + // Confirm the agent landed on the sandbox this call ensured. For a + // targeted spawn, `invocation.node` and `placement.node` echo the + // requested roster node, so only the engine's dispatch receipt is + // evidence of where the action actually ran. + const sandboxNodeIds = new Set( + [ + ready.nodeId, + ...(invocation.node?.name === ready.nodeName ? [invocation.node.id, invocation.node.nodeId] : []), + ].filter((id): id is string => typeof id === 'string' && id.length > 0) + ); + const receiptNodeIds = [invocation.dispatchedNodeId, invocation.handlerNodeId].filter( + (id): id is string => typeof id === 'string' && id.length > 0 + ); + const landedNodeId = receiptNodeIds.find((id) => !sandboxNodeIds.has(id)); + if (receiptNodeIds.length === 0 || landedNodeId !== undefined) { + // No receipt proves the sandbox node, or one names another node: fail + // closed so an agent routed elsewhere never escapes the sandbox. + await releaseAgent('Fleet sandbox spawn landed on an unverified node').catch((error) => { + deps.warn(`Releasing the unverified agent '${name}' failed: ${errorMessage(error)}`); + }); + throw new FleetSandboxSpawnError( + 'placement_mismatch', + landedNodeId === undefined + ? `Agent '${name}' was dispatched without a node receipt proving sandbox node '${ready.nodeName}' (${ready.nodeId}).` + : `Agent '${name}' landed on node '${landedNodeId}', not sandbox node '${ready.nodeName}' (${ready.nodeId}).` + ); + } + } catch (error) { + if ( + shouldCleanupSandbox && + ready.outcome === 'provisioned' && + !(error instanceof RelayPlacementError && error.state === 'unconfirmed_may_be_running') + ) { + await deleteSandbox(ready).catch((cleanupError) => { + deps.warn( + `Spawn failed and the sandbox could not be cleaned up automatically: ${errorMessage(cleanupError)}` + ); + }); + } + throw error; + } finally { + if (launcherName && workspaceRelay) { + await workspaceRelay.workspace + .release({ + name: launcherName, + reason: 'Temporary fleet spawn launcher completed', + deleteAgent: true, + }) + .catch((error) => { + deps.warn(`Temporary launcher cleanup failed: ${errorMessage(error)}`); + }); + } + } + + let destroyed: Promise | undefined; + let agentReleased = false; + let sandboxDeleted = false; + return { + sandboxId: sandboxIdentity, + nodeId: ready.nodeId, + nodeName: ready.nodeName, + agentName: name, + cloudWorkspaceId: ready.cloudWorkspaceId, + ...(ready.providerId === undefined ? {} : { providerId: ready.providerId }), + ...(workerCwd === undefined ? {} : { workerCwd }), + ...(mountPath === undefined ? {} : { relayfileMountPath: mountPath }), + ownsSandbox, + sandbox: redactFleetSandbox(ready), + invocation, + attach: async (options = {}) => { + if (spawnTransport instanceof Error) throw spawnTransport; + const attachTransport = spawnTransport; + return deps.startFleetNodeAttachProxy({ + node: ready.nodeName, + agent: name, + mode: options.mode ?? 'drive', + // Keep attach on the exact pair the spawn resolved and verified; a + // transport without an origin used Relaycast's canonical default. + pinnedTransport: true, + workspaceKey: attachTransport.workspaceKey, + baseUrl: attachTransport.baseUrl ?? CANONICAL_RELAYCAST_ORIGIN, + }); + }, + destroy: () => + (destroyed ??= (async () => { + // Each step is recorded once it succeeds, so a retry only replays + // the step that failed (a deleted sandbox would otherwise 404). + const failures: string[] = []; + if (!agentReleased) { + await releaseAgent('Fleet sandbox handle destroyed').then( + () => { + agentReleased = true; + }, + (error) => { + failures.push(`agent release failed: ${errorMessage(error)}`); + } + ); + } + if (ownsSandbox && !sandboxDeleted) { + await deleteSandbox(ready).then( + () => { + sandboxDeleted = true; + }, + (error) => { + failures.push(`sandbox deletion failed: ${errorMessage(error)}`); + } + ); + } + if (failures.length > 0) { + destroyed = undefined; + throw new Error(`Fleet sandbox teardown incomplete: ${failures.join('; ')}`); + } + })()), + }; +} diff --git a/packages/sdk/src/fleet.ts b/packages/sdk/src/fleet.ts new file mode 100644 index 0000000000..07ddf2a8e7 --- /dev/null +++ b/packages/sdk/src/fleet.ts @@ -0,0 +1,2 @@ +export * from '@agent-relay/cloud/fleet'; +export * from './fleet-spawn.js'; diff --git a/packages/sdk/tsconfig.build.json b/packages/sdk/tsconfig.build.json index 0c970cfdd8..bd08700afb 100644 --- a/packages/sdk/tsconfig.build.json +++ b/packages/sdk/tsconfig.build.json @@ -28,7 +28,10 @@ "src/messaging/**/*.ts", "src/delivery/**/*.ts", "src/actions/**/*.ts", - "src/session/**/*.ts" + "src/session/**/*.ts", + "src/attach.ts", + "src/fleet.ts", + "src/fleet-spawn.ts" ], "exclude": ["src/__tests__/**"] } diff --git a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs index 916cd2edb4..8065facf04 100644 --- a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs +++ b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs @@ -270,6 +270,9 @@ test('fleet spawn --sandbox replays and retains an exact provider-attributed ide spawn: vi.fn(async () => ({ invocationId: 'inv_relayflow', node: { name: REPLAY_SANDBOX_NAME }, + // The engine's dispatch receipt: sandbox spawns fail closed + // (placement_mismatch) unless it names the ensured sandbox node. + dispatchedNodeId: 'node-relayflow', })), }, }, diff --git a/tests/relayflows/cases/1744-explicit-provider-workload-profile/run.mjs b/tests/relayflows/cases/1744-explicit-provider-workload-profile/run.mjs index 468b88e766..2a43fa6b52 100644 --- a/tests/relayflows/cases/1744-explicit-provider-workload-profile/run.mjs +++ b/tests/relayflows/cases/1744-explicit-provider-workload-profile/run.mjs @@ -194,6 +194,8 @@ test('explicit provider selection emits the provider-compatible Cloud workload p spawn: vi.fn(async () => ({ invocationId: 'inv_' + provider, node: { name: provider + '-worker' }, + // Sandbox spawns fail closed unless the dispatch receipt names the sandbox node. + dispatchedNodeId: 'node-' + provider, })), }, }, diff --git a/tests/relayflows/cases/1746-sandbox-launcher-token-authority/run.mjs b/tests/relayflows/cases/1746-sandbox-launcher-token-authority/run.mjs index 300507cde2..fff4acc93f 100644 --- a/tests/relayflows/cases/1746-sandbox-launcher-token-authority/run.mjs +++ b/tests/relayflows/cases/1746-sandbox-launcher-token-authority/run.mjs @@ -120,6 +120,8 @@ test('sandbox dispatch constructs an agent client with exactly one authority', a spawn: vi.fn(async () => ({ invocationId: 'inv_relayflow_1746', node: { name: 'daytona-worker' }, + // Sandbox spawns fail closed unless the dispatch receipt names the sandbox node. + dispatchedNodeId: 'node-daytona', })), }; let constructorOptions: Record | undefined; diff --git a/tests/relayflows/cases/sandbox-readonly-paths/case.json b/tests/relayflows/cases/sandbox-readonly-paths/case.json new file mode 100644 index 0000000000..fd0d6c2997 --- /dev/null +++ b/tests/relayflows/cases/sandbox-readonly-paths/case.json @@ -0,0 +1,20 @@ +{ + "version": 1, + "id": "sandbox-readonly-paths", + "kind": "feature", + "title": "Forward read-only Relayfile subtree requests to Cloud", + "runner": { + "command": ["node", "tests/relayflows/cases/sandbox-readonly-paths/run.mjs"] + }, + "timeoutSeconds": 900, + "expected": { + "base": { + "outcome": "absent", + "signature": "sandbox_readonly_paths_not_forwarded" + }, + "head": { + "outcome": "fixed", + "signature": "sandbox_readonly_paths_forwarded" + } + } +} diff --git a/tests/relayflows/cases/sandbox-readonly-paths/run.mjs b/tests/relayflows/cases/sandbox-readonly-paths/run.mjs new file mode 100644 index 0000000000..693fdc8eea --- /dev/null +++ b/tests/relayflows/cases/sandbox-readonly-paths/run.mjs @@ -0,0 +1,264 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { lstat, mkdir, open, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const CASE_ID = 'sandbox-readonly-paths'; +const COMMAND_TIMEOUT_MS = 5 * 60 * 1000; +const targetDir = requiredDirectory('RELAY_PR_PROOF_TARGET_DIR'); +const harnessDir = requiredDirectory('RELAY_PR_PROOF_HARNESS_DIR'); +const resultPath = requiredValue('RELAY_PR_PROOF_RESULT_PATH'); +const arm = requiredValue('RELAY_PR_PROOF_ARM'); + +if (arm !== 'base' && arm !== 'head') { + throw new Error(`RELAY_PR_PROOF_ARM must be base or head, received ${JSON.stringify(arm)}.`); +} + +const expectedSha = + arm === 'base' ? process.env.RELAY_PR_PROOF_BASE_SHA : process.env.RELAY_PR_PROOF_HEAD_SHA; +if (!expectedSha) throw new Error(`Missing expected ${arm} SHA.`); +const targetSha = execFileSync('git', ['-C', targetDir, 'rev-parse', 'HEAD'], { + encoding: 'utf8', +}).trim(); +if (targetSha !== expectedSha) { + throw new Error(`Target checkout ${targetSha} does not match exact ${arm} SHA ${expectedSha}.`); +} + +const runnerPath = fileURLToPath(import.meta.url); +if (!isWithin(harnessDir, runnerPath)) { + throw new Error('The RelayFlow runner must execute from the exact-head harness checkout.'); +} + +const probePath = path.join(targetDir, 'packages/cloud/src/.relayflow-sandbox-readonly-paths.test.ts'); +const probeObservationPath = path.join(targetDir, '.relayflow-sandbox-readonly-paths-observation.json'); +const probeConfigPath = path.join(targetDir, '.relayflow-sandbox-readonly-paths.vitest.config.mjs'); + +const probeSource = String.raw`import { afterEach, expect, test, vi } from 'vitest'; +import { writeFile } from 'node:fs/promises'; + +const mocks = vi.hoisted(() => ({ + ensureCloudSession: vi.fn(), + authorizedApiFetch: vi.fn(), +})); + +vi.mock('./auth.js', () => ({ + ensureCloudSession: mocks.ensureCloudSession, + authorizedApiFetch: mocks.authorizedApiFetch, +})); + +import { ensureCloudFleetSandbox } from './fleet-sandbox.js'; + +const auth = { + accessToken: 'relayflow-probe-access', + refreshToken: 'relayflow-probe-refresh', + accessTokenExpiresAt: '2099-01-01T00:00:00Z', + apiUrl: 'https://relayflow.invalid', +}; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +test('observes exact Relayfile subtree forwarding', async () => { + const observationPath = process.env.RELAY_READONLY_OBSERVATION_PATH; + if (!observationPath) throw new Error('Missing RELAY_READONLY_OBSERVATION_PATH.'); + + mocks.ensureCloudSession.mockResolvedValue({ auth, client: {} }); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99' }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeId: 'node-relayflow', + nodeName: 'scoped-relayflow', + sandboxId: 'sandbox-relayflow', + relayWorkspaceId: 'rw_relayflow', + relayfileMounted: true, + providerId: 'daytona', + providerSandboxId: '223e4567-e89b-42d3-a456-426614174000', + }, + { status: 201 } + ), + auth, + }); + + await ensureCloudFleetSandbox({ + workspaceId: 'rw_relayflow', + requiredCapability: 'spawn:claude', + mountRelayfile: true, + readonlyPaths: ['/live-review/run-123/**', '/live-review/run-123/reviews/**'], + }); + + const ensureRequest = mocks.authorizedApiFetch.mock.calls[1]?.[2]; + expect(ensureRequest?.body).toEqual(expect.any(String)); + const ensureBody = JSON.parse(ensureRequest.body); + + await writeFile( + observationPath, + JSON.stringify({ readonlyPaths: ensureBody.readonlyPaths ?? null }), + 'utf8' + ); +}); +`; + +const probeConfigSource = `export default { + test: { + environment: 'node', + include: ['packages/cloud/src/.relayflow-sandbox-readonly-paths.test.ts'], + setupFiles: [], + }, +};\n`; + +try { + run('npm', ['ci', '--ignore-scripts'], targetDir, 'workspace dependency installation'); + run('npm', ['run', 'build:session'], targetDir, 'session package build'); + run('npm', ['run', 'build:config'], targetDir, 'configuration package build'); + run('npm', ['run', 'build:cloud'], targetDir, 'Cloud package build'); + run('npm', ['run', 'build:utils'], targetDir, 'utilities package build'); + run('npm', ['run', 'build:policy'], targetDir, 'policy package build'); + run('npm', ['run', 'build:sdk'], targetDir, 'SDK package build'); + run('npm', ['run', 'build:harness-driver'], targetDir, 'harness driver package build'); + run('npm', ['run', 'build:harnesses'], targetDir, 'harnesses package build'); + run('npm', ['run', 'build:fleet'], targetDir, 'fleet package build'); + run('npm', ['run', 'build:cli-surface'], targetDir, 'CLI surface package build'); + run('npm', ['run', 'build:cli'], targetDir, 'CLI package build'); + + await writeGeneratedFile(probePath, probeSource); + await writeGeneratedFile(probeConfigPath, probeConfigSource); + run( + 'npm', + ['exec', '--', 'vitest', 'run', '--config', path.relative(targetDir, probeConfigPath)], + targetDir, + 'Relayfile subtree forwarding probe', + { RELAY_READONLY_OBSERVATION_PATH: probeObservationPath } + ); + + const observation = JSON.parse(await readFile(probeObservationPath, 'utf8')); + const cliHelp = runCapture( + process.execPath, + [path.join(targetDir, 'packages/cli/dist/cli/index.js'), 'fleet', 'spawn', '--help'], + targetDir, + 'compiled fleet spawn help' + ); + const cliHasScopedMountFlag = cliHelp.includes('--sandbox-readonly-path '); + const requestedPaths = ['/live-review/run-123/**', '/live-review/run-123/reviews/**']; + const baseObserved = observation.readonlyPaths === null && !cliHasScopedMountFlag; + const headObserved = + JSON.stringify(observation.readonlyPaths) === JSON.stringify(requestedPaths) && cliHasScopedMountFlag; + + let outcome; + let signature; + let details; + if (baseObserved) { + outcome = 'absent'; + signature = 'sandbox_readonly_paths_not_forwarded'; + details = + 'The base Cloud client dropped requested Relayfile subtrees and the fleet spawn CLI exposed no read-only-mount flag.'; + } else if (headObserved) { + outcome = 'fixed'; + signature = 'sandbox_readonly_paths_forwarded'; + details = + 'The head Cloud client forwarded the exact requested Relayfile subtrees and the fleet spawn CLI exposed the read-only-mount flag.'; + } else { + throw new Error( + `Unexpected read-only Relayfile mount observation: ${JSON.stringify({ + ...observation, + cliHasScopedMountFlag, + })}.` + ); + } + + await mkdir(path.dirname(resultPath), { recursive: true }); + await writeFile( + resultPath, + `${JSON.stringify({ version: 1, caseId: CASE_ID, arm, outcome, signature, details })}\n`, + 'utf8' + ); +} finally { + await rm(probePath, { force: true }); + await rm(probeConfigPath, { force: true }); + await rm(probeObservationPath, { force: true }); +} + +function requiredValue(name) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`Missing required environment variable ${name}.`); + return value; +} + +function requiredDirectory(name) { + return path.resolve(requiredValue(name)); +} + +function isWithin(directory, candidate) { + const relative = path.relative(directory, candidate); + return ( + relative === '' || + (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) + ); +} + +async function writeGeneratedFile(targetPath, source) { + try { + const existing = await lstat(targetPath); + if (!existing.isFile()) { + throw new Error(`Refusing to replace non-regular generated file ${targetPath}.`); + } + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + + const temporaryPath = `${targetPath}.tmp-${process.pid}-${randomUUID()}`; + try { + const handle = await open(temporaryPath, 'wx', 0o600); + try { + await handle.writeFile(source, 'utf8'); + } finally { + await handle.close(); + } + await rename(temporaryPath, targetPath); + } finally { + await rm(temporaryPath, { force: true }); + } +} + +function run(command, args, cwd, label, extraEnv = {}) { + const completed = spawnSync(command, args, { + cwd, + env: { ...process.env, ...extraEnv }, + stdio: ['ignore', 'inherit', 'inherit'], + timeout: COMMAND_TIMEOUT_MS, + }); + if (completed.error) throw new Error(`${label} could not start: ${completed.error.message}`); + if (completed.status !== 0) { + throw new Error( + `${label} failed with ${ + completed.signal ? `signal ${completed.signal}` : `exit code ${completed.status ?? 'unknown'}` + }.` + ); + } +} + +function runCapture(command, args, cwd, label, extraEnv = {}) { + const completed = spawnSync(command, args, { + cwd, + env: { ...process.env, ...extraEnv }, + encoding: 'utf8', + timeout: COMMAND_TIMEOUT_MS, + }); + if (completed.error) throw new Error(`${label} could not start: ${completed.error.message}`); + if (completed.status !== 0) { + throw new Error( + `${label} failed with ${ + completed.signal ? `signal ${completed.signal}` : `exit code ${completed.status ?? 'unknown'}` + }: ${completed.stderr?.slice(-2000) ?? ''}` + ); + } + return completed.stdout; +}