From eff5f79926c0804bb6d939c13509c3fada613d34 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 11:04:00 +0200 Subject: [PATCH 01/50] fix(fleet): preserve Cloud sandbox identity across cutover Session-Id: 01a08039-c131-7032-9591-9077c44d4569 Session-Id: 01a08039-c131-7032-9591-9077c44d4569 Session-Id: 01a08039-c131-7032-9591-9077c44d4569 Session-Id: 01a08039-c131-7032-9591-9077c44d4569 --- CHANGELOG.md | 6 +- packages/cli/README.md | 8 +- packages/cli/src/cli/commands/fleet.test.ts | 135 +++++++++++++++- packages/cli/src/cli/commands/fleet.ts | 31 +++- packages/cloud/src/fleet-sandbox.test.ts | 150 +++++++++++++++++- packages/cloud/src/fleet-sandbox.ts | 90 ++++++++++- .../case.json | 6 +- .../1656-long-running-agent37-sandbox/run.mjs | 98 +++++++++--- 8 files changed, 469 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cbd402e81..877d61184a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Patch] + +### Changed + +- `agent-relay fleet spawn --sandbox` now forwards a one-to-one `sbx_` identity with a deterministic `fleet-sandbox-` node name for Cloud long-running workloads, rejects arbitrary `--sandbox-name` values, and supports `--sandbox-id` replay after an unknown provisioning outcome. ## [11.10.4] - 2026-09-08 diff --git a/packages/cli/README.md b/packages/cli/README.md index 5065737f7f..06dfffe3e1 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -179,7 +179,13 @@ sees the same synced Relayfile workspace. Use `--sandbox-provider daytona` or `--sandbox-provider e2b` to require an operator-enabled provider; omit the flag to let Cloud's sandbox router choose. Pass `--no-sandbox-relayfile` only when a deliberately bare sandbox is desired. If provisioning times out or the spawn -fails, Relay asks Cloud to delete the newly created sandbox. +fails, Relay asks Cloud to delete the newly created sandbox. Each generation +uses one `sbx_` identity and the matching `fleet-sandbox-` node +name; a custom `--sandbox-name` is rejected unless it is that exact generated +name. If provisioning ends with an unknown outcome, rerun the command with the +warning's `--sandbox-id ` (and optionally its matching +`--sandbox-name`) to replay the same Cloud identity instead of adopting another +fleet node. Large workspaces should select only the live subtree an agent needs. Pass one or more explicit directory roots after `--sandbox-relayfile-path`; Cloud diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index a02af8a467..8e9ed7adc3 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -46,6 +46,9 @@ vi.mock('@agent-relay/harness-driver', async (importOriginal) => ({ import { registerFleetCommands } from './fleet.js'; import { writeProjectWorkspaceKey } from '../lib/project-workspace-key.js'; +const REPLAY_SANDBOX_ID = 'sbx_123e4567-e89b-42d3-a456-426614174000'; +const REPLAY_SANDBOX_NAME = 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000'; + const LIVE_AGENT_CAPABILITY_NAME = 'relay:live-agents:v1'; const liveAgentCapabilities = (...names: string[]) => [ { @@ -668,6 +671,7 @@ describe('fleet command support', () => { nodeId: 'node-1', nodeName: 'e2b-codex', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, relayfileMountPath: '/workspace', @@ -701,10 +705,12 @@ describe('fleet command support', () => { '--sandbox', '--sandbox-provider', 'e2b', + '--sandbox-id', + REPLAY_SANDBOX_ID, + '--sandbox-name', + REPLAY_SANDBOX_NAME, '--sandbox-relayfile-path', '/live-review/run-123/**', - '--sandbox-name', - 'e2b-codex', '--name', 'sandbox-worker', '--task', @@ -714,18 +720,21 @@ describe('fleet command support', () => { ], { from: 'user' } ); - expect(ensureCloudFleetSandbox).toHaveBeenCalledWith({ + const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0]; + expect(ensureInput).toEqual({ workspaceId: 'rw_abc', requiredCapability: 'spawn:codex', maxAgents: 1, mountRelayfile: true, relayfilePaths: ['/live-review/run-123/**'], + sandboxId: REPLAY_SANDBOX_ID, forceProvision: true, providerId: 'e2b', workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, - name: 'e2b-codex', + name: REPLAY_SANDBOX_NAME, }); + expect(ensureInput?.name).toBe(`fleet-sandbox-${ensureInput?.sandboxId?.slice('sbx_'.length)}`); expect(register).toHaveBeenCalledWith( expect.objectContaining({ name: expect.stringMatching(/^fleet-spawn-launcher-[a-f0-9]{8}$/), @@ -772,6 +781,7 @@ describe('fleet command support', () => { nodeId: 'node-1', nodeName: 'agent37-codex', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, relayfileMountPath: '/workspace', @@ -834,6 +844,111 @@ describe('fleet command support', () => { }); }); + it('rejects a custom sandbox name before provisioning because identity is one-to-one', async () => { + const ensureCloudFleetSandbox = vi.fn(); + const errors: string[] = []; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: (...args: unknown[]) => errors.push(args.join(' ')), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-name', + 'custom-node', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(ensureCloudFleetSandbox).not.toHaveBeenCalled(); + expect(errors.join('\n')).toContain('--sandbox-name'); + expect(errors.join('\n')).toContain('one-to-one sandbox identity'); + }); + + it('rejects an invalid replay sandbox ID before provisioning', async () => { + const ensureCloudFleetSandbox = vi.fn(); + const errors: string[] = []; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: (...args: unknown[]) => errors.push(args.join(' ')), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-id', + 'sandbox-1', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(ensureCloudFleetSandbox).not.toHaveBeenCalled(); + expect(errors.join('\n')).toContain('--sandbox-id must match sbx_'); + }); + it('cleans up when Cloud reports a post-provision response failure with a sandbox ID', async () => { const deleteCloudFleetSandbox = vi.fn(async () => undefined); const program = new Command(); @@ -873,8 +988,10 @@ describe('fleet command support', () => { 'spawn', 'codex', '--sandbox', + '--sandbox-id', + REPLAY_SANDBOX_ID, '--sandbox-name', - 'daytona-codex', + REPLAY_SANDBOX_NAME, '--name', 'sandbox-worker', '--task', @@ -936,8 +1053,6 @@ describe('fleet command support', () => { '--sandbox', '--sandbox-provider', 'e2b', - '--sandbox-name', - 'e2b-codex', '--name', 'sandbox-worker', '--task', @@ -997,8 +1112,10 @@ describe('fleet command support', () => { 'spawn', 'codex', '--sandbox', + '--sandbox-id', + REPLAY_SANDBOX_ID, '--sandbox-name', - 'daytona-codex', + REPLAY_SANDBOX_NAME, '--name', 'sandbox-worker', '--task', @@ -1014,6 +1131,7 @@ describe('fleet command support', () => { expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); expect(warnings.join('\n')).toContain("check Cloud Fleet for node 'daytona-codex'"); + expect(warnings.join('\n')).toContain(`--sandbox-id '${REPLAY_SANDBOX_ID}'`); }); it('warns when an unmounted sandbox cannot be cleaned up automatically', async () => { @@ -1040,6 +1158,7 @@ describe('fleet command support', () => { nodeId: 'node-1', nodeName: 'daytona-codex', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: false, providerId: 'agent37' as const, diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 2eebd392ee..e4b200a0a0 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -47,6 +47,8 @@ const SERVE_REPLACEMENT_MESSAGE = "for Cloud-managed nodes run 'relay cloud enroll --token ' first."; const FLEET_CLIS = new Set(['claude', 'codex', 'gemini', 'aider', 'goose', 'grok', 'opencode']); +const CLOUD_SANDBOX_ID_PATTERN = + /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; export interface FleetCommandDependencies { core: CoreDependencies; @@ -161,7 +163,11 @@ export function registerFleetCommands( '--sandbox', 'Provision a fresh Cloud sandbox node, mount this Relayfile workspace, and spawn there' ) - .option('--sandbox-name ', 'Name for the provisioned sandbox fleet node') + .option( + '--sandbox-name ', + 'Explicit sandbox node name (must match the generated fleet-sandbox- identity)' + ) + .option('--sandbox-id ', 'Reuse a caller-declared sbx_ identity for an exact replay') .option('--sandbox-provider ', 'Sandbox provider: daytona or e2b') .option( '--sandbox-relayfile-path ', @@ -196,6 +202,10 @@ export function registerFleetCommands( let targetNode = optionalText(options.targetNode, 'Target node') ?? optionalText(options.node, 'Node'); const useSandbox = options.sandbox === true; const sandboxName = optionalText(options.sandboxName, 'Sandbox name'); + const sandboxIdOption = optionalText(options.sandboxId, 'Sandbox ID'); + if (sandboxIdOption !== undefined && !CLOUD_SANDBOX_ID_PATTERN.test(sandboxIdOption)) { + throw new Error('--sandbox-id must match sbx_ using an RFC 4122 UUID.'); + } const sandboxProviderText = optionalText(options.sandboxProvider, 'Sandbox provider'); const sandboxProvider: CloudFleetSandboxProviderId | undefined = sandboxProviderText === undefined @@ -214,6 +224,9 @@ export function registerFleetCommands( if (!useSandbox && sandboxName) { throw new Error('--sandbox-name requires --sandbox.'); } + if (!useSandbox && sandboxIdOption) { + throw new Error('--sandbox-id requires --sandbox.'); + } if (!useSandbox && sandboxProvider) { throw new Error('--sandbox-provider requires --sandbox.'); } @@ -254,7 +267,14 @@ export function registerFleetCommands( if (!relayWorkspaceId) { throw new Error('The current Relay workspace did not report an ID for Cloud provisioning.'); } - const requestedSandboxName = sandboxName ?? `fleet-sandbox-${randomUUID().slice(0, 8)}`; + const sandboxId = sandboxIdOption ?? `sbx_${randomUUID()}`; + const sandboxGenerationUuid = sandboxId.slice('sbx_'.length); + const deterministicSandboxName = `fleet-sandbox-${sandboxGenerationUuid}`; + if (sandboxName !== undefined && sandboxName !== deterministicSandboxName) { + throw new Error( + `--sandbox-name must be '${deterministicSandboxName}' for a long-running sandbox; custom names cannot preserve the one-to-one sandbox identity.` + ); + } try { sandbox = await deps.ensureCloudFleetSandbox({ workspaceId: relayWorkspaceId, @@ -262,11 +282,12 @@ export function registerFleetCommands( maxAgents: 1, mountRelayfile: mountSandboxRelayfile, ...(sandboxRelayfilePaths === undefined ? {} : { relayfilePaths: sandboxRelayfilePaths }), + sandboxId, forceProvision: true, ...(sandboxProvider === undefined ? {} : { providerId: sandboxProvider }), workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, - name: requestedSandboxName, + name: deterministicSandboxName, }); } catch (error) { if (error instanceof CloudFleetSandboxProvisionError && error.cloudWorkspaceId && error.sandboxId) { @@ -286,8 +307,8 @@ export function registerFleetCommands( } else if (error instanceof CloudFleetSandboxProvisionError && error.outcomeUnknown) { deps.warn( `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ - error.nodeName ?? requestedSandboxName - }' before retrying so a sandbox is not left running.` + error.nodeName ?? deterministicSandboxName + }' before retrying with --sandbox-id '${sandboxId}' so a sandbox is not left running.` ); } throw error; diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index c9348c0174..b56ffef07f 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -24,6 +24,8 @@ const auth = { }; const refreshedAuth = { ...auth, accessToken: 'refreshed' }; const CLOUD_WORKSPACE_ID = '50587328-441d-4acb-b8f3-dbe1b3c5de99'; +const SANDBOX_ID = 'sbx_123e4567-e89b-42d3-a456-426614174000'; +const SANDBOX_NAME = 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000'; describe('Cloud fleet sandbox client', () => { beforeEach(() => { @@ -47,7 +49,8 @@ describe('Cloud fleet sandbox client', () => { outcome: 'provisioned', nodeId: 'node-1', nodeName: 'daytona-codex', - sandboxId: 'sandbox-1', + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, relayfileMountPath: '/workspace', @@ -60,7 +63,8 @@ describe('Cloud fleet sandbox client', () => { const result = await ensureCloudFleetSandbox({ workspaceId: 'rw_abc', - name: 'daytona-codex', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, requiredCapability: 'spawn:codex', maxAgents: 1, mountRelayfile: true, @@ -81,7 +85,8 @@ describe('Cloud fleet sandbox client', () => { expect(ensureCall?.[1]).toBe('/api/v1/fleet/nodes/sandbox/ensure'); expect(JSON.parse(String(ensureCall?.[2]?.body))).toEqual({ workspaceId: CLOUD_WORKSPACE_ID, - name: 'daytona-codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, requiredCapability: 'spawn:codex', maxAgents: 1, mountRelayfile: true, @@ -94,7 +99,8 @@ describe('Cloud fleet sandbox client', () => { cloudWorkspaceId: CLOUD_WORKSPACE_ID, nodeId: 'node-1', nodeName: 'daytona-codex', - sandboxId: 'sandbox-1', + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, relayfileMountPath: '/workspace', @@ -102,6 +108,98 @@ describe('Cloud fleet sandbox client', () => { }); }); + it('rejects an invalid sandbox identity before contacting Cloud', async () => { + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: 'sandbox-1', + forceProvision: true, + }) + ).rejects.toThrow('sandboxId must match sbx_'); + + expect(mocks.ensureCloudSession).not.toHaveBeenCalled(); + expect(mocks.authorizedApiFetch).not.toHaveBeenCalled(); + }); + + it('requires a stable identity and deterministic name for long-running requests', async () => { + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + forceProvision: true, + workloadProfile: 'long-running-agent', + }) + ).rejects.toThrow('require sandboxId'); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: 'custom-name', + forceProvision: true, + workloadProfile: 'long-running-agent', + }) + ).rejects.toThrow(SANDBOX_NAME); + + expect(mocks.ensureCloudSession).not.toHaveBeenCalled(); + expect(mocks.authorizedApiFetch).not.toHaveBeenCalled(); + }); + + it.each(['provisioned', 'provisioning_timeout'] as const)( + 'fails closed when Cloud echoes a different sandbox identity for %s', + async (outcome) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + outcome === 'provisioned' + ? { + outcome, + nodeId: 'node-other', + nodeName: SANDBOX_NAME, + sandboxId: 'sbx_223e4567-e89b-42d3-a456-426614174000', + providerSandboxId: 'provider-sandbox-other', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + } + : { + outcome, + sandboxId: 'sbx_223e4567-e89b-42d3-a456-426614174000', + providerSandboxId: 'provider-sandbox-other', + relayWorkspaceId: 'rw_abc', + nodeName: SANDBOX_NAME, + waitedMs: 90_000, + }, + { status: outcome === 'provisioned' ? 201 : 202 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + outcomeUnknown: true, + sandboxId: undefined, + }); + expect(String(error)).toContain(`instead of requested sandboxId ${SANDBOX_ID}`); + } + ); + it('forwards a repos list into the ensure request body when the caller opts in', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -115,6 +213,7 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'jit-repos-node', sandboxId: 'sandbox-9', + providerSandboxId: 'provider-sandbox-9', relayWorkspaceId: 'rw_abc', relayfileMounted: true, }, @@ -152,6 +251,7 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'scoped-reviewer', sandboxId: 'sandbox-scoped', + providerSandboxId: 'provider-sandbox-scoped', relayWorkspaceId: 'rw_abc', relayfileMounted: true, }, @@ -203,6 +303,7 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-e2b', nodeName: 'e2b-reviewer', sandboxId: 'sandbox-e2b', + providerSandboxId: 'provider-sandbox-e2b', relayWorkspaceId: 'rw_abc', relayfileMounted: true, }, @@ -274,6 +375,7 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-future', nodeName: 'future-reviewer', sandboxId: 'sandbox-future', + providerSandboxId: 'provider-sandbox-future', relayWorkspaceId: 'rw_abc', relayfileMounted: true, }, @@ -309,6 +411,7 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'wrong-provider', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, }, @@ -351,6 +454,7 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-e2b', nodeName: 'e2b-reviewer', sandboxId: 'sandbox-e2b', + providerSandboxId: 'provider-sandbox-e2b', relayWorkspaceId: 'rw_abc', relayfileMounted: true, ...(providerId === undefined ? {} : { providerId }), @@ -393,6 +497,7 @@ describe('Cloud fleet sandbox client', () => { { outcome: 'provisioning_timeout', sandboxId: 'sandbox-e2b', + providerSandboxId: 'provider-sandbox-e2b', relayWorkspaceId: 'rw_abc', nodeName: 'e2b-reviewer', waitedMs: 90_000, @@ -424,7 +529,7 @@ describe('Cloud fleet sandbox client', () => { ['missing', undefined], ['invalid', 'modal'], ])( - 'preserves the requested provider for cleanup when a %s provider proof arrives on a non-OK response', + 'preserves the public cleanup identity when a %s provider proof arrives on a non-OK response', async (_case, providerId) => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -437,6 +542,7 @@ describe('Cloud fleet sandbox client', () => { error: 'provider rejected request', nodeName: 'e2b-reviewer', sandboxId: 'sandbox-e2b', + providerSandboxId: 'provider-sandbox-e2b', ...(providerId === undefined ? {} : { providerId }), }, { status: 502 } @@ -473,6 +579,7 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'bare-node', sandboxId: 'sandbox-10', + providerSandboxId: 'provider-sandbox-10', relayWorkspaceId: 'rw_abc', relayfileMounted: true, }, @@ -504,6 +611,7 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'bare-node', sandboxId: 'sandbox-11', + providerSandboxId: 'provider-sandbox-11', relayWorkspaceId: 'rw_abc', relayfileMounted: true, }, @@ -534,6 +642,7 @@ describe('Cloud fleet sandbox client', () => { { outcome: 'provisioning_timeout', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', nodeName: 'daytona-codex', waitedMs: 90_000, @@ -552,6 +661,7 @@ describe('Cloud fleet sandbox client', () => { ).resolves.toMatchObject({ outcome: 'provisioning_timeout', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', nodeName: 'daytona-codex', waitedMs: 90_000, }); @@ -578,6 +688,7 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'daytona-codex', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, providerId: 'daytona', @@ -617,6 +728,7 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-1', nodeName: 'daytona-codex', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, providerId: 'daytona', @@ -676,6 +788,7 @@ describe('Cloud fleet sandbox client', () => { outcome: 'provisioned', nodeName: 'daytona-codex', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, }, @@ -698,6 +811,32 @@ describe('Cloud fleet sandbox client', () => { }); }); + it('does not copy a caller sandbox identity into an unknown-response error', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockRejectedValueOnce(new Error('request timed out')); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + outcomeUnknown: true, + }); + expect(error).toMatchObject({ sandboxId: undefined }); + }); + it('rejects a timeout response that omits waitedMs', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -709,6 +848,7 @@ describe('Cloud fleet sandbox client', () => { { outcome: 'provisioning_timeout', sandboxId: 'sandbox-1', + providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', nodeName: 'daytona-codex', }, diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 51f6702b8c..1e2940fc73 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -5,6 +5,8 @@ import { defaultApiUrl } from './types.js'; type JsonRecord = Record; const CLOUD_WORKSPACE_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const CLOUD_SANDBOX_ID_PATTERN = + /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; const DEFAULT_RESOLUTION_TIMEOUT_MS = 120_000; // Mounted provisioning can spend up to 240s completing the initial Relayfile // sync, then up to 90s waiting for the enrolled node to report ready. Leave a @@ -60,9 +62,13 @@ export class CloudFleetSandboxProvisionError extends Error { } } +class CloudFleetSandboxIdentityMismatchError extends Error {} + export type EnsureCloudFleetSandboxInput = { /** Cloud UUID or unified rw_* workspace id. */ workspaceId: string; + /** Caller-declared one-time Cloud identity used to resume a cut-off provision. */ + sandboxId?: string; name?: string; requiredCapability: string; maxAgents?: number; @@ -110,6 +116,7 @@ export type CloudFleetSandboxReady = { nodeId: string; nodeName: string; sandboxId: string; + providerSandboxId: string; relayWorkspaceId: string; relayfileMounted: boolean; relayfileMountPath?: string; @@ -131,6 +138,7 @@ export type CloudFleetSandboxProvisioningTimeout = { outcome: 'provisioning_timeout'; cloudWorkspaceId: string; sandboxId: string; + providerSandboxId: string; relayWorkspaceId: string; nodeName: string; waitedMs: number; @@ -214,6 +222,51 @@ function requiredString(payload: JsonRecord, key: string, context: string): stri return value; } +function validateSandboxIdentity(input: EnsureCloudFleetSandboxInput): { + sandboxId?: string; + name?: string; +} { + if (input.sandboxId !== undefined && typeof input.sandboxId !== 'string') { + throw new Error('Cloud fleet sandbox sandboxId must be a string.'); + } + if (input.name !== undefined && typeof input.name !== 'string') { + throw new Error('Cloud fleet sandbox name must be a string.'); + } + const sandboxId = input.sandboxId?.trim(); + const name = input.name?.trim(); + if (input.sandboxId !== undefined && (!sandboxId || !CLOUD_SANDBOX_ID_PATTERN.test(sandboxId))) { + throw new Error('Cloud fleet sandbox sandboxId must match sbx_ using an RFC 4122 UUID.'); + } + if (sandboxId !== undefined && input.forceProvision !== true) { + throw new Error('Cloud fleet sandbox sandboxId requires forceProvision: true.'); + } + if (sandboxId !== undefined && !name) { + throw new Error('Cloud fleet sandbox sandboxId requires a node name.'); + } + + const longRunning = + input.workloadProfile === 'long-running-agent' || input.workloadProfile === 'standard-long-running-agent'; + if (longRunning) { + if (sandboxId === undefined) { + throw new Error('Long-running Cloud fleet sandbox requests require sandboxId.'); + } + if (input.forceProvision !== true) { + throw new Error('Long-running Cloud fleet sandbox requests require forceProvision: true.'); + } + const expectedName = `fleet-sandbox-${sandboxId.slice('sbx_'.length)}`; + if (name !== expectedName) { + throw new Error( + `Long-running Cloud fleet sandbox requests require name '${expectedName}' to preserve the one-to-one sandbox identity.` + ); + } + } + + return { + ...(sandboxId === undefined ? {} : { sandboxId }), + ...(name === undefined ? {} : { name }), + }; +} + async function resolveCloudWorkspaceId( workspaceId: string, auth: Awaited>['auth'], @@ -270,6 +323,7 @@ function cleanupProviderId( function normalizeEnsureResult( payload: unknown, cloudWorkspaceId: string, + expectedSandboxId?: string, requestedProviderId?: CloudFleetSandboxProviderId ): EnsureCloudFleetSandboxResult { if (!isObject(payload)) throw new Error('Cloud fleet sandbox response was not valid JSON.'); @@ -288,12 +342,19 @@ function normalizeEnsureResult( if (typeof payload.relayfileMounted !== 'boolean') { throw new Error('Cloud fleet sandbox response is missing relayfileMounted.'); } + const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); + if (expectedSandboxId !== undefined && sandboxId !== expectedSandboxId) { + throw new CloudFleetSandboxIdentityMismatchError( + `Cloud returned sandboxId ${sandboxId} instead of requested sandboxId ${expectedSandboxId}.` + ); + } return { outcome, cloudWorkspaceId, nodeId: requiredString(payload, 'nodeId', 'Cloud fleet sandbox'), nodeName, - sandboxId: requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'), + sandboxId, + providerSandboxId: requiredString(payload, 'providerSandboxId', 'Cloud fleet sandbox'), relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), relayfileMounted: payload.relayfileMounted, ...(providerId === undefined ? {} : { providerId }), @@ -317,10 +378,17 @@ function normalizeEnsureResult( } if (outcome === 'provisioning_timeout') { + const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); + if (expectedSandboxId !== undefined && sandboxId !== expectedSandboxId) { + throw new CloudFleetSandboxIdentityMismatchError( + `Cloud returned sandboxId ${sandboxId} instead of requested sandboxId ${expectedSandboxId}.` + ); + } return { outcome, cloudWorkspaceId, - sandboxId: requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'), + sandboxId, + providerSandboxId: requiredString(payload, 'providerSandboxId', 'Cloud fleet sandbox'), relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), nodeName, waitedMs: requiredNumber(payload, 'waitedMs', 'Cloud fleet sandbox'), @@ -340,6 +408,7 @@ export async function ensureCloudFleetSandbox( const requiredCapability = input.requiredCapability.trim(); if (!workspaceId) throw new Error('A workspace ID is required to provision a fleet sandbox.'); if (!requiredCapability) throw new Error('A spawn capability is required to provision a fleet sandbox.'); + const sandboxIdentity = validateSandboxIdentity(input); if (input.relayfilePaths !== undefined && input.relayfilePaths.length === 0) { throw new Error('At least one Relayfile subtree path is required when relayfilePaths is provided.'); } @@ -362,7 +431,8 @@ export async function ensureCloudFleetSandbox( body: JSON.stringify({ workspaceId: resolved.cloudWorkspaceId, requiredCapability, - ...(input.name ? { name: input.name } : {}), + ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), + ...(sandboxIdentity.name === undefined ? {} : { name: sandboxIdentity.name }), ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), ...(input.relayfilePaths === undefined ? {} : { relayfilePaths: [...input.relayfilePaths] }), @@ -407,16 +477,22 @@ export async function ensureCloudFleetSandbox( throw error; } try { - return normalizeEnsureResult(payload, resolved.cloudWorkspaceId, input.providerId); + return normalizeEnsureResult( + payload, + resolved.cloudWorkspaceId, + sandboxIdentity.sandboxId, + input.providerId + ); } catch (error) { const providerId = isObject(payload) ? cleanupProviderId(payload, input.providerId) : input.providerId; + const identityMismatch = error instanceof CloudFleetSandboxIdentityMismatchError; + const cleanupSandboxId = + !identityMismatch && isObject(payload) ? readString(payload, 'sandboxId') : undefined; throw new CloudFleetSandboxProvisionError( error instanceof Error ? error.message : 'Cloud fleet sandbox response was invalid.', { cloudWorkspaceId: resolved.cloudWorkspaceId, - ...(isObject(payload) && readString(payload, 'sandboxId') - ? { sandboxId: readString(payload, 'sandboxId') } - : {}), + ...(cleanupSandboxId === undefined ? {} : { sandboxId: cleanupSandboxId }), ...(isObject(payload) && (readString(payload, 'nodeName') ?? input.name) ? { nodeName: readString(payload, 'nodeName') ?? input.name } : {}), diff --git a/tests/relayflows/cases/1656-long-running-agent37-sandbox/case.json b/tests/relayflows/cases/1656-long-running-agent37-sandbox/case.json index be05a4b080..5d19739175 100644 --- a/tests/relayflows/cases/1656-long-running-agent37-sandbox/case.json +++ b/tests/relayflows/cases/1656-long-running-agent37-sandbox/case.json @@ -2,7 +2,7 @@ "version": 1, "id": "1656-long-running-agent37-sandbox", "kind": "feature", - "title": "Route unpinned fleet sandboxes with long-running semantics and preserve Agent37 attribution", + "title": "Replay fleet sandbox identities with long-running semantics and preserve Agent37 attribution", "runner": { "command": ["node", "tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs"] }, @@ -10,11 +10,11 @@ "expected": { "base": { "outcome": "absent", - "signature": "long_running_profile_and_agent37_attribution_absent" + "signature": "long_running_profile_replay_identity_absent_agent37_attribution_preserved" }, "head": { "outcome": "fixed", - "signature": "long_running_profile_and_agent37_attribution_preserved" + "signature": "long_running_profile_replay_identity_and_agent37_attribution_preserved" } } } diff --git a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs index 0f330762ae..42fea222ac 100644 --- a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs +++ b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs @@ -1,14 +1,14 @@ /** * relay#1656 — `agent-relay fleet spawn --sandbox` must ask Cloud for - * long-running semantics and must clean up against the provider Cloud actually - * chose. + * long-running semantics with a one-to-one sandbox identity and must clean up + * against the provider Cloud actually chose. * * The claim has two halves and they live in different packages, so the probe * runs them as one chain rather than as two independent assertions: * * `fleet spawn --sandbox` (packages/cli) * -> ensureCloudFleetSandbox (packages/cloud, REAL) - * -> POST /fleet/nodes/sandbox/ensure <- workloadProfile observed here + * -> POST /fleet/nodes/sandbox/ensure <- identity/profile observed here * <- Cloud answers `providerId: agent37` * -> deleteCloudFleetSandbox (packages/cloud, REAL) * -> DELETE /fleet/nodes/sandbox/ <- providerId observed here @@ -32,10 +32,13 @@ * Dispatch is then failed on purpose, because the cleanup call is what carries * the attribution. * - * Base: the CLI sends no workload profile, and `agent37` is not a provider the - * Cloud client will parse, so it is dropped and cleanup names no provider. - * Head: the profile reaches the ensure body and `agent37` reaches the delete - * body. + * Base: the CLI sends no sandbox identity, while the existing long-running + * profile and Cloud-selected `agent37` response are present; the older client + * drops provider attribution during cleanup. Head: an explicit replay + * `sbx_` identity, deterministic node name, and long-running profile + * reach the ensure body, while the exact public identity reaches cleanup and + * the separate physical provider sandbox ID is returned as evidence. Base does + * not know the replay flags, so it remains the negative arm. */ import { execFileSync, spawnSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; @@ -45,6 +48,9 @@ import process from 'node:process'; import { fileURLToPath } from 'node:url'; const CASE_ID = '1656-long-running-agent37-sandbox'; +const REPLAY_SANDBOX_ID = 'sbx_123e4567-e89b-42d3-a456-426614174000'; +const REPLAY_SANDBOX_NAME = 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000'; +const PROVIDER_SANDBOX_ID = 'provider-sandbox-relayflow'; // Separate budgets, both inside case.json's 900s per-arm deadline. The install // and the probe are very different jobs and a shared cap sizes neither: a // timeout on either is an INFRASTRUCTURE failure, which cannot report red or @@ -171,6 +177,9 @@ import { deleteCloudFleetSandbox, ensureCloudFleetSandbox } from './fleet-sandbo import { registerFleetCommands } from '../../cli/src/cli/commands/fleet.js'; const CLOUD_WORKSPACE_ID = '50587328-441d-4acb-b8f3-dbe1b3c5de99'; +const REPLAY_SANDBOX_ID = 'sbx_123e4567-e89b-42d3-a456-426614174000'; +const REPLAY_SANDBOX_NAME = 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000'; +const PROVIDER_SANDBOX_ID = 'provider-sandbox-relayflow'; const auth = { accessToken: 'relayflow-probe-access', refreshToken: 'relayflow-probe-refresh', @@ -178,11 +187,22 @@ const auth = { apiUrl: 'https://relayflow.invalid', }; -test('fleet spawn --sandbox reaches Cloud with a profile and cleans up by returned provider', async () => { +test('fleet spawn --sandbox replays an exact identity and cleans up by returned provider', async () => { const output = process.env.RELAY_PR1656_OBSERVATION_PATH; if (!output) throw new Error('Missing RELAY_PR1656_OBSERVATION_PATH.'); mocks.ensureCloudSession.mockResolvedValue({ auth, client: {} }); + const provisionedResponse = { + outcome: 'provisioned', + nodeId: 'node-relayflow', + nodeName: 'agent37-relayflow', + sandboxId: REPLAY_SANDBOX_ID, + providerSandboxId: PROVIDER_SANDBOX_ID, + relayWorkspaceId: 'rw_relayflow', + relayfileMounted: true, + relayfileMountPath: '/workspace', + providerId: 'agent37', + }; mocks.authorizedApiFetch // 1. workspace resolution .mockResolvedValueOnce({ response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), auth }) @@ -190,27 +210,22 @@ test('fleet spawn --sandbox reaches Cloud with a profile and cleans up by return // never names a provider, which is the whole point of the feature. .mockResolvedValueOnce({ response: Response.json( - { - outcome: 'provisioned', - nodeId: 'node-relayflow', - nodeName: 'agent37-relayflow', - sandboxId: 'sandbox-relayflow', - relayWorkspaceId: 'rw_relayflow', - relayfileMounted: true, - relayfileMountPath: '/workspace', - providerId: 'agent37', - }, + provisionedResponse, { status: 201 } ), auth, }) // 3. cleanup, triggered by the deliberate dispatch failure below .mockResolvedValueOnce({ - response: Response.json({ sandboxId: 'sandbox-relayflow', deleted: true }), + response: Response.json({ sandboxId: PROVIDER_SANDBOX_ID, deleted: true }), auth, }); const errors: string[] = []; + const replayArgs = + process.env.RELAY_PR_PROOF_ARM === 'head' + ? ['--sandbox-id', REPLAY_SANDBOX_ID, '--sandbox-name', REPLAY_SANDBOX_NAME] + : []; const program = new Command(); program.exitOverride(); registerFleetCommands(program, { @@ -251,6 +266,7 @@ test('fleet spawn --sandbox reaches Cloud with a profile and cleans up by return 'spawn', 'codex', '--sandbox', + ...replayArgs, '--name', 'sandbox-worker', '--task', @@ -279,7 +295,16 @@ test('fleet spawn --sandbox reaches Cloud with a profile and cleans up by return output, JSON.stringify({ ensureWorkloadProfile: ensureBody.workloadProfile ?? null, + ensureSandboxId: ensureBody.sandboxId ?? null, + ensureForceProvision: ensureBody.forceProvision ?? null, + ensureName: ensureBody.name ?? null, ensureProviderId: ensureBody.providerId ?? null, + responseSandboxId: provisionedResponse.sandboxId, + responseProviderSandboxId: provisionedResponse.providerSandboxId, + responseProviderId: provisionedResponse.providerId, + deleteSandboxId: decodeURIComponent( + String(mocks.authorizedApiFetch.mock.calls[2]?.[1] ?? '').split('/').pop() ?? '' + ) || null, deleteProviderId: deleteBody.providerId ?? null, dispatchFailureObserved: errors.join('\n').includes('dispatch failed'), }), @@ -323,23 +348,46 @@ try { ); } - const baseObserved = observation.ensureWorkloadProfile === null && observation.deleteProviderId === null; + const baseObserved = + observation.ensureSandboxId === null && + observation.ensureWorkloadProfile === 'long-running-agent' && + observation.responseSandboxId === REPLAY_SANDBOX_ID && + observation.responseProviderSandboxId === PROVIDER_SANDBOX_ID && + observation.responseProviderId === 'agent37' && + observation.deleteSandboxId === REPLAY_SANDBOX_ID && + observation.deleteProviderId === 'agent37'; + const sandboxIdentityObserved = + typeof observation.ensureSandboxId === 'string' && + /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( + observation.ensureSandboxId + ) && + observation.ensureName === `fleet-sandbox-${observation.ensureSandboxId.slice('sbx_'.length)}` && + observation.ensureForceProvision === true; + const replayIdentityObserved = + observation.ensureSandboxId === REPLAY_SANDBOX_ID && observation.ensureName === REPLAY_SANDBOX_NAME; const headObserved = - observation.ensureWorkloadProfile === 'long-running-agent' && observation.deleteProviderId === 'agent37'; + observation.ensureWorkloadProfile === 'long-running-agent' && + sandboxIdentityObserved && + replayIdentityObserved && + observation.responseSandboxId === REPLAY_SANDBOX_ID && + observation.responseProviderSandboxId === PROVIDER_SANDBOX_ID && + observation.responseProviderId === 'agent37' && + observation.deleteSandboxId === REPLAY_SANDBOX_ID && + observation.deleteProviderId === 'agent37'; let outcome; let signature; let details; if (baseObserved) { outcome = 'absent'; - signature = 'long_running_profile_and_agent37_attribution_absent'; + signature = 'long_running_profile_replay_identity_absent_agent37_attribution_preserved'; details = - 'fleet spawn --sandbox reached Cloud with no workload profile in the ensure request, and the agent37 provider Cloud returned was dropped, so cleanup named no provider.'; + "fleet spawn --sandbox reached Cloud with the existing long-running profile and Cloud's agent37 response, but without a caller-declared sandbox identity; only the one-to-one replay identity is absent while provider attribution remains preserved."; } else if (headObserved) { outcome = 'fixed'; - signature = 'long_running_profile_and_agent37_attribution_preserved'; + signature = 'long_running_profile_replay_identity_and_agent37_attribution_preserved'; details = - "fleet spawn --sandbox reached Cloud with workloadProfile 'long-running-agent' in the ensure request without pinning a provider, and the agent37 provider Cloud returned was carried into the cleanup request."; + "fleet spawn --sandbox replayed the exact caller-declared sbx_/fleet-sandbox- identity with workloadProfile 'long-running-agent' and forceProvision true without pinning a provider; Cloud echoed the public identity, returned a separate providerSandboxId, and agent37 attribution reached cleanup by public sandbox ID."; } else { throw new Error(`Unexpected long-running Agent37 observation: ${JSON.stringify(observation)}.`); } From 4256581c6f9dbbb4dc36dacce8ae4ae4ff2375c5 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 11:50:01 +0200 Subject: [PATCH 02/50] fix(cloud): reject untrusted sandbox cleanup identity --- packages/cloud/src/fleet-sandbox.test.ts | 75 +++++++++++++++++++ packages/cloud/src/fleet-sandbox.ts | 42 ++++++++--- .../1656-long-running-agent37-sandbox/run.mjs | 6 +- 3 files changed, 110 insertions(+), 13 deletions(-) diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index b56ffef07f..de104f4ed4 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -200,6 +200,81 @@ describe('Cloud fleet sandbox client', () => { } ); + it.each([ + ['malformed success', { outcome: 'provisioned' }], + ['unknown success outcome', { outcome: 'future_cloud_outcome', nodeName: SANDBOX_NAME }], + ])('rejects a mismatched sandbox identity before parsing a %s response', async (_case, responseFields) => { + const returnedSandboxId = 'sbx_223e4567-e89b-42d3-a456-426614174000'; + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ ...responseFields, sandboxId: returnedSandboxId }, { status: 201 }), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + outcomeUnknown: true, + sandboxId: undefined, + }); + expect(String(error)).toContain(`instead of requested sandboxId ${SANDBOX_ID}`); + }); + + it('does not trust a mismatched sandbox identity from a non-OK response for cleanup', async () => { + const returnedSandboxId = 'sbx_223e4567-e89b-42d3-a456-426614174000'; + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + error: 'provider rejected request', + nodeName: 'untrusted-node-name', + sandboxId: returnedSandboxId, + providerSandboxId: 'untrusted-provider-sandbox', + providerId: 'agent37', + }, + { status: 502 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + outcomeUnknown: true, + sandboxId: undefined, + providerId: undefined, + }); + expect(String(error)).toContain(`instead of requested sandboxId ${SANDBOX_ID}`); + }); + it('forwards a repos list into the ensure request body when the caller opts in', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 1e2940fc73..aedc5bcf68 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -320,6 +320,15 @@ function cleanupProviderId( : requestedProviderId; } +function assertExpectedSandboxIdentity(payload: JsonRecord, expectedSandboxId: string): void { + const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); + if (sandboxId !== expectedSandboxId) { + throw new CloudFleetSandboxIdentityMismatchError( + `Cloud returned sandboxId ${sandboxId} instead of requested sandboxId ${expectedSandboxId}.` + ); + } +} + function normalizeEnsureResult( payload: unknown, cloudWorkspaceId: string, @@ -327,6 +336,12 @@ function normalizeEnsureResult( requestedProviderId?: CloudFleetSandboxProviderId ): EnsureCloudFleetSandboxResult { if (!isObject(payload)) throw new Error('Cloud fleet sandbox response was not valid JSON.'); + // A caller-declared identity is the cleanup authority. Validate it before + // reading any other response field so malformed and future outcomes cannot + // make an untrusted public ID eligible for automatic deletion. + if (expectedSandboxId !== undefined) { + assertExpectedSandboxIdentity(payload, expectedSandboxId); + } const outcome = readString(payload, 'outcome'); const nodeName = requiredString(payload, 'nodeName', 'Cloud fleet sandbox'); const providerId = readProviderId(payload, requestedProviderId !== undefined); @@ -343,11 +358,6 @@ function normalizeEnsureResult( throw new Error('Cloud fleet sandbox response is missing relayfileMounted.'); } const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); - if (expectedSandboxId !== undefined && sandboxId !== expectedSandboxId) { - throw new CloudFleetSandboxIdentityMismatchError( - `Cloud returned sandboxId ${sandboxId} instead of requested sandboxId ${expectedSandboxId}.` - ); - } return { outcome, cloudWorkspaceId, @@ -379,11 +389,6 @@ function normalizeEnsureResult( if (outcome === 'provisioning_timeout') { const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); - if (expectedSandboxId !== undefined && sandboxId !== expectedSandboxId) { - throw new CloudFleetSandboxIdentityMismatchError( - `Cloud returned sandboxId ${sandboxId} instead of requested sandboxId ${expectedSandboxId}.` - ); - } return { outcome, cloudWorkspaceId, @@ -463,6 +468,23 @@ export async function ensureCloudFleetSandbox( } const payload = await readJson(response); if (!response.ok) { + const returnedSandboxId = isObject(payload) ? readString(payload, 'sandboxId') : undefined; + if ( + sandboxIdentity.sandboxId !== undefined && + returnedSandboxId !== undefined && + returnedSandboxId !== sandboxIdentity.sandboxId + ) { + const mismatch = new CloudFleetSandboxIdentityMismatchError( + `Cloud returned sandboxId ${returnedSandboxId} instead of requested sandboxId ${sandboxIdentity.sandboxId}.` + ); + throw new CloudFleetSandboxProvisionError(mismatch.message, { + cloudWorkspaceId: resolved.cloudWorkspaceId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), + outcomeUnknown: true, + cause: mismatch, + }); + } const error = endpointError('provision the fleet sandbox', response, payload); if (isObject(payload) && readString(payload, 'sandboxId')) { const providerId = cleanupProviderId(payload, input.providerId); diff --git a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs index 42fea222ac..998919d551 100644 --- a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs +++ b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs @@ -33,8 +33,8 @@ * the attribution. * * Base: the CLI sends no sandbox identity, while the existing long-running - * profile and Cloud-selected `agent37` response are present; the older client - * drops provider attribution during cleanup. Head: an explicit replay + * profile and Cloud-selected `agent37` response and cleanup attribution are + * present. Head: an explicit replay * `sbx_` identity, deterministic node name, and long-running profile * reach the ensure body, while the exact public identity reaches cleanup and * the separate physical provider sandbox ID is returned as evidence. Base does @@ -217,7 +217,7 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned }) // 3. cleanup, triggered by the deliberate dispatch failure below .mockResolvedValueOnce({ - response: Response.json({ sandboxId: PROVIDER_SANDBOX_ID, deleted: true }), + response: Response.json({ sandboxId: REPLAY_SANDBOX_ID, deleted: true }), auth, }); From 0763beba518f3afdcac7cef8fdfe54532e263d62 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 12:04:37 +0200 Subject: [PATCH 03/50] fix(fleet): preserve legacy sandbox naming compatibility Session-Id: 01a08039-c131-7032-9591-9077c44d4569 --- CHANGELOG.md | 4 +- packages/cli/README.md | 14 +- packages/cli/src/cli/commands/fleet.test.ts | 142 +++++++++++++++++++- packages/cli/src/cli/commands/fleet.ts | 27 ++-- packages/cloud/src/fleet-sandbox.test.ts | 51 ++++++- packages/cloud/src/fleet-sandbox.ts | 7 +- 6 files changed, 215 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 877d61184a..b39aa4095b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,11 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased - Patch] +## [Unreleased] ### Changed -- `agent-relay fleet spawn --sandbox` now forwards a one-to-one `sbx_` identity with a deterministic `fleet-sandbox-` node name for Cloud long-running workloads, rejects arbitrary `--sandbox-name` values, and supports `--sandbox-id` replay after an unknown provisioning outcome. +- Operators can keep using custom `--sandbox-name` values with `agent-relay fleet spawn --sandbox`; launches without a custom name generate a stable `sbx_` identity automatically, while only `--sandbox-id` replay requires the matching deterministic lowercase `fleet-sandbox-` name. ## [11.10.4] - 2026-09-08 diff --git a/packages/cli/README.md b/packages/cli/README.md index 06dfffe3e1..01990324f6 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -179,12 +179,14 @@ sees the same synced Relayfile workspace. Use `--sandbox-provider daytona` or `--sandbox-provider e2b` to require an operator-enabled provider; omit the flag to let Cloud's sandbox router choose. Pass `--no-sandbox-relayfile` only when a deliberately bare sandbox is desired. If provisioning times out or the spawn -fails, Relay asks Cloud to delete the newly created sandbox. Each generation -uses one `sbx_` identity and the matching `fleet-sandbox-` node -name; a custom `--sandbox-name` is rejected unless it is that exact generated -name. If provisioning ends with an unknown outcome, rerun the command with the -warning's `--sandbox-id ` (and optionally its matching -`--sandbox-name`) to replay the same Cloud identity instead of adopting another +fails, Relay asks Cloud to delete the newly created sandbox. Runs without a +custom name use one `sbx_` identity and the matching +`fleet-sandbox-` node name. Legacy custom `--sandbox-name` values remain supported when no +`--sandbox-id` is supplied; in that mode Cloud receives no sandbox identity. +When replaying with `--sandbox-id ` (lowercase RFC 4122 UUID), pass +its matching deterministic `--sandbox-name` or let Relay derive it. If +provisioning ends with an unknown outcome, rerun the command with the warning's +`--sandbox-id` to replay the same Cloud identity instead of adopting another fleet node. Large workspaces should select only the live subtree an agent needs. Pass one diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 8e9ed7adc3..ecd0999be1 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -844,7 +844,141 @@ describe('fleet command support', () => { }); }); - it('rejects a custom sandbox name before provisioning because identity is one-to-one', async () => { + it('preserves legacy custom sandbox names without sending a sandbox identity', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', undefined); + const placement = { + spawn: vi.fn(async () => ({ + invocationId: 'inv_legacy_sandbox', + node: { name: 'custom-node' }, + })), + }; + const register = vi.fn(async () => ({ token: 'at_live_launcher' })); + const release = vi.fn(async () => ({ released: true, deleted: true })); + const ensureCloudFleetSandbox = vi.fn(async () => ({ + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-legacy', + nodeName: 'custom-node', + sandboxId: 'legacy-public-sandbox', + providerSandboxId: 'legacy-provider-sandbox', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + relayfileMountPath: '/workspace', + })); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })), register, release }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + ensureCloudFleetSandbox, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-name', + 'custom-node', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + + const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0]; + expect(ensureInput).toMatchObject({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + forceProvision: true, + workloadProfile: 'long-running-agent', + name: 'custom-node', + }); + expect(ensureInput).not.toHaveProperty('sandboxId'); + }); + + it('generates a stable identity and matching name when neither option is supplied', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', undefined); + const ensureCloudFleetSandbox = vi.fn(async () => ({ + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-generated', + nodeName: 'generated-node', + sandboxId: 'generated-public-sandbox', + providerSandboxId: 'generated-provider-sandbox', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + relayfileMountPath: '/workspace', + })); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn(() => ({ + messaging: { placement: { spawn: vi.fn(async () => ({ invocationId: 'inv_generated' })) } }, + })) as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { + info: vi.fn(async () => ({ id: 'rw_abc' })), + register: vi.fn(async () => ({ token: 'at_live_launcher' })), + release: vi.fn(async () => ({ released: true, deleted: true })), + }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + ensureCloudFleetSandbox, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + + const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0]; + expect(ensureInput?.sandboxId).toMatch( + /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ + ); + expect(ensureInput?.name).toBe(`fleet-sandbox-${ensureInput?.sandboxId?.slice('sbx_'.length)}`); + }); + + it('rejects a replay sandbox name that does not match its identity', async () => { const ensureCloudFleetSandbox = vi.fn(); const errors: string[] = []; const program = new Command(); @@ -877,6 +1011,8 @@ describe('fleet command support', () => { 'spawn', 'codex', '--sandbox', + '--sandbox-id', + REPLAY_SANDBOX_ID, '--sandbox-name', 'custom-node', '--name', @@ -894,7 +1030,7 @@ describe('fleet command support', () => { expect(ensureCloudFleetSandbox).not.toHaveBeenCalled(); expect(errors.join('\n')).toContain('--sandbox-name'); - expect(errors.join('\n')).toContain('one-to-one sandbox identity'); + expect(errors.join('\n')).toContain('when --sandbox-id is supplied'); }); it('rejects an invalid replay sandbox ID before provisioning', async () => { @@ -946,7 +1082,7 @@ describe('fleet command support', () => { ).rejects.toThrow('__exit__'); expect(ensureCloudFleetSandbox).not.toHaveBeenCalled(); - expect(errors.join('\n')).toContain('--sandbox-id must match sbx_'); + expect(errors.join('\n')).toContain('--sandbox-id must match lowercase sbx_'); }); it('cleans up when Cloud reports a post-provision response failure with a sandbox ID', async () => { diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index e4b200a0a0..146dd6a7b1 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -204,7 +204,7 @@ export function registerFleetCommands( const sandboxName = optionalText(options.sandboxName, 'Sandbox name'); const sandboxIdOption = optionalText(options.sandboxId, 'Sandbox ID'); if (sandboxIdOption !== undefined && !CLOUD_SANDBOX_ID_PATTERN.test(sandboxIdOption)) { - throw new Error('--sandbox-id must match sbx_ using an RFC 4122 UUID.'); + throw new Error('--sandbox-id must match lowercase sbx_ using an RFC 4122 UUID.'); } const sandboxProviderText = optionalText(options.sandboxProvider, 'Sandbox provider'); const sandboxProvider: CloudFleetSandboxProviderId | undefined = @@ -267,14 +267,19 @@ export function registerFleetCommands( if (!relayWorkspaceId) { throw new Error('The current Relay workspace did not report an ID for Cloud provisioning.'); } - const sandboxId = sandboxIdOption ?? `sbx_${randomUUID()}`; - const sandboxGenerationUuid = sandboxId.slice('sbx_'.length); - const deterministicSandboxName = `fleet-sandbox-${sandboxGenerationUuid}`; - if (sandboxName !== undefined && sandboxName !== deterministicSandboxName) { + const sandboxId = sandboxIdOption ?? (sandboxName === undefined ? `sbx_${randomUUID()}` : undefined); + const deterministicSandboxName = + sandboxId === undefined ? undefined : `fleet-sandbox-${sandboxId.slice('sbx_'.length)}`; + if ( + sandboxIdOption !== undefined && + sandboxName !== undefined && + sandboxName !== deterministicSandboxName + ) { throw new Error( - `--sandbox-name must be '${deterministicSandboxName}' for a long-running sandbox; custom names cannot preserve the one-to-one sandbox identity.` + `--sandbox-name must be '${deterministicSandboxName}' when --sandbox-id is supplied; custom names cannot preserve the one-to-one sandbox identity.` ); } + const effectiveSandboxName = deterministicSandboxName ?? sandboxName; try { sandbox = await deps.ensureCloudFleetSandbox({ workspaceId: relayWorkspaceId, @@ -282,12 +287,12 @@ export function registerFleetCommands( maxAgents: 1, mountRelayfile: mountSandboxRelayfile, ...(sandboxRelayfilePaths === undefined ? {} : { relayfilePaths: sandboxRelayfilePaths }), - sandboxId, + ...(sandboxId === undefined ? {} : { sandboxId }), forceProvision: true, ...(sandboxProvider === undefined ? {} : { providerId: sandboxProvider }), workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, - name: deterministicSandboxName, + ...(effectiveSandboxName === undefined ? {} : { name: effectiveSandboxName }), }); } catch (error) { if (error instanceof CloudFleetSandboxProvisionError && error.cloudWorkspaceId && error.sandboxId) { @@ -307,8 +312,10 @@ export function registerFleetCommands( } else if (error instanceof CloudFleetSandboxProvisionError && error.outcomeUnknown) { deps.warn( `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ - error.nodeName ?? deterministicSandboxName - }' before retrying with --sandbox-id '${sandboxId}' so a sandbox is not left running.` + error.nodeName ?? effectiveSandboxName ?? 'the requested sandbox' + }'${ + sandboxId === undefined ? '' : ` before retrying with --sandbox-id '${sandboxId}'` + } so a sandbox is not left running.` ); } throw error; diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index de104f4ed4..1887a784fe 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -116,22 +116,65 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-1', forceProvision: true, }) - ).rejects.toThrow('sandboxId must match sbx_'); + ).rejects.toThrow('sandboxId must match lowercase sbx_'); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: 'sbx_123E4567-e89b-42d3-a456-426614174000', + forceProvision: true, + }) + ).rejects.toThrow('sandboxId must match lowercase sbx_'); expect(mocks.ensureCloudSession).not.toHaveBeenCalled(); expect(mocks.authorizedApiFetch).not.toHaveBeenCalled(); }); - it('requires a stable identity and deterministic name for long-running requests', async () => { + it('preserves legacy long-running requests with a custom name and no sandbox identity', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeId: 'node-legacy', + nodeName: 'legacy-custom-node', + sandboxId: 'legacy-public-sandbox', + providerSandboxId: 'legacy-provider-sandbox', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + }, + { status: 201 } + ), + auth, + }); + await expect( ensureCloudFleetSandbox({ workspaceId: 'rw_abc', requiredCapability: 'spawn:codex', - forceProvision: true, + name: 'legacy-custom-node', workloadProfile: 'long-running-agent', }) - ).rejects.toThrow('require sandboxId'); + ).resolves.toMatchObject({ + outcome: 'provisioned', + sandboxId: 'legacy-public-sandbox', + providerSandboxId: 'legacy-provider-sandbox', + }); + + const ensureBody = JSON.parse(String(mocks.authorizedApiFetch.mock.calls[1]?.[2]?.body)); + expect(ensureBody).toMatchObject({ + name: 'legacy-custom-node', + workloadProfile: 'long-running-agent', + }); + expect(ensureBody).not.toHaveProperty('sandboxId'); + }); + it('requires a deterministic name whenever a sandbox identity is supplied', async () => { await expect( ensureCloudFleetSandbox({ workspaceId: 'rw_abc', diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index aedc5bcf68..bf0e71cebb 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -235,7 +235,7 @@ function validateSandboxIdentity(input: EnsureCloudFleetSandboxInput): { const sandboxId = input.sandboxId?.trim(); const name = input.name?.trim(); if (input.sandboxId !== undefined && (!sandboxId || !CLOUD_SANDBOX_ID_PATTERN.test(sandboxId))) { - throw new Error('Cloud fleet sandbox sandboxId must match sbx_ using an RFC 4122 UUID.'); + throw new Error('Cloud fleet sandbox sandboxId must match lowercase sbx_ using an RFC 4122 UUID.'); } if (sandboxId !== undefined && input.forceProvision !== true) { throw new Error('Cloud fleet sandbox sandboxId requires forceProvision: true.'); @@ -246,10 +246,7 @@ function validateSandboxIdentity(input: EnsureCloudFleetSandboxInput): { const longRunning = input.workloadProfile === 'long-running-agent' || input.workloadProfile === 'standard-long-running-agent'; - if (longRunning) { - if (sandboxId === undefined) { - throw new Error('Long-running Cloud fleet sandbox requests require sandboxId.'); - } + if (longRunning && sandboxId !== undefined) { if (input.forceProvision !== true) { throw new Error('Long-running Cloud fleet sandbox requests require forceProvision: true.'); } From 0eb71ff2be56c2ee949958804a4fb1c2f49f038a Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 12:07:36 +0200 Subject: [PATCH 04/50] fix(cloud): verify stable sandbox node names Session-Id: 01a08039-c131-7032-9591-9077c44d4569 --- packages/cloud/src/fleet-sandbox.test.ts | 57 ++++++++++++++++++- packages/cloud/src/fleet-sandbox.ts | 7 +++ .../1656-long-running-agent37-sandbox/run.mjs | 5 +- 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 1887a784fe..31eb48e5b4 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -48,7 +48,7 @@ describe('Cloud fleet sandbox client', () => { { outcome: 'provisioned', nodeId: 'node-1', - nodeName: 'daytona-codex', + nodeName: SANDBOX_NAME, sandboxId: SANDBOX_ID, providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', @@ -98,7 +98,7 @@ describe('Cloud fleet sandbox client', () => { outcome: 'provisioned', cloudWorkspaceId: CLOUD_WORKSPACE_ID, nodeId: 'node-1', - nodeName: 'daytona-codex', + nodeName: SANDBOX_NAME, sandboxId: SANDBOX_ID, providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', @@ -243,6 +243,59 @@ describe('Cloud fleet sandbox client', () => { } ); + it.each(['provisioned', 'provisioning_timeout'] as const)( + 'fails closed when Cloud echoes a different deterministic node name for %s', + async (outcome) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + outcome === 'provisioned' + ? { + outcome, + nodeId: 'node-other', + nodeName: 'fleet-sandbox-other', + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-sandbox-other', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + } + : { + outcome, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-sandbox-other', + relayWorkspaceId: 'rw_abc', + nodeName: 'fleet-sandbox-other', + waitedMs: 90_000, + }, + { status: outcome === 'provisioned' ? 201 : 202 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: 'fleet-sandbox-other', + outcomeUnknown: true, + sandboxId: undefined, + }); + expect(String(error)).toContain(`instead of requested nodeName ${SANDBOX_NAME}`); + } + ); + it.each([ ['malformed success', { outcome: 'provisioned' }], ['unknown success outcome', { outcome: 'future_cloud_outcome', nodeName: SANDBOX_NAME }], diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index bf0e71cebb..028b080675 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -330,6 +330,7 @@ function normalizeEnsureResult( payload: unknown, cloudWorkspaceId: string, expectedSandboxId?: string, + expectedNodeName?: string, requestedProviderId?: CloudFleetSandboxProviderId ): EnsureCloudFleetSandboxResult { if (!isObject(payload)) throw new Error('Cloud fleet sandbox response was not valid JSON.'); @@ -341,6 +342,11 @@ function normalizeEnsureResult( } const outcome = readString(payload, 'outcome'); const nodeName = requiredString(payload, 'nodeName', 'Cloud fleet sandbox'); + if (expectedSandboxId !== undefined && expectedNodeName !== undefined && nodeName !== expectedNodeName) { + throw new CloudFleetSandboxIdentityMismatchError( + `Cloud returned nodeName ${nodeName} instead of requested nodeName ${expectedNodeName}.` + ); + } const providerId = readProviderId(payload, requestedProviderId !== undefined); if (requestedProviderId !== undefined && providerId !== requestedProviderId) { throw new Error( @@ -500,6 +506,7 @@ export async function ensureCloudFleetSandbox( payload, resolved.cloudWorkspaceId, sandboxIdentity.sandboxId, + sandboxIdentity.name, input.providerId ); } catch (error) { diff --git a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs index 998919d551..494134dc3a 100644 --- a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs +++ b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs @@ -195,7 +195,7 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned const provisionedResponse = { outcome: 'provisioned', nodeId: 'node-relayflow', - nodeName: 'agent37-relayflow', + nodeName: REPLAY_SANDBOX_NAME, sandboxId: REPLAY_SANDBOX_ID, providerSandboxId: PROVIDER_SANDBOX_ID, relayWorkspaceId: 'rw_relayflow', @@ -302,6 +302,7 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned responseSandboxId: provisionedResponse.sandboxId, responseProviderSandboxId: provisionedResponse.providerSandboxId, responseProviderId: provisionedResponse.providerId, + responseNodeName: provisionedResponse.nodeName, deleteSandboxId: decodeURIComponent( String(mocks.authorizedApiFetch.mock.calls[2]?.[1] ?? '').split('/').pop() ?? '' ) || null, @@ -354,6 +355,7 @@ try { observation.responseSandboxId === REPLAY_SANDBOX_ID && observation.responseProviderSandboxId === PROVIDER_SANDBOX_ID && observation.responseProviderId === 'agent37' && + observation.responseNodeName === REPLAY_SANDBOX_NAME && observation.deleteSandboxId === REPLAY_SANDBOX_ID && observation.deleteProviderId === 'agent37'; const sandboxIdentityObserved = @@ -372,6 +374,7 @@ try { observation.responseSandboxId === REPLAY_SANDBOX_ID && observation.responseProviderSandboxId === PROVIDER_SANDBOX_ID && observation.responseProviderId === 'agent37' && + observation.responseNodeName === REPLAY_SANDBOX_NAME && observation.deleteSandboxId === REPLAY_SANDBOX_ID && observation.deleteProviderId === 'agent37'; From b09797c20e982778d19c8e8a2462e49c03431d59 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 12:11:21 +0200 Subject: [PATCH 05/50] fix(pr-proof): retain Cloud failure diagnostics Session-Id: 01a08039-c131-7032-9591-9077c44d4569 --- scripts/pr-proof/run-cloud.mjs | 53 +++++++++++++++++++++++- tests/fixtures/pr-proof-contract.test.ts | 17 ++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 00cd83d22f..60b98b6e13 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -17,6 +17,7 @@ const LEGACY_REFRESHABLE_AUTH_KEYS = [ ]; const MAX_CAPTURE_BYTES = 2 * 1024 * 1024; const MAX_LIVE_OUTPUT_BYTES = 256 * 1024; +const MAX_DIAGNOSTIC_BYTES = 64 * 1024; const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60_000; const PREPARED_RUN_ID_MARKER = 'AGENT_RELAY_CLOUD_PREPARED_RUN_ID='; const RUN_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; @@ -54,6 +55,33 @@ function parseJsonOutput(output, label) { } } +function boundedDiagnostic(value) { + const text = String(value ?? ''); + if (Buffer.byteLength(text, 'utf8') <= MAX_DIAGNOSTIC_BYTES) return text; + return `${text.slice(-MAX_DIAGNOSTIC_BYTES)}\n[... diagnostic output truncated ...]`; +} + +export function formatCloudRunDiagnostics({ + runId, + terminalStatus, + lastStatusOutput, + statusPollFailures, + logs, +}) { + const logOutput = `${logs?.stdout ?? ''}${logs?.stderr ?? ''}`; + return [ + 'Cloud RelayFlow diagnostics', + `run_id=${runId}`, + `terminal_status=${terminalStatus ?? 'unknown'}`, + `status_poll_failures=${statusPollFailures}`, + `last_status_response=${boundedDiagnostic(lastStatusOutput) || ''}`, + `cloud_logs_exit_code=${logs?.exitCode ?? 'unknown'}`, + `cloud_logs_timed_out=${logs?.timedOut === true}`, + `cloud_logs_output=${logOutput ? 'present' : 'empty'}`, + '', + ].join('\n'); +} + function statusFrom(payload) { for (const candidate of [payload.status, payload.run?.status, payload.workflowRun?.status]) { if (typeof candidate === 'string') return candidate.toLowerCase(); @@ -149,6 +177,8 @@ export async function main() { let shuttingDown = false; let activeCommandController = null; let launchProgressError = null; + let lastStatusOutput = ''; + let statusPollFailures = 0; const notePreparedRunId = (preparedRunId) => { try { @@ -252,12 +282,20 @@ export async function main() { timeoutMs: commandTimeoutMs, }); if (statusResult.timedOut) { + lastStatusOutput = statusResult.stderr.trim() || statusResult.stdout.trim(); throw new Error(`Cloud status command timed out for run ${runId}`); } if (statusResult.exitCode !== 0) { - console.warn(`Cloud status poll failed (${statusResult.exitCode}); retrying`); + statusPollFailures += 1; + lastStatusOutput = statusResult.stderr.trim() || statusResult.stdout.trim(); + console.warn( + `Cloud status poll failed (${statusResult.exitCode}); retrying${ + lastStatusOutput ? `: ${boundedDiagnostic(lastStatusOutput)}` : '' + }` + ); continue; } + lastStatusOutput = statusResult.stdout.trim(); const status = statusFrom(parseJsonOutput(statusResult.stdout, 'Cloud status')); console.log(`Cloud RelayFlow status: ${status}`); if (TERMINAL_SUCCESS.has(status) || TERMINAL_FAILURE.has(status)) { @@ -278,7 +316,18 @@ export async function main() { quiet: true, timeoutMs: commandTimeoutMs, }); - await writeFile(logsPath, logs.stdout + logs.stderr); + await writeFile( + logsPath, + formatCloudRunDiagnostics({ + runId, + terminalStatus, + lastStatusOutput, + statusPollFailures, + logs, + }) + + logs.stdout + + logs.stderr + ); if (logs.stdout) process.stdout.write(logs.stdout); if (logs.stderr) process.stderr.write(logs.stderr); if (logs.timedOut) throw new Error(`Cloud log retrieval timed out for run ${runId}`); diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index f3f107b892..0a4af3cb83 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -47,6 +47,7 @@ import { boundedDuration, createPreparedRunProgressParser, createCliApiKeyEnvironment, + formatCloudRunDiagnostics, preparedRunIdFromOutput, } from '../../scripts/pr-proof/run-cloud.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. @@ -438,6 +439,22 @@ describe('Cloud dispatcher API key lifecycle', () => { ); }); + it('retains terminal status diagnostics when Cloud logs are empty', () => { + const diagnostics = formatCloudRunDiagnostics({ + runId: 'cloud-run-123', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed","error":"step timeout"}', + statusPollFailures: 2, + logs: { stdout: '', stderr: '', exitCode: 0, timedOut: false }, + }); + + expect(diagnostics).toContain('run_id=cloud-run-123'); + expect(diagnostics).toContain('terminal_status=failed'); + expect(diagnostics).toContain('status_poll_failures=2'); + expect(diagnostics).toContain('step timeout'); + expect(diagnostics).toContain('cloud_logs_output=empty'); + }); + it('waits for a complete prepared-run progress line split across stderr chunks', () => { const runIds: string[] = []; const parser = createPreparedRunProgressParser((runId: string) => runIds.push(runId)); From c8afa977cd96713ae9d8804f3a2c97d57b5eb9f9 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 12:12:09 +0200 Subject: [PATCH 06/50] chore: restore unreleased patch metadata Session-Id: 01a08039-c131-7032-9591-9077c44d4569 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b39aa4095b..8fffefc137 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Patch] ### Changed From cddb9d966b4fb755f9ebd795268fcfe2a645e95a Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 12:15:08 +0200 Subject: [PATCH 07/50] chore: mark sandbox replay as minor --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fffefc137..837f434e10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased - Patch] +## [Unreleased - Minor] ### Changed From 155af7925d339cdace6d61c12f912980e8b14589 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 12:21:43 +0200 Subject: [PATCH 08/50] fix(pr-proof): bound multibyte diagnostics --- scripts/pr-proof/run-cloud.mjs | 15 ++++++++++++--- tests/fixtures/pr-proof-contract.test.ts | 8 ++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 60b98b6e13..e86b10ad62 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -55,10 +55,19 @@ function parseJsonOutput(output, label) { } } -function boundedDiagnostic(value) { +export function boundedDiagnostic(value) { const text = String(value ?? ''); - if (Buffer.byteLength(text, 'utf8') <= MAX_DIAGNOSTIC_BYTES) return text; - return `${text.slice(-MAX_DIAGNOSTIC_BYTES)}\n[... diagnostic output truncated ...]`; + const bytes = Buffer.from(text, 'utf8'); + if (bytes.length <= MAX_DIAGNOSTIC_BYTES) return text; + + const marker = '\n[... diagnostic output truncated ...]'; + const tailBudget = MAX_DIAGNOSTIC_BYTES - Buffer.byteLength(marker, 'utf8'); + let tail = bytes.subarray(bytes.length - tailBudget).toString('utf8'); + // A byte slice can begin in the middle of a multi-byte code point. Removing + // the replacement character (or another leading code point if needed) keeps + // the final diagnostic, including its marker, within the byte contract. + while (Buffer.byteLength(tail, 'utf8') > tailBudget) tail = tail.slice(1); + return `${tail}${marker}`; } export function formatCloudRunDiagnostics({ diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 0a4af3cb83..dacaf27559 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -44,6 +44,7 @@ import { } from '../../scripts/pr-proof/prepare.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. import { + boundedDiagnostic, boundedDuration, createPreparedRunProgressParser, createCliApiKeyEnvironment, @@ -455,6 +456,13 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(diagnostics).toContain('cloud_logs_output=empty'); }); + it('bounds multibyte Cloud diagnostics by UTF-8 bytes', () => { + const diagnostics = boundedDiagnostic('😀'.repeat(100_000)); + + expect(Buffer.byteLength(diagnostics, 'utf8')).toBeLessThanOrEqual(64 * 1024); + expect(diagnostics).toContain('[... diagnostic output truncated ...]'); + }); + it('waits for a complete prepared-run progress line split across stderr chunks', () => { const runIds: string[] = []; const parser = createPreparedRunProgressParser((runId: string) => runIds.push(runId)); From 1b160f35bd40b91a61f7e078561033591fdb88cf Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 12:41:43 +0200 Subject: [PATCH 09/50] fix(cloud): retain replay identity after failed provisioning --- packages/cli/src/cli/commands/fleet.ts | 2 +- packages/cloud/src/fleet-sandbox.test.ts | 111 +++++++++++++++++++---- packages/cloud/src/fleet-sandbox.ts | 29 +----- scripts/pr-proof/run-cloud.mjs | 26 ++++++ tests/fixtures/pr-proof-contract.test.ts | 23 +++++ 5 files changed, 149 insertions(+), 42 deletions(-) diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 146dd6a7b1..f6f34b910d 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -165,7 +165,7 @@ export function registerFleetCommands( ) .option( '--sandbox-name ', - 'Explicit sandbox node name (must match the generated fleet-sandbox- identity)' + 'Explicit sandbox node name (custom unless --sandbox-id requires matching fleet-sandbox-)' ) .option('--sandbox-id ', 'Reuse a caller-declared sbx_ identity for an exact replay') .option('--sandbox-provider ', 'Sandbox provider: daytona or e2b') diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 31eb48e5b4..207985e32d 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -288,7 +288,7 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ cloudWorkspaceId: CLOUD_WORKSPACE_ID, - nodeName: 'fleet-sandbox-other', + nodeName: SANDBOX_NAME, outcomeUnknown: true, sandboxId: undefined, }); @@ -371,6 +371,82 @@ describe('Cloud fleet sandbox client', () => { expect(String(error)).toContain(`instead of requested sandboxId ${SANDBOX_ID}`); }); + it('does not expose a matching sandbox identity from a non-OK response for cleanup', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + error: 'provider request failed after allocation', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-sandbox-1', + providerId: 'agent37', + }, + { status: 502 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + outcomeUnknown: true, + sandboxId: undefined, + providerId: undefined, + }); + }); + + it('does not expose a matching sandbox identity from a malformed success for cleanup', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + }, + { status: 201 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + outcomeUnknown: true, + sandboxId: undefined, + providerId: undefined, + }); + }); + it('forwards a repos list into the ensure request body when the caller opts in', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -568,7 +644,7 @@ describe('Cloud fleet sandbox client', () => { ); }); - it('rejects and preserves cleanup identity when Cloud cannot prove the requested provider', async () => { + it('rejects without exposing cleanup identity when Cloud cannot prove the requested provider', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), @@ -599,9 +675,9 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ - sandboxId: 'sandbox-1', - nodeName: 'wrong-provider', - providerId: 'daytona', + sandboxId: undefined, + nodeName: undefined, + providerId: 'e2b', outcomeUnknown: true, }); expect(String(error)).toContain('instead of requested provider e2b'); @@ -611,7 +687,7 @@ describe('Cloud fleet sandbox client', () => { ['missing', undefined], ['invalid', 'modal'], ])( - 'preserves the requested provider for cleanup when a %s provider proof arrives on a 201 response', + 'rejects without cleanup identity when a %s provider proof arrives on a 201 response', async (_case, providerId) => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -644,8 +720,8 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ cloudWorkspaceId: CLOUD_WORKSPACE_ID, - sandboxId: 'sandbox-e2b', - nodeName: 'e2b-reviewer', + sandboxId: undefined, + nodeName: undefined, providerId: 'e2b', outcomeUnknown: true, }); @@ -656,7 +732,7 @@ describe('Cloud fleet sandbox client', () => { ['missing', undefined], ['invalid', 'modal'], ])( - 'preserves the requested provider for cleanup when a %s provider proof arrives on a 202 timeout response', + 'rejects without cleanup identity when a %s provider proof arrives on a 202 timeout response', async (_case, providerId) => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -688,8 +764,8 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ cloudWorkspaceId: CLOUD_WORKSPACE_ID, - sandboxId: 'sandbox-e2b', - nodeName: 'e2b-reviewer', + sandboxId: undefined, + nodeName: undefined, providerId: 'e2b', outcomeUnknown: true, }); @@ -700,7 +776,7 @@ describe('Cloud fleet sandbox client', () => { ['missing', undefined], ['invalid', 'modal'], ])( - 'preserves the public cleanup identity when a %s provider proof arrives on a non-OK response', + 'rejects without cleanup identity when a %s provider proof arrives on a non-OK response', async (_case, providerId) => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -730,9 +806,10 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ cloudWorkspaceId: CLOUD_WORKSPACE_ID, - sandboxId: 'sandbox-e2b', - nodeName: 'e2b-reviewer', + sandboxId: undefined, + nodeName: undefined, providerId: 'e2b', + outcomeUnknown: true, }); } ); @@ -947,7 +1024,7 @@ describe('Cloud fleet sandbox client', () => { expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(1); }); - it('preserves the sandbox identity from a malformed successful response', async () => { + it('does not expose an unrequested sandbox identity from a malformed successful response', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), @@ -976,8 +1053,8 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ cloudWorkspaceId: CLOUD_WORKSPACE_ID, - sandboxId: 'sandbox-1', - nodeName: 'daytona-codex', + sandboxId: undefined, + nodeName: undefined, outcomeUnknown: true, }); }); diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 028b080675..90201d8cae 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -306,17 +306,6 @@ function readProviderId( return undefined; } -function cleanupProviderId( - payload: JsonRecord, - requestedProviderId?: CloudFleetSandboxProviderId -): CloudFleetSandboxProviderId | undefined { - const payloadProviderId = readString(payload, 'providerId'); - return payloadProviderId && - CLOUD_FLEET_SANDBOX_PROVIDER_IDS.includes(payloadProviderId as CloudFleetSandboxProviderId) - ? (payloadProviderId as CloudFleetSandboxProviderId) - : requestedProviderId; -} - function assertExpectedSandboxIdentity(payload: JsonRecord, expectedSandboxId: string): void { const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); if (sandboxId !== expectedSandboxId) { @@ -490,12 +479,11 @@ export async function ensureCloudFleetSandbox( } const error = endpointError('provision the fleet sandbox', response, payload); if (isObject(payload) && readString(payload, 'sandboxId')) { - const providerId = cleanupProviderId(payload, input.providerId); throw new CloudFleetSandboxProvisionError(error.message, { cloudWorkspaceId: resolved.cloudWorkspaceId, - sandboxId: readString(payload, 'sandboxId'), - nodeName: readString(payload, 'nodeName') ?? input.name, - ...(providerId === undefined ? {} : { providerId }), + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), + outcomeUnknown: true, cause: error, }); } @@ -510,19 +498,12 @@ export async function ensureCloudFleetSandbox( input.providerId ); } catch (error) { - const providerId = isObject(payload) ? cleanupProviderId(payload, input.providerId) : input.providerId; - const identityMismatch = error instanceof CloudFleetSandboxIdentityMismatchError; - const cleanupSandboxId = - !identityMismatch && isObject(payload) ? readString(payload, 'sandboxId') : undefined; throw new CloudFleetSandboxProvisionError( error instanceof Error ? error.message : 'Cloud fleet sandbox response was invalid.', { cloudWorkspaceId: resolved.cloudWorkspaceId, - ...(cleanupSandboxId === undefined ? {} : { sandboxId: cleanupSandboxId }), - ...(isObject(payload) && (readString(payload, 'nodeName') ?? input.name) - ? { nodeName: readString(payload, 'nodeName') ?? input.name } - : {}), - ...(providerId === undefined ? {} : { providerId }), + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), outcomeUnknown: true, cause: error, } diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index e86b10ad62..181ca2a3a9 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -91,6 +91,25 @@ export function formatCloudRunDiagnostics({ ].join('\n'); } +export async function writeStatusPollTimeoutDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, +}) { + await mkdir(path.dirname(logsPath), { recursive: true }); + await writeFile( + logsPath, + formatCloudRunDiagnostics({ + runId, + terminalStatus: 'status_poll_timeout', + lastStatusOutput, + statusPollFailures, + logs: { stdout: '', stderr: '', exitCode: 'unknown', timedOut: true }, + }) + ); +} + function statusFrom(payload) { for (const candidate of [payload.status, payload.run?.status, payload.workflowRun?.status]) { if (typeof candidate === 'string') return candidate.toLowerCase(); @@ -291,7 +310,14 @@ export async function main() { timeoutMs: commandTimeoutMs, }); if (statusResult.timedOut) { + statusPollFailures += 1; lastStatusOutput = statusResult.stderr.trim() || statusResult.stdout.trim(); + await writeStatusPollTimeoutDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, + }); throw new Error(`Cloud status command timed out for run ${runId}`); } if (statusResult.exitCode !== 0) { diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index dacaf27559..d287e1c601 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -50,6 +50,7 @@ import { createCliApiKeyEnvironment, formatCloudRunDiagnostics, preparedRunIdFromOutput, + writeStatusPollTimeoutDiagnostics, } from '../../scripts/pr-proof/run-cloud.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. import { @@ -456,6 +457,28 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(diagnostics).toContain('cloud_logs_output=empty'); }); + it('persists bounded diagnostics when a Cloud status poll times out', async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'relay-pr-proof-status-timeout-')); + const logsPath = path.join(root, 'nested', 'cloud.log'); + try { + await writeStatusPollTimeoutDiagnostics({ + logsPath, + runId: 'cloud-run-timeout', + lastStatusOutput: '😀'.repeat(100_000), + statusPollFailures: 3, + }); + + const diagnostics = await readFile(logsPath, 'utf8'); + expect(diagnostics).toContain('run_id=cloud-run-timeout'); + expect(diagnostics).toContain('terminal_status=status_poll_timeout'); + expect(diagnostics).toContain('status_poll_failures=3'); + expect(diagnostics).toContain('cloud_logs_timed_out=true'); + expect(Buffer.byteLength(diagnostics, 'utf8')).toBeLessThanOrEqual(65 * 1024); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + it('bounds multibyte Cloud diagnostics by UTF-8 bytes', () => { const diagnostics = boundedDiagnostic('😀'.repeat(100_000)); From 814579a157676f2827806535bd3c92c623e395ce Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 12:59:04 +0200 Subject: [PATCH 10/50] fix(pr-proof): sanitize Cloud status diagnostics --- scripts/pr-proof/run-cloud.mjs | 102 ++++++++++++++++++++++- tests/fixtures/pr-proof-contract.test.ts | 74 ++++++++++++++++ 2 files changed, 173 insertions(+), 3 deletions(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 181ca2a3a9..f137a808b2 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -21,6 +21,27 @@ const MAX_DIAGNOSTIC_BYTES = 64 * 1024; const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60_000; const PREPARED_RUN_ID_MARKER = 'AGENT_RELAY_CLOUD_PREPARED_RUN_ID='; const RUN_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; +const LIVE_CREDENTIAL_RE = + /(rk_live_|rjt_live_|at_live_|nt_live_|ot_live_|cld_at_|rth_at_|ocl_node_enr_|br_)([A-Za-z0-9_%-]+(?:\.[A-Za-z0-9_%-]+)*)/g; +const STATUS_DIAGNOSTIC_FIELDS = [ + 'runId', + 'status', + 'sandboxId', + 'dispatchType', + 'relayflowVersion', + 'createdAt', + 'updatedAt', + 'error', + 'message', +]; +const STATUS_FAILURE_DIAGNOSTIC_FIELDS = [ + 'phase', + 'code', + 'message', + 'dispatchType', + 'sandboxId', + 'occurredAt', +]; function run(command, args, options = {}) { return runBoundedProcess(command, args, { @@ -70,12 +91,79 @@ export function boundedDiagnostic(value) { return `${tail}${marker}`; } +function redactDiagnosticText(value, secretValues = []) { + let text = String(value ?? ''); + for (const secretValue of secretValues) { + const secret = typeof secretValue === 'string' ? secretValue : ''; + if (secret) text = text.split(secret).join('[redacted]'); + } + return text.replace(LIVE_CREDENTIAL_RE, (_match, prefix) => `${prefix}…`); +} + +function diagnosticRecord(value, secretValues) { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const diagnostic = {}; + for (const field of STATUS_DIAGNOSTIC_FIELDS) { + if (typeof value[field] === 'string') { + diagnostic[field] = redactDiagnosticText(value[field], secretValues); + } + } + if (value.failure && typeof value.failure === 'object' && !Array.isArray(value.failure)) { + const failure = {}; + for (const field of STATUS_FAILURE_DIAGNOSTIC_FIELDS) { + if (typeof value.failure[field] === 'string') { + failure[field] = redactDiagnosticText(value.failure[field], secretValues); + } + } + if (Object.keys(failure).length > 0) diagnostic.failure = failure; + } + return diagnostic; +} + +/** + * Reduce `cloud status --json` to the structural fields useful for triage. + * Workflow source, result payloads, nested errors, and cause chains are never + * copied because they can contain arbitrary workflow-provided credentials. + */ +export function sanitizeCloudStatusDiagnostic(output, secretValues = []) { + const text = String(output ?? '').trim(); + if (!text) return ''; + try { + const payload = parseJsonOutput(text, 'Cloud status diagnostic'); + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + return ''; + } + const source = + typeof payload.status === 'string' + ? payload + : payload.run && typeof payload.run === 'object' && !Array.isArray(payload.run) + ? payload.run + : payload.workflowRun && + typeof payload.workflowRun === 'object' && + !Array.isArray(payload.workflowRun) + ? payload.workflowRun + : payload; + const diagnostic = diagnosticRecord(source, secretValues); + return boundedDiagnostic( + diagnostic && Object.keys(diagnostic).length > 0 + ? JSON.stringify(diagnostic) + : '' + ); + } catch { + if (text.includes('{') || text.includes('}')) { + return ''; + } + return boundedDiagnostic(redactDiagnosticText(text, secretValues)); + } +} + export function formatCloudRunDiagnostics({ runId, terminalStatus, lastStatusOutput, statusPollFailures, logs, + diagnosticSecretValues = [], }) { const logOutput = `${logs?.stdout ?? ''}${logs?.stderr ?? ''}`; return [ @@ -83,7 +171,9 @@ export function formatCloudRunDiagnostics({ `run_id=${runId}`, `terminal_status=${terminalStatus ?? 'unknown'}`, `status_poll_failures=${statusPollFailures}`, - `last_status_response=${boundedDiagnostic(lastStatusOutput) || ''}`, + `last_status_response=${ + sanitizeCloudStatusDiagnostic(lastStatusOutput, diagnosticSecretValues) || '' + }`, `cloud_logs_exit_code=${logs?.exitCode ?? 'unknown'}`, `cloud_logs_timed_out=${logs?.timedOut === true}`, `cloud_logs_output=${logOutput ? 'present' : 'empty'}`, @@ -96,6 +186,7 @@ export async function writeStatusPollTimeoutDiagnostics({ runId, lastStatusOutput, statusPollFailures, + diagnosticSecretValues = [], }) { await mkdir(path.dirname(logsPath), { recursive: true }); await writeFile( @@ -106,6 +197,7 @@ export async function writeStatusPollTimeoutDiagnostics({ lastStatusOutput, statusPollFailures, logs: { stdout: '', stderr: '', exitCode: 'unknown', timedOut: true }, + diagnosticSecretValues, }) ); } @@ -173,7 +265,7 @@ export function createCliApiKeyEnvironment(env = process.env) { const cliEnv = { ...env, CLOUD_API_URL: apiUrl, CLOUD_API_KEY: apiKey }; for (const key of LEGACY_REFRESHABLE_AUTH_KEYS) delete cliEnv[key]; - return { cliEnv }; + return { cliEnv, diagnosticSecretValues: [apiKey] }; } export async function main() { @@ -317,6 +409,7 @@ export async function main() { runId, lastStatusOutput, statusPollFailures, + diagnosticSecretValues: auth.diagnosticSecretValues, }); throw new Error(`Cloud status command timed out for run ${runId}`); } @@ -325,7 +418,9 @@ export async function main() { lastStatusOutput = statusResult.stderr.trim() || statusResult.stdout.trim(); console.warn( `Cloud status poll failed (${statusResult.exitCode}); retrying${ - lastStatusOutput ? `: ${boundedDiagnostic(lastStatusOutput)}` : '' + lastStatusOutput + ? `: ${sanitizeCloudStatusDiagnostic(lastStatusOutput, auth.diagnosticSecretValues)}` + : '' }` ); continue; @@ -359,6 +454,7 @@ export async function main() { lastStatusOutput, statusPollFailures, logs, + diagnosticSecretValues: auth.diagnosticSecretValues, }) + logs.stdout + logs.stderr diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index d287e1c601..5ea8dc4251 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -50,6 +50,7 @@ import { createCliApiKeyEnvironment, formatCloudRunDiagnostics, preparedRunIdFromOutput, + sanitizeCloudStatusDiagnostic, writeStatusPollTimeoutDiagnostics, } from '../../scripts/pr-proof/run-cloud.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. @@ -457,6 +458,78 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(diagnostics).toContain('cloud_logs_output=empty'); }); + it('allowlists status diagnostics without retaining nested workflow output', () => { + const rawStatus = JSON.stringify({ + runId: 'cloud-run-123', + status: 'failed', + updatedAt: '2026-09-08T10:00:00.000Z', + workflow: 'return process.env.SECRET', + error: { + message: 'nested-error-must-not-survive', + apiKey: 'ci-key', + }, + result: { + error: { + token: 'rk_live_0123456789abcdef', + detail: 'nested-result-must-not-survive', + }, + }, + failure: { + phase: 'launch', + code: 'workflow_launch_failed', + message: 'request ci-key failed for rk_live_0123456789abcdef', + causeChain: ['nested-cause-must-not-survive'], + }, + }); + const diagnostic = sanitizeCloudStatusDiagnostic(`status response follows\n${rawStatus}`, ['ci-key']); + + expect(JSON.parse(diagnostic)).toEqual({ + runId: 'cloud-run-123', + status: 'failed', + updatedAt: '2026-09-08T10:00:00.000Z', + failure: { + phase: 'launch', + code: 'workflow_launch_failed', + message: 'request [redacted] failed for rk_live_…', + }, + }); + expect(diagnostic).not.toContain('nested-result-must-not-survive'); + expect(diagnostic).not.toContain('nested-error-must-not-survive'); + expect(diagnostic).not.toContain('nested-cause-must-not-survive'); + expect(diagnostic).not.toContain('0123456789abcdef'); + + const persisted = formatCloudRunDiagnostics({ + runId: 'cloud-run-123', + terminalStatus: 'failed', + lastStatusOutput: rawStatus, + statusPollFailures: 0, + logs: { stdout: '', stderr: '', exitCode: 0, timedOut: false }, + diagnosticSecretValues: ['ci-key'], + }); + expect(persisted).toContain('workflow_launch_failed'); + expect(persisted).not.toContain('nested-result-must-not-survive'); + expect(persisted).not.toContain('nested-error-must-not-survive'); + expect(persisted).not.toContain('ci-key'); + }); + + it('redacts configured short secrets from non-JSON status errors', () => { + const diagnostic = sanitizeCloudStatusDiagnostic('Status request failed: upstream rejected short-key', [ + 'short-key', + ]); + + expect(diagnostic).toBe('Status request failed: upstream rejected [redacted]'); + expect(diagnostic).not.toContain('short-key'); + }); + + it('omits malformed JSON status payloads instead of falling back to raw output', () => { + const diagnostic = sanitizeCloudStatusDiagnostic( + '{"status":"failed","result":{"token":"unknown-secret"}' + ); + + expect(diagnostic).toBe(''); + expect(diagnostic).not.toContain('unknown-secret'); + }); + it('persists bounded diagnostics when a Cloud status poll times out', async () => { const root = await mkdtemp(path.join(os.tmpdir(), 'relay-pr-proof-status-timeout-')); const logsPath = path.join(root, 'nested', 'cloud.log'); @@ -507,6 +580,7 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(auth.cliEnv.CLOUD_API_URL).toBe(credentialEnv.CLOUD_API_URL); expect(auth.cliEnv.CLOUD_API_KEY).toBe(credentialEnv.CLOUD_API_KEY); + expect(auth.diagnosticSecretValues).toEqual([credentialEnv.CLOUD_API_KEY]); expect(auth.cliEnv.CLOUD_API_ACCESS_TOKEN).toBeUndefined(); expect(auth.cliEnv.CLOUD_API_REFRESH_TOKEN).toBeUndefined(); expect(auth.cliEnv.CLOUD_API_ACCESS_TOKEN_EXPIRES_AT).toBeUndefined(); From 1925c1a7d2c4f7dd9739eb708dc016fe6c29b915 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 13:15:42 +0200 Subject: [PATCH 11/50] fix(pr-proof): redact all Cloud diagnostics --- scripts/pr-proof/run-cloud.mjs | 118 +++++++++++++++++------ tests/fixtures/pr-proof-contract.test.ts | 71 +++++++++++--- 2 files changed, 147 insertions(+), 42 deletions(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index f137a808b2..2baa3478a7 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -9,6 +9,14 @@ import { runBoundedProcess } from './process-runner.mjs'; const TERMINAL_SUCCESS = new Set(['completed', 'succeeded', 'success']); const TERMINAL_FAILURE = new Set(['failed', 'cancelled', 'canceled', 'timed_out', 'error']); +const CLOUD_RUN_STATUSES = new Set([ + 'pending', + 'queued', + 'launching', + 'running', + ...TERMINAL_SUCCESS, + ...TERMINAL_FAILURE, +]); const LEGACY_REFRESHABLE_AUTH_KEYS = [ 'CLOUD_API_ACCESS_TOKEN', 'CLOUD_API_REFRESH_TOKEN', @@ -21,6 +29,8 @@ const MAX_DIAGNOSTIC_BYTES = 64 * 1024; const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60_000; const PREPARED_RUN_ID_MARKER = 'AGENT_RELAY_CLOUD_PREPARED_RUN_ID='; const RUN_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; +const DIAGNOSTIC_TOKEN_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/; +const ISO_TIMESTAMP_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/; const LIVE_CREDENTIAL_RE = /(rk_live_|rjt_live_|at_live_|nt_live_|ot_live_|cld_at_|rth_at_|ocl_node_enr_|br_)([A-Za-z0-9_%-]+(?:\.[A-Za-z0-9_%-]+)*)/g; const STATUS_DIAGNOSTIC_FIELDS = [ @@ -31,17 +41,8 @@ const STATUS_DIAGNOSTIC_FIELDS = [ 'relayflowVersion', 'createdAt', 'updatedAt', - 'error', - 'message', -]; -const STATUS_FAILURE_DIAGNOSTIC_FIELDS = [ - 'phase', - 'code', - 'message', - 'dispatchType', - 'sandboxId', - 'occurredAt', ]; +const STATUS_FAILURE_DIAGNOSTIC_FIELDS = ['phase', 'code', 'dispatchType', 'sandboxId', 'occurredAt']; function run(command, args, options = {}) { return runBoundedProcess(command, args, { @@ -91,7 +92,7 @@ export function boundedDiagnostic(value) { return `${tail}${marker}`; } -function redactDiagnosticText(value, secretValues = []) { +export function sanitizeCloudCommandOutput(value, secretValues = []) { let text = String(value ?? ''); for (const secretValue of secretValues) { const secret = typeof secretValue === 'string' ? secretValue : ''; @@ -100,19 +101,36 @@ function redactDiagnosticText(value, secretValues = []) { return text.replace(LIVE_CREDENTIAL_RE, (_match, prefix) => `${prefix}…`); } +function structuralDiagnosticValue(field, value, secretValues) { + const sanitized = sanitizeCloudCommandOutput(value, secretValues); + if (field === 'status') return recognizedCloudRunStatus(sanitized); + if (field === 'runId' || field === 'sandboxId') { + return RUN_ID_RE.test(sanitized) ? sanitized : null; + } + if (field === 'relayflowVersion') { + return sanitized === 'v1' || sanitized === 'v2' ? sanitized : null; + } + if (field === 'createdAt' || field === 'updatedAt' || field === 'occurredAt') { + return ISO_TIMESTAMP_RE.test(sanitized) && Number.isFinite(Date.parse(sanitized)) ? sanitized : null; + } + return DIAGNOSTIC_TOKEN_RE.test(sanitized) ? sanitized : null; +} + function diagnosticRecord(value, secretValues) { if (!value || typeof value !== 'object' || Array.isArray(value)) return null; const diagnostic = {}; for (const field of STATUS_DIAGNOSTIC_FIELDS) { if (typeof value[field] === 'string') { - diagnostic[field] = redactDiagnosticText(value[field], secretValues); + const structuralValue = structuralDiagnosticValue(field, value[field], secretValues); + if (structuralValue) diagnostic[field] = structuralValue; } } if (value.failure && typeof value.failure === 'object' && !Array.isArray(value.failure)) { const failure = {}; for (const field of STATUS_FAILURE_DIAGNOSTIC_FIELDS) { if (typeof value.failure[field] === 'string') { - failure[field] = redactDiagnosticText(value.failure[field], secretValues); + const structuralValue = structuralDiagnosticValue(field, value.failure[field], secretValues); + if (structuralValue) failure[field] = structuralValue; } } if (Object.keys(failure).length > 0) diagnostic.failure = failure; @@ -153,7 +171,7 @@ export function sanitizeCloudStatusDiagnostic(output, secretValues = []) { if (text.includes('{') || text.includes('}')) { return ''; } - return boundedDiagnostic(redactDiagnosticText(text, secretValues)); + return ''; } } @@ -168,8 +186,8 @@ export function formatCloudRunDiagnostics({ const logOutput = `${logs?.stdout ?? ''}${logs?.stderr ?? ''}`; return [ 'Cloud RelayFlow diagnostics', - `run_id=${runId}`, - `terminal_status=${terminalStatus ?? 'unknown'}`, + `run_id=${sanitizeCloudCommandOutput(runId, diagnosticSecretValues)}`, + `terminal_status=${sanitizeCloudCommandOutput(terminalStatus ?? 'unknown', diagnosticSecretValues)}`, `status_poll_failures=${statusPollFailures}`, `last_status_response=${ sanitizeCloudStatusDiagnostic(lastStatusOutput, diagnosticSecretValues) || '' @@ -181,6 +199,14 @@ export function formatCloudRunDiagnostics({ ].join('\n'); } +export function formatCloudRunArtifact(input) { + return ( + formatCloudRunDiagnostics(input) + + sanitizeCloudCommandOutput(input.logs?.stdout, input.diagnosticSecretValues) + + sanitizeCloudCommandOutput(input.logs?.stderr, input.diagnosticSecretValues) + ); +} + export async function writeStatusPollTimeoutDiagnostics({ logsPath, runId, @@ -202,9 +228,15 @@ export async function writeStatusPollTimeoutDiagnostics({ ); } +export function recognizedCloudRunStatus(value) { + if (typeof value !== 'string') return null; + const status = value.toLowerCase(); + return CLOUD_RUN_STATUSES.has(status) ? status : null; +} + function statusFrom(payload) { for (const candidate of [payload.status, payload.run?.status, payload.workflowRun?.status]) { - if (typeof candidate === 'string') return candidate.toLowerCase(); + if (typeof candidate === 'string') return recognizedCloudRunStatus(candidate); } throw new Error('Cloud status response did not contain a status'); } @@ -332,14 +364,24 @@ export async function main() { const cancelRemote = async (reason) => { if (!runId || terminal) return; cancelPromise ??= (async () => { - console.warn(`Cancelling Cloud RelayFlow run ${runId} (${reason})`); + console.warn( + `Cancelling Cloud RelayFlow run ${sanitizeCloudCommandOutput( + runId, + auth.diagnosticSecretValues + )} (${reason})` + ); const result = await run(cli, ['cloud', 'cancel', runId, '--json'], { env: auth.cliEnv, quiet: true, timeoutMs: commandTimeoutMs, }); if (result.exitCode !== 0 || result.timedOut) { - console.warn(`Cloud cancellation failed with exit ${result.exitCode}: ${result.stderr.trim()}`); + console.warn( + `Cloud cancellation failed with exit ${result.exitCode}: ${sanitizeCloudCommandOutput( + result.stderr.trim(), + auth.diagnosticSecretValues + )}` + ); } })(); await cancelPromise; @@ -350,7 +392,9 @@ export async function main() { shuttingDown = true; activeCommandController?.abort(); void (async () => { - await cancelRemote(signal).catch((error) => console.warn(error.message)); + await cancelRemote(signal).catch((error) => + console.warn(sanitizeCloudCommandOutput(error.message, auth.diagnosticSecretValues)) + ); process.exit(signal === 'SIGINT' ? 130 : 143); })(); }; @@ -377,7 +421,7 @@ export async function main() { ); } if (launch.exitCode !== 0) { - process.stderr.write(launch.stderr); + process.stderr.write(sanitizeCloudCommandOutput(launch.stderr, auth.diagnosticSecretValues)); throw new Error(`Cloud workflow submission failed with exit ${launch.exitCode}`); } const launchPayload = parseJsonOutput(launch.stdout, 'Cloud run'); @@ -389,7 +433,7 @@ export async function main() { throw new Error(`Cloud prepare/run ID mismatch: ${runId} != ${launchedRunId}`); } runId = launchedRunId; - console.log(`Cloud RelayFlow run: ${runId}`); + console.log(`Cloud RelayFlow run: ${sanitizeCloudCommandOutput(runId, auth.diagnosticSecretValues)}`); if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `run_id=${runId}\n`); const deadline = Date.now() + timeoutMs; @@ -427,6 +471,11 @@ export async function main() { } lastStatusOutput = statusResult.stdout.trim(); const status = statusFrom(parseJsonOutput(statusResult.stdout, 'Cloud status')); + if (!status) { + statusPollFailures += 1; + console.warn('Cloud RelayFlow status: '); + continue; + } console.log(`Cloud RelayFlow status: ${status}`); if (TERMINAL_SUCCESS.has(status) || TERMINAL_FAILURE.has(status)) { terminalStatus = status; @@ -448,19 +497,21 @@ export async function main() { }); await writeFile( logsPath, - formatCloudRunDiagnostics({ + formatCloudRunArtifact({ runId, terminalStatus, lastStatusOutput, statusPollFailures, logs, diagnosticSecretValues: auth.diagnosticSecretValues, - }) + - logs.stdout + - logs.stderr + }) ); - if (logs.stdout) process.stdout.write(logs.stdout); - if (logs.stderr) process.stderr.write(logs.stderr); + if (logs.stdout) { + process.stdout.write(sanitizeCloudCommandOutput(logs.stdout, auth.diagnosticSecretValues)); + } + if (logs.stderr) { + process.stderr.write(sanitizeCloudCommandOutput(logs.stderr, auth.diagnosticSecretValues)); + } if (logs.timedOut) throw new Error(`Cloud log retrieval timed out for run ${runId}`); if (logs.exitCode !== 0) throw new Error(`Cloud log retrieval failed with exit ${logs.exitCode}`); @@ -470,7 +521,10 @@ export async function main() { if (process.env.GITHUB_STEP_SUMMARY) { await appendFile( process.env.GITHUB_STEP_SUMMARY, - `\n- Cloud run: \`${runId}\`\n- Cloud status: **${terminalStatus}**\n` + `\n- Cloud run: \`${sanitizeCloudCommandOutput( + runId, + auth.diagnosticSecretValues + )}\`\n- Cloud status: **${terminalStatus}**\n` ); } } finally { @@ -478,13 +532,15 @@ export async function main() { process.removeListener('SIGINT', signalHandler); process.removeListener('SIGTERM', signalHandler); if (runId && !terminal) - await cancelRemote('dispatcher exiting').catch((error) => console.warn(error.message)); + await cancelRemote('dispatcher exiting').catch((error) => + console.warn(sanitizeCloudCommandOutput(error.message, auth.diagnosticSecretValues)) + ); } } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { main().catch((error) => { - console.error(error.message); + console.error(sanitizeCloudCommandOutput(error.message, [process.env.CLOUD_API_KEY])); process.exitCode = 1; }); } diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 5ea8dc4251..a96afe3b1c 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -48,8 +48,11 @@ import { boundedDuration, createPreparedRunProgressParser, createCliApiKeyEnvironment, + formatCloudRunArtifact, formatCloudRunDiagnostics, preparedRunIdFromOutput, + recognizedCloudRunStatus, + sanitizeCloudCommandOutput, sanitizeCloudStatusDiagnostic, writeStatusPollTimeoutDiagnostics, } from '../../scripts/pr-proof/run-cloud.mjs'; @@ -454,7 +457,7 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(diagnostics).toContain('run_id=cloud-run-123'); expect(diagnostics).toContain('terminal_status=failed'); expect(diagnostics).toContain('status_poll_failures=2'); - expect(diagnostics).toContain('step timeout'); + expect(diagnostics).not.toContain('step timeout'); expect(diagnostics).toContain('cloud_logs_output=empty'); }); @@ -464,10 +467,9 @@ describe('Cloud dispatcher API key lifecycle', () => { status: 'failed', updatedAt: '2026-09-08T10:00:00.000Z', workflow: 'return process.env.SECRET', - error: { - message: 'nested-error-must-not-survive', - apiKey: 'ci-key', - }, + error: 'top-level-error-must-not-survive ci-key', + message: 'top-level-message-must-not-survive', + dispatchType: 'arbitrary free text must not survive', result: { error: { token: 'rk_live_0123456789abcdef', @@ -477,7 +479,8 @@ describe('Cloud dispatcher API key lifecycle', () => { failure: { phase: 'launch', code: 'workflow_launch_failed', - message: 'request ci-key failed for rk_live_0123456789abcdef', + message: 'failure-message-must-not-survive ci-key rk_live_0123456789abcdef', + sandboxId: 'sandbox id with arbitrary free text', causeChain: ['nested-cause-must-not-survive'], }, }); @@ -490,11 +493,13 @@ describe('Cloud dispatcher API key lifecycle', () => { failure: { phase: 'launch', code: 'workflow_launch_failed', - message: 'request [redacted] failed for rk_live_…', }, }); expect(diagnostic).not.toContain('nested-result-must-not-survive'); - expect(diagnostic).not.toContain('nested-error-must-not-survive'); + expect(diagnostic).not.toContain('top-level-error-must-not-survive'); + expect(diagnostic).not.toContain('top-level-message-must-not-survive'); + expect(diagnostic).not.toContain('failure-message-must-not-survive'); + expect(diagnostic).not.toContain('arbitrary free text'); expect(diagnostic).not.toContain('nested-cause-must-not-survive'); expect(diagnostic).not.toContain('0123456789abcdef'); @@ -508,19 +513,54 @@ describe('Cloud dispatcher API key lifecycle', () => { }); expect(persisted).toContain('workflow_launch_failed'); expect(persisted).not.toContain('nested-result-must-not-survive'); - expect(persisted).not.toContain('nested-error-must-not-survive'); + expect(persisted).not.toContain('top-level-error-must-not-survive'); + expect(persisted).not.toContain('top-level-message-must-not-survive'); + expect(persisted).not.toContain('failure-message-must-not-survive'); + expect(persisted).not.toContain('arbitrary free text'); expect(persisted).not.toContain('ci-key'); }); - it('redacts configured short secrets from non-JSON status errors', () => { + it('omits non-JSON status errors even when they contain configured secrets', () => { const diagnostic = sanitizeCloudStatusDiagnostic('Status request failed: upstream rejected short-key', [ 'short-key', ]); - expect(diagnostic).toBe('Status request failed: upstream rejected [redacted]'); + expect(diagnostic).toBe(''); expect(diagnostic).not.toContain('short-key'); }); + it('accepts only known Cloud run statuses', () => { + expect(recognizedCloudRunStatus('RUNNING')).toBe('running'); + expect(recognizedCloudRunStatus('failed')).toBe('failed'); + expect(recognizedCloudRunStatus('running-opaque-secret')).toBeNull(); + expect(recognizedCloudRunStatus(null)).toBeNull(); + }); + + it('redacts configured and recognized credentials from raw command output and artifacts', () => { + const raw = 'stdout ci-api-key rk_live_0123456789abcdef'; + expect(sanitizeCloudCommandOutput(raw, ['ci-api-key'])).toBe('stdout [redacted] rk_live_…'); + + const artifact = formatCloudRunArtifact({ + runId: 'cloud-run-123', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed","error":"opaque-status-secret"}', + statusPollFailures: 0, + logs: { + stdout: `workflow output ci-api-key\n`, + stderr: 'failure rth_at_0123456789abcdef\n', + exitCode: 0, + timedOut: false, + }, + diagnosticSecretValues: ['ci-api-key'], + }); + + expect(artifact).toContain('workflow output [redacted]'); + expect(artifact).toContain('failure rth_at_…'); + expect(artifact).not.toContain('ci-api-key'); + expect(artifact).not.toContain('0123456789abcdef'); + expect(artifact).not.toContain('opaque-status-secret'); + }); + it('omits malformed JSON status payloads instead of falling back to raw output', () => { const diagnostic = sanitizeCloudStatusDiagnostic( '{"status":"failed","result":{"token":"unknown-secret"}' @@ -1845,6 +1885,15 @@ describe('trusted dispatcher source contract', () => { expect(source).toContain("requiredCredential(env, 'CLOUD_API_KEY')"); expect(source).not.toContain("path.join(authDir, 'cloud-auth.json')"); expect(source).not.toContain('CLOUD_API_REFRESH_TOKEN='); + expect(source).toContain('formatCloudRunArtifact({'); + expect(source).toContain('sanitizeCloudCommandOutput(launch.stderr'); + expect(source).toContain('sanitizeCloudCommandOutput(logs.stdout'); + expect(source).toContain('sanitizeCloudCommandOutput(logs.stderr'); + expect(source).toContain('sanitizeCloudCommandOutput(error.message'); + expect(source).toContain("console.warn('Cloud RelayFlow status: ')"); + expect(source).not.toContain('process.stderr.write(launch.stderr)'); + expect(source).not.toContain('process.stdout.write(logs.stdout)'); + expect(source).not.toContain('process.stderr.write(logs.stderr)'); }); it('emits the prepared Cloud run id before upload and final submission', async () => { From e22d2b353bbf1dbf69fe770c4a6d9f8fb02ba5e7 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 13:25:57 +0200 Subject: [PATCH 12/50] fix(pr-proof): hide malformed status payloads --- scripts/pr-proof/run-cloud.mjs | 10 ++++++++-- tests/fixtures/pr-proof-contract.test.ts | 9 +++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 2baa3478a7..94aeb00537 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -66,13 +66,19 @@ export function boundedDuration(value, { fallback, minimum, maximum, label }) { return parsed; } -function parseJsonOutput(output, label) { +export function parseJsonOutput(output, label) { try { return JSON.parse(output); } catch { const first = output.indexOf('{'); const last = output.lastIndexOf('}'); - if (first >= 0 && last > first) return JSON.parse(output.slice(first, last + 1)); + if (first >= 0 && last > first) { + try { + return JSON.parse(output.slice(first, last + 1)); + } catch { + // Fall through to the fixed error below without exposing payload excerpts. + } + } throw new Error(`${label} did not return JSON`); } } diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index a96afe3b1c..277d99f22b 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -50,6 +50,7 @@ import { createCliApiKeyEnvironment, formatCloudRunArtifact, formatCloudRunDiagnostics, + parseJsonOutput, preparedRunIdFromOutput, recognizedCloudRunStatus, sanitizeCloudCommandOutput, @@ -570,6 +571,14 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(diagnostic).not.toContain('unknown-secret'); }); + it('uses a fixed error when malformed JSON contains unrecognized secrets', () => { + const malformed = 'status response follows\n{"status":"failed","token":unknown-secret}'; + + expect(() => parseJsonOutput(malformed, 'Cloud status')).toThrow( + new Error('Cloud status did not return JSON') + ); + }); + it('persists bounded diagnostics when a Cloud status poll times out', async () => { const root = await mkdtemp(path.join(os.tmpdir(), 'relay-pr-proof-status-timeout-')); const logsPath = path.join(root, 'nested', 'cloud.log'); From 02155c463fd5faaa3370bce05c1dead7fabbc24b Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 13:32:36 +0200 Subject: [PATCH 13/50] fix(proof): retry malformed cloud status polls --- scripts/pr-proof/run-cloud.mjs | 10 +++++++++- tests/fixtures/pr-proof-contract.test.ts | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 94aeb00537..81a1ff31aa 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -247,6 +247,14 @@ function statusFrom(payload) { throw new Error('Cloud status response did not contain a status'); } +export function recognizedCloudStatusFromOutput(output) { + try { + return statusFrom(parseJsonOutput(output, 'Cloud status')); + } catch { + return null; + } +} + function requiredCredential(env, name) { const value = env[name]?.trim(); if (!value) throw new Error(`${name} is required`); @@ -476,7 +484,7 @@ export async function main() { continue; } lastStatusOutput = statusResult.stdout.trim(); - const status = statusFrom(parseJsonOutput(statusResult.stdout, 'Cloud status')); + const status = recognizedCloudStatusFromOutput(statusResult.stdout); if (!status) { statusPollFailures += 1; console.warn('Cloud RelayFlow status: '); diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 277d99f22b..600ff71ad2 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -52,6 +52,7 @@ import { formatCloudRunDiagnostics, parseJsonOutput, preparedRunIdFromOutput, + recognizedCloudStatusFromOutput, recognizedCloudRunStatus, sanitizeCloudCommandOutput, sanitizeCloudStatusDiagnostic, @@ -537,6 +538,12 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(recognizedCloudRunStatus(null)).toBeNull(); }); + it('treats malformed and status-less successful poll output as retryable', () => { + expect(recognizedCloudStatusFromOutput('{"status":unknown-secret}')).toBeNull(); + expect(recognizedCloudStatusFromOutput('{"runId":"still-running"}')).toBeNull(); + expect(recognizedCloudStatusFromOutput('{"workflowRun":{"status":"running"}}')).toBe('running'); + }); + it('redacts configured and recognized credentials from raw command output and artifacts', () => { const raw = 'stdout ci-api-key rk_live_0123456789abcdef'; expect(sanitizeCloudCommandOutput(raw, ['ci-api-key'])).toBe('stdout [redacted] rk_live_…'); From a787b1f2790b5d1dfb2375474f6c2d459786c1cb Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 13:44:49 +0200 Subject: [PATCH 14/50] fix(cloud): preserve legacy sandbox responses --- packages/cloud/src/fleet-sandbox.test.ts | 56 +++++++++++++++++-- packages/cloud/src/fleet-sandbox.ts | 10 ++-- scripts/pr-proof/run-cloud.mjs | 28 ++++++++++ tests/fixtures/pr-proof-contract.test.ts | 27 +++++++++ .../1656-long-running-agent37-sandbox/run.mjs | 46 ++++++++------- 5 files changed, 140 insertions(+), 27 deletions(-) diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 207985e32d..1e3570d622 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -174,6 +174,58 @@ describe('Cloud fleet sandbox client', () => { expect(ensureBody).not.toHaveProperty('sandboxId'); }); + it.each(['provisioned', 'provisioning_timeout'] as const)( + 'accepts an older Cloud %s response without providerSandboxId when the exact public identity matches', + async (outcome) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + outcome === 'provisioned' + ? { + outcome, + nodeId: 'node-legacy', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + providerId: 'agent37', + } + : { + outcome, + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + waitedMs: 90_000, + providerId: 'agent37', + }, + { status: outcome === 'provisioned' ? 201 : 202 } + ), + auth, + }); + + const result = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }); + + expect(result).toMatchObject({ + outcome, + sandboxId: SANDBOX_ID, + nodeName: SANDBOX_NAME, + providerId: 'agent37', + }); + expect(result).not.toHaveProperty('providerSandboxId'); + } + ); + it('requires a deterministic name whenever a sandbox identity is supplied', async () => { await expect( ensureCloudFleetSandbox({ @@ -206,14 +258,12 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-other', nodeName: SANDBOX_NAME, sandboxId: 'sbx_223e4567-e89b-42d3-a456-426614174000', - providerSandboxId: 'provider-sandbox-other', relayWorkspaceId: 'rw_abc', relayfileMounted: true, } : { outcome, sandboxId: 'sbx_223e4567-e89b-42d3-a456-426614174000', - providerSandboxId: 'provider-sandbox-other', relayWorkspaceId: 'rw_abc', nodeName: SANDBOX_NAME, waitedMs: 90_000, @@ -259,14 +309,12 @@ describe('Cloud fleet sandbox client', () => { nodeId: 'node-other', nodeName: 'fleet-sandbox-other', sandboxId: SANDBOX_ID, - providerSandboxId: 'provider-sandbox-other', relayWorkspaceId: 'rw_abc', relayfileMounted: true, } : { outcome, sandboxId: SANDBOX_ID, - providerSandboxId: 'provider-sandbox-other', relayWorkspaceId: 'rw_abc', nodeName: 'fleet-sandbox-other', waitedMs: 90_000, diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 90201d8cae..294f91237b 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -116,7 +116,7 @@ export type CloudFleetSandboxReady = { nodeId: string; nodeName: string; sandboxId: string; - providerSandboxId: string; + providerSandboxId?: string; relayWorkspaceId: string; relayfileMounted: boolean; relayfileMountPath?: string; @@ -138,7 +138,7 @@ export type CloudFleetSandboxProvisioningTimeout = { outcome: 'provisioning_timeout'; cloudWorkspaceId: string; sandboxId: string; - providerSandboxId: string; + providerSandboxId?: string; relayWorkspaceId: string; nodeName: string; waitedMs: number; @@ -350,13 +350,14 @@ function normalizeEnsureResult( throw new Error('Cloud fleet sandbox response is missing relayfileMounted.'); } const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); + const providerSandboxId = readString(payload, 'providerSandboxId'); return { outcome, cloudWorkspaceId, nodeId: requiredString(payload, 'nodeId', 'Cloud fleet sandbox'), nodeName, sandboxId, - providerSandboxId: requiredString(payload, 'providerSandboxId', 'Cloud fleet sandbox'), + ...(providerSandboxId === undefined ? {} : { providerSandboxId }), relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), relayfileMounted: payload.relayfileMounted, ...(providerId === undefined ? {} : { providerId }), @@ -381,11 +382,12 @@ function normalizeEnsureResult( if (outcome === 'provisioning_timeout') { const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); + const providerSandboxId = readString(payload, 'providerSandboxId'); return { outcome, cloudWorkspaceId, sandboxId, - providerSandboxId: requiredString(payload, 'providerSandboxId', 'Cloud fleet sandbox'), + ...(providerSandboxId === undefined ? {} : { providerSandboxId }), relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), nodeName, waitedMs: requiredNumber(payload, 'waitedMs', 'Cloud fleet sandbox'), diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 81a1ff31aa..7420f79f11 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -234,6 +234,27 @@ export async function writeStatusPollTimeoutDiagnostics({ ); } +export async function writeStatusPollDeadlineDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, + diagnosticSecretValues = [], +}) { + await mkdir(path.dirname(logsPath), { recursive: true }); + await writeFile( + logsPath, + formatCloudRunDiagnostics({ + runId, + terminalStatus: 'status_poll_deadline_exceeded', + lastStatusOutput, + statusPollFailures, + logs: { stdout: '', stderr: '', exitCode: 'unknown', timedOut: false }, + diagnosticSecretValues, + }) + ); +} + export function recognizedCloudRunStatus(value) { if (typeof value !== 'string') return null; const status = value.toLowerCase(); @@ -498,6 +519,13 @@ export async function main() { } } if (!terminalStatus) { + await writeStatusPollDeadlineDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, + diagnosticSecretValues: auth.diagnosticSecretValues, + }); await cancelRemote('deadline exceeded'); terminal = true; throw new Error(`Cloud RelayFlow exceeded ${timeoutMs}ms`); diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 600ff71ad2..5f25cb8721 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -56,6 +56,7 @@ import { recognizedCloudRunStatus, sanitizeCloudCommandOutput, sanitizeCloudStatusDiagnostic, + writeStatusPollDeadlineDiagnostics, writeStatusPollTimeoutDiagnostics, } from '../../scripts/pr-proof/run-cloud.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. @@ -608,6 +609,32 @@ describe('Cloud dispatcher API key lifecycle', () => { } }); + it.each([ + ['nonzero', 'Status request failed: unknown-secret'], + ['malformed', '{"status":unknown-secret}'], + ['status-less', '{"runId":"cloud-run-deadline","result":{"token":"unknown-secret"}}'], + ])('persists safe bounded diagnostics when %s polls exhaust the deadline', async (_case, output) => { + const root = await mkdtemp(path.join(os.tmpdir(), 'relay-pr-proof-status-deadline-')); + const logsPath = path.join(root, 'nested', 'cloud.log'); + try { + await writeStatusPollDeadlineDiagnostics({ + logsPath, + runId: 'cloud-run-deadline', + lastStatusOutput: output, + statusPollFailures: 4, + }); + + const diagnostics = await readFile(logsPath, 'utf8'); + expect(diagnostics).toContain('run_id=cloud-run-deadline'); + expect(diagnostics).toContain('terminal_status=status_poll_deadline_exceeded'); + expect(diagnostics).toContain('status_poll_failures=4'); + expect(diagnostics).not.toContain('unknown-secret'); + expect(Buffer.byteLength(diagnostics, 'utf8')).toBeLessThanOrEqual(65 * 1024); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + it('bounds multibyte Cloud diagnostics by UTF-8 bytes', () => { const diagnostics = boundedDiagnostic('😀'.repeat(100_000)); diff --git a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs index 494134dc3a..cdb79c2307 100644 --- a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs +++ b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs @@ -29,8 +29,9 @@ * * The probe deliberately requests NO `--sandbox-provider`. That is the feature: * Cloud picks the provider, and `agent37` must survive back out into cleanup. - * Dispatch is then failed on purpose, because the cleanup call is what carries - * the attribution. + * After dispatch succeeds, the probe captures the CLI's JSON output and makes + * that output sink fail on purpose. This observes the normalized Cloud response + * and drives the same command into cleanup, where provider attribution is used. * * Base: the CLI sends no sandbox identity, while the existing long-running * profile and Cloud-selected `agent37` response and cleanup attribution are @@ -215,13 +216,14 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned ), auth, }) - // 3. cleanup, triggered by the deliberate dispatch failure below + // 3. cleanup, triggered by the deliberate output-sink failure below .mockResolvedValueOnce({ response: Response.json({ sandboxId: REPLAY_SANDBOX_ID, deleted: true }), auth, }); const errors: string[] = []; + const cliOutput: string[] = []; const replayArgs = process.env.RELAY_PR_PROOF_ARM === 'head' ? ['--sandbox-id', REPLAY_SANDBOX_ID, '--sandbox-name', REPLAY_SANDBOX_NAME] @@ -230,14 +232,13 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned program.exitOverride(); registerFleetCommands(program, { sdk: { - // Dispatch fails on purpose: the cleanup call is what carries provider - // attribution, and it only happens on this path. createAgentRelay: vi.fn(() => ({ messaging: { placement: { - spawn: vi.fn(async () => { - throw new Error('dispatch failed'); - }), + spawn: vi.fn(async () => ({ + invocationId: 'inv_relayflow', + node: { name: REPLAY_SANDBOX_NAME }, + })), }, }, })) as never, @@ -245,7 +246,12 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned workspace: { info: vi.fn(async () => ({ id: 'rw_relayflow' })) }, })) as never, createWorkspace: vi.fn() as never, - log: vi.fn(), + // Capture the real CLI serialization, then fail so this same invocation + // exercises cleanup without sourcing evidence from the network mock. + log: (...args: unknown[]) => { + cliOutput.push(args.join(' ')); + throw new Error('CLI output sink failed after capture'); + }, error: (...args: unknown[]) => errors.push(args.join(' ')), exit: (() => { throw new Error('__exit__'); @@ -280,9 +286,10 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned ) ).rejects.toThrow('__exit__'); - // The command really ran and really failed dispatch; without this a probe - // that never reached the sandbox path could report a false base. - expect(errors.join('\n')).toContain('dispatch failed'); + // The command really reached successful dispatch and serialized its result; + // without this a probe that never reached the sandbox path could report a false base. + expect(errors.join('\n')).toContain('CLI output sink failed after capture'); + expect(cliOutput).toHaveLength(1); const ensureRequest = mocks.authorizedApiFetch.mock.calls[1]?.[2]; const deleteRequest = mocks.authorizedApiFetch.mock.calls[2]?.[2]; @@ -290,6 +297,7 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned expect(deleteRequest?.body).toEqual(expect.any(String)); const ensureBody = JSON.parse(ensureRequest.body); const deleteBody = JSON.parse(deleteRequest.body); + const cliResult = JSON.parse(cliOutput[0]); await writeFile( output, @@ -299,15 +307,15 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned ensureForceProvision: ensureBody.forceProvision ?? null, ensureName: ensureBody.name ?? null, ensureProviderId: ensureBody.providerId ?? null, - responseSandboxId: provisionedResponse.sandboxId, - responseProviderSandboxId: provisionedResponse.providerSandboxId, - responseProviderId: provisionedResponse.providerId, - responseNodeName: provisionedResponse.nodeName, + responseSandboxId: cliResult.sandbox?.sandboxId ?? null, + responseProviderSandboxId: cliResult.sandbox?.providerSandboxId ?? null, + responseProviderId: cliResult.sandbox?.providerId ?? null, + responseNodeName: cliResult.sandbox?.nodeName ?? null, deleteSandboxId: decodeURIComponent( String(mocks.authorizedApiFetch.mock.calls[2]?.[1] ?? '').split('/').pop() ?? '' ) || null, deleteProviderId: deleteBody.providerId ?? null, - dispatchFailureObserved: errors.join('\n').includes('dispatch failed'), + outputFailureObserved: errors.join('\n').includes('CLI output sink failed after capture'), }), 'utf8' ); @@ -338,8 +346,8 @@ try { ); const observation = JSON.parse(await readFile(observationPath, 'utf8')); - if (observation.dispatchFailureObserved !== true) { - throw new Error('The probe did not reach the sandbox dispatch path, so it observed nothing.'); + if (observation.outputFailureObserved !== true) { + throw new Error('The probe did not serialize the CLI sandbox result, so it observed nothing.'); } // The command line named no provider on either arm. If this ever stops being // true the case is proving provider pinning, not capability routing. From 17b075f61793cc64ba8624be5dddc8ccd00b3231 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 13:48:52 +0200 Subject: [PATCH 15/50] fix(pr-proof): redact credential prefixes safely --- scripts/pr-proof/run-cloud.mjs | 4 ++-- tests/fixtures/pr-proof-contract.test.ts | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 7420f79f11..c516e7a04a 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -99,12 +99,12 @@ export function boundedDiagnostic(value) { } export function sanitizeCloudCommandOutput(value, secretValues = []) { - let text = String(value ?? ''); + let text = String(value ?? '').replace(LIVE_CREDENTIAL_RE, (_match, prefix) => `${prefix}…`); for (const secretValue of secretValues) { const secret = typeof secretValue === 'string' ? secretValue : ''; if (secret) text = text.split(secret).join('[redacted]'); } - return text.replace(LIVE_CREDENTIAL_RE, (_match, prefix) => `${prefix}…`); + return text; } function structuralDiagnosticValue(field, value, secretValues) { diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 5f25cb8721..6d4c21f160 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -570,6 +570,30 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(artifact).not.toContain('opaque-status-secret'); }); + it('redacts complete credentials before configured prefix secrets in console and artifacts', () => { + const credentialSuffix = '0123456789abcdef'; + const consoleOutput = sanitizeCloudCommandOutput(`launch rk_live_${credentialSuffix}`, ['rk_live_']); + expect(consoleOutput).toBe('launch [redacted]…'); + expect(consoleOutput).not.toContain(credentialSuffix); + + const artifact = formatCloudRunArtifact({ + runId: 'cloud-run-prefix-secret', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed"}', + statusPollFailures: 0, + logs: { + stdout: `workflow output rk_live_${credentialSuffix}\n`, + stderr: '', + exitCode: 0, + timedOut: false, + }, + diagnosticSecretValues: ['rk_live_'], + }); + + expect(artifact).toContain('workflow output [redacted]…'); + expect(artifact).not.toContain(credentialSuffix); + }); + it('omits malformed JSON status payloads instead of falling back to raw output', () => { const diagnostic = sanitizeCloudStatusDiagnostic( '{"status":"failed","result":{"token":"unknown-secret"}' From 2dcc6cbcf22b82582f4a71f8db3de11dd23614bc Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 14:03:29 +0200 Subject: [PATCH 16/50] test: align Agent37 base proof with legacy response --- .../cases/1656-long-running-agent37-sandbox/run.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs index cdb79c2307..4ed5dcd4bf 100644 --- a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs +++ b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs @@ -361,7 +361,7 @@ try { observation.ensureSandboxId === null && observation.ensureWorkloadProfile === 'long-running-agent' && observation.responseSandboxId === REPLAY_SANDBOX_ID && - observation.responseProviderSandboxId === PROVIDER_SANDBOX_ID && + observation.responseProviderSandboxId === null && observation.responseProviderId === 'agent37' && observation.responseNodeName === REPLAY_SANDBOX_NAME && observation.deleteSandboxId === REPLAY_SANDBOX_ID && @@ -393,7 +393,7 @@ try { outcome = 'absent'; signature = 'long_running_profile_replay_identity_absent_agent37_attribution_preserved'; details = - "fleet spawn --sandbox reached Cloud with the existing long-running profile and Cloud's agent37 response, but without a caller-declared sandbox identity; only the one-to-one replay identity is absent while provider attribution remains preserved."; + "fleet spawn --sandbox reached Cloud with the existing long-running profile and Cloud's agent37 response, but without a caller-declared sandbox identity; the one-to-one replay identity and separately observed providerSandboxId are absent while agent37 cleanup attribution remains preserved."; } else if (headObserved) { outcome = 'fixed'; signature = 'long_running_profile_replay_identity_and_agent37_attribution_preserved'; From 02c907bd0a943a1d20757da010f800b118342be2 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 14:09:19 +0200 Subject: [PATCH 17/50] test: harden Agent37 proof diagnostics --- scripts/pr-proof/run-cloud.mjs | 45 +++++++++++++------ .../1656-long-running-agent37-sandbox/run.mjs | 35 ++++++++++++--- 2 files changed, 61 insertions(+), 19 deletions(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index c516e7a04a..ef95109d7e 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -213,11 +213,13 @@ export function formatCloudRunArtifact(input) { ); } -export async function writeStatusPollTimeoutDiagnostics({ +async function writeStatusPollDiagnostics({ logsPath, runId, lastStatusOutput, statusPollFailures, + terminalStatus, + logsTimedOut, diagnosticSecretValues = [], }) { await mkdir(path.dirname(logsPath), { recursive: true }); @@ -225,15 +227,33 @@ export async function writeStatusPollTimeoutDiagnostics({ logsPath, formatCloudRunDiagnostics({ runId, - terminalStatus: 'status_poll_timeout', + terminalStatus, lastStatusOutput, statusPollFailures, - logs: { stdout: '', stderr: '', exitCode: 'unknown', timedOut: true }, + logs: { stdout: '', stderr: '', exitCode: 'unknown', timedOut: logsTimedOut }, diagnosticSecretValues, }) ); } +export async function writeStatusPollTimeoutDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, + diagnosticSecretValues = [], +}) { + await writeStatusPollDiagnostics({ + logsPath, + runId, + lastStatusOutput, + statusPollFailures, + terminalStatus: 'status_poll_timeout', + logsTimedOut: true, + diagnosticSecretValues, + }); +} + export async function writeStatusPollDeadlineDiagnostics({ logsPath, runId, @@ -241,18 +261,15 @@ export async function writeStatusPollDeadlineDiagnostics({ statusPollFailures, diagnosticSecretValues = [], }) { - await mkdir(path.dirname(logsPath), { recursive: true }); - await writeFile( + await writeStatusPollDiagnostics({ logsPath, - formatCloudRunDiagnostics({ - runId, - terminalStatus: 'status_poll_deadline_exceeded', - lastStatusOutput, - statusPollFailures, - logs: { stdout: '', stderr: '', exitCode: 'unknown', timedOut: false }, - diagnosticSecretValues, - }) - ); + runId, + lastStatusOutput, + statusPollFailures, + terminalStatus: 'status_poll_deadline_exceeded', + logsTimedOut: false, + diagnosticSecretValues, + }); } export function recognizedCloudRunStatus(value) { diff --git a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs index 4ed5dcd4bf..e0551c3dc7 100644 --- a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs +++ b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs @@ -223,7 +223,9 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned }); const errors: string[] = []; - const cliOutput: string[] = []; + // A progress line ahead of the JSON result proves the observation does not + // depend on the result being the first (or only) CLI output call. + const cliOutput: string[] = ['Preparing sandbox dispatch']; const replayArgs = process.env.RELAY_PR_PROOF_ARM === 'head' ? ['--sandbox-id', REPLAY_SANDBOX_ID, '--sandbox-name', REPLAY_SANDBOX_NAME] @@ -249,8 +251,17 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned // Capture the real CLI serialization, then fail so this same invocation // exercises cleanup without sourcing evidence from the network mock. log: (...args: unknown[]) => { - cliOutput.push(args.join(' ')); - throw new Error('CLI output sink failed after capture'); + const line = args.join(' '); + cliOutput.push(line); + let candidate: { sandbox?: { sandboxId?: unknown } } | null = null; + try { + candidate = JSON.parse(line); + } catch { + // Auxiliary CLI output is allowed; only the sandbox result drives cleanup. + } + if (candidate?.sandbox?.sandboxId === REPLAY_SANDBOX_ID) { + throw new Error('CLI output sink failed after capture'); + } }, error: (...args: unknown[]) => errors.push(args.join(' ')), exit: (() => { @@ -289,7 +300,21 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned // The command really reached successful dispatch and serialized its result; // without this a probe that never reached the sandbox path could report a false base. expect(errors.join('\n')).toContain('CLI output sink failed after capture'); - expect(cliOutput).toHaveLength(1); + const sandboxResults = cliOutput.flatMap((line) => { + try { + const candidate = JSON.parse(line); + return candidate?.sandbox?.sandboxId === REPLAY_SANDBOX_ID ? [candidate] : []; + } catch { + return []; + } + }); + if (sandboxResults.length !== 1) { + throw new Error( + 'Expected exactly one serialized sandbox result, found ' + + String(sandboxResults.length) + + '.' + ); + } const ensureRequest = mocks.authorizedApiFetch.mock.calls[1]?.[2]; const deleteRequest = mocks.authorizedApiFetch.mock.calls[2]?.[2]; @@ -297,7 +322,7 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned expect(deleteRequest?.body).toEqual(expect.any(String)); const ensureBody = JSON.parse(ensureRequest.body); const deleteBody = JSON.parse(deleteRequest.body); - const cliResult = JSON.parse(cliOutput[0]); + const [cliResult] = sandboxResults; await writeFile( output, From ee274438af80b6787835410f425e6572170e7bf3 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 14:49:51 +0200 Subject: [PATCH 18/50] fix: harden sandbox failure and proof redaction --- packages/cli/src/cli/commands/fleet.test.ts | 5 +- packages/cloud/src/fleet-sandbox.test.ts | 55 +++++++ packages/cloud/src/fleet-sandbox.ts | 13 ++ scripts/pr-proof/process-runner.mjs | 52 +++--- scripts/pr-proof/run-cloud.mjs | 168 ++++++++++++++++++-- tests/fixtures/pr-proof-contract.test.ts | 27 ++++ 6 files changed, 286 insertions(+), 34 deletions(-) diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index ecd0999be1..1d27b2ce38 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1209,7 +1209,7 @@ describe('fleet command support', () => { }); }); - it('identifies the requested node when the provisioning response is interrupted', async () => { + it('keeps the caller sandbox ID for replay guidance after an identifier-less server failure', async () => { const warnings: string[] = []; const deleteCloudFleetSandbox = vi.fn(); const program = new Command(); @@ -1231,6 +1231,7 @@ describe('fleet command support', () => { throw new CloudFleetSandboxProvisionError('request interrupted', { cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', nodeName: 'daytona-codex', + providerId: 'e2b', outcomeUnknown: true, }); }), @@ -1248,6 +1249,8 @@ describe('fleet command support', () => { 'spawn', 'codex', '--sandbox', + '--sandbox-provider', + 'e2b', '--sandbox-id', REPLAY_SANDBOX_ID, '--sandbox-name', diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 1e3570d622..ff4846e21d 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -458,6 +458,61 @@ describe('Cloud fleet sandbox client', () => { }); }); + it('marks an identifier-less server failure unknown without exposing a cleanup ID', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'provider timed out after allocation' }, { status: 504 }), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + providerId: 'e2b', + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + providerId: 'e2b', + outcomeUnknown: true, + sandboxId: undefined, + }); + expect(String(error)).toContain('provider timed out after allocation'); + }); + + it('keeps a proven client error ordinary when a stable sandbox ID was supplied', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'invalid capability' }, { status: 422 }), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + }).catch((caught: unknown) => caught); + + expect(error).not.toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ message: expect.stringContaining('invalid capability') }); + }); + it('does not expose a matching sandbox identity from a malformed success for cleanup', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 294f91237b..f932dc5d03 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -480,6 +480,19 @@ export async function ensureCloudFleetSandbox( }); } const error = endpointError('provision the fleet sandbox', response, payload); + if ( + response.status >= 500 && + sandboxIdentity.sandboxId !== undefined && + returnedSandboxId === undefined + ) { + throw new CloudFleetSandboxProvisionError(error.message, { + cloudWorkspaceId: resolved.cloudWorkspaceId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), + outcomeUnknown: true, + cause: error, + }); + } if (isObject(payload) && readString(payload, 'sandboxId')) { throw new CloudFleetSandboxProvisionError(error.message, { cloudWorkspaceId: resolved.cloudWorkspaceId, diff --git a/scripts/pr-proof/process-runner.mjs b/scripts/pr-proof/process-runner.mjs index 44a95e32f5..a29413f180 100644 --- a/scripts/pr-proof/process-runner.mjs +++ b/scripts/pr-proof/process-runner.mjs @@ -104,6 +104,14 @@ export function runBoundedProcess(command, args, options = {}) { const stdoutDecoder = new StringDecoder('utf8'); const stderrDecoder = new StringDecoder('utf8'); + // Transforms run before capture, live output, and callbacks. A transform + // may retain a small streaming boundary and is called once more with + // `final=true` after the decoder has been flushed. + const transformOutput = (stream, text, final = false) => { + if (typeof options.transformChunk !== 'function') return text; + return options.transformChunk(text, stream, final) ?? ''; + }; + const forceKill = () => { if (forced) return; forced = true; @@ -152,20 +160,22 @@ export function runBoundedProcess(command, args, options = {}) { } }; - child.stdout.on('data', (chunk) => { - const text = stdoutDecoder.write(chunk); - if (!text) return; - stdout = appendBounded(stdout, text, maximum); - options.onStdout?.(text); - writeLiveOutput(process.stdout, text); - }); - child.stderr.on('data', (chunk) => { - const text = stderrDecoder.write(chunk); - if (!text) return; - stderr = appendBounded(stderr, text, maximum); - options.onStderr?.(text); - writeLiveOutput(process.stderr, text); - }); + const consumeOutput = (stream, text, final = false) => { + const transformed = transformOutput(stream, text, final); + if (!transformed) return; + if (stream === 'stdout') { + stdout = appendBounded(stdout, transformed, maximum); + options.onStdout?.(transformed); + writeLiveOutput(process.stdout, transformed); + } else { + stderr = appendBounded(stderr, transformed, maximum); + options.onStderr?.(transformed); + writeLiveOutput(process.stderr, transformed); + } + }; + + child.stdout.on('data', (chunk) => consumeOutput('stdout', stdoutDecoder.write(chunk))); + child.stderr.on('data', (chunk) => consumeOutput('stderr', stderrDecoder.write(chunk))); child.on('error', (error) => { if (settled) return; settled = true; @@ -182,16 +192,10 @@ export function runBoundedProcess(command, args, options = {}) { cleanup(); const stdoutTail = stdoutDecoder.end(); const stderrTail = stderrDecoder.end(); - stdout = appendBounded(stdout, stdoutTail, maximum); - stderr = appendBounded(stderr, stderrTail, maximum); - if (stdoutTail) { - options.onStdout?.(stdoutTail); - writeLiveOutput(process.stdout, stdoutTail); - } - if (stderrTail) { - options.onStderr?.(stderrTail); - writeLiveOutput(process.stderr, stderrTail); - } + consumeOutput('stdout', stdoutTail); + consumeOutput('stderr', stderrTail); + consumeOutput('stdout', '', true); + consumeOutput('stderr', '', true); resolve({ exitCode: code ?? 1, signal, stdout, stderr, timedOut, aborted }); }); }); diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index ef95109d7e..d87d71fbdc 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -31,8 +31,17 @@ const PREPARED_RUN_ID_MARKER = 'AGENT_RELAY_CLOUD_PREPARED_RUN_ID='; const RUN_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; const DIAGNOSTIC_TOKEN_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/; const ISO_TIMESTAMP_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/; -const LIVE_CREDENTIAL_RE = - /(rk_live_|rjt_live_|at_live_|nt_live_|ot_live_|cld_at_|rth_at_|ocl_node_enr_|br_)([A-Za-z0-9_%-]+(?:\.[A-Za-z0-9_%-]+)*)/g; +const LIVE_CREDENTIAL_PREFIXES = [ + 'rk_live_', + 'rjt_live_', + 'at_live_', + 'nt_live_', + 'ot_live_', + 'cld_at_', + 'rth_at_', + 'ocl_node_enr_', + 'br_', +]; const STATUS_DIAGNOSTIC_FIELDS = [ 'runId', 'status', @@ -45,6 +54,9 @@ const STATUS_DIAGNOSTIC_FIELDS = [ const STATUS_FAILURE_DIAGNOSTIC_FIELDS = ['phase', 'code', 'dispatchType', 'sandboxId', 'occurredAt']; function run(command, args, options = {}) { + const diagnosticSecretValues = options.diagnosticSecretValues ?? []; + const stdoutRedactor = createCredentialRedactor(diagnosticSecretValues); + const stderrRedactor = createCredentialRedactor(diagnosticSecretValues); return runBoundedProcess(command, args, { env: options.env, echo: !options.quiet, @@ -54,6 +66,8 @@ function run(command, args, options = {}) { signal: options.signal, onStdout: options.onStdout, onStderr: options.onStderr, + transformChunk: (text, stream, final) => + (stream === 'stdout' ? stdoutRedactor : stderrRedactor).push(text, final), }); } @@ -98,13 +112,144 @@ export function boundedDiagnostic(value) { return `${tail}${marker}`; } -export function sanitizeCloudCommandOutput(value, secretValues = []) { - let text = String(value ?? '').replace(LIVE_CREDENTIAL_RE, (_match, prefix) => `${prefix}…`); - for (const secretValue of secretValues) { - const secret = typeof secretValue === 'string' ? secretValue : ''; - if (secret) text = text.split(secret).join('[redacted]'); +function longestSuffixThatStartsSecret(value, secrets) { + const maximum = Math.min(value.length, Math.max(...secrets.map((secret) => secret.length), 0) - 1); + for (let length = maximum; length > 0; length -= 1) { + const suffix = value.slice(value.length - length); + if (secrets.some((secret) => secret.startsWith(suffix))) return length; } - return text; + return 0; +} + +function createCredentialPrefixRedactor() { + let pending = ''; + let active = null; + + return { + push(value, final = false) { + const input = pending + String(value ?? ''); + pending = ''; + let output = ''; + let index = 0; + + while (index < input.length) { + if (active) { + if (!active.emitted) { + if (/[A-Za-z0-9_%-]/.test(input[index])) { + output += `${active.prefix}…`; + active.emitted = true; + } else { + output += active.prefix; + active = null; + continue; + } + } + while (index < input.length && /[A-Za-z0-9_%.%-]/.test(input[index])) index += 1; + if (index === input.length) { + if (final) active = null; + break; + } + active = null; + continue; + } + + let prefixIndex = -1; + let prefix; + for (let candidateIndex = index; candidateIndex < input.length; candidateIndex += 1) { + const candidate = LIVE_CREDENTIAL_PREFIXES.find((value) => input.startsWith(value, candidateIndex)); + if (candidate) { + prefixIndex = candidateIndex; + prefix = candidate; + break; + } + } + if (prefix) { + output += input.slice(index, prefixIndex); + index = prefixIndex; + active = { prefix, emitted: false }; + index += prefix.length; + continue; + } + + if (!final) { + const suffixLength = longestSuffixThatStartsSecret(input.slice(index), LIVE_CREDENTIAL_PREFIXES); + if (suffixLength > 0) { + const end = input.length - suffixLength; + output += input.slice(index, end); + pending = input.slice(end); + break; + } + } + output += input[index]; + index += 1; + } + + if (final && active && !active.emitted) output += active.prefix; + if (final) { + active = null; + if (pending) { + output += pending; + pending = ''; + } + } + return output; + }, + }; +} + +function createConfiguredSecretRedactor(secretValues) { + const secrets = [...new Set(secretValues.filter((value) => typeof value === 'string' && value))]; + let pending = ''; + + return { + push(value, final = false) { + if (secrets.length === 0) return String(value ?? ''); + const input = pending + String(value ?? ''); + pending = ''; + let output = ''; + let index = 0; + while (index < input.length) { + const secret = secrets.find((candidate) => input.startsWith(candidate, index)); + if (secret) { + output += '[redacted]'; + index += secret.length; + continue; + } + if (!final) { + const suffixLength = longestSuffixThatStartsSecret(input.slice(index), secrets); + if (suffixLength > 0) { + const end = input.length - suffixLength; + output += input.slice(index, end); + pending = input.slice(end); + break; + } + } + output += input[index]; + index += 1; + } + if (final) { + output += pending; + pending = ''; + } + return output; + }, + }; +} + +/** Redact credentials across subprocess chunks before bounded capture. */ +export function createCredentialRedactor(secretValues = []) { + const prefixRedactor = createCredentialPrefixRedactor(); + const secretRedactor = createConfiguredSecretRedactor(secretValues); + return { + push(value, final = false) { + const prefixed = prefixRedactor.push(value, final); + return secretRedactor.push(prefixed, final); + }, + }; +} + +export function sanitizeCloudCommandOutput(value, secretValues = []) { + return createCredentialRedactor(secretValues).push(value, true); } function structuralDiagnosticValue(field, value, secretValues) { @@ -407,7 +552,11 @@ export async function main() { const controller = new AbortController(); activeCommandController = controller; try { - return await run(command, args, { ...options, signal: controller.signal }); + return await run(command, args, { + ...options, + diagnosticSecretValues: auth.diagnosticSecretValues, + signal: controller.signal, + }); } finally { if (activeCommandController === controller) activeCommandController = null; } @@ -426,6 +575,7 @@ export async function main() { env: auth.cliEnv, quiet: true, timeoutMs: commandTimeoutMs, + diagnosticSecretValues: auth.diagnosticSecretValues, }); if (result.exitCode !== 0 || result.timedOut) { console.warn( diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 6d4c21f160..4cc5a5fdcd 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -47,6 +47,7 @@ import { boundedDiagnostic, boundedDuration, createPreparedRunProgressParser, + createCredentialRedactor, createCliApiKeyEnvironment, formatCloudRunArtifact, formatCloudRunDiagnostics, @@ -60,6 +61,8 @@ import { writeStatusPollTimeoutDiagnostics, } from '../../scripts/pr-proof/run-cloud.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. +import { runBoundedProcess } from '../../scripts/pr-proof/process-runner.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. import { openVerifiedBrokerExecutable, probeLandlockedProcessSupport, @@ -594,6 +597,30 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(artifact).not.toContain(credentialSuffix); }); + it('redacts credential prefixes and configured secrets across subprocess chunk boundaries', async () => { + const redactor = createCredentialRedactor(['split-secret']); + const sanitized = + redactor.push('prefix rk_', false) + + redactor.push('live_0123456789 split-', false) + + redactor.push('secret tail', true); + expect(sanitized).toBe('prefix rk_live_… [redacted] tail'); + + const longCredential = 'a'.repeat(200_000); + const capture = await runBoundedProcess( + process.execPath, + ['-e', `process.stdout.write('head rk_live_${longCredential} tail')`], + { + echo: false, + maxCaptureBytes: 128, + maxLiveOutputBytes: 128, + transformChunk: (text, stream, final) => redactor.push(text, final), + } + ); + + expect(capture.stdout).toBe('head rk_live_… tail'); + expect(capture.stdout).not.toContain(longCredential); + }); + it('omits malformed JSON status payloads instead of falling back to raw output', () => { const diagnostic = sanitizeCloudStatusDiagnostic( '{"status":"failed","result":{"token":"unknown-secret"}' From 462907f9de3125223559fb7e753c104e273dfcca Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 14:54:56 +0200 Subject: [PATCH 19/50] fix: keep proof redaction linear --- scripts/pr-proof/run-cloud.mjs | 74 +++++++++++++----------- tests/fixtures/pr-proof-contract.test.ts | 5 ++ 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index d87d71fbdc..f175bdd533 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -42,6 +42,10 @@ const LIVE_CREDENTIAL_PREFIXES = [ 'ocl_node_enr_', 'br_', ]; +const LIVE_CREDENTIAL_PREFIX_RE = new RegExp( + `(${LIVE_CREDENTIAL_PREFIXES.map((prefix) => prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|')})`, + 'g' +); const STATUS_DIAGNOSTIC_FIELDS = [ 'runId', 'status', @@ -153,17 +157,11 @@ function createCredentialPrefixRedactor() { continue; } - let prefixIndex = -1; - let prefix; - for (let candidateIndex = index; candidateIndex < input.length; candidateIndex += 1) { - const candidate = LIVE_CREDENTIAL_PREFIXES.find((value) => input.startsWith(value, candidateIndex)); - if (candidate) { - prefixIndex = candidateIndex; - prefix = candidate; - break; - } - } - if (prefix) { + LIVE_CREDENTIAL_PREFIX_RE.lastIndex = index; + const match = LIVE_CREDENTIAL_PREFIX_RE.exec(input); + if (match) { + const prefixIndex = match.index; + const prefix = match[0]; output += input.slice(index, prefixIndex); index = prefixIndex; active = { prefix, emitted: false }; @@ -171,17 +169,15 @@ function createCredentialPrefixRedactor() { continue; } - if (!final) { - const suffixLength = longestSuffixThatStartsSecret(input.slice(index), LIVE_CREDENTIAL_PREFIXES); - if (suffixLength > 0) { - const end = input.length - suffixLength; - output += input.slice(index, end); - pending = input.slice(end); - break; - } + const suffixLength = final + ? 0 + : longestSuffixThatStartsSecret(input.slice(index), LIVE_CREDENTIAL_PREFIXES); + const end = input.length - suffixLength; + output += input.slice(index, end); + if (suffixLength > 0) { + pending = input.slice(end); } - output += input[index]; - index += 1; + break; } if (final && active && !active.emitted) output += active.prefix; @@ -209,23 +205,33 @@ function createConfiguredSecretRedactor(secretValues) { let output = ''; let index = 0; while (index < input.length) { - const secret = secrets.find((candidate) => input.startsWith(candidate, index)); - if (secret) { + let secret; + let secretIndex = -1; + for (const candidate of secrets) { + const candidateIndex = input.indexOf(candidate, index); + if ( + candidateIndex !== -1 && + (secretIndex === -1 || + candidateIndex < secretIndex || + (candidateIndex === secretIndex && candidate.length > (secret?.length ?? 0))) + ) { + secret = candidate; + secretIndex = candidateIndex; + } + } + if (secret !== undefined) { + output += input.slice(index, secretIndex); output += '[redacted]'; - index += secret.length; + index = secretIndex + secret.length; continue; } - if (!final) { - const suffixLength = longestSuffixThatStartsSecret(input.slice(index), secrets); - if (suffixLength > 0) { - const end = input.length - suffixLength; - output += input.slice(index, end); - pending = input.slice(end); - break; - } + const suffixLength = final ? 0 : longestSuffixThatStartsSecret(input.slice(index), secrets); + const end = input.length - suffixLength; + output += input.slice(index, end); + if (suffixLength > 0) { + pending = input.slice(end); } - output += input[index]; - index += 1; + break; } if (final) { output += pending; diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 4cc5a5fdcd..3ea5eee95a 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -621,6 +621,11 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(capture.stdout).not.toContain(longCredential); }); + it('preserves large credential-free output without pathological rescanning', () => { + const cleanOutput = `head ${'ordinary-output '.repeat(20_000)}tail`; + expect(sanitizeCloudCommandOutput(cleanOutput, ['configured-secret'])).toBe(cleanOutput); + }); + it('omits malformed JSON status payloads instead of falling back to raw output', () => { const diagnostic = sanitizeCloudStatusDiagnostic( '{"status":"failed","result":{"token":"unknown-secret"}' From 56c5da5e00e9070c3d3242b2e71c70805fdb7e43 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 15:07:12 +0200 Subject: [PATCH 20/50] fix: contain proof output transform failures --- scripts/pr-proof/process-runner.mjs | 49 +++++++++++++++++------- scripts/pr-proof/run-cloud.mjs | 17 ++++---- tests/fixtures/pr-proof-contract.test.ts | 38 +++++++++++++++++- 3 files changed, 81 insertions(+), 23 deletions(-) diff --git a/scripts/pr-proof/process-runner.mjs b/scripts/pr-proof/process-runner.mjs index a29413f180..7b61b6a3cb 100644 --- a/scripts/pr-proof/process-runner.mjs +++ b/scripts/pr-proof/process-runner.mjs @@ -138,6 +138,14 @@ export function runBoundedProcess(command, args, options = {}) { options.signal?.removeEventListener('abort', abortHandler); }; + const fail = (error) => { + if (settled) return; + settled = true; + cleanup(); + forceKill(); + reject(error); + }; + const writeLiveOutput = (stream, text) => { if (options.echo === false || !text) return; if (liveOutputBytes >= maximumLiveOutput) { @@ -174,28 +182,41 @@ export function runBoundedProcess(command, args, options = {}) { } }; - child.stdout.on('data', (chunk) => consumeOutput('stdout', stdoutDecoder.write(chunk))); - child.stderr.on('data', (chunk) => consumeOutput('stderr', stderrDecoder.write(chunk))); - child.on('error', (error) => { - if (settled) return; - settled = true; - cleanup(); - reject(error); + child.stdout.on('data', (chunk) => { + try { + consumeOutput('stdout', stdoutDecoder.write(chunk)); + } catch (error) { + fail(error); + } + }); + child.stderr.on('data', (chunk) => { + try { + consumeOutput('stderr', stderrDecoder.write(chunk)); + } catch (error) { + fail(error); + } }); + child.on('error', fail); child.on('close', (code, signal) => { if (settled) return; - settled = true; // The process-group leader can exit after SIGTERM while a descendant // with detached stdio remains alive. Force-kill the group before // clearing the grace timer so that descendant cannot escape cleanup. if (timedOut || aborted) forceKill(); cleanup(); - const stdoutTail = stdoutDecoder.end(); - const stderrTail = stderrDecoder.end(); - consumeOutput('stdout', stdoutTail); - consumeOutput('stderr', stderrTail); - consumeOutput('stdout', '', true); - consumeOutput('stderr', '', true); + try { + const stdoutTail = stdoutDecoder.end(); + const stderrTail = stderrDecoder.end(); + consumeOutput('stdout', stdoutTail); + consumeOutput('stderr', stderrTail); + consumeOutput('stdout', '', true); + consumeOutput('stderr', '', true); + } catch (error) { + settled = true; + reject(error); + return; + } + settled = true; resolve({ exitCode: code ?? 1, signal, stdout, stderr, timedOut, aborted }); }); }); diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index f175bdd533..79b44a6309 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -359,8 +359,10 @@ export function formatCloudRunDiagnostics({ export function formatCloudRunArtifact(input) { return ( formatCloudRunDiagnostics(input) + - sanitizeCloudCommandOutput(input.logs?.stdout, input.diagnosticSecretValues) + - sanitizeCloudCommandOutput(input.logs?.stderr, input.diagnosticSecretValues) + sanitizeCloudCommandOutput( + `${input.logs?.stdout ?? ''}${input.logs?.stderr ?? ''}`, + input.diagnosticSecretValues + ) ); } @@ -721,11 +723,12 @@ export async function main() { diagnosticSecretValues: auth.diagnosticSecretValues, }) ); - if (logs.stdout) { - process.stdout.write(sanitizeCloudCommandOutput(logs.stdout, auth.diagnosticSecretValues)); - } - if (logs.stderr) { - process.stderr.write(sanitizeCloudCommandOutput(logs.stderr, auth.diagnosticSecretValues)); + const sanitizedLogs = sanitizeCloudCommandOutput( + `${logs.stdout ?? ''}${logs.stderr ?? ''}`, + auth.diagnosticSecretValues + ); + if (sanitizedLogs) { + process.stdout.write(sanitizedLogs); } if (logs.timedOut) throw new Error(`Cloud log retrieval timed out for run ${runId}`); if (logs.exitCode !== 0) throw new Error(`Cloud log retrieval failed with exit ${logs.exitCode}`); diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 3ea5eee95a..208e562ecc 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -597,6 +597,29 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(artifact).not.toContain(credentialSuffix); }); + it('redacts credentials reconstructed across captured stdout and stderr', () => { + const configuredArtifact = formatCloudRunArtifact({ + runId: 'cloud-run-split-secret', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed"}', + statusPollFailures: 0, + logs: { stdout: 'split-', stderr: 'secret', exitCode: 1, timedOut: false }, + diagnosticSecretValues: ['split-secret'], + }); + expect(configuredArtifact).toContain('[redacted]'); + expect(configuredArtifact).not.toContain('split-secret'); + + const prefixedArtifact = formatCloudRunArtifact({ + runId: 'cloud-run-split-prefix', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed"}', + statusPollFailures: 0, + logs: { stdout: 'rk_', stderr: 'live_token', exitCode: 1, timedOut: false }, + }); + expect(prefixedArtifact).toContain('rk_live_…'); + expect(prefixedArtifact).not.toContain('rk_live_token'); + }); + it('redacts credential prefixes and configured secrets across subprocess chunk boundaries', async () => { const redactor = createCredentialRedactor(['split-secret']); const sanitized = @@ -626,6 +649,17 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(sanitizeCloudCommandOutput(cleanOutput, ['configured-secret'])).toBe(cleanOutput); }); + it('rejects and terminates when an output transform throws', async () => { + await expect( + runBoundedProcess(process.execPath, ['-e', "process.stdout.write('trigger')"], { + echo: false, + transformChunk: () => { + throw new Error('transform failed'); + }, + }) + ).rejects.toThrow('transform failed'); + }); + it('omits malformed JSON status payloads instead of falling back to raw output', () => { const diagnostic = sanitizeCloudStatusDiagnostic( '{"status":"failed","result":{"token":"unknown-secret"}' @@ -1986,8 +2020,8 @@ describe('trusted dispatcher source contract', () => { expect(source).not.toContain('CLOUD_API_REFRESH_TOKEN='); expect(source).toContain('formatCloudRunArtifact({'); expect(source).toContain('sanitizeCloudCommandOutput(launch.stderr'); - expect(source).toContain('sanitizeCloudCommandOutput(logs.stdout'); - expect(source).toContain('sanitizeCloudCommandOutput(logs.stderr'); + expect(source).toContain('const sanitizedLogs = sanitizeCloudCommandOutput('); + expect(source).toContain("`${logs.stdout ?? ''}${logs.stderr ?? ''}`"); expect(source).toContain('sanitizeCloudCommandOutput(error.message'); expect(source).toContain("console.warn('Cloud RelayFlow status: ')"); expect(source).not.toContain('process.stderr.write(launch.stderr)'); From 99424641bdefc42a97fef7827822dc3d7a16a30d Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 15:10:54 +0200 Subject: [PATCH 21/50] fix: preserve transform failure rejection --- scripts/pr-proof/process-runner.mjs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/pr-proof/process-runner.mjs b/scripts/pr-proof/process-runner.mjs index 7b61b6a3cb..3f1343c442 100644 --- a/scripts/pr-proof/process-runner.mjs +++ b/scripts/pr-proof/process-runner.mjs @@ -142,7 +142,14 @@ export function runBoundedProcess(command, args, options = {}) { if (settled) return; settled = true; cleanup(); - forceKill(); + try { + forceKill(); + } catch { + // Preserve the transform/spawn failure as the rejection reason. The + // parent-side pipes still must close even if process signaling fails. + child.stdout.destroy(); + child.stderr.destroy(); + } reject(error); }; From a5dca892406fefa6c9dfcc8c8888802688bbbf46 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 15:13:31 +0200 Subject: [PATCH 22/50] fix: kill descendants on final transform failure --- scripts/pr-proof/process-runner.mjs | 6 ++-- tests/fixtures/pr-proof-contract.test.ts | 43 ++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/scripts/pr-proof/process-runner.mjs b/scripts/pr-proof/process-runner.mjs index 3f1343c442..7028d806c4 100644 --- a/scripts/pr-proof/process-runner.mjs +++ b/scripts/pr-proof/process-runner.mjs @@ -219,8 +219,10 @@ export function runBoundedProcess(command, args, options = {}) { consumeOutput('stdout', '', true); consumeOutput('stderr', '', true); } catch (error) { - settled = true; - reject(error); + // A descendant can keep running after the process-group leader closes + // its inherited pipes. Use the same terminal cleanup path as a + // transform failure observed during a data event. + fail(error); return; } settled = true; diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 208e562ecc..a63ceebb87 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -660,6 +660,49 @@ describe('Cloud dispatcher API key lifecycle', () => { ).rejects.toThrow('transform failed'); }); + it.skipIf(process.platform === 'win32' || !PS_PATH)( + 'kills same-group descendants when an output transform fails during final flush', + async () => { + const script = [ + "const { spawn } = require('node:child_process');", + "const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });", + "process.stdout.write(String(child.pid) + '\\n');", + 'child.unref();', + ].join(''); + let transformed = ''; + await expect( + runBoundedProcess(process.execPath, ['-e', script], { + echo: false, + transformChunk: (text, _stream, final) => { + if (final) throw new Error('final transform failed'); + transformed += text; + return text; + }, + }) + ).rejects.toThrow('final transform failed'); + + const descendantPid = Number(transformed.trim()); + expect(descendantPid).toBeGreaterThan(0); + const deadline = Date.now() + 2_000; + let running = true; + while (running && Date.now() < deadline) { + let processState = ''; + try { + processState = execFileSync(PS_PATH!, ['-o', 'stat=', '-p', String(descendantPid)], { + encoding: 'utf8', + }).trim(); + } catch (error) { + const status = (error as { status?: number }).status; + if (status !== 1) throw error; + } + running = processState.length > 0 && !processState.startsWith('Z'); + if (running) await new Promise((resolve) => setTimeout(resolve, 20)); + } + if (running) process.kill(descendantPid, 'SIGKILL'); + expect(running).toBe(false); + } + ); + it('omits malformed JSON status payloads instead of falling back to raw output', () => { const diagnostic = sanitizeCloudStatusDiagnostic( '{"status":"failed","result":{"token":"unknown-secret"}' From 7703e7fbb22466997d17522e162ecf01f2e1f04d Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 15:35:13 +0200 Subject: [PATCH 23/50] test: generate long proof credential in child --- tests/fixtures/pr-proof-contract.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index a63ceebb87..31ec31106e 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -631,7 +631,7 @@ describe('Cloud dispatcher API key lifecycle', () => { const longCredential = 'a'.repeat(200_000); const capture = await runBoundedProcess( process.execPath, - ['-e', `process.stdout.write('head rk_live_${longCredential} tail')`], + ['-e', "process.stdout.write('head rk_live_' + 'a'.repeat(200_000) + ' tail')"], { echo: false, maxCaptureBytes: 128, From 0bfc175fc9b579067273504dc612838d466230ec Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 15:43:53 +0200 Subject: [PATCH 24/50] fix: preserve unknown sandbox replay identity Session-Id: 01a08135-507d-73a2-a689-6125e136d7d3 --- packages/cloud/src/fleet-sandbox.test.ts | 61 +++++++++++++----------- packages/cloud/src/fleet-sandbox.ts | 10 ++-- scripts/pr-proof/run-cloud.mjs | 6 +++ tests/fixtures/pr-proof-contract.test.ts | 45 +++++++++++++++-- 4 files changed, 85 insertions(+), 37 deletions(-) diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index ff4846e21d..23b14d196a 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -458,37 +458,40 @@ describe('Cloud fleet sandbox client', () => { }); }); - it('marks an identifier-less server failure unknown without exposing a cleanup ID', async () => { - mocks.authorizedApiFetch - .mockResolvedValueOnce({ - response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), - auth, - }) - .mockResolvedValueOnce({ - response: Response.json({ error: 'provider timed out after allocation' }, { status: 504 }), - auth, - }); + it.each([500, 502, 503, 504])( + 'marks an identifier-less server failure (%s) unknown without exposing a cleanup ID', + async (status) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'provider timed out after allocation' }, { status }), + auth, + }); - const error = await ensureCloudFleetSandbox({ - workspaceId: 'rw_abc', - requiredCapability: 'spawn:codex', - sandboxId: SANDBOX_ID, - name: SANDBOX_NAME, - providerId: 'e2b', - forceProvision: true, - workloadProfile: 'long-running-agent', - }).catch((caught: unknown) => caught); + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + providerId: 'e2b', + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); - expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); - expect(error).toMatchObject({ - cloudWorkspaceId: CLOUD_WORKSPACE_ID, - nodeName: SANDBOX_NAME, - providerId: 'e2b', - outcomeUnknown: true, - sandboxId: undefined, - }); - expect(String(error)).toContain('provider timed out after allocation'); - }); + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + nodeName: SANDBOX_NAME, + providerId: 'e2b', + outcomeUnknown: true, + sandboxId: undefined, + }); + expect(String(error)).toContain('provider timed out after allocation'); + } + ); it('keeps a proven client error ordinary when a stable sandbox ID was supplied', async () => { mocks.authorizedApiFetch diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index f932dc5d03..625153e624 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -480,11 +480,11 @@ export async function ensureCloudFleetSandbox( }); } const error = endpointError('provision the fleet sandbox', response, payload); - if ( - response.status >= 500 && - sandboxIdentity.sandboxId !== undefined && - returnedSandboxId === undefined - ) { + // Gateway/server failures can arrive after Cloud accepted the ensure + // request but before it could return an identity. Keep every 5xx failure + // replayable as an unknown outcome, even for legacy custom-name callers; + // never copy an unverified response ID into cleanup authority. + if (response.status >= 500) { throw new CloudFleetSandboxProvisionError(error.message, { cloudWorkspaceId: resolved.cloudWorkspaceId, ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 79b44a6309..3479390fca 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -41,6 +41,12 @@ const LIVE_CREDENTIAL_PREFIXES = [ 'rth_at_', 'ocl_node_enr_', 'br_', + 'github_pat_', + 'ghp_', + 'gho_', + 'ghu_', + 'ghs_', + 'ghr_', ]; const LIVE_CREDENTIAL_PREFIX_RE = new RegExp( `(${LIVE_CREDENTIAL_PREFIXES.map((prefix) => prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|')})`, diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 31ec31106e..242fe5b117 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -551,6 +551,8 @@ describe('Cloud dispatcher API key lifecycle', () => { it('redacts configured and recognized credentials from raw command output and artifacts', () => { const raw = 'stdout ci-api-key rk_live_0123456789abcdef'; expect(sanitizeCloudCommandOutput(raw, ['ci-api-key'])).toBe('stdout [redacted] rk_live_…'); + const githubToken = 'github_pat_0123456789abcdef0123456789abcdef'; + expect(sanitizeCloudCommandOutput(`status ${githubToken}`)).toBe('status github_pat_…'); const artifact = formatCloudRunArtifact({ runId: 'cloud-run-123', @@ -628,10 +630,10 @@ describe('Cloud dispatcher API key lifecycle', () => { redactor.push('secret tail', true); expect(sanitized).toBe('prefix rk_live_… [redacted] tail'); - const longCredential = 'a'.repeat(200_000); + const longCredentialLength = 200_000; const capture = await runBoundedProcess( process.execPath, - ['-e', "process.stdout.write('head rk_live_' + 'a'.repeat(200_000) + ' tail')"], + ['-e', `process.stdout.write('head rk_live_' + 'a'.repeat(${longCredentialLength}) + ' tail')`], { echo: false, maxCaptureBytes: 128, @@ -641,7 +643,44 @@ describe('Cloud dispatcher API key lifecycle', () => { ); expect(capture.stdout).toBe('head rk_live_… tail'); - expect(capture.stdout).not.toContain(longCredential); + expect(capture.stdout).not.toContain('a'.repeat(longCredentialLength)); + + const captureLimit = 2 * 1024 * 1024; + const boundaryCredential = '0123456789abcdef'.repeat(128); + const boundaryRedactor = createCredentialRedactor(); + const boundaryCapture = await runBoundedProcess( + process.execPath, + [ + '-e', + [ + `process.stdout.write('x'.repeat(${captureLimit - 'rk_live_'.length}))`, + "process.stdout.write('rk_')", + `setImmediate(() => process.stdout.write('live_' + ${JSON.stringify(boundaryCredential)} + ' tail'))`, + ].join(';'), + ], + { + echo: false, + maxCaptureBytes: captureLimit, + maxLiveOutputBytes: 128, + transformChunk: (text, stream, final) => { + // Force the credential prefix to cross a redactor boundary even if + // the OS coalesces the two writes into one pipe chunk. + const prefixIndex = text.indexOf('rk_live_'); + if (prefixIndex >= 0) { + const split = prefixIndex + 'rk_'.length; + return ( + boundaryRedactor.push(text.slice(0, split), false) + + boundaryRedactor.push(text.slice(split), final) + ); + } + return boundaryRedactor.push(text, final); + }, + } + ); + + expect(Buffer.byteLength(boundaryCapture.stdout, 'utf8')).toBeLessThanOrEqual(captureLimit); + expect(boundaryCapture.stdout).toContain('rk_live_…'); + expect(boundaryCapture.stdout).not.toContain('0123456789abcdef'); }); it('preserves large credential-free output without pathological rescanning', () => { From 0a41237d97fe4164e81703ffdfa8d23bc4c1302e Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 16:32:25 +0200 Subject: [PATCH 25/50] test: avoid dynamic code in redaction proof --- tests/fixtures/pr-proof-contract.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 242fe5b117..d2010c38fb 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -655,11 +655,12 @@ describe('Cloud dispatcher API key lifecycle', () => { [ `process.stdout.write('x'.repeat(${captureLimit - 'rk_live_'.length}))`, "process.stdout.write('rk_')", - `setImmediate(() => process.stdout.write('live_' + ${JSON.stringify(boundaryCredential)} + ' tail'))`, + "setImmediate(() => process.stdout.write('live_' + process.env.BOUNDARY_CREDENTIAL + ' tail'))", ].join(';'), ], { echo: false, + env: { ...process.env, BOUNDARY_CREDENTIAL: boundaryCredential }, maxCaptureBytes: captureLimit, maxLiveOutputBytes: 128, transformChunk: (text, stream, final) => { From 515d54325edc2b3880756af5e5e6a7a041f1e995 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 16:56:21 +0200 Subject: [PATCH 26/50] fix: redact cross-stream credential fragments --- scripts/pr-proof/run-cloud.mjs | 41 +++++++++++++++--------- tests/fixtures/pr-proof-contract.test.ts | 30 +++++++++++++++++ 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 3479390fca..c4b3b06eb3 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -65,8 +65,12 @@ const STATUS_FAILURE_DIAGNOSTIC_FIELDS = ['phase', 'code', 'dispatchType', 'sand function run(command, args, options = {}) { const diagnosticSecretValues = options.diagnosticSecretValues ?? []; - const stdoutRedactor = createCredentialRedactor(diagnosticSecretValues); - const stderrRedactor = createCredentialRedactor(diagnosticSecretValues); + // One ordering-aware redactor covers both pipes. If the OS reports the two + // streams in the opposite order, conservatively mask any trailing fragment + // that could begin a credential instead of releasing it at finalization. + const outputRedactor = createCredentialRedactor(diagnosticSecretValues, { + maskPendingOnFinal: true, + }); return runBoundedProcess(command, args, { env: options.env, echo: !options.quiet, @@ -76,8 +80,7 @@ function run(command, args, options = {}) { signal: options.signal, onStdout: options.onStdout, onStderr: options.onStderr, - transformChunk: (text, stream, final) => - (stream === 'stdout' ? stdoutRedactor : stderrRedactor).push(text, final), + transformChunk: (text, _stream, final) => outputRedactor.push(text, final), }); } @@ -131,7 +134,7 @@ function longestSuffixThatStartsSecret(value, secrets) { return 0; } -function createCredentialPrefixRedactor() { +function createCredentialPrefixRedactor(maskPendingOnFinal = false) { let pending = ''; let active = null; @@ -175,9 +178,10 @@ function createCredentialPrefixRedactor() { continue; } - const suffixLength = final - ? 0 - : longestSuffixThatStartsSecret(input.slice(index), LIVE_CREDENTIAL_PREFIXES); + const suffixLength = + final && !maskPendingOnFinal + ? 0 + : longestSuffixThatStartsSecret(input.slice(index), LIVE_CREDENTIAL_PREFIXES); const end = input.length - suffixLength; output += input.slice(index, end); if (suffixLength > 0) { @@ -186,11 +190,13 @@ function createCredentialPrefixRedactor() { break; } - if (final && active && !active.emitted) output += active.prefix; + if (final && active && !active.emitted) { + output += maskPendingOnFinal ? '[redacted]' : active.prefix; + } if (final) { active = null; if (pending) { - output += pending; + output += maskPendingOnFinal ? '[redacted]' : pending; pending = ''; } } @@ -199,7 +205,7 @@ function createCredentialPrefixRedactor() { }; } -function createConfiguredSecretRedactor(secretValues) { +function createConfiguredSecretRedactor(secretValues, maskPendingOnFinal = false) { const secrets = [...new Set(secretValues.filter((value) => typeof value === 'string' && value))]; let pending = ''; @@ -231,7 +237,10 @@ function createConfiguredSecretRedactor(secretValues) { index = secretIndex + secret.length; continue; } - const suffixLength = final ? 0 : longestSuffixThatStartsSecret(input.slice(index), secrets); + const suffixLength = + final && !maskPendingOnFinal + ? 0 + : longestSuffixThatStartsSecret(input.slice(index), secrets); const end = input.length - suffixLength; output += input.slice(index, end); if (suffixLength > 0) { @@ -240,7 +249,7 @@ function createConfiguredSecretRedactor(secretValues) { break; } if (final) { - output += pending; + output += pending && maskPendingOnFinal ? '[redacted]' : pending; pending = ''; } return output; @@ -249,9 +258,9 @@ function createConfiguredSecretRedactor(secretValues) { } /** Redact credentials across subprocess chunks before bounded capture. */ -export function createCredentialRedactor(secretValues = []) { - const prefixRedactor = createCredentialPrefixRedactor(); - const secretRedactor = createConfiguredSecretRedactor(secretValues); +export function createCredentialRedactor(secretValues = [], { maskPendingOnFinal = false } = {}) { + const prefixRedactor = createCredentialPrefixRedactor(maskPendingOnFinal); + const secretRedactor = createConfiguredSecretRedactor(secretValues, maskPendingOnFinal); return { push(value, final = false) { const prefixed = prefixRedactor.push(value, final); diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index d2010c38fb..0c83fc7d82 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -622,6 +622,36 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(prefixedArtifact).not.toContain('rk_live_token'); }); + it('never releases cross-stream credential fragments at finalization', () => { + const redactor = createCredentialRedactor(['split-secret'], { + maskPendingOnFinal: true, + }); + + // Model the OS reporting stdout before an earlier stderr write. The suffix + // cannot be recognized on its own, but the trailing prefix must not be + // released when the shared stream closes. + const sanitized = + redactor.push('secret', false) + + redactor.push('split-', false) + + redactor.push('', true); + + expect(sanitized).toBe('secret[redacted]'); + expect(sanitized).not.toContain('split-'); + expect(sanitized).not.toContain('split-secret'); + + const prefixRedactor = createCredentialRedactor([], { + maskPendingOnFinal: true, + }); + const sanitizedPrefix = + prefixRedactor.push('live_token', false) + + prefixRedactor.push('rk_', false) + + prefixRedactor.push('', true); + + expect(sanitizedPrefix).toBe('live_token[redacted]'); + expect(sanitizedPrefix).not.toContain('rk_'); + expect(sanitizedPrefix).not.toContain('rk_live_token'); + }); + it('redacts credential prefixes and configured secrets across subprocess chunk boundaries', async () => { const redactor = createCredentialRedactor(['split-secret']); const sanitized = From afadcf6872c0678d0910ade30640f55c1633300c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 8 Sep 2026 14:57:41 +0000 Subject: [PATCH 27/50] style: auto-format with Prettier --- scripts/pr-proof/run-cloud.mjs | 4 +--- tests/fixtures/pr-proof-contract.test.ts | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index c4b3b06eb3..687015c2e9 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -238,9 +238,7 @@ function createConfiguredSecretRedactor(secretValues, maskPendingOnFinal = false continue; } const suffixLength = - final && !maskPendingOnFinal - ? 0 - : longestSuffixThatStartsSecret(input.slice(index), secrets); + final && !maskPendingOnFinal ? 0 : longestSuffixThatStartsSecret(input.slice(index), secrets); const end = input.length - suffixLength; output += input.slice(index, end); if (suffixLength > 0) { diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 0c83fc7d82..0ead90de99 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -631,9 +631,7 @@ describe('Cloud dispatcher API key lifecycle', () => { // cannot be recognized on its own, but the trailing prefix must not be // released when the shared stream closes. const sanitized = - redactor.push('secret', false) + - redactor.push('split-', false) + - redactor.push('', true); + redactor.push('secret', false) + redactor.push('split-', false) + redactor.push('', true); expect(sanitized).toBe('secret[redacted]'); expect(sanitized).not.toContain('split-'); From cf64da470f44c713611c6a650272d132bb144e9b Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 8 Sep 2026 17:20:00 +0200 Subject: [PATCH 28/50] fix: mask cross-stream credential fragments atomically --- scripts/pr-proof/run-cloud.mjs | 34 +++++++++++- tests/fixtures/pr-proof-contract.test.ts | 70 ++++++++++++++++-------- 2 files changed, 80 insertions(+), 24 deletions(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 687015c2e9..9124861a59 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -63,7 +63,7 @@ const STATUS_DIAGNOSTIC_FIELDS = [ ]; const STATUS_FAILURE_DIAGNOSTIC_FIELDS = ['phase', 'code', 'dispatchType', 'sandboxId', 'occurredAt']; -function run(command, args, options = {}) { +async function run(command, args, options = {}) { const diagnosticSecretValues = options.diagnosticSecretValues ?? []; // One ordering-aware redactor covers both pipes. If the OS reports the two // streams in the opposite order, conservatively mask any trailing fragment @@ -71,7 +71,7 @@ function run(command, args, options = {}) { const outputRedactor = createCredentialRedactor(diagnosticSecretValues, { maskPendingOnFinal: true, }); - return runBoundedProcess(command, args, { + const result = await runBoundedProcess(command, args, { env: options.env, echo: !options.quiet, maxCaptureBytes: MAX_CAPTURE_BYTES, @@ -82,6 +82,22 @@ function run(command, args, options = {}) { onStderr: options.onStderr, transformChunk: (text, _stream, final) => outputRedactor.push(text, final), }); + + // A trailing fragment means the other half may already have been captured + // from the other pipe in an order that cannot be reconstructed safely. + // Fail closed by replacing every non-empty captured stream wholesale. + return maskCapturedCommandOutput(result, outputRedactor); +} + +export function maskCapturedCommandOutput(result, outputRedactor) { + if (outputRedactor.requiresCapturedOutputMask()) { + return { + ...result, + stdout: result.stdout ? '[redacted]' : '', + stderr: result.stderr ? '[redacted]' : '', + }; + } + return result; } export function boundedDuration(value, { fallback, minimum, maximum, label }) { @@ -137,6 +153,7 @@ function longestSuffixThatStartsSecret(value, secrets) { function createCredentialPrefixRedactor(maskPendingOnFinal = false) { let pending = ''; let active = null; + let maskedPendingOnFinal = false; return { push(value, final = false) { @@ -191,23 +208,29 @@ function createCredentialPrefixRedactor(maskPendingOnFinal = false) { } if (final && active && !active.emitted) { + if (maskPendingOnFinal) maskedPendingOnFinal = true; output += maskPendingOnFinal ? '[redacted]' : active.prefix; } if (final) { active = null; if (pending) { + if (maskPendingOnFinal) maskedPendingOnFinal = true; output += maskPendingOnFinal ? '[redacted]' : pending; pending = ''; } } return output; }, + maskedPendingOnFinal() { + return maskedPendingOnFinal; + }, }; } function createConfiguredSecretRedactor(secretValues, maskPendingOnFinal = false) { const secrets = [...new Set(secretValues.filter((value) => typeof value === 'string' && value))]; let pending = ''; + let maskedPendingOnFinal = false; return { push(value, final = false) { @@ -247,11 +270,15 @@ function createConfiguredSecretRedactor(secretValues, maskPendingOnFinal = false break; } if (final) { + if (pending && maskPendingOnFinal) maskedPendingOnFinal = true; output += pending && maskPendingOnFinal ? '[redacted]' : pending; pending = ''; } return output; }, + maskedPendingOnFinal() { + return maskedPendingOnFinal; + }, }; } @@ -264,6 +291,9 @@ export function createCredentialRedactor(secretValues = [], { maskPendingOnFinal const prefixed = prefixRedactor.push(value, final); return secretRedactor.push(prefixed, final); }, + requiresCapturedOutputMask() { + return prefixRedactor.maskedPendingOnFinal() || secretRedactor.maskedPendingOnFinal(); + }, }; } diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 0ead90de99..5a72d29d83 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -51,6 +51,7 @@ import { createCliApiKeyEnvironment, formatCloudRunArtifact, formatCloudRunDiagnostics, + maskCapturedCommandOutput, parseJsonOutput, preparedRunIdFromOutput, recognizedCloudStatusFromOutput, @@ -622,32 +623,57 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(prefixedArtifact).not.toContain('rk_live_token'); }); - it('never releases cross-stream credential fragments at finalization', () => { - const redactor = createCredentialRedactor(['split-secret'], { - maskPendingOnFinal: true, - }); + it('never releases cross-stream credential fragments in captures or artifacts', async () => { + const captureWithRedaction = async (code: string, secretValues: string[]) => { + const redactor = createCredentialRedactor(secretValues, { maskPendingOnFinal: true }); + const capture = await runBoundedProcess(process.execPath, ['-e', code], { + echo: false, + transformChunk: (text, _stream, final) => redactor.push(text, final), + }); + return maskCapturedCommandOutput(capture, redactor); + }; - // Model the OS reporting stdout before an earlier stderr write. The suffix - // cannot be recognized on its own, but the trailing prefix must not be - // released when the shared stream closes. - const sanitized = - redactor.push('secret', false) + redactor.push('split-', false) + redactor.push('', true); + const configuredCapture = await captureWithRedaction( + "process.stdout.write('secret'); setTimeout(() => process.stderr.write('split-'), 25)", + ['split-secret'] + ); + const configuredArtifact = formatCloudRunArtifact({ + runId: 'cloud-run-redaction-a', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed"}', + statusPollFailures: 0, + logs: configuredCapture, + diagnosticSecretValues: ['split-secret'], + }); - expect(sanitized).toBe('secret[redacted]'); - expect(sanitized).not.toContain('split-'); - expect(sanitized).not.toContain('split-secret'); + for (const value of [configuredCapture.stdout, configuredCapture.stderr, configuredArtifact]) { + expect(value).not.toContain('secret'); + expect(value).not.toContain('split-'); + expect(value).not.toContain('split-secret'); + } + expect(configuredCapture.stdout + configuredCapture.stderr).toContain('[redacted]'); + expect(configuredArtifact).toContain('[redacted]'); - const prefixRedactor = createCredentialRedactor([], { - maskPendingOnFinal: true, + const prefixCapture = await captureWithRedaction( + "process.stdout.write('live_token'); setTimeout(() => process.stderr.write('rk_'), 25)", + [] + ); + const prefixArtifact = formatCloudRunArtifact({ + runId: 'cloud-run-redaction-b', + terminalStatus: 'failed', + lastStatusOutput: '{"status":"failed"}', + statusPollFailures: 0, + logs: prefixCapture, + diagnosticSecretValues: [], }); - const sanitizedPrefix = - prefixRedactor.push('live_token', false) + - prefixRedactor.push('rk_', false) + - prefixRedactor.push('', true); - - expect(sanitizedPrefix).toBe('live_token[redacted]'); - expect(sanitizedPrefix).not.toContain('rk_'); - expect(sanitizedPrefix).not.toContain('rk_live_token'); + + for (const value of [prefixCapture.stdout, prefixCapture.stderr, prefixArtifact]) { + expect(value).not.toContain('live_token'); + expect(value).not.toContain('rk_'); + expect(value).not.toContain('rk_live_token'); + } + expect(prefixCapture.stdout + prefixCapture.stderr).toContain('[redacted]'); + expect(prefixArtifact).toContain('[redacted]'); }); it('redacts credential prefixes and configured secrets across subprocess chunk boundaries', async () => { From f7163e71bd966c646f07f3fdc2d221f05d135f92 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Wed, 9 Sep 2026 12:49:11 +0200 Subject: [PATCH 29/50] feat: route Agent37 sandboxes through isolated Relaycast --- .github/workflows/package-validation.yml | 1 - .github/workflows/publish.yml | 2 - .../cli/src/cli/ci-standalone-smoke.test.ts | 128 ++++++---- packages/cli/src/cli/commands/fleet.test.ts | 229 +++++++++++++++++- packages/cli/src/cli/commands/fleet.ts | 141 ++++++++++- .../cli/src/cli/commands/local-agent.test.ts | 23 ++ packages/cli/src/cli/commands/local-agent.ts | 4 + .../cli/src/cli/lib/attach-fleet-node.test.ts | 30 ++- .../cli/src/cli/lib/broker-lifecycle.test.ts | 40 +++ packages/cli/src/cli/lib/broker-lifecycle.ts | 10 +- .../cli/src/cli/lib/enrollment-pin.test.ts | 9 +- packages/cli/src/cli/lib/enrollment-pin.ts | 4 +- .../cli/src/cli/lib/project-workspace-key.ts | 2 + packages/cli/src/cli/lib/sdk-client.test.ts | 37 ++- packages/cli/src/cli/lib/sdk-client.ts | 114 ++++++++- .../cli/src/cli/lib/workspace-session.test.ts | 12 +- packages/cli/src/cli/lib/workspace-session.ts | 20 +- packages/cloud/src/fleet-sandbox.test.ts | 211 ++++++++++++++-- packages/cloud/src/fleet-sandbox.ts | 111 ++++++++- packages/cloud/src/index.ts | 5 + .../cloud/src/project-workspace-key.test.ts | 22 ++ packages/cloud/src/project-workspace-key.ts | 110 ++++++++- packages/cloud/src/workspace-key.ts | 2 + scripts/ci-standalone-smoke.sh | 93 ++++++- 24 files changed, 1248 insertions(+), 112 deletions(-) diff --git a/.github/workflows/package-validation.yml b/.github/workflows/package-validation.yml index 3b3126795a..f09aa05d77 100644 --- a/.github/workflows/package-validation.yml +++ b/.github/workflows/package-validation.yml @@ -354,5 +354,4 @@ jobs: - name: Smoke standalone lifecycle env: AGENT_RELAY_STARTUP_DEBUG: 1 - RELAY_WORKSPACE_KEY: ${{ secrets.RELAY_CI_WORKSPACE_KEY }} run: bash scripts/ci-standalone-smoke.sh "$STANDALONE_CLI" "$STANDALONE_BROKER" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 52f8d317fc..0f7ef3bc63 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1521,8 +1521,6 @@ jobs: echo "✓ Uncompressed binary verified" - name: Smoke standalone lifecycle - env: - RELAY_WORKSPACE_KEY: ${{ secrets.RELAY_CI_WORKSPACE_KEY }} run: | if [ "$(uname -m)" != "${{ matrix.expected_arch }}" ]; then echo "Skipping lifecycle smoke for ${{ matrix.expected_arch }} on $(uname -m) host" diff --git a/packages/cli/src/cli/ci-standalone-smoke.test.ts b/packages/cli/src/cli/ci-standalone-smoke.test.ts index ed6e4d4d7e..54b4a19e44 100644 --- a/packages/cli/src/cli/ci-standalone-smoke.test.ts +++ b/packages/cli/src/cli/ci-standalone-smoke.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from 'node:child_process'; -import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; @@ -15,12 +15,59 @@ function makeExecutable(directory: string, name: string, contents: string): stri return path; } -function createFakeBinaries(): { cli: string; broker: string; invocationLog: string } { +function createFakeBinaries(): { cli: string; broker: string; invocationLog: string; toolsPath: string } { const directory = mkdtempSync(join(tmpdir(), 'relay-standalone-smoke-test-')); temporaryDirectories.push(directory); const invocationLog = join(directory, 'invocations.log'); writeFileSync(invocationLog, ''); + const toolsPath = join(directory, 'tools'); + mkdirSync(toolsPath); + makeExecutable( + toolsPath, + 'curl', + `#!/usr/bin/env bash +set -euo pipefail +output=/dev/null +method=GET +url= +while [ "\$#" -gt 0 ]; do + case "\$1" in + --output) output="\$2"; shift 2 ;; + --request) method="\$2"; shift 2 ;; + --write-out) shift 2 ;; + --data|--header) shift 2 ;; + *) url="\$1"; shift ;; + esac +done +if [[ "$method" = POST && "$url" = */v1/workspaces ]]; then + printf '%s' '{"data":{"api_key":"rk_live_fake_smoke_key","workspace_id":"rw_fake_smoke"}}' > "\$output" + echo 201 +elif [[ "$method" = DELETE && "$url" = */v1/workspace ]]; then + echo 204 +elif [[ "$method" = GET && "$url" = */v1/workspace ]]; then + echo 401 +else + echo 500 +fi +` + ); + makeExecutable( + toolsPath, + 'jq', + `#!/usr/bin/env bash +set -euo pipefail +if [ "\${1:-}" = "-cn" ]; then + echo '{"name":"fake","expires_in_seconds":60}' +elif [ "\${1:-}" = "-er" ] && [[ "\${2:-}" = *api_key* ]]; then + echo 'rk_live_fake_smoke_key' +elif [ "\${1:-}" = "-er" ] && [[ "\${2:-}" = *workspace_id* ]]; then + echo 'rw_fake_smoke' +else + exit 1 +fi +` + ); const broker = makeExecutable( directory, 'broker', @@ -48,8 +95,8 @@ case "\${2:-}" in echo "Cleaned up (was not running)" ;; up) - if [ -z "\${RELAY_WORKSPACE_KEY:-}" ]; then - echo "workspace key missing" >&2 + if [ "\${RELAY_BASE_URL:-}" != "https://agent37-cast.agentrelay.com" ]; then + echo "trusted Relaycast base URL missing" >&2 exit 65 fi if [ -n "\${RELAY_WORKSPACES_JSON:-}" ]; then @@ -82,7 +129,7 @@ esac ` ); - return { cli, broker, invocationLog }; + return { cli, broker, invocationLog, toolsPath }; } afterEach(() => { @@ -103,11 +150,15 @@ describe('ci-standalone-smoke workspace reuse', () => { expect(cleanupSubshellIndex).toBeGreaterThan(trapDisarmIndex); }); - it('injects the same dedicated secret at every workflow call site', () => { + it('creates an ephemeral workspace on the trusted shard and wires its base URL explicitly', () => { + const script = readFileSync(smokeScript, 'utf8'); + expect(script).toContain('TRUSTED_RELAY_BASE_URL="https://agent37-cast.agentrelay.com"'); + expect(script).toContain('expires_in_seconds: 60'); + expect(script).toContain('printf \'::add-mask::%s\\n\' "$WORKSPACE_KEY"'); + expect(script).toContain('--request DELETE'); + expect(script).toContain('Ephemeral workspace deletion verified'); for (const workflow of ['.github/workflows/package-validation.yml', '.github/workflows/publish.yml']) { - expect(readFileSync(resolve(workflow), 'utf8')).toContain( - 'RELAY_WORKSPACE_KEY: ${{ secrets.RELAY_CI_WORKSPACE_KEY }}' - ); + expect(readFileSync(resolve(workflow), 'utf8')).not.toContain('RELAY_CI_WORKSPACE_KEY'); } }); @@ -140,7 +191,6 @@ describe('ci-standalone-smoke workspace reuse', () => { encoding: 'utf8', env: { ...process.env, - RELAY_WORKSPACE_KEY: 'rk_live_test_only', AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS: override, INVOCATION_LOG: invocationLog, }, @@ -152,47 +202,37 @@ describe('ci-standalone-smoke workspace reuse', () => { } }); - it('fails closed before invoking binaries when the dedicated key is missing or whitespace-only', () => { - const unusableKeys: Array<[label: string, value: string | undefined]> = [ - ['unset', undefined], - ['empty', ''], - ['single space', ' '], - ['tab', '\t'], - ]; - - for (const [label, value] of unusableKeys) { - const { cli, broker, invocationLog } = createFakeBinaries(); - const env = { ...process.env }; - if (value === undefined) { - delete env.RELAY_WORKSPACE_KEY; - } else { - env.RELAY_WORKSPACE_KEY = value; - } - env.INVOCATION_LOG = invocationLog; - - const result = spawnSync('bash', [smokeScript, cli, broker], { - encoding: 'utf8', - env, - }); + it('mints and cleans up the ephemeral workspace without printing its key', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + }, + timeout: 10_000, + }); - expect(result.status, `${label}: ${result.stderr}`).toBe(2); - expect(result.stderr, label).toContain('Refusing to start'); - expect(result.stderr, label).toContain('throwaway workspace'); - expect(result.stderr, label).not.toContain('binary not found'); - expect(readFileSync(invocationLog, 'utf8'), label).toBe(''); - } + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('Ephemeral workspace deletion verified'); + // The GitHub workflow command necessarily carries the value once so the + // runner can mask it; all ordinary output must remain free of the key. + const ordinaryOutput = result.stdout.replace(/::add-mask::[^\n]*\n/g, ''); + expect(ordinaryOutput).not.toContain('rk_live_fake_smoke_key'); + expect(result.stderr).not.toContain('rk_live_fake_smoke_key'); }); it('passes the shared key through the isolated lifecycle and joins its workspace', () => { - const { cli, broker, invocationLog } = createFakeBinaries(); + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); const result = spawnSync('bash', [smokeScript, cli, broker], { encoding: 'utf8', env: { ...process.env, - RELAY_WORKSPACE_KEY: 'rk_live_test_only', RELAY_WORKSPACES_JSON: '[{"workspace_id":"rw_wrong","api_key":"rk_wrong"}]', AGENT_RELAY_STANDALONE_BROKER_NAME: 'relay-ci-test-a', INVOCATION_LOG: invocationLog, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, }, timeout: 10_000, }); @@ -203,15 +243,15 @@ describe('ci-standalone-smoke workspace reuse', () => { }); it('rejects a lifecycle that reports a newly created workspace', () => { - const { cli, broker, invocationLog } = createFakeBinaries(); + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); const result = spawnSync('bash', [smokeScript, cli, broker], { encoding: 'utf8', env: { ...process.env, - RELAY_WORKSPACE_KEY: 'rk_live_test_only', AGENT_RELAY_STANDALONE_BROKER_NAME: 'relay-ci-test-b', FAKE_WORKSPACE_MODE: 'created', INVOCATION_LOG: invocationLog, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, }, timeout: 10_000, }); @@ -222,7 +262,7 @@ describe('ci-standalone-smoke workspace reuse', () => { }); it('rejects readiness written after the startup deadline', () => { - const { cli, broker, invocationLog } = createFakeBinaries(); + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); const bashEnv = join(dirname(invocationLog), 'accelerated-clock.sh'); writeFileSync( bashEnv, @@ -232,12 +272,12 @@ describe('ci-standalone-smoke workspace reuse', () => { encoding: 'utf8', env: { ...process.env, - RELAY_WORKSPACE_KEY: 'rk_live_test_only', AGENT_RELAY_STANDALONE_BROKER_NAME: 'relay-ci-test-c', AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS: '50', FAKE_READY_AFTER_SECOND_DOWN: '1', INVOCATION_LOG: invocationLog, BASH_ENV: bashEnv, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, }, timeout: 10_000, }); diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 1d27b2ce38..9e72778ca3 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -48,6 +48,12 @@ import { writeProjectWorkspaceKey } from '../lib/project-workspace-key.js'; const REPLAY_SANDBOX_ID = 'sbx_123e4567-e89b-42d3-a456-426614174000'; const REPLAY_SANDBOX_NAME = 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000'; +const AGENT37_RELAYCAST_TARGET = { + route: 'agent37-isolated' as const, + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_agent37_target', +}; const LIVE_AGENT_CAPABILITY_NAME = 'relay:live-agents:v1'; const liveAgentCapabilities = (...names: string[]) => [ @@ -646,7 +652,7 @@ describe('fleet command support', () => { }); }); - it('fleet spawn --sandbox-provider e2b provisions E2B, mounts Relayfile, and uses a temporary launcher', async () => { + it('fleet spawn --sandbox-provider agent37 provisions the isolated canary and uses a temporary launcher', async () => { vi.stubEnv('RELAY_AGENT_TOKEN', undefined); const placement = { spawn: vi.fn(async () => ({ @@ -666,13 +672,19 @@ describe('fleet command support', () => { const createAgentRelay = vi.fn(() => ({ messaging: { placement } })); const ensureCloudFleetSandbox = vi.fn(async () => ({ outcome: 'provisioned' as const, - providerId: 'e2b' as const, + providerId: 'agent37' as const, cloudWorkspaceId: 'cloud-workspace', nodeId: 'node-1', nodeName: 'e2b-codex', sandboxId: 'sandbox-1', providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_agent37_target', + }, relayfileMounted: true, relayfileMountPath: '/workspace', })); @@ -690,6 +702,13 @@ describe('fleet command support', () => { exit: vi.fn() as never, }, ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/tmp/agent-relay-test/workspace-key.json', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, deleteCloudFleetSandbox, createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, @@ -704,7 +723,7 @@ describe('fleet command support', () => { 'codex', '--sandbox', '--sandbox-provider', - 'e2b', + 'agent37', '--sandbox-id', REPLAY_SANDBOX_ID, '--sandbox-name', @@ -729,7 +748,7 @@ describe('fleet command support', () => { relayfilePaths: ['/live-review/run-123/**'], sandboxId: REPLAY_SANDBOX_ID, forceProvision: true, - providerId: 'e2b', + providerId: 'agent37', workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, name: REPLAY_SANDBOX_NAME, @@ -743,9 +762,9 @@ describe('fleet command support', () => { { strict: true } ); expect(createAgentRelay).toHaveBeenCalledWith({ - workspaceKey: 'rk_live_test', + workspaceKey: 'rk_live_agent37_target', token: 'at_live_launcher', - baseUrl: undefined, + baseUrl: 'https://agent37-cast.agentrelay.com', }); expect(placement.spawn).toHaveBeenCalledWith( expect.objectContaining({ @@ -765,11 +784,163 @@ describe('fleet command support', () => { }) ); expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); + expect(createWorkspaceRelay).toHaveBeenNthCalledWith(1, { + workspaceKey: 'rk_live_agent37_target', + baseUrl: 'https://agent37-cast.agentrelay.com', + }); expect(JSON.parse(logs[0]!)).toMatchObject({ - sandbox: { providerId: 'e2b', nodeName: 'e2b-codex', relayfileMountPath: '/workspace' }, + sandbox: { providerId: 'agent37', nodeName: 'e2b-codex', relayfileMountPath: '/workspace' }, invocation: { invocationId: 'inv_sandbox' }, - attachCommand: "agent-relay node agent attach 'sandbox-worker' --node 'e2b-codex' --mode drive", + attachCommand: + "agent-relay node agent attach 'sandbox-worker' --node 'e2b-codex' --mode drive --base-url 'https://agent37-cast.agentrelay.com'", + }); + }); + + it('fleet spawn --sandbox reuses an Agent37 target without a Relayfile mount', async () => { + const placement = { + spawn: vi.fn(async () => ({ invocationId: 'inv_reused', node: { name: 'agent37-codex' } })), + }; + const register = vi.fn(async () => ({ token: 'at_live_agent37' })); + const release = vi.fn(async () => ({ released: true, deleted: true })); + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })), register, release }, + })); + const ensureCloudFleetSandbox = vi.fn(async () => ({ + outcome: 'reused' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-1', + nodeName: 'agent37-codex', + status: 'online', + activeAgents: 0, + maxAgents: 1, + providerId: 'agent37' as const, + relaycastTarget: AGENT37_RELAYCAST_TARGET, + })); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, + createWorkspaceRelay: createWorkspaceRelay as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/tmp/agent-relay-test/workspace-key.json', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-provider', + 'agent37', + '--no-sandbox-relayfile', + '--workspace-id', + 'rw_abc', + '--name', + 'reused-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + + expect(createWorkspaceRelay).toHaveBeenCalledWith({ + workspaceKey: AGENT37_RELAYCAST_TARGET.relaycastApiKey, + baseUrl: AGENT37_RELAYCAST_TARGET.baseUrl, + }); + expect(register).toHaveBeenCalledWith( + expect.objectContaining({ name: expect.stringMatching(/^fleet-spawn-launcher-/) }), + { strict: true } + ); + expect(release).toHaveBeenCalled(); + }); + + it('fleet spawn --sandbox cleans up when the target cannot be persisted', async () => { + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const ensureCloudFleetSandbox = vi.fn(async () => ({ + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-1', + nodeName: 'agent37-codex', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, + relayfileMounted: true, + })); + const createAgentRelay = vi.fn(); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: createAgentRelay as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn(() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/tmp/agent-relay-test/workspace-key.json', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => false, + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-provider', + 'agent37', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + expect(deleteCloudFleetSandbox).toHaveBeenCalledWith({ + cloudWorkspaceId: 'cloud-workspace', + sandboxId: 'sandbox-1', + }); + expect(createAgentRelay).not.toHaveBeenCalled(); }); it('fleet spawn --sandbox deletes a freshly provisioned sandbox when dispatch fails', async () => { @@ -783,6 +954,7 @@ describe('fleet command support', () => { sandboxId: 'sandbox-1', providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, relayfileMounted: true, relayfileMountPath: '/workspace', providerId: 'agent37' as const, @@ -794,7 +966,11 @@ describe('fleet command support', () => { sdk: { createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, createWorkspaceRelay: vi.fn(() => ({ - workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + workspace: { + info: vi.fn(async () => ({ id: 'rw_abc' })), + register: vi.fn(async () => ({ token: 'at_live_isolated_launcher' })), + release: vi.fn(async () => ({ released: true, deleted: true })), + }, })) as never, createWorkspace: vi.fn() as never, log: vi.fn(), @@ -804,6 +980,13 @@ describe('fleet command support', () => { }) as never, }, ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'flag', + origin: 'test', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, deleteCloudFleetSandbox, createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, @@ -818,6 +1001,8 @@ describe('fleet command support', () => { 'spawn', 'codex', '--sandbox', + '--sandbox-provider', + 'agent37', '--name', 'sandbox-worker', '--task', @@ -862,6 +1047,7 @@ describe('fleet command support', () => { sandboxId: 'legacy-public-sandbox', providerSandboxId: 'legacy-provider-sandbox', relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, relayfileMounted: true, relayfileMountPath: '/workspace', })); @@ -879,6 +1065,13 @@ describe('fleet command support', () => { exit: vi.fn() as never, }, ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'flag', + origin: 'test', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, deleteCloudFleetSandbox: vi.fn(async () => undefined), createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, @@ -925,6 +1118,7 @@ describe('fleet command support', () => { sandboxId: 'generated-public-sandbox', providerSandboxId: 'generated-provider-sandbox', relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, relayfileMounted: true, relayfileMountPath: '/workspace', })); @@ -948,6 +1142,13 @@ describe('fleet command support', () => { exit: vi.fn() as never, }, ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'flag', + origin: 'test', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, deleteCloudFleetSandbox: vi.fn(async () => undefined), createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, @@ -1299,9 +1500,17 @@ describe('fleet command support', () => { sandboxId: 'sandbox-1', providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, relayfileMounted: false, providerId: 'agent37' as const, })), + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'flag', + origin: 'test', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget: () => true, deleteCloudFleetSandbox: vi.fn(async () => Promise.reject(new Error('delete failed'))), createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, @@ -1316,6 +1525,8 @@ describe('fleet command support', () => { 'spawn', 'codex', '--sandbox', + '--sandbox-provider', + 'agent37', '--name', 'sandbox-worker', '--task', diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index f6f34b910d..32af37d47f 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -29,6 +29,8 @@ import { attributableReleaseReason } from '../lib/release-reason.js'; import { resolveAgentToken, resolveBaseUrl, + resolveWorkspaceSelection, + persistWorkspaceRelaycastTarget, resolveWorkspaceKey, resolveWorkspaceKeyWithSource, type SdkClientOptions, @@ -54,6 +56,8 @@ export interface FleetCommandDependencies { core: CoreDependencies; sdk: SdkCommandDeps; createFleetWorkspaceClient: (options: SdkClientOptions) => RelayWorkspaceThinClient; + resolveWorkspaceSelection: typeof resolveWorkspaceSelection; + persistWorkspaceRelaycastTarget: typeof persistWorkspaceRelaycastTarget; ensureCloudFleetSandbox: typeof ensureCloudFleetSandbox; deleteCloudFleetSandbox: typeof deleteCloudFleetSandbox; log: (...args: unknown[]) => void; @@ -73,6 +77,8 @@ function withFleetDefaults(overrides: Partial = {}): F workspaceKey: resolveWorkspaceKey(options), baseUrl: resolveBaseUrl(options), }), + resolveWorkspaceSelection, + persistWorkspaceRelaycastTarget, ensureCloudFleetSandbox, deleteCloudFleetSandbox, log: (...args: unknown[]) => console.log(...args), @@ -168,7 +174,8 @@ export function registerFleetCommands( 'Explicit sandbox node name (custom unless --sandbox-id requires matching fleet-sandbox-)' ) .option('--sandbox-id ', 'Reuse a caller-declared sbx_ identity for an exact replay') - .option('--sandbox-provider ', 'Sandbox provider: daytona or e2b') + .option('--workspace-id ', 'Explicit Relay workspace identity required for sandbox provisioning') + .option('--sandbox-provider ', 'Sandbox provider: daytona, e2b, or agent37') .option( '--sandbox-relayfile-path ', 'Mount only these Relayfile subtrees (each path must end in /**)' @@ -203,6 +210,7 @@ export function registerFleetCommands( const useSandbox = options.sandbox === true; const sandboxName = optionalText(options.sandboxName, 'Sandbox name'); const sandboxIdOption = optionalText(options.sandboxId, 'Sandbox ID'); + const explicitWorkspaceId = optionalText(options.workspaceId, 'Workspace ID'); if (sandboxIdOption !== undefined && !CLOUD_SANDBOX_ID_PATTERN.test(sandboxIdOption)) { throw new Error('--sandbox-id must match lowercase sbx_ using an RFC 4122 UUID.'); } @@ -210,11 +218,13 @@ export function registerFleetCommands( const sandboxProvider: CloudFleetSandboxProviderId | undefined = sandboxProviderText === undefined ? undefined - : sandboxProviderText === 'daytona' || sandboxProviderText === 'e2b' + : sandboxProviderText === 'daytona' || + sandboxProviderText === 'e2b' || + sandboxProviderText === 'agent37' ? sandboxProviderText : undefined; if (sandboxProviderText !== undefined && sandboxProvider === undefined) { - throw new Error('--sandbox-provider must be daytona or e2b.'); + throw new Error('--sandbox-provider must be daytona, e2b, or agent37.'); } const mountSandboxRelayfile = options.sandboxRelayfile !== false; const sandboxRelayfilePaths = optionalTextList(options.sandboxRelayfilePath, 'Sandbox Relayfile path'); @@ -227,6 +237,9 @@ export function registerFleetCommands( if (!useSandbox && sandboxIdOption) { throw new Error('--sandbox-id requires --sandbox.'); } + if (!useSandbox && explicitWorkspaceId) { + throw new Error('--workspace-id requires --sandbox.'); + } if (!useSandbox && sandboxProvider) { throw new Error('--sandbox-provider requires --sandbox.'); } @@ -260,12 +273,29 @@ export function registerFleetCommands( let sandbox: EnsureCloudFleetSandboxResult | undefined; let workspaceRelay: ReturnType | undefined; + let relaycastClientOptions = clientOptions; if (useSandbox) { - workspaceRelay = deps.sdk.createWorkspaceRelay(clientOptions); - const workspaceInfo = await workspaceRelay.workspace.info(); - const relayWorkspaceId = workspaceInfo.id?.trim(); + // Cloud must be the first network authority for a sandbox invocation. + // A canonical Relaycast info call would both leak the workspace key and + // make it impossible to prove that Cloud's isolated target is the one + // subsequently used for registration and dispatch. + const workspaceSelection = deps.resolveWorkspaceSelection(clientOptions); + let relayWorkspaceId = explicitWorkspaceId ?? workspaceSelection?.workspaceId?.trim(); + // Legacy providers remain backward compatible: they may resolve the + // workspace from canonical Relaycast. Agent37 may not, because even a + // read there mutates rate-limit/presence accounting on the shared + // service and defeats the zero-shared-traffic canary proof. + if (!relayWorkspaceId && sandboxProvider !== 'agent37') { + workspaceRelay = deps.sdk.createWorkspaceRelay(clientOptions); + const workspaceInfo = await workspaceRelay.workspace.info(); + relayWorkspaceId = workspaceInfo.id?.trim(); + } if (!relayWorkspaceId) { - throw new Error('The current Relay workspace did not report an ID for Cloud provisioning.'); + throw new Error( + sandboxProvider === 'agent37' + ? 'Agent37 sandbox provisioning requires a persisted Relay workspace identity; run `relay workspace pin` or pass --workspace-id.' + : 'The current Relay workspace did not report an ID for Cloud provisioning.' + ); } const sandboxId = sandboxIdOption ?? (sandboxName === undefined ? `sbx_${randomUUID()}` : undefined); const deterministicSandboxName = @@ -320,6 +350,73 @@ export function registerFleetCommands( } throw error; } + if ( + sandbox.outcome === 'provisioned' || + (sandbox.outcome === 'reused' && sandbox.providerId === 'agent37') + ) { + // A provisioned response must carry a closed, server-owned target. + // Rebuild both credentials and origin before any registration, spawn, + // or launcher release, then prove the authenticated client sees the + // exact workspace Cloud returned. + try { + const target = sandbox.relaycastTarget; + if (!target) { + throw new Error('Cloud provisioned a sandbox without a Relaycast target.'); + } + if ( + (sandboxProvider === 'agent37' && target.route !== 'agent37-isolated') || + target.workspaceId.trim() !== relayWorkspaceId.trim() || + (sandbox.outcome === 'provisioned' && + sandbox.relayWorkspaceId.trim() !== relayWorkspaceId.trim()) + ) { + throw new Error( + sandboxProvider === 'agent37' && target.route !== 'agent37-isolated' + ? 'Explicit Agent37 provisioning requires the isolated Agent37 Relaycast target.' + : 'Cloud returned a Relaycast target for a different workspace.' + ); + } + relaycastClientOptions = { + ...clientOptions, + workspaceKey: target.relaycastApiKey, + baseUrl: target.baseUrl, + }; + workspaceRelay = deps.sdk.createWorkspaceRelay(relaycastClientOptions); + const postEnsureWorkspace = await workspaceRelay.workspace.info(); + const postEnsureWorkspaceId = postEnsureWorkspace.id?.trim(); + if ( + !postEnsureWorkspaceId || + (sandbox.outcome === 'provisioned' && + postEnsureWorkspaceId !== sandbox.relayWorkspaceId.trim()) || + postEnsureWorkspaceId !== target.workspaceId.trim() + ) { + throw new Error( + 'Cloud returned a Relaycast workspace that could not be verified on the selected gateway.' + ); + } + if (!deps.persistWorkspaceRelaycastTarget(workspaceSelection, target)) { + throw new Error( + 'Cloud returned a Relaycast target, but no durable project session is available for follow-up attach.' + ); + } + } catch (error) { + if (sandbox.outcome === 'provisioned') { + await deps + .deleteCloudFleetSandbox({ + cloudWorkspaceId: sandbox.cloudWorkspaceId, + sandboxId: sandbox.sandboxId, + ...(sandbox.providerId === undefined ? {} : { providerId: sandbox.providerId }), + }) + .catch((cleanupError) => { + deps.warn( + `Relaycast workspace verification failed and sandbox cleanup also failed: ${ + cleanupError instanceof Error ? cleanupError.message : String(cleanupError) + }` + ); + }); + } + throw error; + } + } if (sandbox.outcome === 'provisioning_timeout') { await deps .deleteCloudFleetSandbox({ @@ -368,7 +465,14 @@ export function registerFleetCommands( if (targetNode) { let launcherName: string | undefined; try { - let agentToken = resolveAgentToken(clientOptions); + // Agent tokens are scoped to a Relaycast deployment. Never replay a + // canonical token after Cloud has selected the isolated shard; mint + // a temporary launcher on the validated target instead. + let agentToken = + sandbox?.outcome === 'provisioned' || + (sandbox?.outcome === 'reused' && sandbox.providerId === 'agent37') + ? undefined + : resolveAgentToken(clientOptions); if (!agentToken) { workspaceRelay ??= deps.sdk.createWorkspaceRelay(clientOptions); const pendingLauncherName = `fleet-spawn-launcher-${randomUUID().slice(0, 8)}`; @@ -386,7 +490,7 @@ export function registerFleetCommands( } } - const relay = deps.sdk.createAgentRelay({ ...clientOptions, token: agentToken }); + const relay = deps.sdk.createAgentRelay({ ...relaycastClientOptions, token: agentToken }); // Placement alone only proves the node accepted the dispatch. A node // running an obsolete broker advertises `spawn:` capacity, acks // the invocation and launches nothing, which is indistinguishable from @@ -409,13 +513,28 @@ export function registerFleetCommands( ...(sessionRef ? { session_ref: sessionRef } : {}), }, }); + const printableSandbox = + (sandbox?.outcome === 'provisioned' || sandbox?.outcome === 'reused') && sandbox.relaycastTarget + ? { + ...sandbox, + relaycastTarget: { + route: sandbox.relaycastTarget.route, + baseUrl: sandbox.relaycastTarget.baseUrl, + workspaceId: sandbox.relaycastTarget.workspaceId, + }, + } + : sandbox; printJson(deps.sdk, { ...(sandbox ? { - sandbox, + sandbox: printableSandbox, attachCommand: `agent-relay node agent attach ${shellQuote(name)} ` + - `--node ${shellQuote(targetNode)} --mode drive`, + `--node ${shellQuote(targetNode)} --mode drive` + + ((sandbox.outcome === 'provisioned' || sandbox.outcome === 'reused') && + sandbox.relaycastTarget + ? ` --base-url ${shellQuote(sandbox.relaycastTarget.baseUrl)}` + : ''), } : {}), invocation, diff --git a/packages/cli/src/cli/commands/local-agent.test.ts b/packages/cli/src/cli/commands/local-agent.test.ts index eb4f8959c4..22d3ca1ef2 100644 --- a/packages/cli/src/cli/commands/local-agent.test.ts +++ b/packages/cli/src/cli/commands/local-agent.test.ts @@ -171,6 +171,29 @@ describe('local agent subtree', () => { ); }); + it('attach --node forwards an explicit base URL for an isolated fleet workspace', async () => { + const { program, attachNode } = harness(); + await program.parseAsync( + [ + 'local', + 'agent', + 'attach', + 'sandbox-worker', + '--node', + 'agent37-codex', + '--base-url', + 'https://agent37-cast.agentrelay.com', + ], + { from: 'user' } + ); + expect(attachNode).toHaveBeenCalledWith( + 'sandbox-worker', + 'view', + 'agent37-codex', + expect.objectContaining({ baseUrl: 'https://agent37-cast.agentrelay.com' }) + ); + }); + it('attach --node without --workspace-key leaves the precedence ladder to resolve it', async () => { const { program, attachNode } = harness(); await program.parseAsync(['local', 'agent', 'attach', 'lead', '--node', 'sf-mini'], { from: 'user' }); diff --git a/packages/cli/src/cli/commands/local-agent.ts b/packages/cli/src/cli/commands/local-agent.ts index 05bb9f682f..eb39885aae 100644 --- a/packages/cli/src/cli/commands/local-agent.ts +++ b/packages/cli/src/cli/commands/local-agent.ts @@ -88,6 +88,7 @@ export function runAttach(name: string, mode: AttachMode, options: NativeAttachO * rather than accepted and ignored. */ export type FleetNodeAttachCliOptions = Pick & { + baseUrl?: string; workspaceKey?: string; }; @@ -106,6 +107,7 @@ export async function attachFleetNode( agent: name, node, mode, + ...(options.baseUrl === undefined ? {} : { baseUrl: options.baseUrl }), ...(options.workspaceKey === undefined ? {} : { workspaceKey: options.workspaceKey }), }); const jsonWriter = options.json ? createBackpressureAwareWriter(process.stdout) : undefined; @@ -845,6 +847,7 @@ export function registerLocalAgentCommands( .argument('', 'Agent name') .option('--mode ', 'drive | view | passthrough', 'view') .option('--node ', 'Canonical authenticated fleet-node terminal attach (physical or Daytona)') + .option('--base-url ', 'Relaycast API base URL for --node (defaults to the selected workspace route)') .option('--ssh-host ', 'SSH host fallback for a physical fleet node') .option('--broker-url ', 'Broker base URL (overrides RELAY_BROKER_URL and connection.json)') .option('--api-key ', 'Broker API key (overrides RELAY_BROKER_API_KEY and connection.json)') @@ -919,6 +922,7 @@ export function registerLocalAgentCommands( }); } const code = await deps.attachNode(name, mode, node, { + baseUrl: options.baseUrl as string | undefined, workspaceKey: attachWorkspaceKey, json: options.json as boolean | undefined, reasoning: options.reasoning as boolean | undefined, diff --git a/packages/cli/src/cli/lib/attach-fleet-node.test.ts b/packages/cli/src/cli/lib/attach-fleet-node.test.ts index 252894fc61..a7edc522d5 100644 --- a/packages/cli/src/cli/lib/attach-fleet-node.test.ts +++ b/packages/cli/src/cli/lib/attach-fleet-node.test.ts @@ -1121,9 +1121,12 @@ describe('startFleetNodeAttachProxy workspace-key precedence', () => { function capturingTicketFetch(remoteUrl: string): { fetch: typeof globalThis.fetch; authorization: () => string | undefined; + requestUrl: () => string | undefined; } { let seen: string | undefined; - const fetchFn = (async (_url: string, init?: RequestInit) => { + let requestedUrl: string | undefined; + const fetchFn = (async (url: string, init?: RequestInit) => { + requestedUrl = url; const headers = (init?.headers ?? {}) as Record; seen = headers.Authorization; return { @@ -1139,7 +1142,7 @@ describe('startFleetNodeAttachProxy workspace-key precedence', () => { }), } as unknown as Response; }) as unknown as typeof globalThis.fetch; - return { fetch: fetchFn, authorization: () => seen }; + return { fetch: fetchFn, authorization: () => seen, requestUrl: () => requestedUrl }; } it('presents an explicit workspace key ahead of the ambient environment', async () => { @@ -1160,6 +1163,29 @@ describe('startFleetNodeAttachProxy workspace-key precedence', () => { expect(ticket.authorization()).toBe('Bearer rk_live_explicit'); }); + it('uses an explicit isolated base URL instead of the canonical fallback', async () => { + const remote = await startFakeRemote(); + cleanup.push(remote.close); + const ticket = capturingTicketFetch(remote.url); + const proxy = await startFleetNodeAttachProxy({ + agent: 'sandbox-worker', + node: 'agent37-codex', + mode: 'view', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceKey: 'rk_live_agent37_target', + env: { RELAY_WORKSPACE_KEY: 'rk_live_ambient' }, + fetch: ticket.fetch, + }); + cleanup.push(proxy.close); + + expect(ticket.requestUrl()).toBe( + 'https://agent37-cast.agentrelay.com/v1/nodes/agent37-codex/terminal/sessions' + ); + expect(ticket.requestUrl()).not.toBe( + 'https://cast.agentrelay.com/v1/nodes/agent37-codex/terminal/sessions' + ); + }); + it('falls back to the environment when no explicit key is supplied', async () => { const remote = await startFakeRemote(); cleanup.push(remote.close); diff --git a/packages/cli/src/cli/lib/broker-lifecycle.test.ts b/packages/cli/src/cli/lib/broker-lifecycle.test.ts index a83a1e4662..19972961f8 100644 --- a/packages/cli/src/cli/lib/broker-lifecycle.test.ts +++ b/packages/cli/src/cli/lib/broker-lifecycle.test.ts @@ -701,6 +701,46 @@ describe('runUpCommand workspace precedence', () => { expect(readPin(dataDir)).toMatchObject({ workspaceKey: 'rk_test', workspaceId: 'rw_test' }); }); + it('retains the Relaycast target when the broker keeps the pinned workspace', async () => { + const { deps, dataDir } = createUpHarness(); + writeRepositoryPin(dataDir, { + workspaceKey: 'rk_test', + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + + await runUpCommand({}, deps); + + expect(readPin(dataDir)).toMatchObject({ + workspaceKey: 'rk_test', + workspaceId: 'rw_test', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + }); + + it('clears the old Relaycast target when the broker changes workspace', async () => { + const { deps, dataDir, createRelay } = createUpHarness(); + writeRepositoryPin(dataDir, { + workspaceKey: 'rk_old', + workspaceId: 'rw_old', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + createRelay.mockImplementationOnce(async () => ({ + spawn: vi.fn(async () => undefined), + getStatus: vi.fn(async () => ({})), + shutdown: vi.fn(async () => undefined), + workspaceKey: 'rk_new', + workspaceId: 'rw_new', + })); + + await runUpCommand({}, deps); + + expect(readPin(dataDir)).toEqual({ workspaceKey: 'rk_new', workspaceId: 'rw_new' }); + }); + it('never prints workspace key material while reporting the winning source', async () => { const { deps, dataDir, home, log, warn, error } = createUpHarness(); setWorkspaceKey('global', 'rk_global', { AGENT_RELAY_HOME: home }); diff --git a/packages/cli/src/cli/lib/broker-lifecycle.ts b/packages/cli/src/cli/lib/broker-lifecycle.ts index 4fba55ae80..1bfe0a6375 100644 --- a/packages/cli/src/cli/lib/broker-lifecycle.ts +++ b/packages/cli/src/cli/lib/broker-lifecycle.ts @@ -30,7 +30,7 @@ import { startReflexCapture, type RunningReflexCapture } from './reflex-capture. import { readProjectWorkspaceSession, resolveWorkspaceSelection, - writeProjectWorkspaceKey, + writeProjectWorkspaceKeyPreservingSession, type ProjectWorkspaceSession, type WorkspaceSelection, } from './project-workspace-key.js'; @@ -1953,12 +1953,16 @@ export async function runUpCommand(options: UpOptions, deps: CoreDependencies): // workspace. Persistence must never abort startup, so a write failure is // swallowed. try { - writeProjectWorkspaceKey(projectWorkspaceKeyDataDir, relay.workspaceKey ?? undefined, { + const relayWorkspaceKey = relay.workspaceKey ?? undefined; + const sameWorkspace = resumedProjectSession?.workspaceKey === relayWorkspaceKey; + writeProjectWorkspaceKeyPreservingSession(projectWorkspaceKeyDataDir, relayWorkspaceKey, { enrolledNodeId: deps.env.AGENT_RELAY_ENROLLED_NODE_ID ?? resumedProjectSession?.enrolledNodeId, // Recording the resolved workspace id lets the NEXT start detect a // conflicting source (a stored enrollment in another workspace) before // the broker comes up, instead of after agents land in the wrong place. - workspaceId: relay.workspaceId ?? resumedProjectSession?.workspaceId, + ...((relay.workspaceId ?? (sameWorkspace ? resumedProjectSession?.workspaceId : undefined)) + ? { workspaceId: relay.workspaceId ?? resumedProjectSession?.workspaceId } + : {}), }); } catch { // best-effort: a broker that came up should stay up even if the key file diff --git a/packages/cli/src/cli/lib/enrollment-pin.test.ts b/packages/cli/src/cli/lib/enrollment-pin.test.ts index 2be08d64a4..b4a74cb2a7 100644 --- a/packages/cli/src/cli/lib/enrollment-pin.test.ts +++ b/packages/cli/src/cli/lib/enrollment-pin.test.ts @@ -24,7 +24,11 @@ afterEach(() => { describe('linkEnrolledNodeToProjectPin', () => { it('records the enrolled node on a pin that has none', () => { const dataDir = projectDataDir(); - writeProjectWorkspaceKey(dataDir, 'rk_live_pinned'); + writeProjectWorkspaceKey(dataDir, 'rk_live_pinned', { + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); const result = linkEnrolledNodeToProjectPin({ nodeId: 'node_abc', projectDataDir: dataDir }); @@ -32,6 +36,9 @@ describe('linkEnrolledNodeToProjectPin', () => { expect(readProjectWorkspaceSession(dataDir)).toEqual({ workspaceKey: 'rk_live_pinned', enrolledNodeId: 'node_abc', + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', }); }); diff --git a/packages/cli/src/cli/lib/enrollment-pin.ts b/packages/cli/src/cli/lib/enrollment-pin.ts index 3641298952..3c2898c963 100644 --- a/packages/cli/src/cli/lib/enrollment-pin.ts +++ b/packages/cli/src/cli/lib/enrollment-pin.ts @@ -3,7 +3,7 @@ import { getProjectPaths } from '@agent-relay/config'; import { projectWorkspaceKeyPath, readProjectWorkspaceSession, - writeProjectWorkspaceKey, + writeProjectWorkspaceKeyPreservingSession, } from './project-workspace-key.js'; /** Outcome of reconciling a fresh enrollment against the project workspace pin. */ @@ -63,6 +63,6 @@ export function linkEnrolledNodeToProjectPin( return { status: 'conflict', nodeId, pinnedNodeId: session.enrolledNodeId, pinPath }; } - writeProjectWorkspaceKey(dataDir, session.workspaceKey, { enrolledNodeId: nodeId }); + writeProjectWorkspaceKeyPreservingSession(dataDir, session.workspaceKey, { enrolledNodeId: nodeId }); return { status: 'linked', nodeId, pinPath }; } diff --git a/packages/cli/src/cli/lib/project-workspace-key.ts b/packages/cli/src/cli/lib/project-workspace-key.ts index 5d97af820e..3c40b044bb 100644 --- a/packages/cli/src/cli/lib/project-workspace-key.ts +++ b/packages/cli/src/cli/lib/project-workspace-key.ts @@ -7,7 +7,9 @@ export { resolveActiveWorkspaceSelection, resolveWorkspaceSelection, writeProjectWorkspaceKey, + writeProjectWorkspaceKeyPreservingSession, type ProjectWorkspaceSession, + type ProjectWorkspaceSessionMetadata, type WorkspaceKeyFileSystem, type WorkspaceSelection, } from '@agent-relay/cloud/workspace-key'; diff --git a/packages/cli/src/cli/lib/sdk-client.test.ts b/packages/cli/src/cli/lib/sdk-client.test.ts index 65e9474535..13cbbce3ca 100644 --- a/packages/cli/src/cli/lib/sdk-client.test.ts +++ b/packages/cli/src/cli/lib/sdk-client.test.ts @@ -6,13 +6,15 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { createAgentRelay, + persistWorkspaceRelaycastTarget, resolveAgentToken, resolveBaseUrl, resolveWorkspaceKey, resolveWorkspaceKeyWithSource, + resolveWorkspaceSelection, } from './sdk-client.js'; import { setWorkspaceKey } from './workspace-store.js'; -import { writeProjectWorkspaceKey } from './project-workspace-key.js'; +import { readProjectWorkspaceSession, writeProjectWorkspaceKey } from './project-workspace-key.js'; let dir: string; let projectRoot: string; @@ -123,4 +125,37 @@ describe('sdk client option resolution', () => { expect(JSON.stringify(relay)).not.toContain('rk_live_project_owner_secret'); expect(JSON.stringify(relay)).not.toContain('at_live_participant_scoped'); }); + + it('durably records an isolated target while preserving the enrolled node session', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + enrolledNodeId: 'node_1', + }); + const selection = resolveWorkspaceSelection({ env: { AGENT_RELAY_HOME: dir } }); + expect( + persistWorkspaceRelaycastTarget(selection, { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_agent37', + }) + ).toBe(true); + expect(readProjectWorkspaceSession(projectDataDir())).toEqual({ + workspaceKey: 'rk_live_agent37', + workspaceId: 'rw_abc', + enrolledNodeId: 'node_1', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + }); + + it('rejects a persisted route that is not the exact server-owned origin', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_agent37', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://evil.example', + }); + + expect(() => resolveBaseUrl({ env: { AGENT_RELAY_HOME: dir } })).toThrow(/not trusted/); + }); }); diff --git a/packages/cli/src/cli/lib/sdk-client.ts b/packages/cli/src/cli/lib/sdk-client.ts index ae1e2b216a..1348c766f1 100644 --- a/packages/cli/src/cli/lib/sdk-client.ts +++ b/packages/cli/src/cli/lib/sdk-client.ts @@ -1,6 +1,13 @@ +import path from 'node:path'; + import { AgentRelay, type AgentRelayAgent } from '@agent-relay/sdk'; +import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; import { + resolveWorkspaceSelection as resolveCloudWorkspaceSelection, resolveWorkspaceKeyWithSource as resolveCloudWorkspaceKeyWithSource, + readProjectWorkspaceSession, + writeProjectWorkspaceKey, + type WorkspaceSelection, type WorkspaceKeySource, } from '@agent-relay/cloud/workspace-key'; @@ -23,6 +30,15 @@ function trimOrUndefined(value: string | undefined): string | undefined { /** Where a resolved workspace key came from, in precedence order. */ export type { WorkspaceKeySource }; +export type { WorkspaceSelection }; + +/** Resolve the selected key and any previously persisted Relay workspace identity. */ +export function resolveWorkspaceSelection(options: SdkClientOptions = {}): WorkspaceSelection | undefined { + return resolveCloudWorkspaceSelection({ + workspaceKey: options.workspaceKey, + env: env(options), + }); +} /** * Resolve the workspace key and report which source it came from. Precedence: @@ -50,7 +66,103 @@ export function resolveWorkspaceKey(options: SdkClientOptions = {}): string { } export function resolveBaseUrl(options: SdkClientOptions = {}): string | undefined { - return trimOrUndefined(options.baseUrl) ?? trimOrUndefined(env(options).RELAY_BASE_URL); + const selection = resolveWorkspaceSelection(options); + const persisted = validatePersistedRelaycastBaseUrl(selection); + const requested = trimOrUndefined(options.baseUrl) ?? trimOrUndefined(env(options).RELAY_BASE_URL); + if (persisted && requested && requested !== persisted) { + throw new Error('The requested Relaycast base URL does not match the persisted workspace route.'); + } + return persisted ?? requested; +} + +function validatePersistedRelaycastBaseUrl(selection: WorkspaceSelection | undefined): string | undefined { + const baseUrl = trimOrUndefined(selection?.relaycastBaseUrl); + const route = selection?.relaycastRoute; + if (!baseUrl && !route) return undefined; + if (!baseUrl || !route) { + throw new Error('The persisted Relaycast workspace route is incomplete.'); + } + let parsed: URL; + try { + parsed = new URL(baseUrl); + } catch { + throw new Error('The persisted Relaycast workspace route is invalid.'); + } + const expectedOrigin = + route === 'canonical' + ? CANONICAL_RELAYCAST_ORIGIN + : route === 'agent37-isolated' + ? AGENT37_RELAYCAST_ORIGIN + : undefined; + if ( + !expectedOrigin || + parsed.origin !== expectedOrigin || + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') + ) { + throw new Error('The persisted Relaycast workspace route is not trusted.'); + } + return parsed.origin; +} + +/** Persist a server-selected target only for an existing project session. */ +export function persistWorkspaceRelaycastTarget( + selection: WorkspaceSelection | undefined, + target: { + route: 'canonical' | 'agent37-isolated'; + baseUrl: string; + workspaceId: string; + relaycastApiKey: string; + } +): boolean { + const selectionWithProjectDir = selection as (WorkspaceSelection & { projectDataDir?: string }) | undefined; + const dataDir = + selectionWithProjectDir?.projectDataDir ?? + (selection?.source === 'project' && selection.origin ? path.dirname(selection.origin) : undefined); + if (!dataDir) return false; + const existing = readProjectWorkspaceSession(dataDir); + if (!existing) return false; + const restoreExisting = (): void => { + writeProjectWorkspaceKey(dataDir, existing.workspaceKey, { + ...(existing.enrolledNodeId ? { enrolledNodeId: existing.enrolledNodeId } : {}), + ...(existing.workspaceId ? { workspaceId: existing.workspaceId } : {}), + ...(existing.relaycastRoute ? { relaycastRoute: existing.relaycastRoute } : {}), + ...(existing.relaycastBaseUrl ? { relaycastBaseUrl: existing.relaycastBaseUrl } : {}), + }); + }; + try { + writeProjectWorkspaceKey(dataDir, target.relaycastApiKey, { + ...(existing.enrolledNodeId ? { enrolledNodeId: existing.enrolledNodeId } : {}), + workspaceId: target.workspaceId, + relaycastRoute: target.route, + relaycastBaseUrl: target.baseUrl, + }); + const persisted = readProjectWorkspaceSession(dataDir); + if ( + persisted?.workspaceKey === target.relaycastApiKey && + persisted.workspaceId === target.workspaceId && + persisted.relaycastRoute === target.route && + persisted.relaycastBaseUrl === target.baseUrl + ) { + return true; + } + restoreExisting(); + return false; + } catch (error) { + try { + restoreExisting(); + } catch (restoreError) { + throw new Error('Could not restore the prior Relaycast project session after persistence failed.', { + cause: restoreError, + }); + } + throw error; + } } export function resolveAgentToken(options: SdkClientOptions = {}): string | undefined { diff --git a/packages/cli/src/cli/lib/workspace-session.test.ts b/packages/cli/src/cli/lib/workspace-session.test.ts index 9691a535ae..a06a315644 100644 --- a/packages/cli/src/cli/lib/workspace-session.test.ts +++ b/packages/cli/src/cli/lib/workspace-session.test.ts @@ -155,7 +155,12 @@ describe('workspace session persistence', () => { const root = tempRoot(); const projectDataDir = path.join(root, 'project', '.agentworkforce', 'relay'); const env = isolatedEnv(root); - writeProjectWorkspaceKey(projectDataDir, 'rk_live_enrolled', { enrolledNodeId: 'node_abc' }); + writeProjectWorkspaceKey(projectDataDir, 'rk_live_enrolled', { + enrolledNodeId: 'node_abc', + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); const result = persistWorkspaceSession({ workspaceKey: 'rk_live_enrolled', @@ -169,6 +174,9 @@ describe('workspace session persistence', () => { expect(readProjectWorkspaceSession(projectDataDir)).toEqual({ workspaceKey: 'rk_live_enrolled', enrolledNodeId: 'node_abc', + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', }); expect(result.clearedEnrolledNodeId).toBeUndefined(); }); @@ -227,6 +235,8 @@ describe('workspace session persistence', () => { writeProjectWorkspaceKey(projectDataDir, 'rk_live_old', { enrolledNodeId: 'node_old', workspaceId: 'rw_old', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', }); pinProjectWorkspaceSession({ workspaceKey: 'rk_live_default', projectDataDir, env }); diff --git a/packages/cli/src/cli/lib/workspace-session.ts b/packages/cli/src/cli/lib/workspace-session.ts index 4818ce1d4d..a3f440d62d 100644 --- a/packages/cli/src/cli/lib/workspace-session.ts +++ b/packages/cli/src/cli/lib/workspace-session.ts @@ -1,7 +1,11 @@ import { getProjectPaths } from '@agent-relay/config'; import { resolveWorkspaceKeyWithSource } from '@agent-relay/cloud/workspace-key'; -import { readProjectWorkspaceSession, writeProjectWorkspaceKey } from './project-workspace-key.js'; +import { + readProjectWorkspaceSession, + writeProjectWorkspaceKey, + writeProjectWorkspaceKeyPreservingSession, +} from './project-workspace-key.js'; import { setWorkspaceKey, switchWorkspace, validateWorkspaceName } from './workspace-store.js'; export interface WorkspaceSessionOptions { @@ -99,10 +103,16 @@ export function persistWorkspaceSession( // for an ordinary switch/join/create that happens to stay on the same key. const keepsWorkspace = existing?.workspaceKey === workspaceKey; const enrolledNodeId = keepsWorkspace ? existing?.enrolledNodeId : undefined; - writeProjectWorkspaceKey(projectDataDir, workspaceKey, { - ...(enrolledNodeId ? { enrolledNodeId } : {}), - ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), - }); + if (keepsWorkspace) { + writeProjectWorkspaceKeyPreservingSession(projectDataDir, workspaceKey, { + ...(enrolledNodeId ? { enrolledNodeId } : {}), + ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), + }); + } else { + writeProjectWorkspaceKey(projectDataDir, workspaceKey, { + ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), + }); + } if (name) { setWorkspaceKey(name, workspaceKey, options.env); diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 23b14d196a..4589e35508 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -11,9 +11,12 @@ vi.mock('./auth.js', () => ({ })); import { + AGENT37_RELAYCAST_ORIGIN, + CANONICAL_RELAYCAST_ORIGIN, CloudFleetSandboxProvisionError, deleteCloudFleetSandbox, ensureCloudFleetSandbox, + normalizeRelaycastTarget, } from './fleet-sandbox.js'; const auth = { @@ -26,6 +29,12 @@ const refreshedAuth = { ...auth, accessToken: 'refreshed' }; const CLOUD_WORKSPACE_ID = '50587328-441d-4acb-b8f3-dbe1b3c5de99'; const SANDBOX_ID = 'sbx_123e4567-e89b-42d3-a456-426614174000'; const SANDBOX_NAME = 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000'; +const RELAYCAST_TARGET = { + route: 'agent37-isolated' as const, + baseUrl: AGENT37_RELAYCAST_ORIGIN, + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_agent37', +}; describe('Cloud fleet sandbox client', () => { beforeEach(() => { @@ -37,6 +46,115 @@ describe('Cloud fleet sandbox client', () => { vi.restoreAllMocks(); }); + it('normalizes a closed Agent37 Relaycast target', () => { + expect( + normalizeRelaycastTarget({ ...RELAYCAST_TARGET, baseUrl: `${AGENT37_RELAYCAST_ORIGIN}/` }) + ).toEqual(RELAYCAST_TARGET); + expect( + normalizeRelaycastTarget({ + route: 'canonical', + baseUrl: CANONICAL_RELAYCAST_ORIGIN, + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_canonical', + }) + ).toMatchObject({ route: 'canonical', baseUrl: CANONICAL_RELAYCAST_ORIGIN }); + }); + + it.each([ + { route: 'agent37-isolated', baseUrl: 'https://evil.example' }, + { route: 'canonical', baseUrl: AGENT37_RELAYCAST_ORIGIN }, + { route: 'agent37-isolated', baseUrl: 'https://agent37-cast.agentrelay.com/path' }, + { route: 'agent37-isolated', baseUrl: 'https://user:pass@agent37-cast.agentrelay.com' }, + ])('rejects a route mapped to an untrusted Relaycast origin', (target) => { + expect(() => normalizeRelaycastTarget({ ...RELAYCAST_TARGET, ...target })).toThrow(/relaycast/i); + }); + + it.each([ + { ...RELAYCAST_TARGET, relaycastApiKey: 'rk_test_not_live' }, + { ...RELAYCAST_TARGET, workspaceId: '' }, + { ...RELAYCAST_TARGET, relaycastApiKey: undefined }, + ])('rejects an incomplete Relaycast target', (target) => { + expect(() => normalizeRelaycastTarget(target)).toThrow(/Relaycast|workspace/); + }); + + it('rejects the legacy generic API key field', () => { + expect(() => + normalizeRelaycastTarget({ + route: RELAYCAST_TARGET.route, + baseUrl: RELAYCAST_TARGET.baseUrl, + workspaceId: RELAYCAST_TARGET.workspaceId, + apiKey: RELAYCAST_TARGET.relaycastApiKey, + }) + ).toThrow(/API key/); + }); + + it('rejects a provisioned response with an untrusted server-owned Relaycast route', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-1', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relaycastTarget: { ...RELAYCAST_TARGET, baseUrl: 'https://evil.example' }, + relayfileMounted: true, + }), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + workloadProfile: 'long-running-agent', + }) + ).rejects.toThrow(/untrusted relaycastTarget.baseUrl/); + }); + + it('rejects a canonical target for an explicitly requested Agent37 provider', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-1', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relaycastTarget: { + route: 'canonical', + baseUrl: CANONICAL_RELAYCAST_ORIGIN, + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_canonical', + }, + relayfileMounted: false, + }), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId: 'agent37', + mountRelayfile: false, + }) + ).rejects.toThrow(/mapped Agent37 to a non-isolated/); + }); + it('resolves the unified workspace and provisions a ready mounted sandbox', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -52,6 +170,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: SANDBOX_ID, providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, relayfileMountPath: '/workspace', providerId: 'agent37', @@ -102,6 +221,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: SANDBOX_ID, providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, relayfileMountPath: '/workspace', providerId: 'agent37', @@ -131,7 +251,7 @@ describe('Cloud fleet sandbox client', () => { expect(mocks.authorizedApiFetch).not.toHaveBeenCalled(); }); - it('preserves legacy long-running requests with a custom name and no sandbox identity', async () => { + it('fails closed when a newly provisioned response omits the Relaycast target', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), @@ -160,11 +280,7 @@ describe('Cloud fleet sandbox client', () => { name: 'legacy-custom-node', workloadProfile: 'long-running-agent', }) - ).resolves.toMatchObject({ - outcome: 'provisioned', - sandboxId: 'legacy-public-sandbox', - providerSandboxId: 'legacy-provider-sandbox', - }); + ).rejects.toThrow('missing relaycastTarget'); const ensureBody = JSON.parse(String(mocks.authorizedApiFetch.mock.calls[1]?.[2]?.body)); expect(ensureBody).toMatchObject({ @@ -191,6 +307,7 @@ describe('Cloud fleet sandbox client', () => { nodeName: SANDBOX_NAME, sandboxId: SANDBOX_ID, relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, providerId: 'agent37', } @@ -287,7 +404,7 @@ describe('Cloud fleet sandbox client', () => { cloudWorkspaceId: CLOUD_WORKSPACE_ID, nodeName: SANDBOX_NAME, outcomeUnknown: true, - sandboxId: undefined, + sandboxId: SANDBOX_ID, }); expect(String(error)).toContain(`instead of requested sandboxId ${SANDBOX_ID}`); } @@ -338,7 +455,7 @@ describe('Cloud fleet sandbox client', () => { cloudWorkspaceId: CLOUD_WORKSPACE_ID, nodeName: SANDBOX_NAME, outcomeUnknown: true, - sandboxId: undefined, + sandboxId: SANDBOX_ID, }); expect(String(error)).toContain(`instead of requested nodeName ${SANDBOX_NAME}`); } @@ -373,7 +490,7 @@ describe('Cloud fleet sandbox client', () => { cloudWorkspaceId: CLOUD_WORKSPACE_ID, nodeName: SANDBOX_NAME, outcomeUnknown: true, - sandboxId: undefined, + sandboxId: SANDBOX_ID, }); expect(String(error)).toContain(`instead of requested sandboxId ${SANDBOX_ID}`); }); @@ -453,7 +570,7 @@ describe('Cloud fleet sandbox client', () => { cloudWorkspaceId: CLOUD_WORKSPACE_ID, nodeName: SANDBOX_NAME, outcomeUnknown: true, - sandboxId: undefined, + sandboxId: SANDBOX_ID, providerId: undefined, }); }); @@ -487,7 +604,7 @@ describe('Cloud fleet sandbox client', () => { nodeName: SANDBOX_NAME, providerId: 'e2b', outcomeUnknown: true, - sandboxId: undefined, + sandboxId: SANDBOX_ID, }); expect(String(error)).toContain('provider timed out after allocation'); } @@ -548,7 +665,7 @@ describe('Cloud fleet sandbox client', () => { cloudWorkspaceId: CLOUD_WORKSPACE_ID, nodeName: SANDBOX_NAME, outcomeUnknown: true, - sandboxId: undefined, + sandboxId: SANDBOX_ID, providerId: undefined, }); }); @@ -568,6 +685,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-9', providerSandboxId: 'provider-sandbox-9', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -606,6 +724,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-scoped', providerSandboxId: 'provider-sandbox-scoped', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -658,6 +777,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-e2b', providerSandboxId: 'provider-sandbox-e2b', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -714,6 +834,64 @@ describe('Cloud fleet sandbox client', () => { ); }); + it('requires and validates the isolated target for a reused Agent37 sandbox', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ + outcome: 'reused', + providerId: 'agent37', + nodeId: 'node-agent37', + nodeName: 'agent37-reviewer', + status: 'online', + activeAgents: 0, + maxAgents: 1, + relaycastTarget: RELAYCAST_TARGET, + }), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId: 'agent37', + mountRelayfile: false, + }) + ).resolves.toEqual(expect.objectContaining({ outcome: 'reused', relaycastTarget: RELAYCAST_TARGET })); + }); + + it('rejects a reused Agent37 response that omits its isolated target', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ + outcome: 'reused', + providerId: 'agent37', + nodeId: 'node-agent37', + nodeName: 'agent37-reviewer', + status: 'online', + activeAgents: 0, + maxAgents: 1, + }), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId: 'agent37', + }) + ).rejects.toThrow(/missing the Agent37 Relaycast target/); + }); + it('keeps router-selected responses forward compatible when no exact provider was requested', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -730,6 +908,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-future', providerSandboxId: 'provider-sandbox-future', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -766,6 +945,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-1', providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -935,6 +1115,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-10', providerSandboxId: 'provider-sandbox-10', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -967,6 +1148,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-11', providerSandboxId: 'provider-sandbox-11', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -1000,7 +1182,6 @@ describe('Cloud fleet sandbox client', () => { relayWorkspaceId: 'rw_abc', nodeName: 'daytona-codex', waitedMs: 90_000, - providerId: 'agent37', }, { status: 202 } ), @@ -1044,6 +1225,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-1', providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, providerId: 'daytona', }, @@ -1084,6 +1266,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-1', providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, relayfileMounted: true, providerId: 'daytona', }, @@ -1188,7 +1371,7 @@ describe('Cloud fleet sandbox client', () => { nodeName: SANDBOX_NAME, outcomeUnknown: true, }); - expect(error).toMatchObject({ sandboxId: undefined }); + expect(error).toMatchObject({ sandboxId: SANDBOX_ID }); }); it('rejects a timeout response that omits waitedMs', async () => { diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 625153e624..24a5773d05 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -7,6 +7,14 @@ type JsonRecord = Record; const CLOUD_WORKSPACE_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; const CLOUD_SANDBOX_ID_PATTERN = /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +/** + * Cloud may hand back the gateway route owned by a provisioned sandbox. This + * is deliberately an exact-origin allowlist: a route is control-plane input, + * not a caller-controlled SDK override. + */ +export const CANONICAL_RELAYCAST_ORIGIN = 'https://cast.agentrelay.com'; +export const AGENT37_RELAYCAST_ORIGIN = 'https://agent37-cast.agentrelay.com'; +const TRUSTED_RELAYCAST_ORIGINS = new Set([CANONICAL_RELAYCAST_ORIGIN, AGENT37_RELAYCAST_ORIGIN]); const DEFAULT_RESOLUTION_TIMEOUT_MS = 120_000; // Mounted provisioning can spend up to 240s completing the initial Relayfile // sync, then up to 90s waiting for the enrolled node to report ready. Leave a @@ -118,6 +126,8 @@ export type CloudFleetSandboxReady = { sandboxId: string; providerSandboxId?: string; relayWorkspaceId: string; + /** Closed server-owned Relaycast contract for the provisioned node. */ + relaycastTarget: CloudFleetRelaycastTarget; relayfileMounted: boolean; relayfileMountPath?: string; providerId?: CloudFleetSandboxProviderId; @@ -132,6 +142,8 @@ export type CloudFleetSandboxReused = { activeAgents: number | null; maxAgents: number | null; providerId?: CloudFleetSandboxProviderId; + /** Required when Cloud reused an Agent37-isolated node. */ + relaycastTarget?: CloudFleetRelaycastTarget; }; export type CloudFleetSandboxProvisioningTimeout = { @@ -140,6 +152,7 @@ export type CloudFleetSandboxProvisioningTimeout = { sandboxId: string; providerSandboxId?: string; relayWorkspaceId: string; + relaycastTarget?: CloudFleetRelaycastTarget; nodeName: string; waitedMs: number; providerId?: CloudFleetSandboxProviderId; @@ -156,6 +169,15 @@ export type DeleteCloudFleetSandboxInput = { providerId?: CloudFleetSandboxProviderId; }; +export type CloudFleetRelaycastRoute = 'canonical' | 'agent37-isolated'; + +export type CloudFleetRelaycastTarget = { + route: CloudFleetRelaycastRoute; + baseUrl: string; + workspaceId: string; + relaycastApiKey: string; +}; + function isObject(value: unknown): value is JsonRecord { return value !== null && typeof value === 'object' && !Array.isArray(value); } @@ -165,6 +187,56 @@ function readString(payload: JsonRecord, key: string): string | undefined { return typeof value === 'string' && value.trim() ? value.trim() : undefined; } +function normalizeRelaycastOrigin(value: unknown, field: string): string { + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`Cloud fleet sandbox response has an invalid ${field}.`); + } + let parsed: URL; + try { + parsed = new URL(value.trim()); + } catch { + throw new Error(`Cloud fleet sandbox response has an invalid ${field}.`); + } + if ( + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') || + !TRUSTED_RELAYCAST_ORIGINS.has(parsed.origin) + ) { + throw new Error(`Cloud fleet sandbox response has an untrusted ${field}.`); + } + return parsed.origin; +} + +/** Validate Cloud's closed Relaycast route, identity, and scoped credential contract. */ +export function normalizeRelaycastTarget(value: unknown): CloudFleetRelaycastTarget { + if (!isObject(value)) { + throw new Error('Cloud fleet sandbox response is missing relaycastTarget.'); + } + const route = readString(value, 'route'); + if (route !== 'canonical' && route !== 'agent37-isolated') { + throw new Error('Cloud fleet sandbox response has an unknown Relaycast route.'); + } + const baseUrl = normalizeRelaycastOrigin(value.baseUrl, 'relaycastTarget.baseUrl'); + const expectedOrigin = route === 'canonical' ? CANONICAL_RELAYCAST_ORIGIN : AGENT37_RELAYCAST_ORIGIN; + if (baseUrl !== expectedOrigin) { + throw new Error('Cloud fleet sandbox response mapped Relaycast route to the wrong origin.'); + } + const workspaceId = readString(value, 'workspaceId'); + if (!workspaceId) { + throw new Error('Cloud fleet sandbox response is missing relaycastTarget.workspaceId.'); + } + const relaycastApiKey = readString(value, 'relaycastApiKey'); + if (!relaycastApiKey || !/^rk_live_[A-Za-z0-9_-]+$/.test(relaycastApiKey)) { + throw new Error('Cloud fleet sandbox response has an invalid Relaycast API key.'); + } + return { route, baseUrl, workspaceId, relaycastApiKey }; +} + function readNumber(payload: JsonRecord, key: string): number | undefined { const value = payload[key]; return typeof value === 'number' && Number.isFinite(value) ? value : undefined; @@ -176,6 +248,19 @@ function requiredNumber(payload: JsonRecord, key: string, context: string): numb return value; } +function assertProviderRelaycastTarget( + providerId: CloudFleetSandboxProviderId | undefined, + target: CloudFleetRelaycastTarget | undefined +): void { + if (providerId !== 'agent37') return; + if (!target) { + throw new Error('Cloud fleet sandbox response is missing the Agent37 Relaycast target.'); + } + if (target.route !== 'agent37-isolated' || target.baseUrl !== AGENT37_RELAYCAST_ORIGIN) { + throw new Error('Cloud fleet sandbox response mapped Agent37 to a non-isolated Relaycast target.'); + } +} + function boundedSignal(options: CloudFleetSandboxRequestOptions, defaultTimeoutMs: number): AbortSignal { const timeoutMs = options.timeoutMs ?? defaultTimeoutMs; if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { @@ -351,6 +436,12 @@ function normalizeEnsureResult( } const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); const providerSandboxId = readString(payload, 'providerSandboxId'); + const relayWorkspaceId = requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'); + const relaycastTarget = normalizeRelaycastTarget(payload.relaycastTarget); + if (relaycastTarget.workspaceId !== relayWorkspaceId) { + throw new Error('Cloud fleet sandbox response has mismatched Relaycast workspace identities.'); + } + assertProviderRelaycastTarget(providerId, relaycastTarget); return { outcome, cloudWorkspaceId, @@ -358,7 +449,8 @@ function normalizeEnsureResult( nodeName, sandboxId, ...(providerSandboxId === undefined ? {} : { providerSandboxId }), - relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), + relayWorkspaceId, + relaycastTarget, relayfileMounted: payload.relayfileMounted, ...(providerId === undefined ? {} : { providerId }), ...(readString(payload, 'relayfileMountPath') @@ -368,6 +460,9 @@ function normalizeEnsureResult( } if (outcome === 'reused') { + const relaycastTarget = + payload.relaycastTarget === undefined ? undefined : normalizeRelaycastTarget(payload.relaycastTarget); + assertProviderRelaycastTarget(providerId, relaycastTarget); return { outcome, cloudWorkspaceId, @@ -377,18 +472,26 @@ function normalizeEnsureResult( activeAgents: readNumber(payload, 'activeAgents') ?? null, maxAgents: readNumber(payload, 'maxAgents') ?? null, ...(providerId === undefined ? {} : { providerId }), + ...(relaycastTarget === undefined ? {} : { relaycastTarget }), }; } if (outcome === 'provisioning_timeout') { const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); const providerSandboxId = readString(payload, 'providerSandboxId'); + const relayWorkspaceId = requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'); + const relaycastTarget = + payload.relaycastTarget === undefined ? undefined : normalizeRelaycastTarget(payload.relaycastTarget); + if (relaycastTarget !== undefined && relaycastTarget.workspaceId !== relayWorkspaceId) { + throw new Error('Cloud fleet sandbox response has mismatched Relaycast workspace identities.'); + } return { outcome, cloudWorkspaceId, sandboxId, ...(providerSandboxId === undefined ? {} : { providerSandboxId }), - relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), + relayWorkspaceId, + ...(relaycastTarget === undefined ? {} : { relaycastTarget }), nodeName, waitedMs: requiredNumber(payload, 'waitedMs', 'Cloud fleet sandbox'), ...(providerId === undefined ? {} : { providerId }), @@ -453,6 +556,7 @@ export async function ensureCloudFleetSandbox( ), { cloudWorkspaceId: resolved.cloudWorkspaceId, + ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(input.name ? { nodeName: input.name } : {}), ...(input.providerId ? { providerId: input.providerId } : {}), outcomeUnknown: true, @@ -487,6 +591,7 @@ export async function ensureCloudFleetSandbox( if (response.status >= 500) { throw new CloudFleetSandboxProvisionError(error.message, { cloudWorkspaceId: resolved.cloudWorkspaceId, + ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), ...(input.providerId === undefined ? {} : { providerId: input.providerId }), outcomeUnknown: true, @@ -496,6 +601,7 @@ export async function ensureCloudFleetSandbox( if (isObject(payload) && readString(payload, 'sandboxId')) { throw new CloudFleetSandboxProvisionError(error.message, { cloudWorkspaceId: resolved.cloudWorkspaceId, + ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), ...(input.providerId === undefined ? {} : { providerId: input.providerId }), outcomeUnknown: true, @@ -517,6 +623,7 @@ export async function ensureCloudFleetSandbox( error instanceof Error ? error.message : 'Cloud fleet sandbox response was invalid.', { cloudWorkspaceId: resolved.cloudWorkspaceId, + ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), ...(input.providerId === undefined ? {} : { providerId: input.providerId }), outcomeUnknown: true, diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 2b5972facc..cf6450788c 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -81,6 +81,9 @@ export { ensureCloudFleetSandbox, deleteCloudFleetSandbox, CloudFleetSandboxProvisionError, + normalizeRelaycastTarget, + CANONICAL_RELAYCAST_ORIGIN, + AGENT37_RELAYCAST_ORIGIN, type EnsureCloudFleetSandboxInput, type EnsureCloudFleetSandboxResult, type CloudFleetSandboxReady, @@ -90,6 +93,8 @@ export { type CloudFleetSandboxWorkloadProfile, type DeleteCloudFleetSandboxInput, type CloudFleetSandboxRequestOptions, + type CloudFleetRelaycastRoute, + type CloudFleetRelaycastTarget, } from './fleet-sandbox.js'; export { diff --git a/packages/cloud/src/project-workspace-key.test.ts b/packages/cloud/src/project-workspace-key.test.ts index 29a5c1b4fa..fc6e87221c 100644 --- a/packages/cloud/src/project-workspace-key.test.ts +++ b/packages/cloud/src/project-workspace-key.test.ts @@ -106,6 +106,28 @@ describe('workspace precedence ladder diagnostics', () => { ).toBe('rw_pinned'); }); + it('carries a persisted Relaycast target when an explicit key matches the project pin', () => { + writeProjectWorkspaceKey(dataDir, 'rk_agent37', { + workspaceId: 'rw_pinned', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + + expect( + resolveWorkspaceSelection({ + workspaceKey: 'rk_agent37', + projectDataDir: dataDir, + env: { AGENT_RELAY_HOME: home }, + }) + ).toMatchObject({ + key: 'rk_agent37', + source: 'flag', + workspaceId: 'rw_pinned', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + }); + it('names each source without leaking key material', () => { const env = { AGENT_RELAY_HOME: home, AGENT_RELAY_WORKSPACE_KEY: 'rk_env' }; setWorkspaceKey('global', 'rk_global', env); diff --git a/packages/cloud/src/project-workspace-key.ts b/packages/cloud/src/project-workspace-key.ts index cf39845ecf..ae7fb1005e 100644 --- a/packages/cloud/src/project-workspace-key.ts +++ b/packages/cloud/src/project-workspace-key.ts @@ -21,8 +21,13 @@ export interface ProjectWorkspaceSession { * source (a stored Fleet enrollment, say) points at a different workspace. */ workspaceId?: string; + /** Last server-selected Relaycast route for follow-up commands in this session. */ + relaycastRoute?: 'canonical' | 'agent37-isolated'; + relaycastBaseUrl?: string; } +export type ProjectWorkspaceSessionMetadata = Omit; + export type WorkspaceKeySource = 'flag' | 'env' | 'project' | 'store'; export interface ResolveWorkspaceKeyOptions { @@ -53,6 +58,10 @@ export interface WorkspaceSelection { origin: string; /** Workspace id this selection is known to address, when previously recorded. */ workspaceId?: string; + relaycastRoute?: 'canonical' | 'agent37-isolated'; + relaycastBaseUrl?: string; + /** Project session directory that can durably carry a server-selected target. */ + projectDataDir?: string; } /** Absolute path to the workspace key recorded by `agent-relay node up`. */ @@ -80,10 +89,17 @@ export function readProjectWorkspaceSession( if (!workspaceKey) return undefined; const enrolledNodeId = trimOrUndefined(parsed.enrolledNodeId); const workspaceId = trimOrUndefined(parsed.workspaceId); + const relaycastRoute = + parsed.relaycastRoute === 'canonical' || parsed.relaycastRoute === 'agent37-isolated' + ? parsed.relaycastRoute + : undefined; + const relaycastBaseUrl = trimOrUndefined(parsed.relaycastBaseUrl); return { workspaceKey, ...(enrolledNodeId ? { enrolledNodeId } : {}), ...(workspaceId ? { workspaceId } : {}), + ...(relaycastRoute ? { relaycastRoute } : {}), + ...(relaycastBaseUrl ? { relaycastBaseUrl } : {}), }; } catch { return undefined; @@ -97,12 +113,19 @@ export function readProjectWorkspaceSession( export function writeProjectWorkspaceKey( dataDir: string, workspaceKey: string | undefined, - options: { enrolledNodeId?: string; workspaceId?: string } = {} + options: { + enrolledNodeId?: string; + workspaceId?: string; + relaycastRoute?: 'canonical' | 'agent37-isolated'; + relaycastBaseUrl?: string; + } = {} ): void { const key = trimOrUndefined(workspaceKey); if (!key) return; const enrolledNodeId = trimOrUndefined(options.enrolledNodeId); const workspaceId = trimOrUndefined(options.workspaceId); + const relaycastRoute = options.relaycastRoute; + const relaycastBaseUrl = trimOrUndefined(options.relaycastBaseUrl); fs.mkdirSync(dataDir, { recursive: true, mode: 0o700 }); const file = projectWorkspaceKeyPath(dataDir); // Worker threads share a PID, so include a per-write nonce as well as the PID. @@ -112,6 +135,8 @@ export function writeProjectWorkspaceKey( workspaceKey: key, ...(enrolledNodeId ? { enrolledNodeId } : {}), ...(workspaceId ? { workspaceId } : {}), + ...(relaycastRoute ? { relaycastRoute } : {}), + ...(relaycastBaseUrl ? { relaycastBaseUrl } : {}), } satisfies ProjectWorkspaceSession, null, 2 @@ -141,6 +166,48 @@ export function writeProjectWorkspaceKey( } } +/** + * Rewrite a project session while retaining server-selected metadata when the + * workspace key is unchanged. A changed key is an intentional rebind, so no + * metadata from the old workspace is carried across. + * + * The low-level writer above deliberately replaces the complete record. This + * helper is for callers that update one part of an existing session (for + * example, linking an enrolled node) and must not accidentally discard the + * Relaycast target or workspace identity recorded alongside the key. + */ +export function writeProjectWorkspaceKeyPreservingSession( + dataDir: string, + workspaceKey: string | undefined, + options: ProjectWorkspaceSessionMetadata = {} +): void { + const key = trimOrUndefined(workspaceKey); + if (!key) return; + + const existing = readProjectWorkspaceSession(dataDir); + const sameWorkspace = existing?.workspaceKey === key; + const retained: ProjectWorkspaceSessionMetadata = sameWorkspace + ? { + ...(existing?.enrolledNodeId ? { enrolledNodeId: existing.enrolledNodeId } : {}), + ...(existing?.workspaceId ? { workspaceId: existing.workspaceId } : {}), + ...(existing?.relaycastRoute ? { relaycastRoute: existing.relaycastRoute } : {}), + ...(existing?.relaycastBaseUrl ? { relaycastBaseUrl: existing.relaycastBaseUrl } : {}), + } + : {}; + + writeProjectWorkspaceKey(dataDir, key, { + ...retained, + ...(trimOrUndefined(options.enrolledNodeId) + ? { enrolledNodeId: trimOrUndefined(options.enrolledNodeId) } + : {}), + ...(trimOrUndefined(options.workspaceId) ? { workspaceId: trimOrUndefined(options.workspaceId) } : {}), + ...(options.relaycastRoute ? { relaycastRoute: options.relaycastRoute } : {}), + ...(trimOrUndefined(options.relaycastBaseUrl) + ? { relaycastBaseUrl: trimOrUndefined(options.relaycastBaseUrl) } + : {}), + }); +} + /** * Resolve which Relay workspace this process addresses. * @@ -163,22 +230,55 @@ export function resolveWorkspaceSelection( options: ResolveWorkspaceKeyOptions = {} ): WorkspaceSelection | undefined { const env = options.env ?? process.env; + const dataDir = options.projectDataDir ?? projectDataDir(options.projectRoot); + const project = dataDir ? readProjectWorkspaceSession(dataDir, options.fileSystem ?? fs) : undefined; const flag = trimOrUndefined(options.workspaceKey); - if (flag) return { key: flag, source: 'flag', origin: '--workspace-key' }; + if (flag) { + return { + key: flag, + source: 'flag', + origin: '--workspace-key', + ...(project?.workspaceKey === flag && project.workspaceId ? { workspaceId: project.workspaceId } : {}), + ...(project?.workspaceKey === flag && project.relaycastRoute + ? { relaycastRoute: project.relaycastRoute } + : {}), + ...(project?.workspaceKey === flag && project.relaycastBaseUrl + ? { relaycastBaseUrl: project.relaycastBaseUrl } + : {}), + ...(project?.workspaceKey === flag && dataDir ? { projectDataDir: dataDir } : {}), + }; + } for (const name of WORKSPACE_KEY_ENV_VARS) { const envKey = trimOrUndefined(env[name]); - if (envKey) return { key: envKey, source: 'env', origin: `$${name}` }; + if (envKey) { + return { + key: envKey, + source: 'env', + origin: `$${name}`, + ...(project?.workspaceKey === envKey && project.workspaceId + ? { workspaceId: project.workspaceId } + : {}), + ...(project?.workspaceKey === envKey && project.relaycastRoute + ? { relaycastRoute: project.relaycastRoute } + : {}), + ...(project?.workspaceKey === envKey && project.relaycastBaseUrl + ? { relaycastBaseUrl: project.relaycastBaseUrl } + : {}), + ...(project?.workspaceKey === envKey && dataDir ? { projectDataDir: dataDir } : {}), + }; + } } - const dataDir = options.projectDataDir ?? projectDataDir(options.projectRoot); - const project = dataDir ? readProjectWorkspaceSession(dataDir, options.fileSystem ?? fs) : undefined; if (project) { return { key: project.workspaceKey, source: 'project', origin: projectWorkspaceKeyPath(dataDir as string), ...(project.workspaceId ? { workspaceId: project.workspaceId } : {}), + ...(project.relaycastRoute ? { relaycastRoute: project.relaycastRoute } : {}), + ...(project.relaycastBaseUrl ? { relaycastBaseUrl: project.relaycastBaseUrl } : {}), + ...(dataDir ? { projectDataDir: dataDir } : {}), }; } diff --git a/packages/cloud/src/workspace-key.ts b/packages/cloud/src/workspace-key.ts index f102295835..54235b5c02 100644 --- a/packages/cloud/src/workspace-key.ts +++ b/packages/cloud/src/workspace-key.ts @@ -7,7 +7,9 @@ export { resolveWorkspaceKeyWithSource, resolveWorkspaceSelection, writeProjectWorkspaceKey, + writeProjectWorkspaceKeyPreservingSession, type ProjectWorkspaceSession, + type ProjectWorkspaceSessionMetadata, type ResolveWorkspaceKeyOptions, type WorkspaceKeyFileSystem, type WorkspaceKeySource, diff --git a/scripts/ci-standalone-smoke.sh b/scripts/ci-standalone-smoke.sh index 56a797c777..b04c33a35a 100755 --- a/scripts/ci-standalone-smoke.sh +++ b/scripts/ci-standalone-smoke.sh @@ -6,16 +6,16 @@ if [ "$#" -ne 2 ]; then exit 2 fi -if [[ -z "${RELAY_WORKSPACE_KEY:-}" || "${RELAY_WORKSPACE_KEY:-}" =~ ^[[:space:]]+$ ]]; then - echo "ERROR: RELAY_WORKSPACE_KEY must name the dedicated standalone-smoke CI workspace." >&2 - echo "Refusing to start without it because node up would create an undeletable throwaway workspace." >&2 - exit 2 -fi +# The standalone smoke is deliberately pinned to the trusted Agent37 shard. +# Do not make this caller-selectable: the workspace key created below is scoped +# to this origin and must never be sent to an arbitrary endpoint. +TRUSTED_RELAY_BASE_URL="https://agent37-cast.agentrelay.com" # The broker gives a multi-workspace session higher precedence than the single -# key. This smoke intentionally exercises one dedicated workspace, so do not +# key. This smoke intentionally exercises one ephemeral workspace, so do not # let an ambient developer/runner session silently replace the CI credential. -unset RELAY_WORKSPACES_JSON +unset RELAY_WORKSPACES_JSON RELAY_WORKSPACE_KEY AGENT_RELAY_WORKSPACE_KEY RELAY_API_KEY \ + RELAYCAST_BASE_URL RELAY_AGENT_TOKEN RELAY_WORKSPACE # Startup can legitimately consume the broker's 40-second aggregate Relaycast # handshake budget on a loaded macOS runner. Keep the outer supervisor at @@ -74,28 +74,59 @@ validate_binary "BROKER" "$BROKER_BIN" TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/agent-relay-standalone-smoke.XXXXXX")" HOME_DIR="$TMP_ROOT/home" PROJECT_DIR="$TMP_ROOT/project" +WORKSPACE_RESPONSE="$TMP_ROOT/workspace-response.json" mkdir -p "$HOME_DIR" "$PROJECT_DIR" CLEANUP_STARTED=false +WORKSPACE_KEY="" +WORKSPACE_ID="" cleanup() { if [ "$CLEANUP_STARTED" = true ]; then - return + return 0 fi CLEANUP_STARTED=true + local cleanup_status=0 # Disarm before entering the cleanup subshell. Some Bash exit paths can # otherwise inherit this EXIT trap and recursively run node down again. trap - EXIT ( cd "$PROJECT_DIR" HOME="$HOME_DIR" \ + RELAY_BASE_URL="$TRUSTED_RELAY_BASE_URL" \ AGENT_RELAY_BIN="$BROKER_BIN" \ AGENT_RELAY_SKIP_UPDATE_CHECK=1 \ AGENT_RELAY_STARTUP_DEBUG=1 \ AGENT_RELAY_TELEMETRY_DISABLED=1 \ + RELAY_WORKSPACE_KEY="$WORKSPACE_KEY" \ "$CLI_BIN" node down --force --timeout 5000 >/dev/null 2>&1 || true ) + if [ -n "$WORKSPACE_KEY" ]; then + local delete_status verify_status + delete_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --request DELETE \ + --header "Authorization: Bearer $WORKSPACE_KEY" \ + "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" + if [ "$delete_status" != "200" ] && [ "$delete_status" != "204" ]; then + echo "ERROR: ephemeral workspace cleanup returned HTTP ${delete_status:-unknown}." >&2 + echo "The workspace id is ${WORKSPACE_ID:-unknown}; remove it manually from the trusted smoke shard." >&2 + cleanup_status=1 + else + verify_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --request GET \ + --header "Authorization: Bearer $WORKSPACE_KEY" \ + "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" + if [ "$verify_status" != "401" ]; then + echo "ERROR: ephemeral workspace deletion was not proved (follow-up HTTP ${verify_status:-unknown})." >&2 + echo "The workspace id is ${WORKSPACE_ID:-unknown}; remove it manually from the trusted smoke shard." >&2 + cleanup_status=1 + else + echo "Ephemeral workspace deletion verified" + fi + fi + fi rm -rf "$TMP_ROOT" + return "$cleanup_status" } trap cleanup EXIT @@ -104,6 +135,8 @@ run_cli() { ( cd "$PROJECT_DIR" HOME="$HOME_DIR" \ + RELAY_BASE_URL="$TRUSTED_RELAY_BASE_URL" \ + RELAY_WORKSPACE_KEY="$WORKSPACE_KEY" \ AGENT_RELAY_BIN="$BROKER_BIN" \ AGENT_RELAY_SKIP_UPDATE_CHECK=1 \ AGENT_RELAY_STARTUP_DEBUG=1 \ @@ -112,6 +145,46 @@ run_cli() { ) } +if ! command -v curl >/dev/null 2>&1; then + echo "ERROR: curl is required to create and delete the ephemeral smoke workspace." >&2 + exit 2 +fi +if ! command -v jq >/dev/null 2>&1; then + echo "ERROR: jq is required to parse the ephemeral smoke workspace response." >&2 + exit 2 +fi + +WORKSPACE_NAME="relay-standalone-smoke-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-$$" +CREATE_STATUS="$(curl --silent --show-error --output "$WORKSPACE_RESPONSE" --write-out '%{http_code}' \ + --request POST \ + --header 'Content-Type: application/json' \ + --data "$(jq -cn --arg name "$WORKSPACE_NAME" '{name: $name, expires_in_seconds: 60}')" \ + "$TRUSTED_RELAY_BASE_URL/v1/workspaces" 2>/dev/null || true)" +# Mask the key before extracting or using any other response field. Never log +# the response body: it contains the administrative workspace credential. +WORKSPACE_KEY="$(jq -er '.data.api_key // .api_key // empty' "$WORKSPACE_RESPONSE" 2>/dev/null || true)" +if [ -n "$WORKSPACE_KEY" ]; then + printf '::add-mask::%s\n' "$WORKSPACE_KEY" +fi +if [ -z "$WORKSPACE_KEY" ]; then + echo "ERROR: ephemeral smoke workspace response did not contain an API key." >&2 + exit 1 +fi +if [[ ! "$WORKSPACE_KEY" =~ ^rk_live_[A-Za-z0-9_-]+$ ]]; then + echo "ERROR: ephemeral smoke workspace response contained an invalid API key scheme." >&2 + exit 1 +fi +WORKSPACE_ID="$(jq -er '.data.workspace_id // .workspace_id // empty' "$WORKSPACE_RESPONSE" 2>/dev/null || true)" +if [ -z "$WORKSPACE_ID" ]; then + echo "ERROR: ephemeral smoke workspace response did not contain a workspace id." >&2 + exit 1 +fi +if [ "$CREATE_STATUS" != "200" ] && [ "$CREATE_STATUS" != "201" ]; then + echo "ERROR: ephemeral smoke workspace creation returned HTTP ${CREATE_STATUS:-unknown}." >&2 + exit 1 +fi +echo "Ephemeral workspace created on trusted smoke shard (id ${WORKSPACE_ID})" + print_output_excerpt() { local output="$1" local total_lines @@ -240,4 +313,8 @@ if printf '%s\n' "$UP_OUTPUT" | grep -q 'Broker already running for this project exit 1 fi +if ! cleanup; then + echo "Standalone smoke lifecycle passed but ephemeral workspace cleanup was not proved" >&2 + exit 1 +fi echo "Standalone smoke passed" From cbe0ff64ecd7b8d7fd9819491d045241e75f8b25 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 9 Sep 2026 10:55:26 +0000 Subject: [PATCH 30/50] style: auto-format with Prettier --- packages/cli/src/cli/commands/local-agent.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/cli/commands/local-agent.ts b/packages/cli/src/cli/commands/local-agent.ts index eb39885aae..3f00cf945d 100644 --- a/packages/cli/src/cli/commands/local-agent.ts +++ b/packages/cli/src/cli/commands/local-agent.ts @@ -847,7 +847,10 @@ export function registerLocalAgentCommands( .argument('', 'Agent name') .option('--mode ', 'drive | view | passthrough', 'view') .option('--node ', 'Canonical authenticated fleet-node terminal attach (physical or Daytona)') - .option('--base-url ', 'Relaycast API base URL for --node (defaults to the selected workspace route)') + .option( + '--base-url ', + 'Relaycast API base URL for --node (defaults to the selected workspace route)' + ) .option('--ssh-host ', 'SSH host fallback for a physical fleet node') .option('--broker-url ', 'Broker base URL (overrides RELAY_BROKER_URL and connection.json)') .option('--api-key ', 'Broker API key (overrides RELAY_BROKER_API_KEY and connection.json)') From 86f7466789aa720343e5d58b91f901d6c29a7257 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Wed, 9 Sep 2026 13:10:30 +0200 Subject: [PATCH 31/50] ci: verify Agent37 routing after formatting From 97263a80dbb04f2cbb33f311f21b8ab64e41faa3 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Wed, 9 Sep 2026 13:59:11 +0200 Subject: [PATCH 32/50] fix(ci): use resolvable Relaycast smoke origin --- packages/cli/src/cli/ci-standalone-smoke.test.ts | 6 +++--- scripts/ci-standalone-smoke.sh | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/cli/ci-standalone-smoke.test.ts b/packages/cli/src/cli/ci-standalone-smoke.test.ts index 54b4a19e44..0decc33fa0 100644 --- a/packages/cli/src/cli/ci-standalone-smoke.test.ts +++ b/packages/cli/src/cli/ci-standalone-smoke.test.ts @@ -95,7 +95,7 @@ case "\${2:-}" in echo "Cleaned up (was not running)" ;; up) - if [ "\${RELAY_BASE_URL:-}" != "https://agent37-cast.agentrelay.com" ]; then + if [ "\${RELAY_BASE_URL:-}" != "https://cast.agentrelay.com" ]; then echo "trusted Relaycast base URL missing" >&2 exit 65 fi @@ -150,9 +150,9 @@ describe('ci-standalone-smoke workspace reuse', () => { expect(cleanupSubshellIndex).toBeGreaterThan(trapDisarmIndex); }); - it('creates an ephemeral workspace on the trusted shard and wires its base URL explicitly', () => { + it('creates an ephemeral workspace on the trusted engine and wires its base URL explicitly', () => { const script = readFileSync(smokeScript, 'utf8'); - expect(script).toContain('TRUSTED_RELAY_BASE_URL="https://agent37-cast.agentrelay.com"'); + expect(script).toContain('TRUSTED_RELAY_BASE_URL="https://cast.agentrelay.com"'); expect(script).toContain('expires_in_seconds: 60'); expect(script).toContain('printf \'::add-mask::%s\\n\' "$WORKSPACE_KEY"'); expect(script).toContain('--request DELETE'); diff --git a/scripts/ci-standalone-smoke.sh b/scripts/ci-standalone-smoke.sh index b04c33a35a..8e3000fce2 100755 --- a/scripts/ci-standalone-smoke.sh +++ b/scripts/ci-standalone-smoke.sh @@ -6,10 +6,10 @@ if [ "$#" -ne 2 ]; then exit 2 fi -# The standalone smoke is deliberately pinned to the trusted Agent37 shard. -# Do not make this caller-selectable: the workspace key created below is scoped -# to this origin and must never be sent to an arbitrary endpoint. -TRUSTED_RELAY_BASE_URL="https://agent37-cast.agentrelay.com" +# The standalone smoke is deliberately pinned to the trusted hosted Relaycast +# engine. Do not make this caller-selectable: the workspace key created below +# is scoped to this origin and must never be sent to an arbitrary endpoint. +TRUSTED_RELAY_BASE_URL="https://cast.agentrelay.com" # The broker gives a multi-workspace session higher precedence than the single # key. This smoke intentionally exercises one ephemeral workspace, so do not From f93f0504599dafef92782cd4f2c1ed37f3b520f4 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Wed, 9 Sep 2026 14:20:49 +0200 Subject: [PATCH 33/50] fix(security): enforce trusted fleet Relaycast routes --- .../cli/src/cli/lib/attach-fleet-node.test.ts | 77 +++++++----- packages/cli/src/cli/lib/attach-fleet-node.ts | 34 +++++- packages/cloud/src/fleet-sandbox.test.ts | 114 +++++++++++++++++- packages/cloud/src/fleet-sandbox.ts | 22 +++- 4 files changed, 207 insertions(+), 40 deletions(-) diff --git a/packages/cli/src/cli/lib/attach-fleet-node.test.ts b/packages/cli/src/cli/lib/attach-fleet-node.test.ts index a7edc522d5..97ee09fc34 100644 --- a/packages/cli/src/cli/lib/attach-fleet-node.test.ts +++ b/packages/cli/src/cli/lib/attach-fleet-node.test.ts @@ -161,7 +161,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-transient', node: 'node-transient', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, sessionRequest: { @@ -217,7 +217,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-slow-healthy', node: 'node-slow-healthy', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, }); @@ -262,7 +262,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-slow-dead', node: 'node-slow-dead', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, }).catch((error: unknown) => error); @@ -306,7 +306,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-budget-expired', node: 'node-budget-expired', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, sessionRequest: { @@ -351,7 +351,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-dead', node: 'node-dead', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, sessionRequest: { @@ -370,7 +370,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { expect((rejected as Error).message).toContain('resolved node id unavailable'); expect((rejected as Error).message).toContain('attempts 5 (retried 4 times)'); expect((rejected as Error).message).toContain( - 'endpoint "https://fake.example/v1/nodes/node-dead/terminal/sessions"' + 'endpoint "https://cast.agentrelay.com/v1/nodes/node-dead/terminal/sessions"' ); }); @@ -385,7 +385,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-missing-node', node: 'node-missing', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, sessionRequest: { sleep: async () => undefined }, @@ -409,7 +409,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-unknown-completion', node: 'node-unknown-completion', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, sessionRequest: { sleep: async () => undefined }, @@ -436,7 +436,7 @@ describe('startFleetNodeAttachProxy terminal-session request retries', () => { agent: 'agent-malformed', node: 'node-malformed', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fetchFn, sessionRequest: { sleep: async () => undefined }, @@ -467,7 +467,7 @@ describe('startFleetNodeAttachProxy delivery-mode PUT lifecycle', () => { agent: 'agent-a', node: 'node-a', mode: 'drive', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -508,7 +508,7 @@ describe('startFleetNodeAttachProxy delivery-mode PUT lifecycle', () => { agent: 'agent-readiness', node: 'node-readiness', mode: 'drive', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -576,7 +576,7 @@ describe('startFleetNodeAttachProxy delivery-mode PUT lifecycle', () => { agent: 'agent-b', node: 'node-b', mode: 'drive', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -617,7 +617,7 @@ describe('startFleetNodeAttachProxy delivery-mode PUT lifecycle', () => { agent: 'agent-c', node: 'node-c', mode: 'drive', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -653,7 +653,7 @@ describe('startFleetNodeAttachProxy delivery-mode PUT lifecycle', () => { agent: 'agent-d', node: 'node-d', mode: 'drive', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -704,7 +704,7 @@ describe('startFleetNodeAttachProxy view target lifecycle', () => { agent: 'view-delayed-upgrade', node: 'node-delayed-upgrade', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), reconnectDelay: { handshakeTimeoutMs: 10_000, readyTimeoutMs: 4_000 }, @@ -737,7 +737,7 @@ describe('startFleetNodeAttachProxy view target lifecycle', () => { agent: 'view-backoff-snapshot', node: 'node-backoff-snapshot', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), reconnectDelay: { @@ -772,7 +772,7 @@ describe('startFleetNodeAttachProxy view target lifecycle', () => { agent: 'view-target', node: 'view-node', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -826,7 +826,7 @@ describe('startFleetNodeAttachProxy view target lifecycle', () => { agent: 'view-reconnect', node: 'node-reconnect', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), reconnectDelay: { initialMs: 1, maxMs: 1 }, @@ -877,7 +877,7 @@ describe('startFleetNodeAttachProxy view target lifecycle', () => { // path, including a multi-byte UTF-8 boundary. node: `node-exhaust-${'é'.repeat(100)}`, mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), reconnectDelay: { initialMs: 1, maxMs: 1 }, @@ -917,7 +917,7 @@ describe('startFleetNodeAttachProxy view target lifecycle', () => { agent: 'view-stalled-resume', node: 'node-stalled-resume', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), reconnectDelay: { @@ -1002,7 +1002,7 @@ describe('startFleetNodeAttachProxy readiness-gate status mapping', () => { agent, node: 'node-mapping', mode: 'drive', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remoteUrl), }); @@ -1153,7 +1153,7 @@ describe('startFleetNodeAttachProxy workspace-key precedence', () => { agent: 'agent-e', node: 'node-e', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'rk_live_explicit', env: { RELAY_WORKSPACE_KEY: 'rk_live_ambient' }, fetch: ticket.fetch, @@ -1186,6 +1186,29 @@ describe('startFleetNodeAttachProxy workspace-key precedence', () => { ); }); + it.each([ + 'https://evil.example', + 'http://cast.agentrelay.com', + 'https://cast.agentrelay.com.attacker.example', + 'https://cast.agentrelay.com/path', + ])('rejects an untrusted explicit base URL before sending the workspace key', async (baseUrl) => { + const fetch = vi.fn(); + + await expect( + startFleetNodeAttachProxy({ + agent: 'agent-attacker', + node: 'node-attacker', + mode: 'view', + baseUrl, + workspaceKey: 'rk_live_must_not_leave_process', + env: {}, + fetch, + }) + ).rejects.toThrow(/trusted Relaycast origin/); + + expect(fetch).not.toHaveBeenCalled(); + }); + it('falls back to the environment when no explicit key is supplied', async () => { const remote = await startFakeRemote(); cleanup.push(remote.close); @@ -1194,7 +1217,7 @@ describe('startFleetNodeAttachProxy workspace-key precedence', () => { agent: 'agent-f', node: 'node-f', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', env: { RELAY_WORKSPACE_KEY: 'rk_live_ambient' }, fetch: ticket.fetch, }); @@ -1238,7 +1261,7 @@ describe('startFleetNodeAttachProxy flush route', () => { // changing delivery mode, so it must not have to seize the single drive // slot from whoever is attached. mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -1289,7 +1312,7 @@ describe('startFleetNodeAttachProxy flush route', () => { agent: 'agent-missing', node: 'node-remote', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -1330,7 +1353,7 @@ describe('startFleetNodeAttachProxy flush route', () => { agent: 'agent-noid', node: 'node-remote', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); @@ -1387,7 +1410,7 @@ describe('startFleetNodeAttachProxy flush route', () => { agent: 'agent-cross', node: 'node-remote', mode: 'view', - baseUrl: 'https://fake.example', + baseUrl: 'https://cast.agentrelay.com', workspaceKey: 'wk', fetch: fakeTicketFetch(remote.url), }); diff --git a/packages/cli/src/cli/lib/attach-fleet-node.ts b/packages/cli/src/cli/lib/attach-fleet-node.ts index a6375fd40a..420e267f9c 100644 --- a/packages/cli/src/cli/lib/attach-fleet-node.ts +++ b/packages/cli/src/cli/lib/attach-fleet-node.ts @@ -15,6 +15,7 @@ import { randomBytes } from 'node:crypto'; import WebSocket, { WebSocketServer } from 'ws'; +import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; import type { AttachMode } from './attach-mode.js'; import { collectWithRetry } from './collect-with-retry.js'; import { resolveBaseUrl, resolveWorkspaceKey } from './sdk-client.js'; @@ -114,6 +115,33 @@ export class FleetNodeAttachError extends Error { } } +const TRUSTED_RELAYCAST_ORIGINS = new Set([ + CANONICAL_RELAYCAST_ORIGIN, + AGENT37_RELAYCAST_ORIGIN, +]); + +function validateFleetAttachBaseUrl(value: string): string { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new FleetNodeAttachError('Fleet node attach requires a trusted Relaycast origin.'); + } + if ( + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') || + !TRUSTED_RELAYCAST_ORIGINS.has(parsed.origin) + ) { + throw new FleetNodeAttachError('Fleet node attach requires a trusted Relaycast origin.'); + } + return parsed.origin; +} + class TerminalSessionAttemptError extends FleetNodeAttachError { constructor( message: string, @@ -291,10 +319,8 @@ export async function startFleetNodeAttachProxy( const env = options.env ?? process.env; const fetchFn = options.fetch ?? globalThis.fetch; const workspaceKey = options.workspaceKey ?? resolveWorkspaceKey({ env }); - const baseUrl = (options.baseUrl ?? resolveBaseUrl({ env }) ?? 'https://cast.agentrelay.com').replace( - /\/+$/, - '' - ); + const requestedBaseUrl = resolveBaseUrl({ baseUrl: options.baseUrl, env }); + const baseUrl = validateFleetAttachBaseUrl(requestedBaseUrl ?? CANONICAL_RELAYCAST_ORIGIN); const nodePath = safeNodePath(options.node); const sessionEndpoint = `${baseUrl}/v1/nodes/${nodePath}/terminal/sessions`; const sessionRequestTimeoutMs = options.sessionRequest?.timeoutMs ?? SESSION_REQUEST_TIMEOUT_MS; diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 4589e35508..6ae7796e70 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -35,6 +35,16 @@ const RELAYCAST_TARGET = { workspaceId: 'rw_abc', relaycastApiKey: 'rk_live_agent37', }; +const CANONICAL_RELAYCAST_TARGET = { + route: 'canonical' as const, + baseUrl: CANONICAL_RELAYCAST_ORIGIN, + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_canonical', +}; +const NON_AGENT_PROVIDER_IDS = ['daytona', 'e2b', 'vercel', 'freestyle', 'microsandbox'] as const; +const PROVIDER_OUTCOME_MATRIX = NON_AGENT_PROVIDER_IDS.flatMap((providerId) => + (['provisioned', 'reused'] as const).map((outcome) => ({ providerId, outcome })) +); describe('Cloud fleet sandbox client', () => { beforeEach(() => { @@ -777,7 +787,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-e2b', providerSandboxId: 'provider-sandbox-e2b', relayWorkspaceId: 'rw_abc', - relaycastTarget: RELAYCAST_TARGET, + relaycastTarget: CANONICAL_RELAYCAST_TARGET, relayfileMounted: true, }, { status: 201 } @@ -816,6 +826,7 @@ describe('Cloud fleet sandbox client', () => { status: 'online', activeAgents: 0, maxAgents: 1, + relaycastTarget: CANONICAL_RELAYCAST_TARGET, }), auth, }); @@ -834,6 +845,103 @@ describe('Cloud fleet sandbox client', () => { ); }); + it.each(PROVIDER_OUTCOME_MATRIX)( + 'accepts the canonical Relaycast target for the $providerId $outcome result', + async ({ providerId, outcome }) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + outcome === 'provisioned' + ? { + outcome, + providerId, + nodeId: `node-${providerId}`, + nodeName: `${providerId}-reviewer`, + sandboxId: `sandbox-${providerId}`, + providerSandboxId: `provider-sandbox-${providerId}`, + relayWorkspaceId: 'rw_abc', + relaycastTarget: CANONICAL_RELAYCAST_TARGET, + relayfileMounted: true, + } + : { + outcome, + providerId, + nodeId: `node-${providerId}`, + nodeName: `${providerId}-reviewer`, + status: 'online', + activeAgents: 0, + maxAgents: 1, + relaycastTarget: CANONICAL_RELAYCAST_TARGET, + }, + { status: outcome === 'provisioned' ? 201 : 200 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId, + }) + ).resolves.toEqual(expect.objectContaining({ + outcome, + providerId, + relaycastTarget: CANONICAL_RELAYCAST_TARGET, + })); + } + ); + + it.each(PROVIDER_OUTCOME_MATRIX)( + 'rejects an Agent37 Relaycast target for the $providerId $outcome result', + async ({ providerId, outcome }) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + outcome === 'provisioned' + ? { + outcome, + providerId, + nodeId: `node-${providerId}`, + nodeName: `${providerId}-reviewer`, + sandboxId: `sandbox-${providerId}`, + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + } + : { + outcome, + providerId, + nodeId: `node-${providerId}`, + nodeName: `${providerId}-reviewer`, + status: 'online', + activeAgents: 0, + maxAgents: 1, + relaycastTarget: RELAYCAST_TARGET, + }, + { status: outcome === 'provisioned' ? 201 : 200 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId, + }) + ).rejects.toThrow(/non-canonical Relaycast target/); + } + ); + it('requires and validates the isolated target for a reused Agent37 sandbox', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -1225,7 +1333,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-1', providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', - relaycastTarget: RELAYCAST_TARGET, + relaycastTarget: CANONICAL_RELAYCAST_TARGET, relayfileMounted: true, providerId: 'daytona', }, @@ -1266,7 +1374,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-1', providerSandboxId: 'provider-sandbox-1', relayWorkspaceId: 'rw_abc', - relaycastTarget: RELAYCAST_TARGET, + relaycastTarget: CANONICAL_RELAYCAST_TARGET, relayfileMounted: true, providerId: 'daytona', }, diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 24a5773d05..9667dd5fa5 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -142,7 +142,7 @@ export type CloudFleetSandboxReused = { activeAgents: number | null; maxAgents: number | null; providerId?: CloudFleetSandboxProviderId; - /** Required when Cloud reused an Agent37-isolated node. */ + /** Required when Cloud reused a node with a known provider. */ relaycastTarget?: CloudFleetRelaycastTarget; }; @@ -252,12 +252,22 @@ function assertProviderRelaycastTarget( providerId: CloudFleetSandboxProviderId | undefined, target: CloudFleetRelaycastTarget | undefined ): void { - if (providerId !== 'agent37') return; - if (!target) { - throw new Error('Cloud fleet sandbox response is missing the Agent37 Relaycast target.'); + if (providerId === 'agent37') { + if (!target) { + throw new Error('Cloud fleet sandbox response is missing the Agent37 Relaycast target.'); + } + if (target.route !== 'agent37-isolated' || target.baseUrl !== AGENT37_RELAYCAST_ORIGIN) { + throw new Error('Cloud fleet sandbox response mapped Agent37 to a non-isolated Relaycast target.'); + } + return; } - if (target.route !== 'agent37-isolated' || target.baseUrl !== AGENT37_RELAYCAST_ORIGIN) { - throw new Error('Cloud fleet sandbox response mapped Agent37 to a non-isolated Relaycast target.'); + if (providerId !== undefined) { + if (!target) { + throw new Error(`Cloud fleet sandbox response is missing the canonical Relaycast target for ${providerId}.`); + } + if (target.route !== 'canonical' || target.baseUrl !== CANONICAL_RELAYCAST_ORIGIN) { + throw new Error(`Cloud fleet sandbox response mapped ${providerId} to a non-canonical Relaycast target.`); + } } } From 4563d759f4e878a9640cac2b4782def4e16a094d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 9 Sep 2026 12:23:22 +0000 Subject: [PATCH 34/50] style: auto-format with Prettier --- packages/cli/src/cli/lib/attach-fleet-node.ts | 5 +---- packages/cloud/src/fleet-sandbox.test.ts | 12 +++++++----- packages/cloud/src/fleet-sandbox.ts | 8 ++++++-- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/cli/lib/attach-fleet-node.ts b/packages/cli/src/cli/lib/attach-fleet-node.ts index 420e267f9c..db5630fc78 100644 --- a/packages/cli/src/cli/lib/attach-fleet-node.ts +++ b/packages/cli/src/cli/lib/attach-fleet-node.ts @@ -115,10 +115,7 @@ export class FleetNodeAttachError extends Error { } } -const TRUSTED_RELAYCAST_ORIGINS = new Set([ - CANONICAL_RELAYCAST_ORIGIN, - AGENT37_RELAYCAST_ORIGIN, -]); +const TRUSTED_RELAYCAST_ORIGINS = new Set([CANONICAL_RELAYCAST_ORIGIN, AGENT37_RELAYCAST_ORIGIN]); function validateFleetAttachBaseUrl(value: string): string { let parsed: URL; diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 6ae7796e70..fbf4dfb79b 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -888,11 +888,13 @@ describe('Cloud fleet sandbox client', () => { requiredCapability: 'spawn:codex', providerId, }) - ).resolves.toEqual(expect.objectContaining({ - outcome, - providerId, - relaycastTarget: CANONICAL_RELAYCAST_TARGET, - })); + ).resolves.toEqual( + expect.objectContaining({ + outcome, + providerId, + relaycastTarget: CANONICAL_RELAYCAST_TARGET, + }) + ); } ); diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 9667dd5fa5..bd2cf59fd3 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -263,10 +263,14 @@ function assertProviderRelaycastTarget( } if (providerId !== undefined) { if (!target) { - throw new Error(`Cloud fleet sandbox response is missing the canonical Relaycast target for ${providerId}.`); + throw new Error( + `Cloud fleet sandbox response is missing the canonical Relaycast target for ${providerId}.` + ); } if (target.route !== 'canonical' || target.baseUrl !== CANONICAL_RELAYCAST_ORIGIN) { - throw new Error(`Cloud fleet sandbox response mapped ${providerId} to a non-canonical Relaycast target.`); + throw new Error( + `Cloud fleet sandbox response mapped ${providerId} to a non-canonical Relaycast target.` + ); } } } From e80584eb5623793b77aa61329855083c9412bb47 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Wed, 9 Sep 2026 22:08:32 +0200 Subject: [PATCH 35/50] test(relayflow): model isolated Relaycast target --- .../1656-long-running-agent37-sandbox/run.mjs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs index e0551c3dc7..352c42c7df 100644 --- a/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs +++ b/tests/relayflows/cases/1656-long-running-agent37-sandbox/run.mjs @@ -131,6 +131,8 @@ import { Command } from 'commander'; const mocks = vi.hoisted(() => ({ ensureCloudSession: vi.fn(), authorizedApiFetch: vi.fn(), + persistWorkspaceRelaycastTarget: vi.fn(() => true), + resolveWorkspaceSelection: vi.fn(() => ({ workspaceId: 'rw_relayflow' })), })); // The Cloud network boundary -- the only thing this probe stubs. Everything @@ -200,6 +202,12 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned sandboxId: REPLAY_SANDBOX_ID, providerSandboxId: PROVIDER_SANDBOX_ID, relayWorkspaceId: 'rw_relayflow', + relaycastTarget: { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_relayflow', + relaycastApiKey: 'rk_live_relayflow_probe', + }, relayfileMounted: true, relayfileMountPath: '/workspace', providerId: 'agent37', @@ -245,7 +253,11 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned }, })) as never, createWorkspaceRelay: vi.fn(() => ({ - workspace: { info: vi.fn(async () => ({ id: 'rw_relayflow' })) }, + workspace: { + info: vi.fn(async () => ({ id: 'rw_relayflow' })), + register: vi.fn(async () => ({ token: 'at_relayflow_launcher' })), + release: vi.fn(async () => undefined), + }, })) as never, createWorkspace: vi.fn() as never, // Capture the real CLI serialization, then fail so this same invocation @@ -270,6 +282,8 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned }, ensureCloudFleetSandbox, deleteCloudFleetSandbox, + resolveWorkspaceSelection: mocks.resolveWorkspaceSelection, + persistWorkspaceRelaycastTarget: mocks.persistWorkspaceRelaycastTarget, createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, warn: () => undefined, @@ -340,6 +354,7 @@ test('fleet spawn --sandbox replays an exact identity and cleans up by returned String(mocks.authorizedApiFetch.mock.calls[2]?.[1] ?? '').split('/').pop() ?? '' ) || null, deleteProviderId: deleteBody.providerId ?? null, + relaycastTargetPersisted: mocks.persistWorkspaceRelaycastTarget.mock.calls.length === 1, outputFailureObserved: errors.join('\n').includes('CLI output sink failed after capture'), }), 'utf8' @@ -374,6 +389,9 @@ try { if (observation.outputFailureObserved !== true) { throw new Error('The probe did not serialize the CLI sandbox result, so it observed nothing.'); } + if (arm === 'head' && observation.relaycastTargetPersisted !== true) { + throw new Error("The head did not persist Cloud's Relaycast target before dispatch."); + } // The command line named no provider on either arm. If this ever stops being // true the case is proving provider pinning, not capability routing. if (observation.ensureProviderId !== null) { From 794753d89aae4a809c989a1983253e1e954627cb Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 00:27:02 +0200 Subject: [PATCH 36/50] fix(fleet): preserve replay and workspace identity --- CHANGELOG.md | 6 +++ .../cli/src/cli/ci-standalone-smoke.test.ts | 4 +- packages/cli/src/cli/commands/fleet.test.ts | 19 ++++----- packages/cli/src/cli/commands/fleet.ts | 22 ++++++---- packages/cli/src/cli/lib/sdk-client.test.ts | 41 ++++++++++++++++++- packages/cli/src/cli/lib/sdk-client.ts | 27 ++++++++---- .../cloud/src/project-workspace-key.test.ts | 8 ++-- packages/cloud/src/project-workspace-key.ts | 19 +++++++++ scripts/ci-standalone-smoke.sh | 2 +- 9 files changed, 113 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 837f434e10..630f87e2c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Operators can keep using custom `--sandbox-name` values with `agent-relay fleet spawn --sandbox`; launches without a custom name generate a stable `sbx_` identity automatically, while only `--sandbox-id` replay requires the matching deterministic lowercase `fleet-sandbox-` name. +### Fixed + +- Ambiguous Cloud sandbox responses retain their stable `--sandbox-id` for replay instead of automatically deleting an allocation whose outcome is unknown. +- Persisting an Agent37 Relaycast target keeps the canonical Cloud workspace key as the durable selector and stores the route-scoped transport credential separately, so later commands can reuse the original explicit key. +- Standalone package smoke workspaces remain valid for five minutes, leaving enough lease time for the full broker startup budget and cleanup verification. + ## [11.10.4] - 2026-09-08 ### Changed diff --git a/packages/cli/src/cli/ci-standalone-smoke.test.ts b/packages/cli/src/cli/ci-standalone-smoke.test.ts index 0decc33fa0..9c90896ed1 100644 --- a/packages/cli/src/cli/ci-standalone-smoke.test.ts +++ b/packages/cli/src/cli/ci-standalone-smoke.test.ts @@ -58,7 +58,7 @@ fi `#!/usr/bin/env bash set -euo pipefail if [ "\${1:-}" = "-cn" ]; then - echo '{"name":"fake","expires_in_seconds":60}' + echo '{"name":"fake","expires_in_seconds":300}' elif [ "\${1:-}" = "-er" ] && [[ "\${2:-}" = *api_key* ]]; then echo 'rk_live_fake_smoke_key' elif [ "\${1:-}" = "-er" ] && [[ "\${2:-}" = *workspace_id* ]]; then @@ -153,7 +153,7 @@ describe('ci-standalone-smoke workspace reuse', () => { it('creates an ephemeral workspace on the trusted engine and wires its base URL explicitly', () => { const script = readFileSync(smokeScript, 'utf8'); expect(script).toContain('TRUSTED_RELAY_BASE_URL="https://cast.agentrelay.com"'); - expect(script).toContain('expires_in_seconds: 60'); + expect(script).toContain('expires_in_seconds: 300'); expect(script).toContain('printf \'::add-mask::%s\\n\' "$WORKSPACE_KEY"'); expect(script).toContain('--request DELETE'); expect(script).toContain('Ephemeral workspace deletion verified'); diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 9e72778ca3..dfd5132447 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1286,7 +1286,8 @@ describe('fleet command support', () => { expect(errors.join('\n')).toContain('--sandbox-id must match lowercase sbx_'); }); - it('cleans up when Cloud reports a post-provision response failure with a sandbox ID', async () => { + it('preserves the stable sandbox ID for replay when Cloud reports an unknown outcome', async () => { + const warnings: string[] = []; const deleteCloudFleetSandbox = vi.fn(async () => undefined); const program = new Command(); program.exitOverride(); @@ -1306,15 +1307,15 @@ describe('fleet command support', () => { ensureCloudFleetSandbox: vi.fn(async () => { throw new CloudFleetSandboxProvisionError('malformed response', { cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', - sandboxId: 'sandbox-1', - nodeName: 'daytona-codex', + sandboxId: REPLAY_SANDBOX_ID, + nodeName: REPLAY_SANDBOX_NAME, outcomeUnknown: true, }); }), deleteCloudFleetSandbox, createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, - warn: () => undefined, + warn: (...args: unknown[]) => warnings.push(args.join(' ')), error: () => undefined, }); @@ -1342,13 +1343,12 @@ describe('fleet command support', () => { ) ).rejects.toThrow('__exit__'); - expect(deleteCloudFleetSandbox).toHaveBeenCalledWith({ - cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', - sandboxId: 'sandbox-1', - }); + expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); + expect(warnings.join('\n')).toContain(`check Cloud Fleet for node '${REPLAY_SANDBOX_NAME}'`); + expect(warnings.join('\n')).toContain(`--sandbox-id '${REPLAY_SANDBOX_ID}'`); }); - it('pins cleanup to the requested E2B provider when Cloud fails closed after provisioning', async () => { + it('pins cleanup to the requested E2B provider for a known post-provision failure', async () => { const deleteCloudFleetSandbox = vi.fn(async () => undefined); const program = new Command(); program.exitOverride(); @@ -1371,7 +1371,6 @@ describe('fleet command support', () => { sandboxId: 'sandbox-e2b', nodeName: 'e2b-codex', providerId: 'e2b', - outcomeUnknown: true, }); }), deleteCloudFleetSandbox, diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 32af37d47f..14311dac13 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -325,7 +325,19 @@ export function registerFleetCommands( ...(effectiveSandboxName === undefined ? {} : { name: effectiveSandboxName }), }); } catch (error) { - if (error instanceof CloudFleetSandboxProvisionError && error.cloudWorkspaceId && error.sandboxId) { + if (error instanceof CloudFleetSandboxProvisionError && error.outcomeUnknown) { + deps.warn( + `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ + error.nodeName ?? effectiveSandboxName ?? 'the requested sandbox' + }'${ + sandboxId === undefined ? '' : ` before retrying with --sandbox-id '${sandboxId}'` + } so a sandbox is not left running.` + ); + } else if ( + error instanceof CloudFleetSandboxProvisionError && + error.cloudWorkspaceId && + error.sandboxId + ) { await deps .deleteCloudFleetSandbox({ cloudWorkspaceId: error.cloudWorkspaceId, @@ -339,14 +351,6 @@ export function registerFleetCommands( }` ); }); - } else if (error instanceof CloudFleetSandboxProvisionError && error.outcomeUnknown) { - deps.warn( - `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ - error.nodeName ?? effectiveSandboxName ?? 'the requested sandbox' - }'${ - sandboxId === undefined ? '' : ` before retrying with --sandbox-id '${sandboxId}'` - } so a sandbox is not left running.` - ); } throw error; } diff --git a/packages/cli/src/cli/lib/sdk-client.test.ts b/packages/cli/src/cli/lib/sdk-client.test.ts index 13cbbce3ca..b15dd39287 100644 --- a/packages/cli/src/cli/lib/sdk-client.test.ts +++ b/packages/cli/src/cli/lib/sdk-client.test.ts @@ -141,12 +141,51 @@ describe('sdk client option resolution', () => { }) ).toBe(true); expect(readProjectWorkspaceSession(projectDataDir())).toEqual({ - workspaceKey: 'rk_live_agent37', + workspaceKey: 'rk_live_canonical', workspaceId: 'rw_abc', enrolledNodeId: 'node_1', relaycastRoute: 'agent37-isolated', relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', }); + + const replayOptions = { + workspaceKey: 'rk_live_canonical', + env: { AGENT_RELAY_HOME: dir }, + }; + expect(resolveWorkspaceSelection(replayOptions)).toMatchObject({ + key: 'rk_live_canonical', + source: 'flag', + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + expect(resolveWorkspaceKey(replayOptions)).toBe('rk_live_agent37'); + expect(resolveBaseUrl(replayOptions)).toBe('https://agent37-cast.agentrelay.com'); + }); + + it('keeps legacy persisted targets usable when no separate Relaycast key exists', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_legacy_agent37', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + + const options = { env: { AGENT_RELAY_HOME: dir } }; + expect(resolveWorkspaceKey(options)).toBe('rk_live_legacy_agent37'); + expect(resolveBaseUrl(options)).toBe('https://agent37-cast.agentrelay.com'); + }); + + it('rejects a separate Relaycast key without a complete persisted route', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_agent37', + }); + + const options = { env: { AGENT_RELAY_HOME: dir } }; + expect(() => resolveWorkspaceKey(options)).toThrow(/persisted Relaycast workspace route is incomplete/); + expect(() => resolveBaseUrl(options)).toThrow(/persisted Relaycast workspace route is incomplete/); }); it('rejects a persisted route that is not the exact server-owned origin', () => { diff --git a/packages/cli/src/cli/lib/sdk-client.ts b/packages/cli/src/cli/lib/sdk-client.ts index 1348c766f1..8995d9ddd2 100644 --- a/packages/cli/src/cli/lib/sdk-client.ts +++ b/packages/cli/src/cli/lib/sdk-client.ts @@ -4,7 +4,6 @@ import { AgentRelay, type AgentRelayAgent } from '@agent-relay/sdk'; import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; import { resolveWorkspaceSelection as resolveCloudWorkspaceSelection, - resolveWorkspaceKeyWithSource as resolveCloudWorkspaceKeyWithSource, readProjectWorkspaceSession, writeProjectWorkspaceKey, type WorkspaceSelection, @@ -51,11 +50,17 @@ export function resolveWorkspaceKeyWithSource(options: SdkClientOptions = {}): { key: string; source: WorkspaceKeySource; } { - const resolved = resolveCloudWorkspaceKeyWithSource({ - workspaceKey: options.workspaceKey, - env: env(options), - }); - if (resolved) return resolved; + const selection = resolveWorkspaceSelection(options); + if (selection) { + // A Cloud workspace key remains the durable selector and Cloud credential. + // A persisted Relaycast target may carry a different route-scoped transport + // credential for SDK traffic on the same workspace. + validatePersistedRelaycastBaseUrl(selection); + return { + key: trimOrUndefined(selection.relaycastApiKey) ?? selection.key, + source: selection.source, + }; + } throw new Error( 'No workspace key found. Pass --workspace-key, set RELAY_WORKSPACE_KEY, or run `relay workspace set_key `.' ); @@ -78,7 +83,8 @@ export function resolveBaseUrl(options: SdkClientOptions = {}): string | undefin function validatePersistedRelaycastBaseUrl(selection: WorkspaceSelection | undefined): string | undefined { const baseUrl = trimOrUndefined(selection?.relaycastBaseUrl); const route = selection?.relaycastRoute; - if (!baseUrl && !route) return undefined; + const relaycastApiKey = trimOrUndefined(selection?.relaycastApiKey); + if (!baseUrl && !route && !relaycastApiKey) return undefined; if (!baseUrl || !route) { throw new Error('The persisted Relaycast workspace route is incomplete.'); } @@ -133,18 +139,21 @@ export function persistWorkspaceRelaycastTarget( ...(existing.workspaceId ? { workspaceId: existing.workspaceId } : {}), ...(existing.relaycastRoute ? { relaycastRoute: existing.relaycastRoute } : {}), ...(existing.relaycastBaseUrl ? { relaycastBaseUrl: existing.relaycastBaseUrl } : {}), + ...(existing.relaycastApiKey ? { relaycastApiKey: existing.relaycastApiKey } : {}), }); }; try { - writeProjectWorkspaceKey(dataDir, target.relaycastApiKey, { + writeProjectWorkspaceKey(dataDir, existing.workspaceKey, { ...(existing.enrolledNodeId ? { enrolledNodeId: existing.enrolledNodeId } : {}), workspaceId: target.workspaceId, relaycastRoute: target.route, relaycastBaseUrl: target.baseUrl, + relaycastApiKey: target.relaycastApiKey, }); const persisted = readProjectWorkspaceSession(dataDir); if ( - persisted?.workspaceKey === target.relaycastApiKey && + persisted?.workspaceKey === existing.workspaceKey && + persisted.relaycastApiKey === target.relaycastApiKey && persisted.workspaceId === target.workspaceId && persisted.relaycastRoute === target.route && persisted.relaycastBaseUrl === target.baseUrl diff --git a/packages/cloud/src/project-workspace-key.test.ts b/packages/cloud/src/project-workspace-key.test.ts index fc6e87221c..35a613d31c 100644 --- a/packages/cloud/src/project-workspace-key.test.ts +++ b/packages/cloud/src/project-workspace-key.test.ts @@ -107,24 +107,26 @@ describe('workspace precedence ladder diagnostics', () => { }); it('carries a persisted Relaycast target when an explicit key matches the project pin', () => { - writeProjectWorkspaceKey(dataDir, 'rk_agent37', { + writeProjectWorkspaceKey(dataDir, 'rk_canonical', { workspaceId: 'rw_pinned', relaycastRoute: 'agent37-isolated', relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_agent37', }); expect( resolveWorkspaceSelection({ - workspaceKey: 'rk_agent37', + workspaceKey: 'rk_canonical', projectDataDir: dataDir, env: { AGENT_RELAY_HOME: home }, }) ).toMatchObject({ - key: 'rk_agent37', + key: 'rk_canonical', source: 'flag', workspaceId: 'rw_pinned', relaycastRoute: 'agent37-isolated', relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_agent37', }); }); diff --git a/packages/cloud/src/project-workspace-key.ts b/packages/cloud/src/project-workspace-key.ts index ae7fb1005e..5d84715754 100644 --- a/packages/cloud/src/project-workspace-key.ts +++ b/packages/cloud/src/project-workspace-key.ts @@ -24,6 +24,8 @@ export interface ProjectWorkspaceSession { /** Last server-selected Relaycast route for follow-up commands in this session. */ relaycastRoute?: 'canonical' | 'agent37-isolated'; relaycastBaseUrl?: string; + /** Route-scoped transport credential; the canonical Cloud workspace key remains `workspaceKey`. */ + relaycastApiKey?: string; } export type ProjectWorkspaceSessionMetadata = Omit; @@ -60,6 +62,7 @@ export interface WorkspaceSelection { workspaceId?: string; relaycastRoute?: 'canonical' | 'agent37-isolated'; relaycastBaseUrl?: string; + relaycastApiKey?: string; /** Project session directory that can durably carry a server-selected target. */ projectDataDir?: string; } @@ -94,12 +97,14 @@ export function readProjectWorkspaceSession( ? parsed.relaycastRoute : undefined; const relaycastBaseUrl = trimOrUndefined(parsed.relaycastBaseUrl); + const relaycastApiKey = trimOrUndefined(parsed.relaycastApiKey); return { workspaceKey, ...(enrolledNodeId ? { enrolledNodeId } : {}), ...(workspaceId ? { workspaceId } : {}), ...(relaycastRoute ? { relaycastRoute } : {}), ...(relaycastBaseUrl ? { relaycastBaseUrl } : {}), + ...(relaycastApiKey ? { relaycastApiKey } : {}), }; } catch { return undefined; @@ -118,6 +123,7 @@ export function writeProjectWorkspaceKey( workspaceId?: string; relaycastRoute?: 'canonical' | 'agent37-isolated'; relaycastBaseUrl?: string; + relaycastApiKey?: string; } = {} ): void { const key = trimOrUndefined(workspaceKey); @@ -126,6 +132,7 @@ export function writeProjectWorkspaceKey( const workspaceId = trimOrUndefined(options.workspaceId); const relaycastRoute = options.relaycastRoute; const relaycastBaseUrl = trimOrUndefined(options.relaycastBaseUrl); + const relaycastApiKey = trimOrUndefined(options.relaycastApiKey); fs.mkdirSync(dataDir, { recursive: true, mode: 0o700 }); const file = projectWorkspaceKeyPath(dataDir); // Worker threads share a PID, so include a per-write nonce as well as the PID. @@ -137,6 +144,7 @@ export function writeProjectWorkspaceKey( ...(workspaceId ? { workspaceId } : {}), ...(relaycastRoute ? { relaycastRoute } : {}), ...(relaycastBaseUrl ? { relaycastBaseUrl } : {}), + ...(relaycastApiKey ? { relaycastApiKey } : {}), } satisfies ProjectWorkspaceSession, null, 2 @@ -192,6 +200,7 @@ export function writeProjectWorkspaceKeyPreservingSession( ...(existing?.workspaceId ? { workspaceId: existing.workspaceId } : {}), ...(existing?.relaycastRoute ? { relaycastRoute: existing.relaycastRoute } : {}), ...(existing?.relaycastBaseUrl ? { relaycastBaseUrl: existing.relaycastBaseUrl } : {}), + ...(existing?.relaycastApiKey ? { relaycastApiKey: existing.relaycastApiKey } : {}), } : {}; @@ -205,6 +214,9 @@ export function writeProjectWorkspaceKeyPreservingSession( ...(trimOrUndefined(options.relaycastBaseUrl) ? { relaycastBaseUrl: trimOrUndefined(options.relaycastBaseUrl) } : {}), + ...(trimOrUndefined(options.relaycastApiKey) + ? { relaycastApiKey: trimOrUndefined(options.relaycastApiKey) } + : {}), }); } @@ -245,6 +257,9 @@ export function resolveWorkspaceSelection( ...(project?.workspaceKey === flag && project.relaycastBaseUrl ? { relaycastBaseUrl: project.relaycastBaseUrl } : {}), + ...(project?.workspaceKey === flag && project.relaycastApiKey + ? { relaycastApiKey: project.relaycastApiKey } + : {}), ...(project?.workspaceKey === flag && dataDir ? { projectDataDir: dataDir } : {}), }; } @@ -265,6 +280,9 @@ export function resolveWorkspaceSelection( ...(project?.workspaceKey === envKey && project.relaycastBaseUrl ? { relaycastBaseUrl: project.relaycastBaseUrl } : {}), + ...(project?.workspaceKey === envKey && project.relaycastApiKey + ? { relaycastApiKey: project.relaycastApiKey } + : {}), ...(project?.workspaceKey === envKey && dataDir ? { projectDataDir: dataDir } : {}), }; } @@ -278,6 +296,7 @@ export function resolveWorkspaceSelection( ...(project.workspaceId ? { workspaceId: project.workspaceId } : {}), ...(project.relaycastRoute ? { relaycastRoute: project.relaycastRoute } : {}), ...(project.relaycastBaseUrl ? { relaycastBaseUrl: project.relaycastBaseUrl } : {}), + ...(project.relaycastApiKey ? { relaycastApiKey: project.relaycastApiKey } : {}), ...(dataDir ? { projectDataDir: dataDir } : {}), }; } diff --git a/scripts/ci-standalone-smoke.sh b/scripts/ci-standalone-smoke.sh index 8e3000fce2..ade392a152 100755 --- a/scripts/ci-standalone-smoke.sh +++ b/scripts/ci-standalone-smoke.sh @@ -158,7 +158,7 @@ WORKSPACE_NAME="relay-standalone-smoke-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTE CREATE_STATUS="$(curl --silent --show-error --output "$WORKSPACE_RESPONSE" --write-out '%{http_code}' \ --request POST \ --header 'Content-Type: application/json' \ - --data "$(jq -cn --arg name "$WORKSPACE_NAME" '{name: $name, expires_in_seconds: 60}')" \ + --data "$(jq -cn --arg name "$WORKSPACE_NAME" '{name: $name, expires_in_seconds: 300}')" \ "$TRUSTED_RELAY_BASE_URL/v1/workspaces" 2>/dev/null || true)" # Mask the key before extracting or using any other response field. Never log # the response body: it contains the administrative workspace credential. From 9fdc4c68f8e38026f8fae9f8e6ebf3b9d5228cf7 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 00:45:44 +0200 Subject: [PATCH 37/50] test(smoke): bound startup within workspace lease --- CHANGELOG.md | 2 +- packages/cli/src/cli/ci-standalone-smoke.test.ts | 11 ++++++++--- scripts/ci-standalone-smoke.sh | 7 +++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 630f87e2c8..75ddfbe25a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ambiguous Cloud sandbox responses retain their stable `--sandbox-id` for replay instead of automatically deleting an allocation whose outcome is unknown. - Persisting an Agent37 Relaycast target keeps the canonical Cloud workspace key as the durable selector and stores the route-scoped transport credential separately, so later commands can reuse the original explicit key. -- Standalone package smoke workspaces remain valid for five minutes, leaving enough lease time for the full broker startup budget and cleanup verification. +- Standalone package smoke workspaces remain valid for five minutes, and startup overrides are capped at four minutes so shutdown and cleanup verification always retain a full-minute lease margin. ## [11.10.4] - 2026-09-08 diff --git a/packages/cli/src/cli/ci-standalone-smoke.test.ts b/packages/cli/src/cli/ci-standalone-smoke.test.ts index 9c90896ed1..189bd6eec2 100644 --- a/packages/cli/src/cli/ci-standalone-smoke.test.ts +++ b/packages/cli/src/cli/ci-standalone-smoke.test.ts @@ -153,7 +153,8 @@ describe('ci-standalone-smoke workspace reuse', () => { it('creates an ephemeral workspace on the trusted engine and wires its base URL explicitly', () => { const script = readFileSync(smokeScript, 'utf8'); expect(script).toContain('TRUSTED_RELAY_BASE_URL="https://cast.agentrelay.com"'); - expect(script).toContain('expires_in_seconds: 300'); + expect(script).toContain('WORKSPACE_LEASE_SECONDS=300'); + expect(script).toContain('expires_in_seconds: $expires'); expect(script).toContain('printf \'::add-mask::%s\\n\' "$WORKSPACE_KEY"'); expect(script).toContain('--request DELETE'); expect(script).toContain('Ephemeral workspace deletion verified'); @@ -166,6 +167,8 @@ describe('ci-standalone-smoke workspace reuse', () => { const script = readFileSync(smokeScript, 'utf8'); const brokerSession = readFileSync(resolve('crates/broker/src/runtime/session.rs'), 'utf8'); const minimumSeconds = Number(script.match(/MIN_STARTUP_TIMEOUT_SECONDS=([0-9]+)/)?.[1]); + const maximumSeconds = Number(script.match(/MAX_STARTUP_TIMEOUT_SECONDS=([0-9]+)/)?.[1]); + const leaseSeconds = Number(script.match(/WORKSPACE_LEASE_SECONDS=([0-9]+)/)?.[1]); const smokeSeconds = Number( script.match(/AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS:-([0-9]+)}/)?.[1] ); @@ -175,6 +178,7 @@ describe('ci-standalone-smoke workspace reuse', () => { expect(minimumSeconds).toBeGreaterThanOrEqual(brokerSeconds + 10); expect(smokeSeconds).toBeGreaterThanOrEqual(minimumSeconds); + expect(leaseSeconds - maximumSeconds).toBeGreaterThanOrEqual(60); }); it('rejects unsafe startup-timeout overrides before invoking binaries', () => { @@ -183,8 +187,9 @@ describe('ci-standalone-smoke workspace reuse', () => { ['050', 'without leading zeros'], ['060', 'without leading zeros'], ['08', 'without leading zeros'], - ['99999', 'must be no more than 86400s'], - ['9223372036854775808', 'between 50s and 86400s'], + ['241', 'must be no more than 240s'], + ['99999', 'must be no more than 240s'], + ['9223372036854775808', 'between 50s and 240s'], ]) { const { cli, broker, invocationLog } = createFakeBinaries(); const result = spawnSync('bash', [smokeScript, cli, broker], { diff --git a/scripts/ci-standalone-smoke.sh b/scripts/ci-standalone-smoke.sh index ade392a152..1f407cc80c 100755 --- a/scripts/ci-standalone-smoke.sh +++ b/scripts/ci-standalone-smoke.sh @@ -22,7 +22,10 @@ unset RELAY_WORKSPACES_JSON RELAY_WORKSPACE_KEY AGENT_RELAY_WORKSPACE_KEY RELAY_ # least ten seconds above that bound so an override cannot reintroduce the race # this smoke is meant to catch. MIN_STARTUP_TIMEOUT_SECONDS=50 -MAX_STARTUP_TIMEOUT_SECONDS=86400 +# Keep every accepted startup override inside the ephemeral workspace lease, +# with a full minute left for shutdown and deletion verification. +WORKSPACE_LEASE_SECONDS=300 +MAX_STARTUP_TIMEOUT_SECONDS=240 STARTUP_TIMEOUT_SECONDS="${AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS:-60}" if ! [[ "$STARTUP_TIMEOUT_SECONDS" =~ ^[1-9][0-9]{0,4}$ ]]; then echo "ERROR: AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS must be a base-10 integer between ${MIN_STARTUP_TIMEOUT_SECONDS}s and ${MAX_STARTUP_TIMEOUT_SECONDS}s without leading zeros." >&2 @@ -158,7 +161,7 @@ WORKSPACE_NAME="relay-standalone-smoke-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTE CREATE_STATUS="$(curl --silent --show-error --output "$WORKSPACE_RESPONSE" --write-out '%{http_code}' \ --request POST \ --header 'Content-Type: application/json' \ - --data "$(jq -cn --arg name "$WORKSPACE_NAME" '{name: $name, expires_in_seconds: 300}')" \ + --data "$(jq -cn --arg name "$WORKSPACE_NAME" --argjson expires "$WORKSPACE_LEASE_SECONDS" '{name: $name, expires_in_seconds: $expires}')" \ "$TRUSTED_RELAY_BASE_URL/v1/workspaces" 2>/dev/null || true)" # Mask the key before extracting or using any other response field. Never log # the response body: it contains the administrative workspace credential. From 2635252b0469d7fe9d99489574d5d28cd1b6e820 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 01:21:23 +0200 Subject: [PATCH 38/50] fix: bind cloud sandbox routing safely --- CHANGELOG.md | 2 + packages/cli/src/cli/commands/fleet.test.ts | 96 ++++++++++- packages/cli/src/cli/commands/fleet.ts | 23 +-- packages/cli/src/cli/commands/integration.ts | 16 +- .../cli/src/cli/commands/observer.test.ts | 31 ++++ packages/cli/src/cli/commands/observer.ts | 11 +- .../cli/src/cli/lib/attach-fleet-node.test.ts | 75 +++++++++ packages/cli/src/cli/lib/attach-fleet-node.ts | 9 +- packages/cli/src/cli/lib/sdk-client.test.ts | 146 +++++++++++++++++ packages/cli/src/cli/lib/sdk-client.ts | 154 +++++++++++------- packages/cloud/src/fleet-sandbox.test.ts | 93 ++++++++++- packages/cloud/src/fleet-sandbox.ts | 20 +-- .../cloud/src/project-workspace-key.test.ts | 28 ++++ packages/cloud/src/project-workspace-key.ts | 12 +- 14 files changed, 604 insertions(+), 112 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75ddfbe25a..1998862fdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ambiguous Cloud sandbox responses retain their stable `--sandbox-id` for replay instead of automatically deleting an allocation whose outcome is unknown. - Persisting an Agent37 Relaycast target keeps the canonical Cloud workspace key as the durable selector and stores the route-scoped transport credential separately, so later commands can reuse the original explicit key. +- Relaycast credentials and origins now resolve as one transport pair for attach, observer, Fleet, and Relayfile provisioning commands; stale sandbox responses cannot overwrite a project workspace that was rebound while provisioning was in flight. +- Legacy non-Agent37 Cloud sandbox responses remain usable when they omit the newer Relaycast target, while any target Cloud does return is verified and persisted for both newly provisioned and reused providers. - Standalone package smoke workspaces remain valid for five minutes, and startup overrides are capped at four minutes so shutdown and cleanup verification always retain a full-minute lease margin. ## [11.10.4] - 2026-09-08 diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index dfd5132447..bc924efdbe 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -54,6 +54,12 @@ const AGENT37_RELAYCAST_TARGET = { workspaceId: 'rw_abc', relaycastApiKey: 'rk_live_agent37_target', }; +const CANONICAL_RELAYCAST_TARGET = { + route: 'canonical' as const, + baseUrl: 'https://cast.agentrelay.com', + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_canonical_target', +}; const LIVE_AGENT_CAPABILITY_NAME = 'relay:live-agents:v1'; const liveAgentCapabilities = (...names: string[]) => [ @@ -874,6 +880,82 @@ describe('fleet command support', () => { expect(release).toHaveBeenCalled(); }); + it('applies and persists a returned target for a reused non-Agent37 provider', async () => { + const placement = { + spawn: vi.fn(async () => ({ invocationId: 'inv_reused_e2b', node: { name: 'e2b-codex' } })), + }; + const register = vi.fn(async () => ({ token: 'at_live_launcher' })); + const release = vi.fn(async () => ({ released: true, deleted: true })); + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })), register, release }, + })); + const persistWorkspaceRelaycastTarget = vi.fn(() => true); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, + createWorkspaceRelay: createWorkspaceRelay as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => ({ + outcome: 'reused' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-e2b', + nodeName: 'e2b-codex', + status: 'online', + activeAgents: 0, + maxAgents: 1, + providerId: 'e2b' as const, + relaycastTarget: CANONICAL_RELAYCAST_TARGET, + })), + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/tmp/agent-relay-test/workspace-key.json', + workspaceId: 'rw_abc', + }), + persistWorkspaceRelaycastTarget, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-provider', + 'e2b', + '--workspace-id', + 'rw_abc', + '--name', + 'reused-e2b-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + + expect(createWorkspaceRelay).toHaveBeenCalledWith({ + workspaceKey: CANONICAL_RELAYCAST_TARGET.relaycastApiKey, + baseUrl: CANONICAL_RELAYCAST_TARGET.baseUrl, + }); + expect(persistWorkspaceRelaycastTarget).toHaveBeenCalledWith( + expect.objectContaining({ key: 'rk_live_test' }), + CANONICAL_RELAYCAST_TARGET + ); + }); + it('fleet spawn --sandbox cleans up when the target cannot be persisted', async () => { const deleteCloudFleetSandbox = vi.fn(async () => undefined); const ensureCloudFleetSandbox = vi.fn(async () => ({ @@ -1039,15 +1121,19 @@ describe('fleet command support', () => { }; const register = vi.fn(async () => ({ token: 'at_live_launcher' })); const release = vi.fn(async () => ({ released: true, deleted: true })); + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })), register, release }, + })); + const persistWorkspaceRelaycastTarget = vi.fn(() => true); const ensureCloudFleetSandbox = vi.fn(async () => ({ outcome: 'provisioned' as const, + providerId: 'e2b' as const, cloudWorkspaceId: 'cloud-workspace', nodeId: 'node-legacy', nodeName: 'custom-node', sandboxId: 'legacy-public-sandbox', providerSandboxId: 'legacy-provider-sandbox', relayWorkspaceId: 'rw_abc', - relaycastTarget: AGENT37_RELAYCAST_TARGET, relayfileMounted: true, relayfileMountPath: '/workspace', })); @@ -1056,9 +1142,7 @@ describe('fleet command support', () => { registerFleetCommands(program, { sdk: { createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, - createWorkspaceRelay: vi.fn(() => ({ - workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })), register, release }, - })) as never, + createWorkspaceRelay: createWorkspaceRelay as never, createWorkspace: vi.fn() as never, log: vi.fn(), error: vi.fn(), @@ -1071,7 +1155,7 @@ describe('fleet command support', () => { origin: 'test', workspaceId: 'rw_abc', }), - persistWorkspaceRelaycastTarget: () => true, + persistWorkspaceRelaycastTarget, deleteCloudFleetSandbox: vi.fn(async () => undefined), createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, @@ -1098,6 +1182,8 @@ describe('fleet command support', () => { ); const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0]; + expect(persistWorkspaceRelaycastTarget).not.toHaveBeenCalled(); + expect(createWorkspaceRelay).toHaveBeenCalledWith({ workspaceKey: 'rk_live_test' }); expect(ensureInput).toMatchObject({ workspaceId: 'rw_abc', requiredCapability: 'spawn:codex', diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 14311dac13..4b8e695648 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -28,11 +28,10 @@ import { redactSecrets } from '../lib/redact.js'; import { attributableReleaseReason } from '../lib/release-reason.js'; import { resolveAgentToken, - resolveBaseUrl, resolveWorkspaceSelection, persistWorkspaceRelaycastTarget, - resolveWorkspaceKey, resolveWorkspaceKeyWithSource, + resolveWorkspaceTransport, type SdkClientOptions, } from '../lib/sdk-client.js'; import { @@ -72,11 +71,10 @@ function withFleetDefaults(overrides: Partial = {}): F return { core, sdk, - createFleetWorkspaceClient: (options) => - createWorkspaceClient({ - workspaceKey: resolveWorkspaceKey(options), - baseUrl: resolveBaseUrl(options), - }), + createFleetWorkspaceClient: (options) => { + const { workspaceKey, baseUrl } = resolveWorkspaceTransport(options); + return createWorkspaceClient({ workspaceKey, baseUrl }); + }, resolveWorkspaceSelection, persistWorkspaceRelaycastTarget, ensureCloudFleetSandbox, @@ -354,19 +352,14 @@ export function registerFleetCommands( } throw error; } - if ( - sandbox.outcome === 'provisioned' || - (sandbox.outcome === 'reused' && sandbox.providerId === 'agent37') - ) { - // A provisioned response must carry a closed, server-owned target. + if (sandbox.outcome !== 'provisioning_timeout' && sandbox.relaycastTarget) { + // When Cloud returns a closed, server-owned target, apply it for any + // provider and outcome before registration, spawn, or launcher release. // Rebuild both credentials and origin before any registration, spawn, // or launcher release, then prove the authenticated client sees the // exact workspace Cloud returned. try { const target = sandbox.relaycastTarget; - if (!target) { - throw new Error('Cloud provisioned a sandbox without a Relaycast target.'); - } if ( (sandboxProvider === 'agent37' && target.route !== 'agent37-isolated') || target.workspaceId.trim() !== relayWorkspaceId.trim() || diff --git a/packages/cli/src/cli/commands/integration.ts b/packages/cli/src/cli/commands/integration.ts index 2172dff128..9ee581272c 100644 --- a/packages/cli/src/cli/commands/integration.ts +++ b/packages/cli/src/cli/commands/integration.ts @@ -29,7 +29,7 @@ import { assertRelayfileVersion, type RelayfileClientOptions, } from '@relayfile/client'; -import { resolveBaseUrl, resolveWorkspaceKey } from '../lib/sdk-client.js'; +import { resolveBaseUrl, resolveWorkspaceKey, resolveWorkspaceTransport } from '../lib/sdk-client.js'; // Re-export the version gate so existing tests importing it from this module // (and any callers) keep working after it moved to the published client package. @@ -473,8 +473,14 @@ async function createRelayfileInboundTarget( const options = sdkOptionsFromOpts(commandOpts); const authOptions = local && !explicitWorkspaceKey(commandOpts) ? localRetryOptions(options, local) : options; - const workspaceKey = resolveWorkspaceKey(authOptions); - const baseUrl = resolveInboundTargetBaseUrl(options); + // A local broker session may expose its loopback broker URL. That URL is not + // the Relaycast control-plane origin for inbound-target provisioning, so pair + // its workspace selector with the caller's Relaycast URL (or canonical). + const { workspaceKey, baseUrl: selectedBaseUrl } = resolveWorkspaceTransport({ + ...authOptions, + baseUrl: options.baseUrl, + }); + const baseUrl = resolveInboundTargetBaseUrl(selectedBaseUrl); const response = await fetch(new URL('/v1/integrations/relayfile/inbound-target', baseUrl), { method: 'POST', headers: { @@ -525,8 +531,8 @@ function parseRelayfileInboundTargetResponse(body: unknown): { url: string; secr return { url: parsedUrl.toString(), secret }; } -function resolveInboundTargetBaseUrl(options: SdkClientOptions): string { - const baseUrl = resolveBaseUrl(options) ?? 'https://cast.agentrelay.com'; +function resolveInboundTargetBaseUrl(selectedBaseUrl: string | undefined): string { + const baseUrl = selectedBaseUrl ?? 'https://cast.agentrelay.com'; const parsed = new URL(baseUrl); if (parsed.protocol !== 'https:') { throw new Error('Inbound relayfile target provisioning requires an https Relaycast base URL.'); diff --git a/packages/cli/src/cli/commands/observer.test.ts b/packages/cli/src/cli/commands/observer.test.ts index 65ca1c6009..9723d1a82f 100644 --- a/packages/cli/src/cli/commands/observer.test.ts +++ b/packages/cli/src/cli/commands/observer.test.ts @@ -1,8 +1,12 @@ import { Command } from 'commander'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { registerObserverCommands, type ObserverCommandDependencies } from './observer.js'; import { observerUrl, resolveObserverBaseUrl } from '../lib/observer-url.js'; +import { writeProjectWorkspaceKey } from '../lib/project-workspace-key.js'; class ExitSignal extends Error { constructor(public readonly code: number) { @@ -94,6 +98,33 @@ describe('agent-relay observer', () => { expect(logs.join('\n')).not.toContain(WORKSPACE_KEY); }); + it('mints through the persisted credential and origin as one transport pair', async () => { + const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-observer-project-')); + vi.stubEnv('AGENT_RELAY_PROJECT', projectRoot); + writeProjectWorkspaceKey(path.join(projectRoot, '.agentworkforce/relay'), WORKSPACE_KEY, { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_isolated_observer', + }); + const { program, createObserverToken } = setup(); + + try { + await program.parseAsync( + ['observer', '--workspace-key', WORKSPACE_KEY, '--base-url', 'https://agent37-cast.agentrelay.com/'], + { from: 'user' } + ); + + const [call] = createObserverToken.mock.calls as unknown as [[Record]]; + expect(call[0]).toMatchObject({ + workspaceKey: 'rk_live_isolated_observer', + baseUrl: 'https://agent37-cast.agentrelay.com', + }); + } finally { + fs.rmSync(projectRoot, { recursive: true, force: true }); + } + }); + it('narrows to channels and includes DMs when asked', async () => { const { program, createObserverToken } = setup(); diff --git a/packages/cli/src/cli/commands/observer.ts b/packages/cli/src/cli/commands/observer.ts index d76ad6315a..233c6c65f3 100644 --- a/packages/cli/src/cli/commands/observer.ts +++ b/packages/cli/src/cli/commands/observer.ts @@ -25,7 +25,7 @@ import { resolveObserverBaseUrl, } from '../lib/observer-url.js'; import { printJson, runSdk, withSdkDefaults, type SdkCommandDeps } from '../lib/sdk-command.js'; -import { resolveBaseUrl, resolveWorkspaceKey } from '../lib/sdk-client.js'; +import { resolveWorkspaceTransport } from '../lib/sdk-client.js'; const MAX_CHANNEL_FILTERS = 50; @@ -100,11 +100,12 @@ function parseChannels(value: string): string[] { * `undefined` and an absent key differently. */ function connection(options: Record): { workspaceKey: string; baseUrl?: string } { - const baseUrl = resolveBaseUrl({ baseUrl: options.baseUrl as string | undefined }); + const { workspaceKey, baseUrl } = resolveWorkspaceTransport({ + workspaceKey: options.workspaceKey as string | undefined, + baseUrl: options.baseUrl as string | undefined, + }); return { - workspaceKey: resolveWorkspaceKey({ - workspaceKey: options.workspaceKey as string | undefined, - }), + workspaceKey, ...(baseUrl ? { baseUrl } : {}), }; } diff --git a/packages/cli/src/cli/lib/attach-fleet-node.test.ts b/packages/cli/src/cli/lib/attach-fleet-node.test.ts index 97ee09fc34..050b38751a 100644 --- a/packages/cli/src/cli/lib/attach-fleet-node.test.ts +++ b/packages/cli/src/cli/lib/attach-fleet-node.test.ts @@ -6,11 +6,15 @@ * under test is the actual runtime wiring rather than a mocked stand-in. */ import type { AddressInfo } from 'node:net'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { WebSocket as WsClient, WebSocketServer, type WebSocket as WsSocket } from 'ws'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { startFleetNodeAttachProxy, type FleetNodeAttachProxy } from './attach-fleet-node.js'; +import { writeProjectWorkspaceKey } from './project-workspace-key.js'; const SESSION_ID = 'session-under-test'; const RESUME_TOKEN = 'resume-token'; @@ -1186,6 +1190,77 @@ describe('startFleetNodeAttachProxy workspace-key precedence', () => { ); }); + it('binds a matching explicit canonical selector to its persisted isolated key and origin', async () => { + const remote = await startFakeRemote(); + cleanup.push(remote.close); + const ticket = capturingTicketFetch(remote.url); + const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-attach-project-')); + const priorProject = process.env.AGENT_RELAY_PROJECT; + process.env.AGENT_RELAY_PROJECT = projectRoot; + writeProjectWorkspaceKey(path.join(projectRoot, '.agentworkforce/relay'), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_isolated', + }); + + try { + const proxy = await startFleetNodeAttachProxy({ + agent: 'sandbox-worker', + node: 'agent37-codex', + mode: 'view', + baseUrl: 'https://agent37-cast.agentrelay.com/', + workspaceKey: 'rk_live_canonical', + env: {}, + fetch: ticket.fetch, + }); + cleanup.push(proxy.close); + + expect(ticket.authorization()).toBe('Bearer rk_live_isolated'); + expect(ticket.requestUrl()).toBe( + 'https://agent37-cast.agentrelay.com/v1/nodes/agent37-codex/terminal/sessions' + ); + } finally { + if (priorProject === undefined) delete process.env.AGENT_RELAY_PROJECT; + else process.env.AGENT_RELAY_PROJECT = priorProject; + fs.rmSync(projectRoot, { recursive: true, force: true }); + } + }); + + it('does not inherit a persisted route when an explicit key selects a different workspace', async () => { + const remote = await startFakeRemote(); + cleanup.push(remote.close); + const ticket = capturingTicketFetch(remote.url); + const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-attach-project-')); + const priorProject = process.env.AGENT_RELAY_PROJECT; + process.env.AGENT_RELAY_PROJECT = projectRoot; + writeProjectWorkspaceKey(path.join(projectRoot, '.agentworkforce/relay'), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_isolated', + }); + + try { + const proxy = await startFleetNodeAttachProxy({ + agent: 'other-worker', + node: 'other-node', + mode: 'view', + workspaceKey: 'rk_live_other', + env: {}, + fetch: ticket.fetch, + }); + cleanup.push(proxy.close); + + expect(ticket.authorization()).toBe('Bearer rk_live_other'); + expect(ticket.requestUrl()).toBe('https://cast.agentrelay.com/v1/nodes/other-node/terminal/sessions'); + } finally { + if (priorProject === undefined) delete process.env.AGENT_RELAY_PROJECT; + else process.env.AGENT_RELAY_PROJECT = priorProject; + fs.rmSync(projectRoot, { recursive: true, force: true }); + } + }); + it.each([ 'https://evil.example', 'http://cast.agentrelay.com', diff --git a/packages/cli/src/cli/lib/attach-fleet-node.ts b/packages/cli/src/cli/lib/attach-fleet-node.ts index db5630fc78..a8bdefa51a 100644 --- a/packages/cli/src/cli/lib/attach-fleet-node.ts +++ b/packages/cli/src/cli/lib/attach-fleet-node.ts @@ -18,7 +18,7 @@ import WebSocket, { WebSocketServer } from 'ws'; import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; import type { AttachMode } from './attach-mode.js'; import { collectWithRetry } from './collect-with-retry.js'; -import { resolveBaseUrl, resolveWorkspaceKey } from './sdk-client.js'; +import { resolveWorkspaceTransport } from './sdk-client.js'; const MAX_BUFFERED_BYTES = 1024 * 1024; const MAX_WEBSOCKET_CLOSE_REASON_BYTES = 123; @@ -315,8 +315,11 @@ export async function startFleetNodeAttachProxy( ): Promise { const env = options.env ?? process.env; const fetchFn = options.fetch ?? globalThis.fetch; - const workspaceKey = options.workspaceKey ?? resolveWorkspaceKey({ env }); - const requestedBaseUrl = resolveBaseUrl({ baseUrl: options.baseUrl, env }); + const { workspaceKey, baseUrl: requestedBaseUrl } = resolveWorkspaceTransport({ + workspaceKey: options.workspaceKey, + baseUrl: options.baseUrl, + env, + }); const baseUrl = validateFleetAttachBaseUrl(requestedBaseUrl ?? CANONICAL_RELAYCAST_ORIGIN); const nodePath = safeNodePath(options.node); const sessionEndpoint = `${baseUrl}/v1/nodes/${nodePath}/terminal/sessions`; diff --git a/packages/cli/src/cli/lib/sdk-client.test.ts b/packages/cli/src/cli/lib/sdk-client.test.ts index b15dd39287..3ea46ed4f1 100644 --- a/packages/cli/src/cli/lib/sdk-client.test.ts +++ b/packages/cli/src/cli/lib/sdk-client.test.ts @@ -12,6 +12,7 @@ import { resolveWorkspaceKey, resolveWorkspaceKeyWithSource, resolveWorkspaceSelection, + resolveWorkspaceTransport, } from './sdk-client.js'; import { setWorkspaceKey } from './workspace-store.js'; import { readProjectWorkspaceSession, writeProjectWorkspaceKey } from './project-workspace-key.js'; @@ -165,6 +166,109 @@ describe('sdk client option resolution', () => { expect(resolveBaseUrl(replayOptions)).toBe('https://agent37-cast.agentrelay.com'); }); + it.each(['flag', 'env'] as const)( + 'creates a fresh project target pin for an unpinned %s selection', + (source) => { + const selection = resolveWorkspaceSelection({ + ...(source === 'flag' ? { workspaceKey: 'rk_live_fresh' } : {}), + env: { + AGENT_RELAY_HOME: dir, + ...(source === 'env' ? { RELAY_WORKSPACE_KEY: 'rk_live_fresh' } : {}), + }, + }); + expect(selection).toMatchObject({ + key: 'rk_live_fresh', + source, + projectDataDir: projectDataDir(), + }); + + expect( + persistWorkspaceRelaycastTarget(selection, { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_fresh', + relaycastApiKey: 'rk_live_fresh_agent37', + }) + ).toBe(true); + expect(readProjectWorkspaceSession(projectDataDir())).toEqual({ + workspaceKey: 'rk_live_fresh', + workspaceId: 'rw_fresh', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_fresh_agent37', + }); + } + ); + + it('refuses to overwrite a project session rebound after workspace selection', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_original', { workspaceId: 'rw_original' }); + const selection = resolveWorkspaceSelection({ env: { AGENT_RELAY_HOME: dir } }); + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_rebound', { + workspaceId: 'rw_rebound', + enrolledNodeId: 'node_rebound', + }); + + expect( + persistWorkspaceRelaycastTarget(selection, { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_original', + relaycastApiKey: 'rk_live_original_agent37', + }) + ).toBe(false); + expect(readProjectWorkspaceSession(projectDataDir())).toEqual({ + workspaceKey: 'rk_live_rebound', + workspaceId: 'rw_rebound', + enrolledNodeId: 'node_rebound', + }); + }); + + it('does not replace a project pin created while a fresh explicit selection is in flight', () => { + const selection = resolveWorkspaceSelection({ + workspaceKey: 'rk_live_fresh', + env: { AGENT_RELAY_HOME: dir }, + }); + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_concurrent', { + workspaceId: 'rw_concurrent', + enrolledNodeId: 'node_concurrent', + }); + + expect( + persistWorkspaceRelaycastTarget(selection, { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_fresh', + relaycastApiKey: 'rk_live_fresh_agent37', + }) + ).toBe(false); + expect(readProjectWorkspaceSession(projectDataDir())).toEqual({ + workspaceKey: 'rk_live_concurrent', + workspaceId: 'rw_concurrent', + enrolledNodeId: 'node_concurrent', + }); + }); + + it('refuses to overwrite a target changed after workspace selection', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { workspaceId: 'rw_abc' }); + const selection = resolveWorkspaceSelection({ env: { AGENT_RELAY_HOME: dir } }); + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'canonical', + relaycastBaseUrl: 'https://cast.agentrelay.com', + relaycastApiKey: 'rk_live_newer_route', + }); + + expect( + persistWorkspaceRelaycastTarget(selection, { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_abc', + relaycastApiKey: 'rk_live_stale_route', + }) + ).toBe(false); + expect(readProjectWorkspaceSession(projectDataDir())?.relaycastApiKey).toBe('rk_live_newer_route'); + }); + it('keeps legacy persisted targets usable when no separate Relaycast key exists', () => { writeProjectWorkspaceKey(projectDataDir(), 'rk_live_legacy_agent37', { workspaceId: 'rw_abc', @@ -197,4 +301,46 @@ describe('sdk client option resolution', () => { expect(() => resolveBaseUrl({ env: { AGENT_RELAY_HOME: dir } })).toThrow(/not trusted/); }); + + it('normalizes an equivalent requested trailing slash against the persisted route', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + + expect( + resolveWorkspaceTransport({ + baseUrl: 'https://agent37-cast.agentrelay.com/', + env: { AGENT_RELAY_HOME: dir }, + }) + ).toEqual({ + workspaceKey: 'rk_live_agent37', + baseUrl: 'https://agent37-cast.agentrelay.com', + source: 'project', + }); + }); + + it.each([ + 'https://agent37-cast.agentrelay.com/path', + 'https://agent37-cast.agentrelay.com?query=1', + 'https://agent37-cast.agentrelay.com#fragment', + 'https://user:pass@agent37-cast.agentrelay.com', + 'https://agent37-cast.agentrelay.com:443', + 'https://agent37-cast.agentrelay.com:444', + 'https://agent37-cast.agentrelay.com/%2e%2e', + 'https://agent37-cast.agentrelay.com.attacker.example', + ])('rejects an unsafe requested URL before pairing it with a persisted route-scoped key', (baseUrl) => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + + expect(() => resolveWorkspaceTransport({ baseUrl, env: { AGENT_RELAY_HOME: dir } })).toThrow( + /trusted origin|does not match/ + ); + }); }); diff --git a/packages/cli/src/cli/lib/sdk-client.ts b/packages/cli/src/cli/lib/sdk-client.ts index 8995d9ddd2..0fd884cbf4 100644 --- a/packages/cli/src/cli/lib/sdk-client.ts +++ b/packages/cli/src/cli/lib/sdk-client.ts @@ -31,6 +31,12 @@ function trimOrUndefined(value: string | undefined): string | undefined { export type { WorkspaceKeySource }; export type { WorkspaceSelection }; +export type WorkspaceTransport = { + workspaceKey: string; + baseUrl?: string; + source: WorkspaceKeySource; +}; + /** Resolve the selected key and any previously persisted Relay workspace identity. */ export function resolveWorkspaceSelection(options: SdkClientOptions = {}): WorkspaceSelection | undefined { return resolveCloudWorkspaceSelection({ @@ -50,20 +56,8 @@ export function resolveWorkspaceKeyWithSource(options: SdkClientOptions = {}): { key: string; source: WorkspaceKeySource; } { - const selection = resolveWorkspaceSelection(options); - if (selection) { - // A Cloud workspace key remains the durable selector and Cloud credential. - // A persisted Relaycast target may carry a different route-scoped transport - // credential for SDK traffic on the same workspace. - validatePersistedRelaycastBaseUrl(selection); - return { - key: trimOrUndefined(selection.relaycastApiKey) ?? selection.key, - source: selection.source, - }; - } - throw new Error( - 'No workspace key found. Pass --workspace-key, set RELAY_WORKSPACE_KEY, or run `relay workspace set_key `.' - ); + const transport = resolveWorkspaceTransport(options); + return { key: transport.workspaceKey, source: transport.source }; } export function resolveWorkspaceKey(options: SdkClientOptions = {}): string { @@ -72,14 +66,59 @@ export function resolveWorkspaceKey(options: SdkClientOptions = {}): string { export function resolveBaseUrl(options: SdkClientOptions = {}): string | undefined { const selection = resolveWorkspaceSelection(options); + return resolveBaseUrlForSelection(selection, options); +} + +function resolveBaseUrlForSelection( + selection: WorkspaceSelection | undefined, + options: SdkClientOptions +): string | undefined { const persisted = validatePersistedRelaycastBaseUrl(selection); const requested = trimOrUndefined(options.baseUrl) ?? trimOrUndefined(env(options).RELAY_BASE_URL); - if (persisted && requested && requested !== persisted) { - throw new Error('The requested Relaycast base URL does not match the persisted workspace route.'); + if (persisted && requested) { + let parsed: URL; + try { + parsed = new URL(requested); + } catch { + throw new Error('The requested Relaycast base URL is invalid.'); + } + const authority = /^https:\/\/([^/?#]+)/i.exec(requested)?.[1] ?? ''; + if ( + !/^https:\/\/[^/?#]+\/?$/i.test(requested) || + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port || + /:\d+$/.test(authority) || + parsed.search || + parsed.hash || + (parsed.pathname !== '' && parsed.pathname !== '/') + ) { + throw new Error('The requested Relaycast base URL is not a trusted origin.'); + } + if (parsed.origin !== persisted) { + throw new Error('The requested Relaycast base URL does not match the persisted workspace route.'); + } } return persisted ?? requested; } +/** Resolve one credential/origin pair from one workspace selection. */ +export function resolveWorkspaceTransport(options: SdkClientOptions = {}): WorkspaceTransport { + const selection = resolveWorkspaceSelection(options); + if (!selection) { + throw new Error( + 'No workspace key found. Pass --workspace-key, set RELAY_WORKSPACE_KEY, or run `relay workspace set_key `.' + ); + } + const baseUrl = resolveBaseUrlForSelection(selection, options); + return { + workspaceKey: trimOrUndefined(selection.relaycastApiKey) ?? selection.key, + ...(baseUrl ? { baseUrl } : {}), + source: selection.source, + }; +} + function validatePersistedRelaycastBaseUrl(selection: WorkspaceSelection | undefined): string | undefined { const baseUrl = trimOrUndefined(selection?.relaycastBaseUrl); const route = selection?.relaycastRoute; @@ -116,7 +155,7 @@ function validatePersistedRelaycastBaseUrl(selection: WorkspaceSelection | undef return parsed.origin; } -/** Persist a server-selected target only for an existing project session. */ +/** Persist a server-selected target only while the captured project selection is still current. */ export function persistWorkspaceRelaycastTarget( selection: WorkspaceSelection | undefined, target: { @@ -126,52 +165,44 @@ export function persistWorkspaceRelaycastTarget( relaycastApiKey: string; } ): boolean { - const selectionWithProjectDir = selection as (WorkspaceSelection & { projectDataDir?: string }) | undefined; + if (!selection) return false; + const selectionWithProjectDir = selection as WorkspaceSelection & { projectDataDir?: string }; const dataDir = selectionWithProjectDir?.projectDataDir ?? (selection?.source === 'project' && selection.origin ? path.dirname(selection.origin) : undefined); if (!dataDir) return false; - const existing = readProjectWorkspaceSession(dataDir); - if (!existing) return false; - const restoreExisting = (): void => { - writeProjectWorkspaceKey(dataDir, existing.workspaceKey, { - ...(existing.enrolledNodeId ? { enrolledNodeId: existing.enrolledNodeId } : {}), - ...(existing.workspaceId ? { workspaceId: existing.workspaceId } : {}), - ...(existing.relaycastRoute ? { relaycastRoute: existing.relaycastRoute } : {}), - ...(existing.relaycastBaseUrl ? { relaycastBaseUrl: existing.relaycastBaseUrl } : {}), - ...(existing.relaycastApiKey ? { relaycastApiKey: existing.relaycastApiKey } : {}), - }); - }; - try { - writeProjectWorkspaceKey(dataDir, existing.workspaceKey, { - ...(existing.enrolledNodeId ? { enrolledNodeId: existing.enrolledNodeId } : {}), - workspaceId: target.workspaceId, - relaycastRoute: target.route, - relaycastBaseUrl: target.baseUrl, - relaycastApiKey: target.relaycastApiKey, - }); - const persisted = readProjectWorkspaceSession(dataDir); - if ( - persisted?.workspaceKey === existing.workspaceKey && - persisted.relaycastApiKey === target.relaycastApiKey && - persisted.workspaceId === target.workspaceId && - persisted.relaycastRoute === target.route && - persisted.relaycastBaseUrl === target.baseUrl - ) { - return true; - } - restoreExisting(); + const current = readProjectWorkspaceSession(dataDir); + if (selection.projectSessionPresent === false && current) return false; + if ( + current && + (current.workspaceKey !== selection.key || + current.workspaceId !== selection.workspaceId || + current.relaycastRoute !== selection.relaycastRoute || + current.relaycastBaseUrl !== selection.relaycastBaseUrl || + current.relaycastApiKey !== selection.relaycastApiKey) + ) { return false; - } catch (error) { - try { - restoreExisting(); - } catch (restoreError) { - throw new Error('Could not restore the prior Relaycast project session after persistence failed.', { - cause: restoreError, - }); - } - throw error; } + if (!current && (selection.projectSessionPresent === true || selection.source === 'project')) { + return false; + } + writeProjectWorkspaceKey(dataDir, selection.key, { + ...(current?.enrolledNodeId ? { enrolledNodeId: current.enrolledNodeId } : {}), + workspaceId: target.workspaceId, + relaycastRoute: target.route, + relaycastBaseUrl: target.baseUrl, + relaycastApiKey: target.relaycastApiKey, + }); + const persisted = readProjectWorkspaceSession(dataDir); + // Never restore a stale snapshot when verification loses a race: another + // process may have intentionally rebound the project after our write. + return ( + persisted?.workspaceKey === selection.key && + persisted.relaycastApiKey === target.relaycastApiKey && + persisted.workspaceId === target.workspaceId && + persisted.relaycastRoute === target.route && + persisted.relaycastBaseUrl === target.baseUrl + ); } export function resolveAgentToken(options: SdkClientOptions = {}): string | undefined { @@ -180,7 +211,8 @@ export function resolveAgentToken(options: SdkClientOptions = {}): string | unde /** Workspace-scoped client (no agent token). */ export function createWorkspaceRelay(options: SdkClientOptions = {}): AgentRelay { - return new AgentRelay({ workspaceKey: resolveWorkspaceKey(options), baseUrl: resolveBaseUrl(options) }); + const { workspaceKey, baseUrl } = resolveWorkspaceTransport(options); + return new AgentRelay({ workspaceKey, baseUrl }); } /** @@ -200,8 +232,6 @@ export function createAgentRelay(options: SdkClientOptions = {}): AgentRelayAgen baseUrl: resolveBaseUrl(options), }); } - return new AgentRelay({ - workspaceKey: resolveWorkspaceKey(options), - baseUrl: resolveBaseUrl(options), - }); + const { workspaceKey, baseUrl } = resolveWorkspaceTransport(options); + return new AgentRelay({ workspaceKey, baseUrl }); } diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index fbf4dfb79b..4dc9b15cdf 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -261,7 +261,7 @@ describe('Cloud fleet sandbox client', () => { expect(mocks.authorizedApiFetch).not.toHaveBeenCalled(); }); - it('fails closed when a newly provisioned response omits the Relaycast target', async () => { + it('accepts a legacy non-Agent37 provisioned response without a Relaycast target', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), @@ -290,7 +290,13 @@ describe('Cloud fleet sandbox client', () => { name: 'legacy-custom-node', workloadProfile: 'long-running-agent', }) - ).rejects.toThrow('missing relaycastTarget'); + ).resolves.toEqual( + expect.objectContaining({ + outcome: 'provisioned', + nodeName: 'legacy-custom-node', + relayWorkspaceId: 'rw_abc', + }) + ); const ensureBody = JSON.parse(String(mocks.authorizedApiFetch.mock.calls[1]?.[2]?.body)); expect(ensureBody).toMatchObject({ @@ -898,6 +904,54 @@ describe('Cloud fleet sandbox client', () => { } ); + it.each(PROVIDER_OUTCOME_MATRIX)( + 'accepts a legacy $providerId $outcome result without a Relaycast target', + async ({ providerId, outcome }) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + outcome === 'provisioned' + ? { + outcome, + providerId, + nodeId: `node-${providerId}`, + nodeName: `${providerId}-reviewer`, + sandboxId: `sandbox-${providerId}`, + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + } + : { + outcome, + providerId, + nodeId: `node-${providerId}`, + nodeName: `${providerId}-reviewer`, + status: 'online', + activeAgents: 0, + maxAgents: 1, + }, + { status: outcome === 'provisioned' ? 201 : 200 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId, + }) + ).resolves.toEqual( + expect.not.objectContaining({ + relaycastTarget: expect.anything(), + }) + ); + } + ); + it.each(PROVIDER_OUTCOME_MATRIX)( 'rejects an Agent37 Relaycast target for the $providerId $outcome result', async ({ providerId, outcome }) => { @@ -1002,6 +1056,41 @@ describe('Cloud fleet sandbox client', () => { ).rejects.toThrow(/missing the Agent37 Relaycast target/); }); + it('rejects a provisioned Agent37 response that omits its isolated target', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-agent37', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + }, + { status: 201 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + providerId: 'agent37', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + workloadProfile: 'long-running-agent', + }) + ).rejects.toThrow(/missing the Agent37 Relaycast target/); + }); + it('keeps router-selected responses forward compatible when no exact provider was requested', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index bd2cf59fd3..9a5a426867 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -126,8 +126,8 @@ export type CloudFleetSandboxReady = { sandboxId: string; providerSandboxId?: string; relayWorkspaceId: string; - /** Closed server-owned Relaycast contract for the provisioned node. */ - relaycastTarget: CloudFleetRelaycastTarget; + /** Closed server-owned Relaycast contract when Cloud returned one. Required for Agent37. */ + relaycastTarget?: CloudFleetRelaycastTarget; relayfileMounted: boolean; relayfileMountPath?: string; providerId?: CloudFleetSandboxProviderId; @@ -142,7 +142,7 @@ export type CloudFleetSandboxReused = { activeAgents: number | null; maxAgents: number | null; providerId?: CloudFleetSandboxProviderId; - /** Required when Cloud reused a node with a known provider. */ + /** Closed server-owned Relaycast contract when Cloud returned one. Required for Agent37. */ relaycastTarget?: CloudFleetRelaycastTarget; }; @@ -261,12 +261,7 @@ function assertProviderRelaycastTarget( } return; } - if (providerId !== undefined) { - if (!target) { - throw new Error( - `Cloud fleet sandbox response is missing the canonical Relaycast target for ${providerId}.` - ); - } + if (providerId !== undefined && target) { if (target.route !== 'canonical' || target.baseUrl !== CANONICAL_RELAYCAST_ORIGIN) { throw new Error( `Cloud fleet sandbox response mapped ${providerId} to a non-canonical Relaycast target.` @@ -451,8 +446,9 @@ function normalizeEnsureResult( const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); const providerSandboxId = readString(payload, 'providerSandboxId'); const relayWorkspaceId = requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'); - const relaycastTarget = normalizeRelaycastTarget(payload.relaycastTarget); - if (relaycastTarget.workspaceId !== relayWorkspaceId) { + const relaycastTarget = + payload.relaycastTarget === undefined ? undefined : normalizeRelaycastTarget(payload.relaycastTarget); + if (relaycastTarget !== undefined && relaycastTarget.workspaceId !== relayWorkspaceId) { throw new Error('Cloud fleet sandbox response has mismatched Relaycast workspace identities.'); } assertProviderRelaycastTarget(providerId, relaycastTarget); @@ -464,7 +460,7 @@ function normalizeEnsureResult( sandboxId, ...(providerSandboxId === undefined ? {} : { providerSandboxId }), relayWorkspaceId, - relaycastTarget, + ...(relaycastTarget === undefined ? {} : { relaycastTarget }), relayfileMounted: payload.relayfileMounted, ...(providerId === undefined ? {} : { providerId }), ...(readString(payload, 'relayfileMountPath') diff --git a/packages/cloud/src/project-workspace-key.test.ts b/packages/cloud/src/project-workspace-key.test.ts index 35a613d31c..b64680ba6d 100644 --- a/packages/cloud/src/project-workspace-key.test.ts +++ b/packages/cloud/src/project-workspace-key.test.ts @@ -130,6 +130,34 @@ describe('workspace precedence ladder diagnostics', () => { }); }); + it.each(['flag', 'env'] as const)( + 'exposes the empty project directory for a fresh %s selection', + (source) => { + const selection = resolveWorkspaceSelection({ + ...(source === 'flag' ? { workspaceKey: 'rk_fresh' } : {}), + projectDataDir: dataDir, + env: { + AGENT_RELAY_HOME: home, + ...(source === 'env' ? { RELAY_WORKSPACE_KEY: 'rk_fresh' } : {}), + }, + }); + + expect(selection).toMatchObject({ key: 'rk_fresh', source, projectDataDir: dataDir }); + } + ); + + it('does not expose a project directory when an explicit selection conflicts with its pin', () => { + writeProjectWorkspaceKey(dataDir, 'rk_project'); + + expect( + resolveWorkspaceSelection({ + workspaceKey: 'rk_other', + projectDataDir: dataDir, + env: { AGENT_RELAY_HOME: home }, + }) + ).not.toHaveProperty('projectDataDir'); + }); + it('names each source without leaking key material', () => { const env = { AGENT_RELAY_HOME: home, AGENT_RELAY_WORKSPACE_KEY: 'rk_env' }; setWorkspaceKey('global', 'rk_global', env); diff --git a/packages/cloud/src/project-workspace-key.ts b/packages/cloud/src/project-workspace-key.ts index 5d84715754..8a38bdc2f5 100644 --- a/packages/cloud/src/project-workspace-key.ts +++ b/packages/cloud/src/project-workspace-key.ts @@ -65,6 +65,8 @@ export interface WorkspaceSelection { relaycastApiKey?: string; /** Project session directory that can durably carry a server-selected target. */ projectDataDir?: string; + /** Whether that project session existed when this selection was captured. */ + projectSessionPresent?: boolean; } /** Absolute path to the workspace key recorded by `agent-relay node up`. */ @@ -260,7 +262,9 @@ export function resolveWorkspaceSelection( ...(project?.workspaceKey === flag && project.relaycastApiKey ? { relaycastApiKey: project.relaycastApiKey } : {}), - ...(project?.workspaceKey === flag && dataDir ? { projectDataDir: dataDir } : {}), + ...((!project || project.workspaceKey === flag) && dataDir + ? { projectDataDir: dataDir, projectSessionPresent: project !== undefined } + : {}), }; } @@ -283,7 +287,9 @@ export function resolveWorkspaceSelection( ...(project?.workspaceKey === envKey && project.relaycastApiKey ? { relaycastApiKey: project.relaycastApiKey } : {}), - ...(project?.workspaceKey === envKey && dataDir ? { projectDataDir: dataDir } : {}), + ...((!project || project.workspaceKey === envKey) && dataDir + ? { projectDataDir: dataDir, projectSessionPresent: project !== undefined } + : {}), }; } } @@ -297,7 +303,7 @@ export function resolveWorkspaceSelection( ...(project.relaycastRoute ? { relaycastRoute: project.relaycastRoute } : {}), ...(project.relaycastBaseUrl ? { relaycastBaseUrl: project.relaycastBaseUrl } : {}), ...(project.relaycastApiKey ? { relaycastApiKey: project.relaycastApiKey } : {}), - ...(dataDir ? { projectDataDir: dataDir } : {}), + ...(dataDir ? { projectDataDir: dataDir, projectSessionPresent: true } : {}), }; } From 1e1a649bb57859ea8b1a2a2c835d71082e335479 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 01:30:46 +0200 Subject: [PATCH 39/50] fix: serialize project target persistence --- packages/cli/src/cli/lib/sdk-client.ts | 31 +---- packages/cloud/src/index.ts | 1 + packages/cloud/src/project-workspace-key.ts | 132 ++++++++++++++++---- packages/cloud/src/workspace-key.ts | 1 + 4 files changed, 112 insertions(+), 53 deletions(-) diff --git a/packages/cli/src/cli/lib/sdk-client.ts b/packages/cli/src/cli/lib/sdk-client.ts index 0fd884cbf4..3d87d72092 100644 --- a/packages/cli/src/cli/lib/sdk-client.ts +++ b/packages/cli/src/cli/lib/sdk-client.ts @@ -4,8 +4,7 @@ import { AgentRelay, type AgentRelayAgent } from '@agent-relay/sdk'; import { AGENT37_RELAYCAST_ORIGIN, CANONICAL_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; import { resolveWorkspaceSelection as resolveCloudWorkspaceSelection, - readProjectWorkspaceSession, - writeProjectWorkspaceKey, + writeProjectWorkspaceTargetIfSelectionCurrent, type WorkspaceSelection, type WorkspaceKeySource, } from '@agent-relay/cloud/workspace-key'; @@ -171,38 +170,12 @@ export function persistWorkspaceRelaycastTarget( selectionWithProjectDir?.projectDataDir ?? (selection?.source === 'project' && selection.origin ? path.dirname(selection.origin) : undefined); if (!dataDir) return false; - const current = readProjectWorkspaceSession(dataDir); - if (selection.projectSessionPresent === false && current) return false; - if ( - current && - (current.workspaceKey !== selection.key || - current.workspaceId !== selection.workspaceId || - current.relaycastRoute !== selection.relaycastRoute || - current.relaycastBaseUrl !== selection.relaycastBaseUrl || - current.relaycastApiKey !== selection.relaycastApiKey) - ) { - return false; - } - if (!current && (selection.projectSessionPresent === true || selection.source === 'project')) { - return false; - } - writeProjectWorkspaceKey(dataDir, selection.key, { - ...(current?.enrolledNodeId ? { enrolledNodeId: current.enrolledNodeId } : {}), + return writeProjectWorkspaceTargetIfSelectionCurrent(dataDir, selection, { workspaceId: target.workspaceId, relaycastRoute: target.route, relaycastBaseUrl: target.baseUrl, relaycastApiKey: target.relaycastApiKey, }); - const persisted = readProjectWorkspaceSession(dataDir); - // Never restore a stale snapshot when verification loses a race: another - // process may have intentionally rebound the project after our write. - return ( - persisted?.workspaceKey === selection.key && - persisted.relaycastApiKey === target.relaycastApiKey && - persisted.workspaceId === target.workspaceId && - persisted.relaycastRoute === target.route && - persisted.relaycastBaseUrl === target.baseUrl - ); } export function resolveAgentToken(options: SdkClientOptions = {}): string | undefined { diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index cf6450788c..a110409687 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -160,6 +160,7 @@ export { resolveWorkspaceKeyWithSource, resolveWorkspaceSelection, writeProjectWorkspaceKey, + writeProjectWorkspaceTargetIfSelectionCurrent, type ProjectWorkspaceSession, type ResolveWorkspaceKeyOptions, type WorkspaceKeyFileSystem, diff --git a/packages/cloud/src/project-workspace-key.ts b/packages/cloud/src/project-workspace-key.ts index 8a38bdc2f5..8fb3dc5d00 100644 --- a/packages/cloud/src/project-workspace-key.ts +++ b/packages/cloud/src/project-workspace-key.ts @@ -7,6 +7,11 @@ import { getProjectPaths } from '@agent-relay/config'; import { readWorkspaceStore, workspaceStorePath } from './workspace-store.js'; const PROJECT_WORKSPACE_KEY_FILENAME = 'workspace-key.json'; +const PROJECT_WORKSPACE_LOCK_SUFFIX = '.lock'; +const PROJECT_WORKSPACE_LOCK_TIMEOUT_MS = 2_000; +const PROJECT_WORKSPACE_LOCK_STALE_MS = 30_000; +const PROJECT_WORKSPACE_LOCK_RETRY_MS = 10; +const PROJECT_WORKSPACE_LOCK_WAIT = new Int32Array(new SharedArrayBuffer(4)); /** Workspace-key environment aliases, highest precedence first. */ const WORKSPACE_KEY_ENV_VARS = ['RELAY_WORKSPACE_KEY', 'AGENT_RELAY_WORKSPACE_KEY', 'RELAY_API_KEY'] as const; @@ -127,6 +132,16 @@ export function writeProjectWorkspaceKey( relaycastBaseUrl?: string; relaycastApiKey?: string; } = {} +): void { + const key = trimOrUndefined(workspaceKey); + if (!key) return; + withProjectWorkspaceKeyLock(dataDir, () => writeProjectWorkspaceKeyUnlocked(dataDir, key, options)); +} + +function writeProjectWorkspaceKeyUnlocked( + dataDir: string, + workspaceKey: string, + options: ProjectWorkspaceSessionMetadata = {} ): void { const key = trimOrUndefined(workspaceKey); if (!key) return; @@ -176,6 +191,73 @@ export function writeProjectWorkspaceKey( } } +function withProjectWorkspaceKeyLock(dataDir: string, callback: () => T): T { + fs.mkdirSync(dataDir, { recursive: true, mode: 0o700 }); + const lockDir = `${projectWorkspaceKeyPath(dataDir)}${PROJECT_WORKSPACE_LOCK_SUFFIX}`; + const startedAt = Date.now(); + while (true) { + try { + fs.mkdirSync(lockDir, { mode: 0o700 }); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + try { + if (Date.now() - fs.statSync(lockDir).mtimeMs >= PROJECT_WORKSPACE_LOCK_STALE_MS) { + fs.rmSync(lockDir, { recursive: true, force: true }); + continue; + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue; + throw error; + } + if (Date.now() - startedAt >= PROJECT_WORKSPACE_LOCK_TIMEOUT_MS) { + throw new Error(`Timed out waiting for the project workspace lock at ${lockDir}.`); + } + Atomics.wait(PROJECT_WORKSPACE_LOCK_WAIT, 0, 0, PROJECT_WORKSPACE_LOCK_RETRY_MS); + } + try { + return callback(); + } finally { + fs.rmSync(lockDir, { recursive: true, force: true }); + } +} + +/** Atomically persist a Relaycast target only if the captured project selection is still current. */ +export function writeProjectWorkspaceTargetIfSelectionCurrent( + dataDir: string, + selection: WorkspaceSelection, + target: Required< + Pick + > +): boolean { + return withProjectWorkspaceKeyLock(dataDir, () => { + const current = readProjectWorkspaceSession(dataDir); + if (selection.projectSessionPresent === false && current) return false; + if ( + current && + (current.workspaceKey !== selection.key || + current.workspaceId !== selection.workspaceId || + current.relaycastRoute !== selection.relaycastRoute || + current.relaycastBaseUrl !== selection.relaycastBaseUrl || + current.relaycastApiKey !== selection.relaycastApiKey) + ) { + return false; + } + if (!current && (selection.projectSessionPresent === true || selection.source === 'project')) { + return false; + } + writeProjectWorkspaceKeyUnlocked(dataDir, selection.key, { + ...(current?.enrolledNodeId ? { enrolledNodeId: current.enrolledNodeId } : {}), + workspaceId: target.workspaceId, + relaycastRoute: target.relaycastRoute, + relaycastBaseUrl: target.relaycastBaseUrl, + relaycastApiKey: target.relaycastApiKey, + }); + return true; + }); +} + /** * Rewrite a project session while retaining server-selected metadata when the * workspace key is unchanged. A changed key is an intentional rebind, so no @@ -194,31 +276,33 @@ export function writeProjectWorkspaceKeyPreservingSession( const key = trimOrUndefined(workspaceKey); if (!key) return; - const existing = readProjectWorkspaceSession(dataDir); - const sameWorkspace = existing?.workspaceKey === key; - const retained: ProjectWorkspaceSessionMetadata = sameWorkspace - ? { - ...(existing?.enrolledNodeId ? { enrolledNodeId: existing.enrolledNodeId } : {}), - ...(existing?.workspaceId ? { workspaceId: existing.workspaceId } : {}), - ...(existing?.relaycastRoute ? { relaycastRoute: existing.relaycastRoute } : {}), - ...(existing?.relaycastBaseUrl ? { relaycastBaseUrl: existing.relaycastBaseUrl } : {}), - ...(existing?.relaycastApiKey ? { relaycastApiKey: existing.relaycastApiKey } : {}), - } - : {}; + withProjectWorkspaceKeyLock(dataDir, () => { + const existing = readProjectWorkspaceSession(dataDir); + const sameWorkspace = existing?.workspaceKey === key; + const retained: ProjectWorkspaceSessionMetadata = sameWorkspace + ? { + ...(existing?.enrolledNodeId ? { enrolledNodeId: existing.enrolledNodeId } : {}), + ...(existing?.workspaceId ? { workspaceId: existing.workspaceId } : {}), + ...(existing?.relaycastRoute ? { relaycastRoute: existing.relaycastRoute } : {}), + ...(existing?.relaycastBaseUrl ? { relaycastBaseUrl: existing.relaycastBaseUrl } : {}), + ...(existing?.relaycastApiKey ? { relaycastApiKey: existing.relaycastApiKey } : {}), + } + : {}; - writeProjectWorkspaceKey(dataDir, key, { - ...retained, - ...(trimOrUndefined(options.enrolledNodeId) - ? { enrolledNodeId: trimOrUndefined(options.enrolledNodeId) } - : {}), - ...(trimOrUndefined(options.workspaceId) ? { workspaceId: trimOrUndefined(options.workspaceId) } : {}), - ...(options.relaycastRoute ? { relaycastRoute: options.relaycastRoute } : {}), - ...(trimOrUndefined(options.relaycastBaseUrl) - ? { relaycastBaseUrl: trimOrUndefined(options.relaycastBaseUrl) } - : {}), - ...(trimOrUndefined(options.relaycastApiKey) - ? { relaycastApiKey: trimOrUndefined(options.relaycastApiKey) } - : {}), + writeProjectWorkspaceKeyUnlocked(dataDir, key, { + ...retained, + ...(trimOrUndefined(options.enrolledNodeId) + ? { enrolledNodeId: trimOrUndefined(options.enrolledNodeId) } + : {}), + ...(trimOrUndefined(options.workspaceId) ? { workspaceId: trimOrUndefined(options.workspaceId) } : {}), + ...(options.relaycastRoute ? { relaycastRoute: options.relaycastRoute } : {}), + ...(trimOrUndefined(options.relaycastBaseUrl) + ? { relaycastBaseUrl: trimOrUndefined(options.relaycastBaseUrl) } + : {}), + ...(trimOrUndefined(options.relaycastApiKey) + ? { relaycastApiKey: trimOrUndefined(options.relaycastApiKey) } + : {}), + }); }); } diff --git a/packages/cloud/src/workspace-key.ts b/packages/cloud/src/workspace-key.ts index 54235b5c02..79d0761e11 100644 --- a/packages/cloud/src/workspace-key.ts +++ b/packages/cloud/src/workspace-key.ts @@ -8,6 +8,7 @@ export { resolveWorkspaceSelection, writeProjectWorkspaceKey, writeProjectWorkspaceKeyPreservingSession, + writeProjectWorkspaceTargetIfSelectionCurrent, type ProjectWorkspaceSession, type ProjectWorkspaceSessionMetadata, type ResolveWorkspaceKeyOptions, From bfda6e63fe596bedb870b2aa9bba97b7c9929d4f Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 01:59:11 +0200 Subject: [PATCH 40/50] fix(relaycast): retry only precommit failures --- CHANGELOG.md | 1 + crates/broker/src/relaycast/auth.rs | 156 ++++++++++++++++++++++++++-- 2 files changed, 148 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1998862fdc..94516defb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Relaycast credentials and origins now resolve as one transport pair for attach, observer, Fleet, and Relayfile provisioning commands; stale sandbox responses cannot overwrite a project workspace that was rebound while provisioning was in flight. - Legacy non-Agent37 Cloud sandbox responses remain usable when they omit the newer Relaycast target, while any target Cloud does return is verified and persisted for both newly provisioned and reused providers. - Standalone package smoke workspaces remain valid for five minutes, and startup overrides are capped at four minutes so shutdown and cleanup verification always retain a full-minute lease margin. +- Startup retries now require Relaycast's typed pre-commit storage-admission codes instead of replaying every 5xx response from an unkeyed workspace or agent-registration request. ## [11.10.4] - 2026-09-08 diff --git a/crates/broker/src/relaycast/auth.rs b/crates/broker/src/relaycast/auth.rs index 46f7ac196c..d5a0f4408d 100644 --- a/crates/broker/src/relaycast/auth.rs +++ b/crates/broker/src/relaycast/auth.rs @@ -933,17 +933,26 @@ const RELAYCAST_HTTP_TIMEOUT: std::time::Duration = std::time::Duration::from_se /// 34099838274 lost three jobs to exactly that. const TRANSIENT_STARTUP_RETRY_BACKOFFS_MS: [u64; 2] = [200, 400]; -/// The server-side statuses worth replaying: 500, 502, 503, 504. A 501 is a -/// contract mismatch rather than a transient and is deliberately excluded, as -/// are transport errors — a timed-out `POST /v1/agents` may already have -/// created the agent, and re-sending it is the AR-448 duplicate shape. +/// Replay only server failures whose typed error code establishes that the +/// request failed at the storage-admission boundary. Retrying every 5xx by +/// status is unsafe for these unkeyed POSTs: an application-level 503 or a 500 +/// returned after commit could create the AR-448 duplicate shape when replayed. +/// +/// `database_overloaded` is Relaycast's D1 admission failure and +/// `workspace_storage_unavailable` is emitted when workspace persistence never +/// starts. Both are explicitly pre-commit contracts. Transport errors remain +/// terminal because a timed-out request may already have committed. fn is_transient_server_error(error: &RelayError) -> bool { matches!( error, RelayError::Api { + code, status: 500 | 502 | 503 | 504, .. - } + } if matches!( + code.trim(), + "database_overloaded" | "workspace_storage_unavailable" + ) ) } @@ -1535,10 +1544,10 @@ mod tests { use super::{ hash_identity_key, is_agent_token_invalid, is_agent_token_invalid_anyhow, - is_agent_token_invalid_code, reclaim_legacy_identity, relay_error_to_anyhow, - relay_request_with_timeout, resolve_relaycast_base_url, retry_transient_relay_error, - stable_node_identity_key, AuthClient, AuthHttpError, CredentialCache, - AGENT_TOKEN_INVALID_CODE, DEFAULT_RELAYCAST_BASE_URL, + is_agent_token_invalid_code, is_transient_server_error, reclaim_legacy_identity, + relay_error_to_anyhow, relay_request_with_timeout, resolve_relaycast_base_url, + retry_transient_relay_error, stable_node_identity_key, AuthClient, AuthHttpError, + CredentialCache, AGENT_TOKEN_INVALID_CODE, DEFAULT_RELAYCAST_BASE_URL, }; use relaycast::RelayError; @@ -2076,6 +2085,135 @@ mod tests { } } + #[tokio::test] + async fn unknown_application_503_is_not_retried() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let calls = AtomicUsize::new(0); + let error = retry_transient_relay_error("probing an application failure", || { + calls.fetch_add(1, Ordering::SeqCst); + async { + Err::<(), _>(RelayError::api( + "application_temporarily_unavailable", + "the application rejected the request", + 503, + )) + } + }) + .await + .expect_err("an unclassified 503 must not replay an unkeyed POST"); + + assert_eq!(calls.load(Ordering::SeqCst), 1); + match error { + RelayError::Api { + code, + status, + attempts, + .. + } => { + assert_eq!(code, "application_temporarily_unavailable"); + assert_eq!(status, 503); + assert_eq!(attempts, 1); + } + other => panic!("expected the original terminal API error, got {other}"), + } + } + + #[test] + fn transient_retry_requires_both_a_safe_code_and_server_status() { + assert!(is_transient_server_error(&RelayError::api( + "database_overloaded", + "overloaded", + 503, + ))); + assert!(is_transient_server_error(&RelayError::api( + "workspace_storage_unavailable", + "storage unavailable", + 502, + ))); + assert!(!is_transient_server_error(&RelayError::api( + "database_overloaded", + "conflict", + 409, + ))); + assert!(!is_transient_server_error(&RelayError::api( + "unknown_error", + "server failure", + 500, + ))); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn unknown_application_503_exits_registration_without_a_replay() { + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + + use axum::{ + extract::State, http::StatusCode as AxumStatusCode, routing::post, Json, Router, + }; + + async fn reject_registration( + State(attempts): State>, + ) -> (AxumStatusCode, Json) { + attempts.fetch_add(1, Ordering::SeqCst); + ( + AxumStatusCode::SERVICE_UNAVAILABLE, + Json(json!({ + "ok": false, + "error": { + "code": "application_temporarily_unavailable", + "message": "The request could not be completed." + } + })), + ) + } + + let _env_guard = clear_relay_env(); + // SAFETY: test-only, serialized by RELAY_ENV_MUTEX via clear_relay_env. + unsafe { + std::env::set_var("AGENT_RELAY_WORKSPACE_KEY", "rk_live_env"); + } + + let attempts = Arc::new(AtomicUsize::new(0)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server_state = attempts.clone(); + let server = tokio::spawn(async move { + axum::serve( + listener, + Router::new() + .route("/v1/agents", post(reject_registration)) + .with_state(server_state), + ) + .await + }); + + let error = AuthClient::new(Some(format!("http://{address}"))) + .startup_session(Some("lead")) + .await + .expect_err("an unclassified application 503 must be terminal"); + + let message = format!("{error:#}"); + assert!( + message.contains("application_temporarily_unavailable"), + "{message}" + ); + assert!(message.contains("attempts: 1"), "{message}"); + assert_eq!( + attempts.load(Ordering::SeqCst), + 1, + "an unkeyed registration must not be replayed on an unknown 503" + ); + + server.abort(); + // SAFETY: test-only, serialized by RELAY_ENV_MUTEX via clear_relay_env. + unsafe { + std::env::remove_var("AGENT_RELAY_WORKSPACE_KEY"); + } + } + /// The retry budget is bounded, and the terminal error still carries the /// registration diagnostics PR #1673 added. An operator chasing a sandbox /// worker that spawns but never registers (AgentWorkforce/cloud#3401) From 7bc5edf727b5e4b6a1a2067a2bc7951742b5cb73 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 02:37:39 +0200 Subject: [PATCH 41/50] fix: close relayflow transport and proof review gaps --- .../cli/src/cli/ci-standalone-smoke.test.ts | 13 ++ packages/cli/src/cli/commands/fleet.test.ts | 106 ++++++++++++++++- packages/cli/src/cli/commands/fleet.ts | 27 +++-- packages/cli/src/cli/lib/sdk-client.ts | 18 ++- .../cloud/src/project-workspace-key.test.ts | 47 +++++++- packages/cloud/src/project-workspace-key.ts | 111 +++++++++++++++++- scripts/ci-standalone-smoke.sh | 8 +- scripts/pr-proof/run-cloud.mjs | 45 ++++--- tests/fixtures/pr-proof-contract.test.ts | 34 +++--- 9 files changed, 357 insertions(+), 52 deletions(-) diff --git a/packages/cli/src/cli/ci-standalone-smoke.test.ts b/packages/cli/src/cli/ci-standalone-smoke.test.ts index 189bd6eec2..b5d52e6b0e 100644 --- a/packages/cli/src/cli/ci-standalone-smoke.test.ts +++ b/packages/cli/src/cli/ci-standalone-smoke.test.ts @@ -34,6 +34,7 @@ url= while [ "\$#" -gt 0 ]; do case "\$1" in --output) output="\$2"; shift 2 ;; + --connect-timeout|--max-time) shift 2 ;; --request) method="\$2"; shift 2 ;; --write-out) shift 2 ;; --data|--header) shift 2 ;; @@ -154,6 +155,10 @@ describe('ci-standalone-smoke workspace reuse', () => { const script = readFileSync(smokeScript, 'utf8'); expect(script).toContain('TRUSTED_RELAY_BASE_URL="https://cast.agentrelay.com"'); expect(script).toContain('WORKSPACE_LEASE_SECONDS=300'); + expect(script).toContain('CURL_CONNECT_TIMEOUT_SECONDS=10'); + expect(script).toContain('CURL_MAX_TIME_SECONDS=60'); + expect(script).toContain('--connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS"'); + expect(script).toContain('--max-time "$CURL_MAX_TIME_SECONDS"'); expect(script).toContain('expires_in_seconds: $expires'); expect(script).toContain('printf \'::add-mask::%s\\n\' "$WORKSPACE_KEY"'); expect(script).toContain('--request DELETE'); @@ -163,6 +168,14 @@ describe('ci-standalone-smoke workspace reuse', () => { } }); + it('reports the create HTTP status and sanitized error code when no key is returned', () => { + const script = readFileSync(smokeScript, 'utf8'); + expect(script).toContain('CREATE_ERROR_CODE='); + expect(script).toContain('did not contain an API key (HTTP ${CREATE_STATUS:-unknown}'); + expect(script).toContain('error code ${CREATE_ERROR_CODE}'); + expect(script).not.toContain('cat "$WORKSPACE_RESPONSE"'); + }); + it('keeps the outer startup deadline above the broker aggregate handshake budget', () => { const script = readFileSync(smokeScript, 'utf8'); const brokerSession = readFileSync(resolve('crates/broker/src/runtime/session.rs'), 'utf8'); diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index bc924efdbe..b670548df2 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -881,9 +881,11 @@ describe('fleet command support', () => { }); it('applies and persists a returned target for a reused non-Agent37 provider', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', 'at_live_canonical_ambient'); const placement = { spawn: vi.fn(async () => ({ invocationId: 'inv_reused_e2b', node: { name: 'e2b-codex' } })), }; + const createAgentRelay = vi.fn(() => ({ messaging: { placement } })); const register = vi.fn(async () => ({ token: 'at_live_launcher' })); const release = vi.fn(async () => ({ released: true, deleted: true })); const createWorkspaceRelay = vi.fn(() => ({ @@ -894,7 +896,7 @@ describe('fleet command support', () => { program.exitOverride(); registerFleetCommands(program, { sdk: { - createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, + createAgentRelay: createAgentRelay as never, createWorkspaceRelay: createWorkspaceRelay as never, createWorkspace: vi.fn() as never, log: vi.fn(), @@ -934,6 +936,7 @@ describe('fleet command support', () => { '--sandbox', '--sandbox-provider', 'e2b', + '--no-sandbox-relayfile', '--workspace-id', 'rw_abc', '--name', @@ -950,12 +953,106 @@ describe('fleet command support', () => { workspaceKey: CANONICAL_RELAYCAST_TARGET.relaycastApiKey, baseUrl: CANONICAL_RELAYCAST_TARGET.baseUrl, }); + expect(register).toHaveBeenCalledWith( + expect.objectContaining({ name: expect.stringMatching(/^fleet-spawn-launcher-/) }), + { strict: true } + ); + expect(createAgentRelay).toHaveBeenCalledWith({ + workspaceKey: CANONICAL_RELAYCAST_TARGET.relaycastApiKey, + token: 'at_live_launcher', + baseUrl: CANONICAL_RELAYCAST_TARGET.baseUrl, + }); expect(persistWorkspaceRelaycastTarget).toHaveBeenCalledWith( expect.objectContaining({ key: 'rk_live_test' }), CANONICAL_RELAYCAST_TARGET ); }); + it('does not inherit a persisted Agent37 target for a legacy sandbox without a returned target', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', 'at_live_unproven_ambient'); + const placement = { + spawn: vi.fn(async () => ({ invocationId: 'inv_legacy_reused', node: { name: 'e2b-codex' } })), + }; + const createAgentRelay = vi.fn(() => ({ messaging: { placement } })); + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { + register: vi.fn(async () => ({ token: 'at_live_legacy_launcher' })), + release: vi.fn(async () => ({ released: true, deleted: true })), + }, + })); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: createAgentRelay as never, + createWorkspaceRelay: createWorkspaceRelay as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => ({ + outcome: 'reused' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-e2b', + nodeName: 'e2b-codex', + status: 'online', + activeAgents: 0, + maxAgents: 1, + providerId: 'e2b' as const, + })), + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: '/tmp/agent-relay-test/workspace-key.json', + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: AGENT37_RELAYCAST_TARGET.baseUrl, + relaycastApiKey: AGENT37_RELAYCAST_TARGET.relaycastApiKey, + }), + persistWorkspaceRelaycastTarget: vi.fn(() => true), + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-provider', + 'e2b', + '--no-sandbox-relayfile', + '--workspace-id', + 'rw_abc', + '--name', + 'legacy-reused-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + + expect(createWorkspaceRelay).toHaveBeenCalledWith({ + workspaceKey: 'rk_live_test', + token: undefined, + baseUrl: undefined, + ignorePersistedRelaycastTarget: true, + }); + expect(createAgentRelay).toHaveBeenCalledWith({ + workspaceKey: 'rk_live_test', + token: 'at_live_legacy_launcher', + baseUrl: undefined, + ignorePersistedRelaycastTarget: true, + }); + }); + it('fleet spawn --sandbox cleans up when the target cannot be persisted', async () => { const deleteCloudFleetSandbox = vi.fn(async () => undefined); const ensureCloudFleetSandbox = vi.fn(async () => ({ @@ -1183,7 +1280,12 @@ describe('fleet command support', () => { const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0]; expect(persistWorkspaceRelaycastTarget).not.toHaveBeenCalled(); - expect(createWorkspaceRelay).toHaveBeenCalledWith({ workspaceKey: 'rk_live_test' }); + expect(createWorkspaceRelay).toHaveBeenCalledWith({ + workspaceKey: 'rk_live_test', + token: undefined, + baseUrl: undefined, + ignorePersistedRelaycastTarget: true, + }); expect(ensureInput).toMatchObject({ workspaceId: 'rw_abc', requiredCapability: 'spawn:codex', diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 4b8e695648..c7f7907e1d 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -272,19 +272,24 @@ export function registerFleetCommands( let sandbox: EnsureCloudFleetSandboxResult | undefined; let workspaceRelay: ReturnType | undefined; let relaycastClientOptions = clientOptions; + let legacyWorkspaceClientOptions = clientOptions; if (useSandbox) { // Cloud must be the first network authority for a sandbox invocation. // A canonical Relaycast info call would both leak the workspace key and // make it impossible to prove that Cloud's isolated target is the one // subsequently used for registration and dispatch. const workspaceSelection = deps.resolveWorkspaceSelection(clientOptions); + legacyWorkspaceClientOptions = { + ...clientOptions, + ...(sandboxProvider === 'agent37' ? {} : { ignorePersistedRelaycastTarget: true }), + }; let relayWorkspaceId = explicitWorkspaceId ?? workspaceSelection?.workspaceId?.trim(); // Legacy providers remain backward compatible: they may resolve the // workspace from canonical Relaycast. Agent37 may not, because even a // read there mutates rate-limit/presence accounting on the shared // service and defeats the zero-shared-traffic canary proof. if (!relayWorkspaceId && sandboxProvider !== 'agent37') { - workspaceRelay = deps.sdk.createWorkspaceRelay(clientOptions); + workspaceRelay = deps.sdk.createWorkspaceRelay(legacyWorkspaceClientOptions); const workspaceInfo = await workspaceRelay.workspace.info(); relayWorkspaceId = workspaceInfo.id?.trim(); } @@ -413,6 +418,12 @@ export function registerFleetCommands( } throw error; } + } else if (sandbox.outcome !== 'provisioning_timeout') { + // Older non-Agent37 Cloud responses can omit a target. In that + // compatibility case, keep every subsequent client on the canonical + // workspace selection; a stale persisted Agent37 target must not + // leak into registration, dispatch, or launcher release. + relaycastClientOptions = legacyWorkspaceClientOptions; } if (sandbox.outcome === 'provisioning_timeout') { await deps @@ -465,13 +476,15 @@ export function registerFleetCommands( // Agent tokens are scoped to a Relaycast deployment. Never replay a // canonical token after Cloud has selected the isolated shard; mint // a temporary launcher on the validated target instead. - let agentToken = - sandbox?.outcome === 'provisioned' || - (sandbox?.outcome === 'reused' && sandbox.providerId === 'agent37') - ? undefined - : resolveAgentToken(clientOptions); + // A sandbox dispatch always mints a launcher on the transport Cloud + // selected (or the canonical compatibility transport when an older + // non-Agent37 response omitted the target). Ambient agent tokens do + // not carry enough provenance to prove they belong to that transport. + let agentToken = sandbox ? undefined : resolveAgentToken(clientOptions); if (!agentToken) { - workspaceRelay ??= deps.sdk.createWorkspaceRelay(clientOptions); + workspaceRelay ??= deps.sdk.createWorkspaceRelay( + sandbox?.relaycastTarget ? relaycastClientOptions : legacyWorkspaceClientOptions + ); const pendingLauncherName = `fleet-spawn-launcher-${randomUUID().slice(0, 8)}`; const launcher = await workspaceRelay.workspace.register( { diff --git a/packages/cli/src/cli/lib/sdk-client.ts b/packages/cli/src/cli/lib/sdk-client.ts index 3d87d72092..2a0ad66370 100644 --- a/packages/cli/src/cli/lib/sdk-client.ts +++ b/packages/cli/src/cli/lib/sdk-client.ts @@ -15,6 +15,8 @@ export interface SdkClientOptions { token?: string; baseUrl?: string; env?: NodeJS.ProcessEnv; + /** Use the canonical gateway instead of a persisted server-selected route. */ + ignorePersistedRelaycastTarget?: boolean; } function env(options: SdkClientOptions): NodeJS.ProcessEnv { @@ -64,10 +66,22 @@ export function resolveWorkspaceKey(options: SdkClientOptions = {}): string { } export function resolveBaseUrl(options: SdkClientOptions = {}): string | undefined { - const selection = resolveWorkspaceSelection(options); + const selection = selectionForTransport(options); return resolveBaseUrlForSelection(selection, options); } +function selectionForTransport(options: SdkClientOptions): WorkspaceSelection | undefined { + const selection = resolveWorkspaceSelection(options); + if (!selection || !options.ignorePersistedRelaycastTarget) return selection; + const { + relaycastRoute: _relaycastRoute, + relaycastBaseUrl: _relaycastBaseUrl, + relaycastApiKey: _relaycastApiKey, + ...canonicalSelection + } = selection; + return canonicalSelection; +} + function resolveBaseUrlForSelection( selection: WorkspaceSelection | undefined, options: SdkClientOptions @@ -104,7 +118,7 @@ function resolveBaseUrlForSelection( /** Resolve one credential/origin pair from one workspace selection. */ export function resolveWorkspaceTransport(options: SdkClientOptions = {}): WorkspaceTransport { - const selection = resolveWorkspaceSelection(options); + const selection = selectionForTransport(options); if (!selection) { throw new Error( 'No workspace key found. Pass --workspace-key, set RELAY_WORKSPACE_KEY, or run `relay workspace set_key `.' diff --git a/packages/cloud/src/project-workspace-key.test.ts b/packages/cloud/src/project-workspace-key.test.ts index b64680ba6d..2ec4617be9 100644 --- a/packages/cloud/src/project-workspace-key.test.ts +++ b/packages/cloud/src/project-workspace-key.test.ts @@ -2,7 +2,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { projectWorkspaceKeyPath, @@ -36,6 +36,51 @@ describe('project workspace key resolution', () => { expect(fs.statSync(projectWorkspaceKeyPath(dataDir)).mode & 0o777).toBe(0o600); }); + it('does not remove a replacement lock when the original holder finishes', () => { + const lockDir = `${projectWorkspaceKeyPath(dataDir)}.lock`; + let replaced = false; + const originalRename = fs.renameSync.bind(fs); + const rename = vi.spyOn(fs, 'renameSync').mockImplementation((source, destination) => { + originalRename(source, destination); + if (!replaced && destination === projectWorkspaceKeyPath(dataDir)) { + replaced = true; + fs.rmSync(lockDir, { recursive: true, force: true }); + fs.mkdirSync(lockDir, { mode: 0o700 }); + fs.writeFileSync(path.join(lockDir, 'replacement-owner'), '', { mode: 0o600, flag: 'wx' }); + } + }); + + try { + writeProjectWorkspaceKey(dataDir, 'rk_project'); + } finally { + rename.mockRestore(); + } + + expect(fs.existsSync(path.join(lockDir, 'replacement-owner'))).toBe(true); + }); + + it('does not reclaim a stale lock whose recorded owner is still alive', () => { + const lockDir = `${projectWorkspaceKeyPath(dataDir)}.lock`; + const token = 'live-owner-token'; + fs.mkdirSync(lockDir, { recursive: true, mode: 0o700 }); + fs.writeFileSync(path.join(lockDir, token), JSON.stringify({ version: 1, pid: process.pid, token }), { + mode: 0o600, + flag: 'wx', + }); + const staleAt = new Date(Date.now() - 60_000); + fs.utimesSync(lockDir, staleAt, staleAt); + const kill = vi.spyOn(process, 'kill').mockImplementation(() => true); + + try { + expect(() => writeProjectWorkspaceKey(dataDir, 'rk_project')).toThrow(/Timed out waiting/); + expect(fs.existsSync(path.join(lockDir, token))).toBe(true); + expect(kill).toHaveBeenCalledWith(process.pid, 0); + } finally { + kill.mockRestore(); + fs.rmSync(lockDir, { recursive: true, force: true }); + } + }); + it('round-trips an enrolled Fleet identity and clears it on an explicit workspace change', () => { writeProjectWorkspaceKey(dataDir, 'rk_enrolled', { enrolledNodeId: ' node_1 ' }); expect(readProjectWorkspaceSession(dataDir)).toEqual({ diff --git a/packages/cloud/src/project-workspace-key.ts b/packages/cloud/src/project-workspace-key.ts index 8fb3dc5d00..ed623ee6ec 100644 --- a/packages/cloud/src/project-workspace-key.ts +++ b/packages/cloud/src/project-workspace-key.ts @@ -12,6 +12,13 @@ const PROJECT_WORKSPACE_LOCK_TIMEOUT_MS = 2_000; const PROJECT_WORKSPACE_LOCK_STALE_MS = 30_000; const PROJECT_WORKSPACE_LOCK_RETRY_MS = 10; const PROJECT_WORKSPACE_LOCK_WAIT = new Int32Array(new SharedArrayBuffer(4)); +const PROJECT_WORKSPACE_LOCK_OWNER_VERSION = 1; + +interface ProjectWorkspaceLockOwner { + version: number; + pid: number; + token: string; +} /** Workspace-key environment aliases, highest precedence first. */ const WORKSPACE_KEY_ENV_VARS = ['RELAY_WORKSPACE_KEY', 'AGENT_RELAY_WORKSPACE_KEY', 'RELAY_API_KEY'] as const; @@ -194,18 +201,62 @@ function writeProjectWorkspaceKeyUnlocked( function withProjectWorkspaceKeyLock(dataDir: string, callback: () => T): T { fs.mkdirSync(dataDir, { recursive: true, mode: 0o700 }); const lockDir = `${projectWorkspaceKeyPath(dataDir)}${PROJECT_WORKSPACE_LOCK_SUFFIX}`; + const ownerToken = randomUUID(); + const ownerPath = path.join(lockDir, ownerToken); const startedAt = Date.now(); while (true) { try { fs.mkdirSync(lockDir, { mode: 0o700 }); + try { + // The token is a child of this lock directory, so a stale holder can + // release only its own marker even if another writer has already + // removed and reacquired the directory. + fs.writeFileSync( + ownerPath, + JSON.stringify({ + version: PROJECT_WORKSPACE_LOCK_OWNER_VERSION, + pid: process.pid, + token: ownerToken, + }), + { mode: 0o600, flag: 'wx' } + ); + } catch (error) { + // mkdir succeeded, but this invocation never acquired a usable lock. + // Clean up only the directory it just created before propagating. + fs.rmSync(ownerPath, { force: true }); + try { + fs.rmdirSync(lockDir); + } catch (cleanupError) { + if ((cleanupError as NodeJS.ErrnoException).code !== 'ENOENT') throw cleanupError; + } + throw error; + } break; } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; } try { if (Date.now() - fs.statSync(lockDir).mtimeMs >= PROJECT_WORKSPACE_LOCK_STALE_MS) { - fs.rmSync(lockDir, { recursive: true, force: true }); - continue; + const observedLock = inspectProjectWorkspaceLock(lockDir); + if (!observedLock.ownerIsAlive) { + // Reclaim only the exact marker entries observed in this lock. A + // replacement writer can add a different marker concurrently; the + // non-recursive rmdir then leaves that replacement lock untouched. + for (const entry of observedLock.entries) { + fs.rmSync(path.join(lockDir, entry), { force: true }); + } + try { + fs.rmdirSync(lockDir); + } catch (error) { + if ( + (error as NodeJS.ErrnoException).code !== 'ENOENT' && + (error as NodeJS.ErrnoException).code !== 'ENOTEMPTY' + ) { + throw error; + } + } + continue; + } } } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue; @@ -219,8 +270,62 @@ function withProjectWorkspaceKeyLock(dataDir: string, callback: () => T): T { try { return callback(); } finally { - fs.rmSync(lockDir, { recursive: true, force: true }); + // Remove our marker first. If the lock was declared stale and replaced + // while the callback was running, the replacement marker is different; + // rmdir then safely leaves the replacement lock in place. + if (fs.existsSync(ownerPath)) { + fs.rmSync(ownerPath, { force: true }); + try { + fs.rmdirSync(lockDir); + } catch (error) { + if ( + (error as NodeJS.ErrnoException).code !== 'ENOENT' && + (error as NodeJS.ErrnoException).code !== 'ENOTEMPTY' + ) { + throw error; + } + } + } + } +} + +function inspectProjectWorkspaceLock(lockDir: string): { + entries: string[]; + ownerIsAlive: boolean; +} { + const entries = fs.readdirSync(lockDir); + let sawLiveOwner = false; + for (const entry of entries) { + let owner: Partial; + try { + owner = JSON.parse( + fs.readFileSync(path.join(lockDir, entry), 'utf8') + ) as Partial; + } catch { + continue; + } + const pid = owner.pid; + if ( + owner.version !== PROJECT_WORKSPACE_LOCK_OWNER_VERSION || + typeof pid !== 'number' || + !Number.isInteger(pid) || + pid <= 0 || + typeof owner.token !== 'string' || + owner.token !== entry + ) { + continue; + } + try { + process.kill(pid, 0); + sawLiveOwner = true; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + // EPERM means the process exists but is not signalable. Treat all + // errors other than ESRCH conservatively as alive. + if (code !== 'ESRCH') sawLiveOwner = true; + } } + return { entries, ownerIsAlive: sawLiveOwner }; } /** Atomically persist a Relaycast target only if the captured project selection is still current. */ diff --git a/scripts/ci-standalone-smoke.sh b/scripts/ci-standalone-smoke.sh index 1f407cc80c..d433b9fad6 100755 --- a/scripts/ci-standalone-smoke.sh +++ b/scripts/ci-standalone-smoke.sh @@ -26,6 +26,8 @@ MIN_STARTUP_TIMEOUT_SECONDS=50 # with a full minute left for shutdown and deletion verification. WORKSPACE_LEASE_SECONDS=300 MAX_STARTUP_TIMEOUT_SECONDS=240 +CURL_CONNECT_TIMEOUT_SECONDS=10 +CURL_MAX_TIME_SECONDS=60 STARTUP_TIMEOUT_SECONDS="${AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS:-60}" if ! [[ "$STARTUP_TIMEOUT_SECONDS" =~ ^[1-9][0-9]{0,4}$ ]]; then echo "ERROR: AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS must be a base-10 integer between ${MIN_STARTUP_TIMEOUT_SECONDS}s and ${MAX_STARTUP_TIMEOUT_SECONDS}s without leading zeros." >&2 @@ -107,6 +109,7 @@ cleanup() { if [ -n "$WORKSPACE_KEY" ]; then local delete_status verify_status delete_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CURL_MAX_TIME_SECONDS" \ --request DELETE \ --header "Authorization: Bearer $WORKSPACE_KEY" \ "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" @@ -116,6 +119,7 @@ cleanup() { cleanup_status=1 else verify_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CURL_MAX_TIME_SECONDS" \ --request GET \ --header "Authorization: Bearer $WORKSPACE_KEY" \ "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" @@ -159,6 +163,7 @@ fi WORKSPACE_NAME="relay-standalone-smoke-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-$$" CREATE_STATUS="$(curl --silent --show-error --output "$WORKSPACE_RESPONSE" --write-out '%{http_code}' \ + --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CURL_MAX_TIME_SECONDS" \ --request POST \ --header 'Content-Type: application/json' \ --data "$(jq -cn --arg name "$WORKSPACE_NAME" --argjson expires "$WORKSPACE_LEASE_SECONDS" '{name: $name, expires_in_seconds: $expires}')" \ @@ -166,11 +171,12 @@ CREATE_STATUS="$(curl --silent --show-error --output "$WORKSPACE_RESPONSE" --wri # Mask the key before extracting or using any other response field. Never log # the response body: it contains the administrative workspace credential. WORKSPACE_KEY="$(jq -er '.data.api_key // .api_key // empty' "$WORKSPACE_RESPONSE" 2>/dev/null || true)" +CREATE_ERROR_CODE="$(jq -er '.error.code // .code // empty' "$WORKSPACE_RESPONSE" 2>/dev/null || true)" if [ -n "$WORKSPACE_KEY" ]; then printf '::add-mask::%s\n' "$WORKSPACE_KEY" fi if [ -z "$WORKSPACE_KEY" ]; then - echo "ERROR: ephemeral smoke workspace response did not contain an API key." >&2 + echo "ERROR: ephemeral smoke workspace response did not contain an API key (HTTP ${CREATE_STATUS:-unknown}${CREATE_ERROR_CODE:+, error code ${CREATE_ERROR_CODE}})." >&2 exit 1 fi if [[ ! "$WORKSPACE_KEY" =~ ^rk_live_[A-Za-z0-9_-]+$ ]]; then diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 9124861a59..77f2ed549b 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -65,12 +65,11 @@ const STATUS_FAILURE_DIAGNOSTIC_FIELDS = ['phase', 'code', 'dispatchType', 'sand async function run(command, args, options = {}) { const diagnosticSecretValues = options.diagnosticSecretValues ?? []; - // One ordering-aware redactor covers both pipes. If the OS reports the two - // streams in the opposite order, conservatively mask any trailing fragment - // that could begin a credential instead of releasing it at finalization. - const outputRedactor = createCredentialRedactor(diagnosticSecretValues, { - maskPendingOnFinal: true, - }); + // Keep a redaction boundary per pipe. A partial credential suffix from + // stdout must never be prepended to the next stderr chunk (or vice versa). + // Each pipe is finalized independently, so an actually benign suffix can be + // released once that originating stream closes. + const outputRedactors = createCommandOutputRedactors(diagnosticSecretValues); const result = await runBoundedProcess(command, args, { env: options.env, echo: !options.quiet, @@ -80,24 +79,34 @@ async function run(command, args, options = {}) { signal: options.signal, onStdout: options.onStdout, onStderr: options.onStderr, - transformChunk: (text, _stream, final) => outputRedactor.push(text, final), + transformChunk: (text, stream, final) => outputRedactors[stream].push(text, final), }); - // A trailing fragment means the other half may already have been captured - // from the other pipe in an order that cannot be reconstructed safely. - // Fail closed by replacing every non-empty captured stream wholesale. - return maskCapturedCommandOutput(result, outputRedactor); + return maskCapturedCommandOutput(result, outputRedactors); +} + +/** Create independent streaming redactors for subprocess stdout and stderr. */ +export function createCommandOutputRedactors(secretValues = []) { + return { + stdout: createCredentialRedactor(secretValues), + stderr: createCredentialRedactor(secretValues), + }; } export function maskCapturedCommandOutput(result, outputRedactor) { - if (outputRedactor.requiresCapturedOutputMask()) { - return { - ...result, - stdout: result.stdout ? '[redacted]' : '', - stderr: result.stderr ? '[redacted]' : '', - }; + if (outputRedactor && typeof outputRedactor.requiresCapturedOutputMask === 'function') { + return outputRedactor.requiresCapturedOutputMask() + ? { ...result, stdout: result.stdout ? '[redacted]' : '', stderr: result.stderr ? '[redacted]' : '' } + : result; } - return result; + const maskedStreams = Object.entries(outputRedactor ?? {}) + .filter(([, redactor]) => redactor?.requiresCapturedOutputMask?.()) + .map(([stream]) => stream); + if (maskedStreams.length === 0) return result; + return { + ...result, + ...Object.fromEntries(maskedStreams.map((stream) => [stream, result[stream] ? '[redacted]' : ''])), + }; } export function boundedDuration(value, { fallback, minimum, maximum, label }) { diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 5a72d29d83..ea8af193bd 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -46,6 +46,7 @@ import { import { boundedDiagnostic, boundedDuration, + createCommandOutputRedactors, createPreparedRunProgressParser, createCredentialRedactor, createCliApiKeyEnvironment, @@ -623,14 +624,14 @@ describe('Cloud dispatcher API key lifecycle', () => { expect(prefixedArtifact).not.toContain('rk_live_token'); }); - it('never releases cross-stream credential fragments in captures or artifacts', async () => { + it('keeps credential fragments attached to their originating output stream', async () => { const captureWithRedaction = async (code: string, secretValues: string[]) => { - const redactor = createCredentialRedactor(secretValues, { maskPendingOnFinal: true }); + const redactors = createCommandOutputRedactors(secretValues); const capture = await runBoundedProcess(process.execPath, ['-e', code], { echo: false, - transformChunk: (text, _stream, final) => redactor.push(text, final), + transformChunk: (text, stream, final) => redactors[stream].push(text, final), }); - return maskCapturedCommandOutput(capture, redactor); + return maskCapturedCommandOutput(capture, redactors); }; const configuredCapture = await captureWithRedaction( @@ -646,13 +647,9 @@ describe('Cloud dispatcher API key lifecycle', () => { diagnosticSecretValues: ['split-secret'], }); - for (const value of [configuredCapture.stdout, configuredCapture.stderr, configuredArtifact]) { - expect(value).not.toContain('secret'); - expect(value).not.toContain('split-'); - expect(value).not.toContain('split-secret'); - } - expect(configuredCapture.stdout + configuredCapture.stderr).toContain('[redacted]'); - expect(configuredArtifact).toContain('[redacted]'); + expect(configuredCapture.stdout).toBe('secret'); + expect(configuredCapture.stderr).toBe('split-'); + expect(configuredArtifact).not.toContain('split-secret'); const prefixCapture = await captureWithRedaction( "process.stdout.write('live_token'); setTimeout(() => process.stderr.write('rk_'), 25)", @@ -667,13 +664,14 @@ describe('Cloud dispatcher API key lifecycle', () => { diagnosticSecretValues: [], }); - for (const value of [prefixCapture.stdout, prefixCapture.stderr, prefixArtifact]) { - expect(value).not.toContain('live_token'); - expect(value).not.toContain('rk_'); - expect(value).not.toContain('rk_live_token'); - } - expect(prefixCapture.stdout + prefixCapture.stderr).toContain('[redacted]'); - expect(prefixArtifact).toContain('[redacted]'); + expect(prefixCapture.stdout).toBe('live_token'); + expect(prefixCapture.stderr).toBe('rk_'); + expect(prefixArtifact).not.toContain('rk_live_token'); + + const benignTrailingPrefix = createCommandOutputRedactors(); + expect( + benignTrailingPrefix.stdout.push('finished with br', false) + benignTrailingPrefix.stdout.push('', true) + ).toBe('finished with br'); }); it('redacts credential prefixes and configured secrets across subprocess chunk boundaries', async () => { From 5026e73989c1ee8590d26e90cdd5ab775979cf40 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 03:08:10 +0200 Subject: [PATCH 42/50] fix: address final cutover review gaps --- packages/cli/src/cli/lib/sdk-client.test.ts | 67 +++++++++++++++++++ packages/cli/src/cli/lib/sdk-client.ts | 9 ++- .../cloud/src/project-workspace-key.test.ts | 33 +++++++++ packages/cloud/src/project-workspace-key.ts | 16 ++++- scripts/pr-proof/run-cloud.mjs | 23 +++++-- scripts/pr-proof/run-cloud.test.mjs | 34 ++++++++++ 6 files changed, 172 insertions(+), 10 deletions(-) create mode 100644 scripts/pr-proof/run-cloud.test.mjs diff --git a/packages/cli/src/cli/lib/sdk-client.test.ts b/packages/cli/src/cli/lib/sdk-client.test.ts index 3ea46ed4f1..6f329ad5d8 100644 --- a/packages/cli/src/cli/lib/sdk-client.test.ts +++ b/packages/cli/src/cli/lib/sdk-client.test.ts @@ -127,6 +127,46 @@ describe('sdk client option resolution', () => { expect(JSON.stringify(relay)).not.toContain('at_live_participant_scoped'); }); + it('does not inherit a persisted gateway when an explicit agent token is supplied', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + + const relay = createAgentRelay({ token: 'at_live_explicit' }) as unknown as { + messagingOptions: { baseUrl?: string }; + workspaceKey?: string; + }; + expect(relay.messagingOptions.baseUrl).toBeUndefined(); + expect(relay.workspaceKey).toBeUndefined(); + + const canonicalRelay = createAgentRelay({ + token: 'at_live_explicit', + baseUrl: 'https://cast.agentrelay.com', + }) as unknown as { messagingOptions: { baseUrl?: string } }; + expect(canonicalRelay.messagingOptions.baseUrl).toBe('https://cast.agentrelay.com'); + }); + + it('does not inherit a persisted gateway when an environment agent token is supplied', () => { + writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { + workspaceId: 'rw_abc', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_agent37', + }); + + const relay = createAgentRelay({ + env: { RELAY_AGENT_TOKEN: 'at_live_environment' }, + }) as unknown as { + messagingOptions: { baseUrl?: string }; + workspaceKey?: string; + }; + expect(relay.messagingOptions.baseUrl).toBeUndefined(); + expect(relay.workspaceKey).toBeUndefined(); + }); + it('durably records an isolated target while preserving the enrolled node session', () => { writeProjectWorkspaceKey(projectDataDir(), 'rk_live_canonical', { workspaceId: 'rw_abc', @@ -200,6 +240,33 @@ describe('sdk client option resolution', () => { } ); + it('creates a fresh project target pin for an active machine-store selection', () => { + setWorkspaceKey('ops', 'rk_live_store', { AGENT_RELAY_HOME: dir }); + const selection = resolveWorkspaceSelection({ env: { AGENT_RELAY_HOME: dir } }); + expect(selection).toMatchObject({ + key: 'rk_live_store', + source: 'store', + projectDataDir: projectDataDir(), + projectSessionPresent: false, + }); + + expect( + persistWorkspaceRelaycastTarget(selection, { + route: 'agent37-isolated', + baseUrl: 'https://agent37-cast.agentrelay.com', + workspaceId: 'rw_store', + relaycastApiKey: 'rk_live_store_agent37', + }) + ).toBe(true); + expect(readProjectWorkspaceSession(projectDataDir())).toEqual({ + workspaceKey: 'rk_live_store', + workspaceId: 'rw_store', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_store_agent37', + }); + }); + it('refuses to overwrite a project session rebound after workspace selection', () => { writeProjectWorkspaceKey(projectDataDir(), 'rk_live_original', { workspaceId: 'rw_original' }); const selection = resolveWorkspaceSelection({ env: { AGENT_RELAY_HOME: dir } }); diff --git a/packages/cli/src/cli/lib/sdk-client.ts b/packages/cli/src/cli/lib/sdk-client.ts index 2a0ad66370..6188b29ae5 100644 --- a/packages/cli/src/cli/lib/sdk-client.ts +++ b/packages/cli/src/cli/lib/sdk-client.ts @@ -216,7 +216,14 @@ export function createAgentRelay(options: SdkClientOptions = {}): AgentRelayAgen if (token) { return new AgentRelay({ agentToken: token, - baseUrl: resolveBaseUrl(options), + // An agent token is already scoped by the caller, whether supplied by a + // flag or RELAY_AGENT_TOKEN. Do not let a persisted project route + // silently select a different gateway; only an explicit/ambient base URL + // may choose the token's origin. + baseUrl: resolveBaseUrl({ + ...options, + ignorePersistedRelaycastTarget: true, + }), }); } const { workspaceKey, baseUrl } = resolveWorkspaceTransport(options); diff --git a/packages/cloud/src/project-workspace-key.test.ts b/packages/cloud/src/project-workspace-key.test.ts index 2ec4617be9..ea894f1334 100644 --- a/packages/cloud/src/project-workspace-key.test.ts +++ b/packages/cloud/src/project-workspace-key.test.ts @@ -81,6 +81,27 @@ describe('project workspace key resolution', () => { } }); + it('preserves a callback error when lock cleanup also fails', () => { + const originalRename = fs.renameSync.bind(fs); + const rename = vi.spyOn(fs, 'renameSync').mockImplementation((source, destination) => { + originalRename(source, destination); + throw new Error('callback failed'); + }); + const rmdir = vi.spyOn(fs, 'rmdirSync').mockImplementation(() => { + throw new Error('cleanup failed'); + }); + const report = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + expect(() => writeProjectWorkspaceKey(dataDir, 'rk_project')).toThrow('callback failed'); + expect(report).toHaveBeenCalledWith(expect.stringContaining('Failed to release'), expect.any(Error)); + } finally { + rename.mockRestore(); + rmdir.mockRestore(); + report.mockRestore(); + } + }); + it('round-trips an enrolled Fleet identity and clears it on an explicit workspace change', () => { writeProjectWorkspaceKey(dataDir, 'rk_enrolled', { enrolledNodeId: ' node_1 ' }); expect(readProjectWorkspaceSession(dataDir)).toEqual({ @@ -132,6 +153,18 @@ describe('project workspace key resolution', () => { }); }); + it('records the absent project session snapshot for an active-store selection', () => { + const env = { AGENT_RELAY_HOME: home }; + setWorkspaceKey('global', 'rk_global', env); + + expect(resolveWorkspaceSelection({ projectDataDir: dataDir, env })).toMatchObject({ + key: 'rk_global', + source: 'store', + projectDataDir: dataDir, + projectSessionPresent: false, + }); + }); + it('returns undefined when no workspace source exists', () => { expect( resolveWorkspaceKeyWithSource({ projectDataDir: dataDir, env: { AGENT_RELAY_HOME: home } }) diff --git a/packages/cloud/src/project-workspace-key.ts b/packages/cloud/src/project-workspace-key.ts index ed623ee6ec..6dbee8fe0c 100644 --- a/packages/cloud/src/project-workspace-key.ts +++ b/packages/cloud/src/project-workspace-key.ts @@ -267,8 +267,12 @@ function withProjectWorkspaceKeyLock(dataDir: string, callback: () => T): T { } Atomics.wait(PROJECT_WORKSPACE_LOCK_WAIT, 0, 0, PROJECT_WORKSPACE_LOCK_RETRY_MS); } + let callbackFailed = false; try { return callback(); + } catch (error) { + callbackFailed = true; + throw error; } finally { // Remove our marker first. If the lock was declared stale and replaced // while the callback was running, the replacement marker is different; @@ -282,7 +286,11 @@ function withProjectWorkspaceKeyLock(dataDir: string, callback: () => T): T { (error as NodeJS.ErrnoException).code !== 'ENOENT' && (error as NodeJS.ErrnoException).code !== 'ENOTEMPTY' ) { - throw error; + if (callbackFailed) { + console.error(`Failed to release the project workspace lock at ${lockDir}.`, error); + } else { + throw error; + } } } } @@ -496,7 +504,7 @@ export function resolveWorkspaceSelection( }; } - return resolveActiveWorkspaceSelection(env); + return resolveActiveWorkspaceSelection(env, dataDir); } /** @@ -508,7 +516,8 @@ export function resolveWorkspaceSelection( * It is never correct to consult this ahead of steps 1–3. */ export function resolveActiveWorkspaceSelection( - env: NodeJS.ProcessEnv = process.env + env: NodeJS.ProcessEnv = process.env, + projectDataDir?: string ): WorkspaceSelection | undefined { const store = readWorkspaceStore(env); const activeName = trimOrUndefined(store.active); @@ -518,6 +527,7 @@ export function resolveActiveWorkspaceSelection( key: storeKey, source: 'store', origin: `${workspaceStorePath(env)} (active: "${activeName}")`, + ...(projectDataDir ? { projectDataDir, projectSessionPresent: false } : {}), } : undefined; } diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 77f2ed549b..c52ac051fa 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -63,13 +63,15 @@ const STATUS_DIAGNOSTIC_FIELDS = [ ]; const STATUS_FAILURE_DIAGNOSTIC_FIELDS = ['phase', 'code', 'dispatchType', 'sandboxId', 'occurredAt']; -async function run(command, args, options = {}) { +export async function run(command, args, options = {}) { const diagnosticSecretValues = options.diagnosticSecretValues ?? []; // Keep a redaction boundary per pipe. A partial credential suffix from // stdout must never be prepended to the next stderr chunk (or vice versa). // Each pipe is finalized independently, so an actually benign suffix can be // released once that originating stream closes. - const outputRedactors = createCommandOutputRedactors(diagnosticSecretValues); + const outputRedactors = createCommandOutputRedactors(diagnosticSecretValues, { + maskPendingOnFinal: true, + }); const result = await runBoundedProcess(command, args, { env: options.env, echo: !options.quiet, @@ -86,10 +88,10 @@ async function run(command, args, options = {}) { } /** Create independent streaming redactors for subprocess stdout and stderr. */ -export function createCommandOutputRedactors(secretValues = []) { +export function createCommandOutputRedactors(secretValues = [], { maskPendingOnFinal = false } = {}) { return { - stdout: createCredentialRedactor(secretValues), - stderr: createCredentialRedactor(secretValues), + stdout: createCredentialRedactor(secretValues, { maskPendingOnFinal }), + stderr: createCredentialRedactor(secretValues, { maskPendingOnFinal }), }; } @@ -159,6 +161,15 @@ function longestSuffixThatStartsSecret(value, secrets) { return 0; } +function longestSuffixThatMatchesSecret(value, secrets) { + const maximum = Math.min(value.length, Math.max(...secrets.map((secret) => secret.length), 0) - 1); + for (let length = maximum; length > 0; length -= 1) { + const suffix = value.slice(value.length - length); + if (secrets.some((secret) => secret.startsWith(suffix) || secret.endsWith(suffix))) return length; + } + return 0; +} + function createCredentialPrefixRedactor(maskPendingOnFinal = false) { let pending = ''; let active = null; @@ -270,7 +281,7 @@ function createConfiguredSecretRedactor(secretValues, maskPendingOnFinal = false continue; } const suffixLength = - final && !maskPendingOnFinal ? 0 : longestSuffixThatStartsSecret(input.slice(index), secrets); + final && !maskPendingOnFinal ? 0 : longestSuffixThatMatchesSecret(input.slice(index), secrets); const end = input.length - suffixLength; output += input.slice(index, end); if (suffixLength > 0) { diff --git a/scripts/pr-proof/run-cloud.test.mjs b/scripts/pr-proof/run-cloud.test.mjs new file mode 100644 index 0000000000..945451993f --- /dev/null +++ b/scripts/pr-proof/run-cloud.test.mjs @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { createCommandOutputRedactors, run } from './run-cloud.mjs'; + +describe('cloud command output redaction', () => { + it('keeps partial credential suffixes on their originating stream', () => { + const redactors = createCommandOutputRedactors(); + + assert.equal(redactors.stdout.push('stdout: rk_live_', false), 'stdout: '); + assert.equal(redactors.stderr.push('stderr: unrelated', false), 'stderr: unrelated'); + assert.equal(redactors.stdout.push('secret-value', true), 'rk_live_…'); + }); + + it('does not redact or drop a benign trailing prefix fragment at stream end', () => { + const redactors = createCommandOutputRedactors(); + + assert.equal(redactors.stdout.push('finished with br', false), 'finished with '); + assert.equal(redactors.stdout.push('', true), 'br'); + }); + + it('masks configured secrets split across streams before echo and capture', async () => { + const result = await run( + process.execPath, + ['-e', "process.stdout.write('custom-'); process.stderr.write('secret-value')"], + { diagnosticSecretValues: ['custom-secret-value'] } + ); + + assert.equal(result.stdout, '[redacted]'); + assert.equal(result.stderr, '[redacted]'); + assert.doesNotMatch(result.stdout, /custom-/); + assert.doesNotMatch(result.stderr, /secret-value/); + }); +}); From f419ec65f3c7d84f994cc165767f7871ebcf88ca Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 03:29:33 +0200 Subject: [PATCH 43/50] fix(cli): close Agent37 routing review gaps --- packages/cli/src/cli/commands/fleet.test.ts | 116 ++++++++++++++++++ packages/cli/src/cli/commands/fleet.ts | 7 ++ .../cli/src/cli/commands/local-agent.test.ts | 33 +++++ packages/cli/src/cli/commands/local-agent.ts | 4 +- scripts/pr-proof/run-cloud.mjs | 3 +- scripts/pr-proof/run-cloud.test.mjs | 11 ++ 6 files changed, 172 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index b670548df2..ca6e028de4 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1296,6 +1296,122 @@ describe('fleet command support', () => { expect(ensureInput).not.toHaveProperty('sandboxId'); }); + it('rejects an explicit workspace ID that conflicts with the captured selection', async () => { + const ensureCloudFleetSandbox = vi.fn(); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn() as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: 'test', + workspaceId: 'rw_captured', + }), + persistWorkspaceRelaycastTarget: vi.fn(() => true), + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--workspace-id', + 'rw_other', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + expect(ensureCloudFleetSandbox).not.toHaveBeenCalled(); + }); + + it('accepts an explicit workspace ID matching the captured selection', async () => { + const ensureCloudFleetSandbox = vi.fn(async () => ({ + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-generated', + nodeName: 'generated-node', + sandboxId: 'generated-public-sandbox', + relayWorkspaceId: 'rw_captured', + relayfileMounted: true, + })); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn(() => ({ messaging: { placement: { spawn: vi.fn() } } })) as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { + register: vi.fn(async () => ({ token: 'at_live_launcher' })), + release: vi.fn(async () => ({ released: true, deleted: true })), + }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + ensureCloudFleetSandbox, + resolveWorkspaceSelection: () => ({ + key: 'rk_live_test', + source: 'project', + origin: 'test', + workspaceId: 'rw_captured', + }), + persistWorkspaceRelaycastTarget: () => true, + deleteCloudFleetSandbox: vi.fn(async () => undefined), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--no-sandbox-relayfile', + '--workspace-id', + 'rw_captured', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( + expect.objectContaining({ workspaceId: 'rw_captured' }) + ); + }); + it('generates a stable identity and matching name when neither option is supplied', async () => { vi.stubEnv('RELAY_AGENT_TOKEN', undefined); const ensureCloudFleetSandbox = vi.fn(async () => ({ diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index c7f7907e1d..c07a9b0432 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -288,6 +288,13 @@ export function registerFleetCommands( // workspace from canonical Relaycast. Agent37 may not, because even a // read there mutates rate-limit/presence accounting on the shared // service and defeats the zero-shared-traffic canary proof. + if ( + explicitWorkspaceId !== undefined && + workspaceSelection?.workspaceId !== undefined && + explicitWorkspaceId !== workspaceSelection.workspaceId.trim() + ) { + throw new Error('--workspace-id does not match the captured workspace identity.'); + } if (!relayWorkspaceId && sandboxProvider !== 'agent37') { workspaceRelay = deps.sdk.createWorkspaceRelay(legacyWorkspaceClientOptions); const workspaceInfo = await workspaceRelay.workspace.info(); diff --git a/packages/cli/src/cli/commands/local-agent.test.ts b/packages/cli/src/cli/commands/local-agent.test.ts index 22d3ca1ef2..47ecb523f8 100644 --- a/packages/cli/src/cli/commands/local-agent.test.ts +++ b/packages/cli/src/cli/commands/local-agent.test.ts @@ -251,6 +251,39 @@ describe('local agent subtree', () => { expect(JSON.stringify([...log.mock.calls, ...error.mock.calls])).not.toContain('rjt_live_one_time'); }); + it('attach --node redeems an isolated-shard join ticket at the explicit base URL', async () => { + const { program, attachNode, redeemJoinTicket } = harness(); + await program.parseAsync( + [ + 'local', + 'agent', + 'attach', + 'lead', + '--node', + 'agent37-codex', + '--join-ticket', + 'rjt_live_one_time', + '--base-url', + 'https://agent37-cast.agentrelay.com', + ], + { from: 'user' } + ); + + expect(redeemJoinTicket).toHaveBeenCalledWith( + expect.objectContaining({ + ticket: 'rjt_live_one_time', + node: 'agent37-codex', + baseUrl: 'https://agent37-cast.agentrelay.com', + }) + ); + expect(attachNode).toHaveBeenCalledWith( + 'lead', + 'view', + 'agent37-codex', + expect.objectContaining({ baseUrl: 'https://agent37-cast.agentrelay.com' }) + ); + }); + it.each(['expired', 'invalid'])( 'attach --node reports a clear %s join-ticket error instead of falling through to workspace resolution', async (reason) => { diff --git a/packages/cli/src/cli/commands/local-agent.ts b/packages/cli/src/cli/commands/local-agent.ts index 3f00cf945d..0bd385e169 100644 --- a/packages/cli/src/cli/commands/local-agent.ts +++ b/packages/cli/src/cli/commands/local-agent.ts @@ -280,13 +280,14 @@ function resolveAttachCredentialSelection( async function redeemAndPersistAttachCredential( deps: LocalAgentDependencies, - options: { ticket: string; node: string; agent: string; mode: AttachMode } + options: { ticket: string; node: string; agent: string; mode: AttachMode; baseUrl?: string } ): Promise { const redeemed = await deps.redeemJoinTicket({ ticket: options.ticket, node: options.node, agent: options.agent, mode: options.mode, + ...(options.baseUrl === undefined ? {} : { baseUrl: options.baseUrl }), env: deps.env, fetch: deps.fetch, }); @@ -922,6 +923,7 @@ export function registerLocalAgentCommands( node, agent: name, mode, + baseUrl: options.baseUrl as string | undefined, }); } const code = await deps.attachNode(name, mode, node, { diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index c52ac051fa..de20b3bbf1 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -26,6 +26,7 @@ const LEGACY_REFRESHABLE_AUTH_KEYS = [ const MAX_CAPTURE_BYTES = 2 * 1024 * 1024; const MAX_LIVE_OUTPUT_BYTES = 256 * 1024; const MAX_DIAGNOSTIC_BYTES = 64 * 1024; +const MIN_SECRET_FRAGMENT_LENGTH = 4; const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60_000; const PREPARED_RUN_ID_MARKER = 'AGENT_RELAY_CLOUD_PREPARED_RUN_ID='; const RUN_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; @@ -163,7 +164,7 @@ function longestSuffixThatStartsSecret(value, secrets) { function longestSuffixThatMatchesSecret(value, secrets) { const maximum = Math.min(value.length, Math.max(...secrets.map((secret) => secret.length), 0) - 1); - for (let length = maximum; length > 0; length -= 1) { + for (let length = maximum; length >= MIN_SECRET_FRAGMENT_LENGTH; length -= 1) { const suffix = value.slice(value.length - length); if (secrets.some((secret) => secret.startsWith(suffix) || secret.endsWith(suffix))) return length; } diff --git a/scripts/pr-proof/run-cloud.test.mjs b/scripts/pr-proof/run-cloud.test.mjs index 945451993f..6db55c9af1 100644 --- a/scripts/pr-proof/run-cloud.test.mjs +++ b/scripts/pr-proof/run-cloud.test.mjs @@ -19,6 +19,17 @@ describe('cloud command output redaction', () => { assert.equal(redactors.stdout.push('', true), 'br'); }); + for (const fragment of ['e', 'ue', 'lue']) { + it(`does not wipe a stream for the benign ${fragment.length}-character secret fragment`, async () => { + const output = `finished with ${fragment}`; + const result = await run(process.execPath, ['-e', `process.stdout.write(${JSON.stringify(output)})`], { + diagnosticSecretValues: ['custom-secret-value'], + }); + + assert.equal(result.stdout, output); + }); + } + it('masks configured secrets split across streams before echo and capture', async () => { const result = await run( process.execPath, From f7067f32c0552611f1d42f9339bca791ccce4404 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 03:46:29 +0200 Subject: [PATCH 44/50] fix(cli): preserve isolated Agent37 routing --- packages/cli/src/cli/commands/fleet.test.ts | 67 +++++++++++++------ packages/cli/src/cli/commands/fleet.ts | 19 ++++-- .../cli/src/cli/commands/local-agent.test.ts | 31 ++++++++- packages/cli/src/cli/commands/local-agent.ts | 15 ++++- packages/cli/src/cli/lib/attach-fleet-node.ts | 2 +- .../cli/src/cli/lib/workspace-session.test.ts | 22 ++++++ packages/cli/src/cli/lib/workspace-session.ts | 6 ++ scripts/pr-proof/run-cloud.mjs | 15 +++-- scripts/pr-proof/run-cloud.test.mjs | 14 ++++ 9 files changed, 154 insertions(+), 37 deletions(-) diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index ca6e028de4..b272327d0c 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1414,18 +1414,45 @@ describe('fleet command support', () => { it('generates a stable identity and matching name when neither option is supplied', async () => { vi.stubEnv('RELAY_AGENT_TOKEN', undefined); - const ensureCloudFleetSandbox = vi.fn(async () => ({ - outcome: 'provisioned' as const, - cloudWorkspaceId: 'cloud-workspace', - nodeId: 'node-generated', - nodeName: 'generated-node', - sandboxId: 'generated-public-sandbox', - providerSandboxId: 'generated-provider-sandbox', - relayWorkspaceId: 'rw_abc', - relaycastTarget: AGENT37_RELAYCAST_TARGET, - relayfileMounted: true, - relayfileMountPath: '/workspace', - })); + const events: string[] = []; + const ensureCloudFleetSandbox = vi + .fn(async () => ({ + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-generated', + nodeName: 'generated-node', + sandboxId: 'generated-public-sandbox', + providerSandboxId: 'generated-provider-sandbox', + relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, + relayfileMounted: true, + relayfileMountPath: '/workspace', + })) + .mockImplementationOnce(async () => { + events.push('ensure'); + return { + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-generated', + nodeName: 'generated-node', + sandboxId: 'generated-public-sandbox', + providerSandboxId: 'generated-provider-sandbox', + relayWorkspaceId: 'rw_abc', + relaycastTarget: AGENT37_RELAYCAST_TARGET, + relayfileMounted: true, + relayfileMountPath: '/workspace', + }; + }); + const createWorkspaceRelay = vi.fn(() => { + events.push('workspace-relay'); + return { + workspace: { + info: vi.fn(async () => ({ id: 'rw_abc' })), + register: vi.fn(async () => ({ token: 'at_live_launcher' })), + release: vi.fn(async () => ({ released: true, deleted: true })), + }, + }; + }); const program = new Command(); program.exitOverride(); registerFleetCommands(program, { @@ -1433,13 +1460,7 @@ describe('fleet command support', () => { createAgentRelay: vi.fn(() => ({ messaging: { placement: { spawn: vi.fn(async () => ({ invocationId: 'inv_generated' })) } }, })) as never, - createWorkspaceRelay: vi.fn(() => ({ - workspace: { - info: vi.fn(async () => ({ id: 'rw_abc' })), - register: vi.fn(async () => ({ token: 'at_live_launcher' })), - release: vi.fn(async () => ({ released: true, deleted: true })), - }, - })) as never, + createWorkspaceRelay: createWorkspaceRelay as never, createWorkspace: vi.fn() as never, log: vi.fn(), error: vi.fn(), @@ -1477,6 +1498,10 @@ describe('fleet command support', () => { ); const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0]; + expect(events.slice(0, 2)).toEqual(['ensure', 'workspace-relay']); + expect(createWorkspaceRelay).not.toHaveBeenCalledWith( + expect.objectContaining({ ignorePersistedRelaycastTarget: true }) + ); expect(ensureInput?.sandboxId).toMatch( /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ ); @@ -1520,6 +1545,8 @@ describe('fleet command support', () => { REPLAY_SANDBOX_ID, '--sandbox-name', 'custom-node', + '--workspace-id', + 'rw_abc', '--name', 'sandbox-worker', '--task', @@ -1634,6 +1661,8 @@ describe('fleet command support', () => { REPLAY_SANDBOX_ID, '--sandbox-name', REPLAY_SANDBOX_NAME, + '--workspace-id', + 'rw_abc', '--name', 'sandbox-worker', '--task', diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index c07a9b0432..0844ca22e9 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -288,14 +288,12 @@ export function registerFleetCommands( // workspace from canonical Relaycast. Agent37 may not, because even a // read there mutates rate-limit/presence accounting on the shared // service and defeats the zero-shared-traffic canary proof. - if ( - explicitWorkspaceId !== undefined && - workspaceSelection?.workspaceId !== undefined && - explicitWorkspaceId !== workspaceSelection.workspaceId.trim() - ) { - throw new Error('--workspace-id does not match the captured workspace identity.'); + if (!relayWorkspaceId && sandboxProvider === undefined) { + throw new Error( + 'Sandbox provisioning without --sandbox-provider requires a persisted Relay workspace identity; run `relay workspace pin` or pass --workspace-id.' + ); } - if (!relayWorkspaceId && sandboxProvider !== 'agent37') { + if (!relayWorkspaceId && sandboxProvider !== undefined && sandboxProvider !== 'agent37') { workspaceRelay = deps.sdk.createWorkspaceRelay(legacyWorkspaceClientOptions); const workspaceInfo = await workspaceRelay.workspace.info(); relayWorkspaceId = workspaceInfo.id?.trim(); @@ -320,6 +318,13 @@ export function registerFleetCommands( ); } const effectiveSandboxName = deterministicSandboxName ?? sandboxName; + if ( + explicitWorkspaceId !== undefined && + workspaceSelection?.workspaceId !== undefined && + explicitWorkspaceId !== workspaceSelection.workspaceId.trim() + ) { + throw new Error('--workspace-id does not match the captured workspace identity.'); + } try { sandbox = await deps.ensureCloudFleetSandbox({ workspaceId: relayWorkspaceId, diff --git a/packages/cli/src/cli/commands/local-agent.test.ts b/packages/cli/src/cli/commands/local-agent.test.ts index 47ecb523f8..8791bb33ed 100644 --- a/packages/cli/src/cli/commands/local-agent.test.ts +++ b/packages/cli/src/cli/commands/local-agent.test.ts @@ -252,7 +252,7 @@ describe('local agent subtree', () => { }); it('attach --node redeems an isolated-shard join ticket at the explicit base URL', async () => { - const { program, attachNode, redeemJoinTicket } = harness(); + const { program, attachNode, redeemJoinTicket, persistWorkspaceSession } = harness(); await program.parseAsync( [ 'local', @@ -276,6 +276,12 @@ describe('local agent subtree', () => { baseUrl: 'https://agent37-cast.agentrelay.com', }) ); + expect(persistWorkspaceSession).toHaveBeenCalledWith( + expect.objectContaining({ + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }) + ); expect(attachNode).toHaveBeenCalledWith( 'lead', 'view', @@ -284,6 +290,29 @@ describe('local agent subtree', () => { ); }); + it('rejects an untrusted join-ticket base URL before redemption', async () => { + const { program, redeemJoinTicket, attachNode, error } = harness(); + await program.parseAsync( + [ + 'local', + 'agent', + 'attach', + 'lead', + '--node', + 'agent37-codex', + '--join-ticket', + 'rjt_live_one_time', + '--base-url', + 'https://evil.example.com', + ], + { from: 'user' } + ); + + expect(redeemJoinTicket).not.toHaveBeenCalled(); + expect(attachNode).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledWith(expect.stringContaining('trusted Relaycast origin')); + }); + it.each(['expired', 'invalid'])( 'attach --node reports a clear %s join-ticket error instead of falling through to workspace resolution', async (reason) => { diff --git a/packages/cli/src/cli/commands/local-agent.ts b/packages/cli/src/cli/commands/local-agent.ts index 0bd385e169..d9c9055244 100644 --- a/packages/cli/src/cli/commands/local-agent.ts +++ b/packages/cli/src/cli/commands/local-agent.ts @@ -1,5 +1,6 @@ import type { Command } from 'commander'; +import { AGENT37_RELAYCAST_ORIGIN } from '@agent-relay/cloud'; import { HarnessDriverClient } from '@agent-relay/harness-driver'; import type { InboundDeliveryMode, ListAgent, PendingRelayMessage } from '@agent-relay/harness-driver'; import type { HarnessRuntime } from '@agent-relay/harnesses'; @@ -12,7 +13,7 @@ import type { AttachMode } from '../lib/attach-mode.js'; import { attachNative, isNativeHarness, type NativeAttachOptions } from '../lib/attach-native.js'; import { attachPassthrough } from '../lib/attach-passthrough.js'; import { attachRemoteNode, type RemoteNodeAttachOptions } from '../lib/attach-remote-node.js'; -import { startFleetNodeAttachProxy } from '../lib/attach-fleet-node.js'; +import { startFleetNodeAttachProxy, validateFleetAttachBaseUrl } from '../lib/attach-fleet-node.js'; import { attachView } from '../lib/attach-view.js'; import { createBackpressureAwareWriter } from '../lib/attach.js'; import { @@ -294,6 +295,12 @@ async function redeemAndPersistAttachCredential( const persisted = deps.persistWorkspaceSession({ workspaceKey: redeemed.workspaceKey, workspaceId: redeemed.workspaceId, + ...(options.baseUrl + ? { + relaycastRoute: options.baseUrl === AGENT37_RELAYCAST_ORIGIN ? 'agent37-isolated' : 'canonical', + relaycastBaseUrl: options.baseUrl, + } + : {}), projectRoot: deps.cwd(), }); const warning = describeClearedEnrollment(persisted); @@ -914,6 +921,10 @@ export function registerLocalAgentCommands( } try { let attachWorkspaceKey = credential.workspaceKey; + const validatedBaseUrl = + credential.joinTicket && typeof options.baseUrl === 'string' + ? validateFleetAttachBaseUrl(options.baseUrl) + : undefined; if (credential.joinTicket) { // Pass the redeemed key explicitly into the first attach. Merely // writing the project pin would leave this process vulnerable to @@ -923,7 +934,7 @@ export function registerLocalAgentCommands( node, agent: name, mode, - baseUrl: options.baseUrl as string | undefined, + baseUrl: validatedBaseUrl, }); } const code = await deps.attachNode(name, mode, node, { diff --git a/packages/cli/src/cli/lib/attach-fleet-node.ts b/packages/cli/src/cli/lib/attach-fleet-node.ts index a8bdefa51a..634b9aaaed 100644 --- a/packages/cli/src/cli/lib/attach-fleet-node.ts +++ b/packages/cli/src/cli/lib/attach-fleet-node.ts @@ -117,7 +117,7 @@ export class FleetNodeAttachError extends Error { const TRUSTED_RELAYCAST_ORIGINS = new Set([CANONICAL_RELAYCAST_ORIGIN, AGENT37_RELAYCAST_ORIGIN]); -function validateFleetAttachBaseUrl(value: string): string { +export function validateFleetAttachBaseUrl(value: string): string { let parsed: URL; try { parsed = new URL(value); diff --git a/packages/cli/src/cli/lib/workspace-session.test.ts b/packages/cli/src/cli/lib/workspace-session.test.ts index a06a315644..3da66895c6 100644 --- a/packages/cli/src/cli/lib/workspace-session.test.ts +++ b/packages/cli/src/cli/lib/workspace-session.test.ts @@ -181,6 +181,28 @@ describe('workspace session persistence', () => { expect(result.clearedEnrolledNodeId).toBeUndefined(); }); + it('persists a validated Relaycast route for later workspace selection', () => { + const root = tempRoot(); + const projectDataDir = path.join(root, 'project', '.agentworkforce', 'relay'); + const env = isolatedEnv(root); + + persistWorkspaceSession({ + workspaceKey: 'rk_live_agent37', + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + projectDataDir, + env, + }); + + expect(readProjectWorkspaceSession(projectDataDir)).toEqual({ + workspaceKey: 'rk_live_agent37', + workspaceId: 'rw_agent37', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + }); + }); + it('clears the enrolled Fleet node id when moving to a different workspace', () => { const root = tempRoot(); const projectDataDir = path.join(root, 'project', '.agentworkforce', 'relay'); diff --git a/packages/cli/src/cli/lib/workspace-session.ts b/packages/cli/src/cli/lib/workspace-session.ts index a3f440d62d..062a4abd80 100644 --- a/packages/cli/src/cli/lib/workspace-session.ts +++ b/packages/cli/src/cli/lib/workspace-session.ts @@ -18,6 +18,8 @@ export interface PersistWorkspaceSessionOptions extends WorkspaceSessionOptions workspaceKey: string; /** Canonical Relaycast workspace id, when the credential issuer supplies it. */ workspaceId?: string; + relaycastRoute?: 'canonical' | 'agent37-isolated'; + relaycastBaseUrl?: string; /** Named sessions are also stored and selected in the machine-global workspace store. */ name?: string; } @@ -107,10 +109,14 @@ export function persistWorkspaceSession( writeProjectWorkspaceKeyPreservingSession(projectDataDir, workspaceKey, { ...(enrolledNodeId ? { enrolledNodeId } : {}), ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), + ...(options.relaycastRoute ? { relaycastRoute: options.relaycastRoute } : {}), + ...(options.relaycastBaseUrl ? { relaycastBaseUrl: options.relaycastBaseUrl } : {}), }); } else { writeProjectWorkspaceKey(projectDataDir, workspaceKey, { ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), + ...(options.relaycastRoute ? { relaycastRoute: options.relaycastRoute } : {}), + ...(options.relaycastBaseUrl ? { relaycastBaseUrl: options.relaycastBaseUrl } : {}), }); } diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index de20b3bbf1..903290a043 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -26,7 +26,7 @@ const LEGACY_REFRESHABLE_AUTH_KEYS = [ const MAX_CAPTURE_BYTES = 2 * 1024 * 1024; const MAX_LIVE_OUTPUT_BYTES = 256 * 1024; const MAX_DIAGNOSTIC_BYTES = 64 * 1024; -const MIN_SECRET_FRAGMENT_LENGTH = 4; +const MIN_FINAL_MASK_FRAGMENT_LENGTH = 4; const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60_000; const PREPARED_RUN_ID_MARKER = 'AGENT_RELAY_CLOUD_PREPARED_RUN_ID='; const RUN_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; @@ -164,7 +164,7 @@ function longestSuffixThatStartsSecret(value, secrets) { function longestSuffixThatMatchesSecret(value, secrets) { const maximum = Math.min(value.length, Math.max(...secrets.map((secret) => secret.length), 0) - 1); - for (let length = maximum; length >= MIN_SECRET_FRAGMENT_LENGTH; length -= 1) { + for (let length = maximum; length > 0; length -= 1) { const suffix = value.slice(value.length - length); if (secrets.some((secret) => secret.startsWith(suffix) || secret.endsWith(suffix))) return length; } @@ -229,14 +229,13 @@ function createCredentialPrefixRedactor(maskPendingOnFinal = false) { } if (final && active && !active.emitted) { - if (maskPendingOnFinal) maskedPendingOnFinal = true; output += maskPendingOnFinal ? '[redacted]' : active.prefix; } if (final) { active = null; if (pending) { - if (maskPendingOnFinal) maskedPendingOnFinal = true; - output += maskPendingOnFinal ? '[redacted]' : pending; + output += + maskPendingOnFinal && pending.length >= MIN_FINAL_MASK_FRAGMENT_LENGTH ? '[redacted]' : pending; pending = ''; } } @@ -291,8 +290,10 @@ function createConfiguredSecretRedactor(secretValues, maskPendingOnFinal = false break; } if (final) { - if (pending && maskPendingOnFinal) maskedPendingOnFinal = true; - output += pending && maskPendingOnFinal ? '[redacted]' : pending; + output += + pending && maskPendingOnFinal && pending.length >= MIN_FINAL_MASK_FRAGMENT_LENGTH + ? '[redacted]' + : pending; pending = ''; } return output; diff --git a/scripts/pr-proof/run-cloud.test.mjs b/scripts/pr-proof/run-cloud.test.mjs index 6db55c9af1..aa3ad9063c 100644 --- a/scripts/pr-proof/run-cloud.test.mjs +++ b/scripts/pr-proof/run-cloud.test.mjs @@ -19,6 +19,20 @@ describe('cloud command output redaction', () => { assert.equal(redactors.stdout.push('', true), 'br'); }); + for (let boundary = 1; boundary < 'custom-secret-value'.length; boundary += 1) { + it(`redacts a configured secret split at boundary ${boundary}`, () => { + const secret = 'custom-secret-value'; + const redactors = createCommandOutputRedactors([secret], { + maskPendingOnFinal: true, + }); + + assert.equal(redactors.stdout.push(secret.slice(0, boundary), false), ''); + const tail = redactors.stdout.push(secret.slice(boundary), true); + assert.equal(tail, '[redacted]'); + assert.equal(redactors.stdout.requiresCapturedOutputMask(), false); + }); + } + for (const fragment of ['e', 'ue', 'lue']) { it(`does not wipe a stream for the benign ${fragment.length}-character secret fragment`, async () => { const output = `finished with ${fragment}`; From 18efa8d6ccc05bef238f211f2dc8ed0c7d5c53c9 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 04:36:52 +0200 Subject: [PATCH 45/50] fix(cli): pair persisted relaycast route credentials --- .../cli/src/cli/commands/local-agent.test.ts | 1 + packages/cli/src/cli/commands/local-agent.ts | 1 + .../cli/src/cli/lib/workspace-session.test.ts | 30 +++++++++++++++++++ packages/cli/src/cli/lib/workspace-session.ts | 4 +++ 4 files changed, 36 insertions(+) diff --git a/packages/cli/src/cli/commands/local-agent.test.ts b/packages/cli/src/cli/commands/local-agent.test.ts index 8791bb33ed..8e827dc12d 100644 --- a/packages/cli/src/cli/commands/local-agent.test.ts +++ b/packages/cli/src/cli/commands/local-agent.test.ts @@ -280,6 +280,7 @@ describe('local agent subtree', () => { expect.objectContaining({ relaycastRoute: 'agent37-isolated', relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_redeemed', }) ); expect(attachNode).toHaveBeenCalledWith( diff --git a/packages/cli/src/cli/commands/local-agent.ts b/packages/cli/src/cli/commands/local-agent.ts index d9c9055244..9f3ae3599b 100644 --- a/packages/cli/src/cli/commands/local-agent.ts +++ b/packages/cli/src/cli/commands/local-agent.ts @@ -299,6 +299,7 @@ async function redeemAndPersistAttachCredential( ? { relaycastRoute: options.baseUrl === AGENT37_RELAYCAST_ORIGIN ? 'agent37-isolated' : 'canonical', relaycastBaseUrl: options.baseUrl, + relaycastApiKey: redeemed.workspaceKey, } : {}), projectRoot: deps.cwd(), diff --git a/packages/cli/src/cli/lib/workspace-session.test.ts b/packages/cli/src/cli/lib/workspace-session.test.ts index 3da66895c6..7535553f34 100644 --- a/packages/cli/src/cli/lib/workspace-session.test.ts +++ b/packages/cli/src/cli/lib/workspace-session.test.ts @@ -203,6 +203,36 @@ describe('workspace session persistence', () => { }); }); + it('replaces a stale route credential when re-persisting an explicit route', () => { + const root = tempRoot(); + const projectDataDir = path.join(root, 'project', '.agentworkforce', 'relay'); + const env = isolatedEnv(root); + writeProjectWorkspaceKey(projectDataDir, 'rk_live_redeemed', { + workspaceId: 'rw_redeemed', + relaycastRoute: 'canonical', + relaycastBaseUrl: 'https://cast.agentrelay.com', + relaycastApiKey: 'rk_live_stale_canonical', + }); + + persistWorkspaceSession({ + workspaceKey: 'rk_live_redeemed', + workspaceId: 'rw_redeemed', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_redeemed', + projectDataDir, + env, + }); + + expect(readProjectWorkspaceSession(projectDataDir)).toEqual({ + workspaceKey: 'rk_live_redeemed', + workspaceId: 'rw_redeemed', + relaycastRoute: 'agent37-isolated', + relaycastBaseUrl: 'https://agent37-cast.agentrelay.com', + relaycastApiKey: 'rk_live_redeemed', + }); + }); + it('clears the enrolled Fleet node id when moving to a different workspace', () => { const root = tempRoot(); const projectDataDir = path.join(root, 'project', '.agentworkforce', 'relay'); diff --git a/packages/cli/src/cli/lib/workspace-session.ts b/packages/cli/src/cli/lib/workspace-session.ts index 062a4abd80..65def6f681 100644 --- a/packages/cli/src/cli/lib/workspace-session.ts +++ b/packages/cli/src/cli/lib/workspace-session.ts @@ -20,6 +20,8 @@ export interface PersistWorkspaceSessionOptions extends WorkspaceSessionOptions workspaceId?: string; relaycastRoute?: 'canonical' | 'agent37-isolated'; relaycastBaseUrl?: string; + /** Route-scoped Relaycast credential paired with the persisted route. */ + relaycastApiKey?: string; /** Named sessions are also stored and selected in the machine-global workspace store. */ name?: string; } @@ -111,12 +113,14 @@ export function persistWorkspaceSession( ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), ...(options.relaycastRoute ? { relaycastRoute: options.relaycastRoute } : {}), ...(options.relaycastBaseUrl ? { relaycastBaseUrl: options.relaycastBaseUrl } : {}), + ...(options.relaycastApiKey ? { relaycastApiKey: options.relaycastApiKey } : {}), }); } else { writeProjectWorkspaceKey(projectDataDir, workspaceKey, { ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), ...(options.relaycastRoute ? { relaycastRoute: options.relaycastRoute } : {}), ...(options.relaycastBaseUrl ? { relaycastBaseUrl: options.relaycastBaseUrl } : {}), + ...(options.relaycastApiKey ? { relaycastApiKey: options.relaycastApiKey } : {}), }); } From dce672c810708d1a8fc29fba65e8c3b79345fdc7 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 06:24:37 +0200 Subject: [PATCH 46/50] fix(ci): prove workspace deletion after ambiguous response --- .../cli/src/cli/ci-standalone-smoke.test.ts | 78 ++++++++++++++++++- scripts/ci-standalone-smoke.sh | 48 ++++++++---- 2 files changed, 107 insertions(+), 19 deletions(-) diff --git a/packages/cli/src/cli/ci-standalone-smoke.test.ts b/packages/cli/src/cli/ci-standalone-smoke.test.ts index b5d52e6b0e..4c6f5a8bcf 100644 --- a/packages/cli/src/cli/ci-standalone-smoke.test.ts +++ b/packages/cli/src/cli/ci-standalone-smoke.test.ts @@ -45,9 +45,13 @@ if [[ "$method" = POST && "$url" = */v1/workspaces ]]; then printf '%s' '{"data":{"api_key":"rk_live_fake_smoke_key","workspace_id":"rw_fake_smoke"}}' > "\$output" echo 201 elif [[ "$method" = DELETE && "$url" = */v1/workspace ]]; then - echo 204 + delete_status="\${FAKE_DELETE_STATUS:-204}" + if [ "$delete_status" != 200 ] && [ "$delete_status" != 204 ]; then + printf '%s' '{"error":{"code":"internal_error","message":"rk_live_delete_body_must_not_print"}}' > "\$output" + fi + echo "$delete_status" elif [[ "$method" = GET && "$url" = */v1/workspace ]]; then - echo 401 + echo "\${FAKE_VERIFY_STATUS:-401}" else echo 500 fi @@ -64,6 +68,8 @@ elif [ "\${1:-}" = "-er" ] && [[ "\${2:-}" = *api_key* ]]; then echo 'rk_live_fake_smoke_key' elif [ "\${1:-}" = "-er" ] && [[ "\${2:-}" = *workspace_id* ]]; then echo 'rw_fake_smoke' +elif [ "\${1:-}" = "-er" ] && [[ "\${2:-}" = *error.code* ]] && [ -n "\${FAKE_DELETE_ERROR_CODE:-}" ]; then + echo "\$FAKE_DELETE_ERROR_CODE" else exit 1 fi @@ -241,6 +247,74 @@ describe('ci-standalone-smoke workspace reuse', () => { expect(result.stderr).not.toContain('rk_live_fake_smoke_key'); }); + it('accepts an ambiguous delete response only when the follow-up proves absence', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + FAKE_DELETE_STATUS: '500', + FAKE_DELETE_ERROR_CODE: 'internal_error', + FAKE_VERIFY_STATUS: '401', + }, + timeout: 10_000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain( + 'Ephemeral workspace deletion verified after ambiguous DELETE HTTP 500, error code internal_error' + ); + expect(result.stdout).not.toContain('rk_live_delete_body_must_not_print'); + expect(result.stderr).not.toContain('rk_live_delete_body_must_not_print'); + }); + + it('classifies an unsafe delete error code instead of logging it', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + FAKE_DELETE_STATUS: '500', + FAKE_DELETE_ERROR_CODE: 'rk_live_untrusted_code', + FAKE_VERIFY_STATUS: '401', + }, + timeout: 10_000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('ambiguous DELETE HTTP 500, error code other'); + expect(result.stdout).not.toContain('rk_live_untrusted_code'); + expect(result.stderr).not.toContain('rk_live_untrusted_code'); + }); + + it('fails closed when an ambiguous delete leaves the workspace readable', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + FAKE_DELETE_STATUS: '500', + FAKE_DELETE_ERROR_CODE: 'internal_error', + FAKE_VERIFY_STATUS: '200', + }, + timeout: 10_000, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + 'ephemeral workspace cleanup returned HTTP 500, error code internal_error' + ); + expect(result.stderr).toContain('ephemeral workspace deletion was not proved (follow-up HTTP 200)'); + expect(result.stdout).not.toContain('rk_live_delete_body_must_not_print'); + expect(result.stderr).not.toContain('rk_live_delete_body_must_not_print'); + }); + it('passes the shared key through the isolated lifecycle and joins its workspace', () => { const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); const result = spawnSync('bash', [smokeScript, cli, broker], { diff --git a/scripts/ci-standalone-smoke.sh b/scripts/ci-standalone-smoke.sh index d433b9fad6..a4524ab13f 100755 --- a/scripts/ci-standalone-smoke.sh +++ b/scripts/ci-standalone-smoke.sh @@ -80,6 +80,7 @@ TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/agent-relay-standalone-smoke.XXXXXX")" HOME_DIR="$TMP_ROOT/home" PROJECT_DIR="$TMP_ROOT/project" WORKSPACE_RESPONSE="$TMP_ROOT/workspace-response.json" +DELETE_RESPONSE="$TMP_ROOT/delete-response.json" mkdir -p "$HOME_DIR" "$PROJECT_DIR" @@ -107,29 +108,42 @@ cleanup() { "$CLI_BIN" node down --force --timeout 5000 >/dev/null 2>&1 || true ) if [ -n "$WORKSPACE_KEY" ]; then - local delete_status verify_status - delete_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + local delete_status delete_error_code delete_error_code_raw verify_status + delete_status="$(curl --silent --show-error --output "$DELETE_RESPONSE" --write-out '%{http_code}' \ --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CURL_MAX_TIME_SECONDS" \ --request DELETE \ --header "Authorization: Bearer $WORKSPACE_KEY" \ "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" - if [ "$delete_status" != "200" ] && [ "$delete_status" != "204" ]; then - echo "ERROR: ephemeral workspace cleanup returned HTTP ${delete_status:-unknown}." >&2 - echo "The workspace id is ${WORKSPACE_ID:-unknown}; remove it manually from the trusted smoke shard." >&2 - cleanup_status=1 - else - verify_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ - --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CURL_MAX_TIME_SECONDS" \ - --request GET \ - --header "Authorization: Bearer $WORKSPACE_KEY" \ - "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" - if [ "$verify_status" != "401" ]; then - echo "ERROR: ephemeral workspace deletion was not proved (follow-up HTTP ${verify_status:-unknown})." >&2 - echo "The workspace id is ${WORKSPACE_ID:-unknown}; remove it manually from the trusted smoke shard." >&2 - cleanup_status=1 - else + delete_error_code_raw="$(jq -er '.error.code // .code // empty' "$DELETE_RESPONSE" 2>/dev/null || true)" + delete_error_code="" + case "$delete_error_code_raw" in + internal_error|workspace_storage_unavailable|database_overloaded|file_storage_delete_unsupported) + delete_error_code="$delete_error_code_raw" + ;; + ?*) delete_error_code="other" ;; + esac + verify_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CURL_MAX_TIME_SECONDS" \ + --request GET \ + --header "Authorization: Bearer $WORKSPACE_KEY" \ + "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" + # A 5xx response can be lost after the database commit. The workspace key + # was proven valid by the lifecycle above, so 401 from the same key is the + # authoritative absence check. Any readable or unverifiable state remains + # a hard failure. + if [ "$verify_status" = "401" ]; then + if [ "$delete_status" = "200" ] || [ "$delete_status" = "204" ]; then echo "Ephemeral workspace deletion verified" + else + echo "Ephemeral workspace deletion verified after ambiguous DELETE HTTP ${delete_status:-unknown}${delete_error_code:+, error code $delete_error_code}" + fi + else + if [ "$delete_status" != "200" ] && [ "$delete_status" != "204" ]; then + echo "ERROR: ephemeral workspace cleanup returned HTTP ${delete_status:-unknown}${delete_error_code:+, error code $delete_error_code}." >&2 fi + echo "ERROR: ephemeral workspace deletion was not proved (follow-up HTTP ${verify_status:-unknown})." >&2 + echo "The workspace id is ${WORKSPACE_ID:-unknown}; remove it manually from the trusted smoke shard." >&2 + cleanup_status=1 fi fi rm -rf "$TMP_ROOT" From 3a3b7c4d363ff8a149bb66415b3ae1bc69aea9e3 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 06:58:13 +0200 Subject: [PATCH 47/50] fix(ci): trust committed workspace deletion --- .../cli/src/cli/ci-standalone-smoke.test.ts | 41 +++++++++++++++++++ scripts/ci-standalone-smoke.sh | 38 ++++++++--------- 2 files changed, 60 insertions(+), 19 deletions(-) diff --git a/packages/cli/src/cli/ci-standalone-smoke.test.ts b/packages/cli/src/cli/ci-standalone-smoke.test.ts index 4c6f5a8bcf..33f17d8c56 100644 --- a/packages/cli/src/cli/ci-standalone-smoke.test.ts +++ b/packages/cli/src/cli/ci-standalone-smoke.test.ts @@ -247,6 +247,25 @@ describe('ci-standalone-smoke workspace reuse', () => { expect(result.stderr).not.toContain('rk_live_fake_smoke_key'); }); + it('accepts a successful delete without adding a follow-up read dependency', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + FAKE_DELETE_STATUS: '204', + FAKE_VERIFY_STATUS: '503', + }, + timeout: 10_000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('Ephemeral workspace deletion verified'); + expect(result.stderr).not.toContain('follow-up HTTP'); + }); + it('accepts an ambiguous delete response only when the follow-up proves absence', () => { const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); const result = spawnSync('bash', [smokeScript, cli, broker], { @@ -315,6 +334,28 @@ describe('ci-standalone-smoke workspace reuse', () => { expect(result.stderr).not.toContain('rk_live_delete_body_must_not_print'); }); + it('fails closed when an ambiguous delete cannot be verified', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + FAKE_DELETE_STATUS: '500', + FAKE_DELETE_ERROR_CODE: 'internal_error', + FAKE_VERIFY_STATUS: '503', + }, + timeout: 10_000, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + 'ephemeral workspace cleanup returned HTTP 500, error code internal_error' + ); + expect(result.stderr).toContain('ephemeral workspace deletion was not proved (follow-up HTTP 503)'); + }); + it('passes the shared key through the isolated lifecycle and joins its workspace', () => { const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); const result = spawnSync('bash', [smokeScript, cli, broker], { diff --git a/scripts/ci-standalone-smoke.sh b/scripts/ci-standalone-smoke.sh index a4524ab13f..e77195e9ef 100755 --- a/scripts/ci-standalone-smoke.sh +++ b/scripts/ci-standalone-smoke.sh @@ -122,28 +122,28 @@ cleanup() { ;; ?*) delete_error_code="other" ;; esac - verify_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ - --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CURL_MAX_TIME_SECONDS" \ - --request GET \ - --header "Authorization: Bearer $WORKSPACE_KEY" \ - "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" - # A 5xx response can be lost after the database commit. The workspace key - # was proven valid by the lifecycle above, so 401 from the same key is the - # authoritative absence check. Any readable or unverifiable state remains - # a hard failure. - if [ "$verify_status" = "401" ]; then - if [ "$delete_status" = "200" ] || [ "$delete_status" = "204" ]; then - echo "Ephemeral workspace deletion verified" - else - echo "Ephemeral workspace deletion verified after ambiguous DELETE HTTP ${delete_status:-unknown}${delete_error_code:+, error code $delete_error_code}" - fi + if [ "$delete_status" = "200" ] || [ "$delete_status" = "204" ]; then + # The engine only returns success after its atomic deletion batch commits. + # A second database read would add an unrelated availability dependency. + echo "Ephemeral workspace deletion verified" else - if [ "$delete_status" != "200" ] && [ "$delete_status" != "204" ]; then + verify_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CURL_MAX_TIME_SECONDS" \ + --request GET \ + --header "Authorization: Bearer $WORKSPACE_KEY" \ + "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" + # A 5xx response can be lost after the database commit. The workspace key + # was proven valid by the lifecycle above, so 401 from the same key is the + # authoritative absence check. Any readable or unverifiable state remains + # a hard failure. + if [ "$verify_status" = "401" ]; then + echo "Ephemeral workspace deletion verified after ambiguous DELETE HTTP ${delete_status:-unknown}${delete_error_code:+, error code $delete_error_code}" + else echo "ERROR: ephemeral workspace cleanup returned HTTP ${delete_status:-unknown}${delete_error_code:+, error code $delete_error_code}." >&2 + echo "ERROR: ephemeral workspace deletion was not proved (follow-up HTTP ${verify_status:-unknown})." >&2 + echo "The workspace id is ${WORKSPACE_ID:-unknown}; remove it manually from the trusted smoke shard." >&2 + cleanup_status=1 fi - echo "ERROR: ephemeral workspace deletion was not proved (follow-up HTTP ${verify_status:-unknown})." >&2 - echo "The workspace id is ${WORKSPACE_ID:-unknown}; remove it manually from the trusted smoke shard." >&2 - cleanup_status=1 fi fi rm -rf "$TMP_ROOT" From 9c8d1f55ff31414b841757b479da55c1adccb385 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 08:43:26 +0200 Subject: [PATCH 48/50] fix(ci): retry transient cleanup verification --- .../cli/src/cli/ci-standalone-smoke.test.ts | 52 ++++++++++++++++++- scripts/ci-standalone-smoke.sh | 52 ++++++++++++++++--- 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/cli/ci-standalone-smoke.test.ts b/packages/cli/src/cli/ci-standalone-smoke.test.ts index 33f17d8c56..f92f3ba3ef 100644 --- a/packages/cli/src/cli/ci-standalone-smoke.test.ts +++ b/packages/cli/src/cli/ci-standalone-smoke.test.ts @@ -29,11 +29,13 @@ function createFakeBinaries(): { cli: string; broker: string; invocationLog: str `#!/usr/bin/env bash set -euo pipefail output=/dev/null +headers=/dev/null method=GET url= while [ "\$#" -gt 0 ]; do case "\$1" in --output) output="\$2"; shift 2 ;; + --dump-header) headers="\$2"; shift 2 ;; --connect-timeout|--max-time) shift 2 ;; --request) method="\$2"; shift 2 ;; --write-out) shift 2 ;; @@ -51,10 +53,35 @@ elif [[ "$method" = DELETE && "$url" = */v1/workspace ]]; then fi echo "$delete_status" elif [[ "$method" = GET && "$url" = */v1/workspace ]]; then - echo "\${FAKE_VERIFY_STATUS:-401}" + verify_status="\${FAKE_VERIFY_STATUS:-401}" + if [ -n "\${FAKE_VERIFY_STATUSES:-}" ]; then + verify_count_file="\${INVOCATION_LOG}.verify-count" + verify_count=0 + if [ -f "\$verify_count_file" ]; then + verify_count="\$(<"\$verify_count_file")" + fi + IFS=',' read -r -a verify_statuses <<< "\$FAKE_VERIFY_STATUSES" + verify_index="\$verify_count" + if [ "\$verify_index" -ge "\${#verify_statuses[@]}" ]; then + verify_index="\$((\${#verify_statuses[@]} - 1))" + fi + verify_status="\${verify_statuses[\$verify_index]}" + printf '%s\n' "\$((verify_count + 1))" > "\$verify_count_file" + fi + if [ -n "\${FAKE_VERIFY_RETRY_AFTER:-}" ]; then + printf 'HTTP/2 %s\r\nRetry-After: %s\r\n\r\n' "\$verify_status" "\$FAKE_VERIFY_RETRY_AFTER" > "\$headers" + fi + echo "\$verify_status" else echo 500 fi +` + ); + makeExecutable( + toolsPath, + 'sleep', + `#!/usr/bin/env bash +printf 'sleep %s\n' "\${1:-}" >> "$INVOCATION_LOG" ` ); makeExecutable( @@ -289,6 +316,29 @@ describe('ci-standalone-smoke workspace reuse', () => { expect(result.stderr).not.toContain('rk_live_delete_body_must_not_print'); }); + it('retries a transient verification read and honors bounded Retry-After', () => { + const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); + const result = spawnSync('bash', [smokeScript, cli, broker], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${toolsPath}:${process.env.PATH ?? ''}`, + INVOCATION_LOG: invocationLog, + FAKE_DELETE_STATUS: '500', + FAKE_DELETE_ERROR_CODE: 'internal_error', + FAKE_VERIFY_STATUSES: '503,401', + FAKE_VERIFY_RETRY_AFTER: '3', + }, + timeout: 10_000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain( + 'Ephemeral workspace deletion verified after ambiguous DELETE HTTP 500, error code internal_error' + ); + expect(readFileSync(invocationLog, 'utf8')).toContain('sleep 3'); + }); + it('classifies an unsafe delete error code instead of logging it', () => { const { cli, broker, invocationLog, toolsPath } = createFakeBinaries(); const result = spawnSync('bash', [smokeScript, cli, broker], { diff --git a/scripts/ci-standalone-smoke.sh b/scripts/ci-standalone-smoke.sh index e77195e9ef..6c9937112c 100755 --- a/scripts/ci-standalone-smoke.sh +++ b/scripts/ci-standalone-smoke.sh @@ -28,6 +28,9 @@ WORKSPACE_LEASE_SECONDS=300 MAX_STARTUP_TIMEOUT_SECONDS=240 CURL_CONNECT_TIMEOUT_SECONDS=10 CURL_MAX_TIME_SECONDS=60 +CLEANUP_VERIFY_MAX_ATTEMPTS=3 +CLEANUP_VERIFY_MAX_TIME_SECONDS=10 +CLEANUP_VERIFY_FALLBACK_DELAY_SECONDS=2 STARTUP_TIMEOUT_SECONDS="${AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS:-60}" if ! [[ "$STARTUP_TIMEOUT_SECONDS" =~ ^[1-9][0-9]{0,4}$ ]]; then echo "ERROR: AGENT_RELAY_STANDALONE_STARTUP_TIMEOUT_SECONDS must be a base-10 integer between ${MIN_STARTUP_TIMEOUT_SECONDS}s and ${MAX_STARTUP_TIMEOUT_SECONDS}s without leading zeros." >&2 @@ -81,6 +84,7 @@ HOME_DIR="$TMP_ROOT/home" PROJECT_DIR="$TMP_ROOT/project" WORKSPACE_RESPONSE="$TMP_ROOT/workspace-response.json" DELETE_RESPONSE="$TMP_ROOT/delete-response.json" +VERIFY_HEADERS="$TMP_ROOT/verify-headers.txt" mkdir -p "$HOME_DIR" "$PROJECT_DIR" @@ -108,7 +112,7 @@ cleanup() { "$CLI_BIN" node down --force --timeout 5000 >/dev/null 2>&1 || true ) if [ -n "$WORKSPACE_KEY" ]; then - local delete_status delete_error_code delete_error_code_raw verify_status + local delete_status delete_error_code delete_error_code_raw verify_status verify_attempt verify_delay delete_status="$(curl --silent --show-error --output "$DELETE_RESPONSE" --write-out '%{http_code}' \ --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CURL_MAX_TIME_SECONDS" \ --request DELETE \ @@ -127,15 +131,47 @@ cleanup() { # A second database read would add an unrelated availability dependency. echo "Ephemeral workspace deletion verified" else - verify_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ - --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CURL_MAX_TIME_SECONDS" \ - --request GET \ - --header "Authorization: Bearer $WORKSPACE_KEY" \ - "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" + verify_attempt=1 + while [ "$verify_attempt" -le "$CLEANUP_VERIFY_MAX_ATTEMPTS" ]; do + verify_status="$(curl --silent --show-error --output /dev/null --dump-header "$VERIFY_HEADERS" \ + --write-out '%{http_code}' \ + --connect-timeout "$CURL_CONNECT_TIMEOUT_SECONDS" --max-time "$CLEANUP_VERIFY_MAX_TIME_SECONDS" \ + --request GET \ + --header "Authorization: Bearer $WORKSPACE_KEY" \ + "$TRUSTED_RELAY_BASE_URL/v1/workspace" 2>/dev/null || true)" + case "$verify_status" in + 401) break ;; + 000|429|5??) + if [ "$verify_attempt" -lt "$CLEANUP_VERIFY_MAX_ATTEMPTS" ]; then + # Relaycast advertises integer Retry-After values from 2–8s for + # database overload. Accept only that bounded vocabulary; a + # malformed, date-form, or excessive value falls back to 2s. + verify_delay="$(awk ' + BEGIN { IGNORECASE = 1 } + /^Retry-After:[[:space:]]*/ { + gsub(/\r/, "") + sub(/^[^:]*:[[:space:]]*/, "") + value = $0 + } + END { print value } + ' "$VERIFY_HEADERS" 2>/dev/null || true)" + case "$verify_delay" in + 0|1|2|3|4|5|6|7|8) ;; + *) verify_delay="$CLEANUP_VERIFY_FALLBACK_DELAY_SECONDS" ;; + esac + sleep "$verify_delay" + verify_attempt=$((verify_attempt + 1)) + continue + fi + ;; + esac + break + done # A 5xx response can be lost after the database commit. The workspace key # was proven valid by the lifecycle above, so 401 from the same key is the - # authoritative absence check. Any readable or unverifiable state remains - # a hard failure. + # authoritative absence check. Transient verification reads get a bounded + # retry window; any readable or still-unverifiable state remains a hard + # failure. if [ "$verify_status" = "401" ]; then echo "Ephemeral workspace deletion verified after ambiguous DELETE HTTP ${delete_status:-unknown}${delete_error_code:+, error code $delete_error_code}" else From 2f2701c1cc79f87ce02b4df1010bc7a1ff4f2dd2 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 09:19:08 +0200 Subject: [PATCH 49/50] fix(broker): honor registration cooldown --- CHANGELOG.md | 1 + crates/broker/src/relaycast/ws.rs | 276 ++++++++++++++++++++++++++++-- 2 files changed, 265 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94516defb2..af25ae67fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Legacy non-Agent37 Cloud sandbox responses remain usable when they omit the newer Relaycast target, while any target Cloud does return is verified and persisted for both newly provisioned and reused providers. - Standalone package smoke workspaces remain valid for five minutes, and startup overrides are capped at four minutes so shutdown and cleanup verification always retain a full-minute lease margin. - Startup retries now require Relaycast's typed pre-commit storage-admission codes instead of replaying every 5xx response from an unkeyed workspace or agent-registration request. +- Agent registration retries now honor Relaycast's typed cooldown (capped at one minute) instead of immediately retrying through the same write-capacity window, while a three-minute aggregate deadline prevents hung requests from stranding callers. ## [11.10.4] - 2026-09-08 diff --git a/crates/broker/src/relaycast/ws.rs b/crates/broker/src/relaycast/ws.rs index 297557005b..a82ef52cc1 100644 --- a/crates/broker/src/relaycast/ws.rs +++ b/crates/broker/src/relaycast/ws.rs @@ -6,12 +6,12 @@ use std::{ use anyhow::{Context, Result}; use relaycast::{ - agent::DmOptions, format_registration_error, - retry_agent_registration as sdk_retry_agent_registration, ActionDefinition, ActionInvocation, - AgentClient, AgentIdentityRecoveryResponse, AgentRegistrationClient, AgentRegistrationError, - AgentRegistrationRetryOutcome, CompleteInvocationRequest, CreateObserverTokenRequest, - EmitSessionEventRequest, MessageListQuery, ObserverToken, RegisterActionRequest, RelayCast, - RelayCastOptions, RelayError, ReleaseAgentRequest, TakeOverAgentRequest, UpdateAgentRequest, + agent::DmOptions, format_registration_error, registration_is_retryable, ActionDefinition, + ActionInvocation, AgentClient, AgentIdentityRecoveryResponse, AgentRegistrationClient, + AgentRegistrationError, AgentRegistrationRetryOutcome, CompleteInvocationRequest, + CreateObserverTokenRequest, EmitSessionEventRequest, MessageListQuery, ObserverToken, + RegisterActionRequest, RelayCast, RelayCastOptions, RelayError, ReleaseAgentRequest, + TakeOverAgentRequest, UpdateAgentRequest, }; use serde_json::Value; @@ -49,8 +49,6 @@ pub struct RelaycastHttpClient { pub type RelaycastRegistrationError = AgentRegistrationError; pub type RegRetryOutcome = AgentRegistrationRetryOutcome; -#[cfg(test)] -pub(crate) use relaycast::registration_is_retryable; pub(crate) use relaycast::registration_retry_after_secs; /// Why the broker is asking `register_agent_token` for a token. @@ -1442,7 +1440,87 @@ pub fn format_worker_preregistration_error( format_registration_error(name, error).replace("register agent", "pre-register worker") } +const MAX_AGENT_REGISTRATION_ATTEMPTS: usize = 3; +const DEFAULT_AGENT_REGISTRATION_RETRY_DELAY: Duration = Duration::from_secs(2); +const MAX_AGENT_REGISTRATION_RETRY_DELAY: Duration = Duration::from_secs(60); +const MAX_AGENT_REGISTRATION_ELAPSED: Duration = Duration::from_secs(180); + +fn agent_registration_retry_delay(error: &RelaycastRegistrationError) -> Duration { + registration_retry_after_secs(error) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_AGENT_REGISTRATION_RETRY_DELAY) + .min(MAX_AGENT_REGISTRATION_RETRY_DELAY) +} + +async fn retry_agent_registration_with( + mut request: F, + mut sleep: S, +) -> Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, + S: FnMut(Duration) -> SleepFut, + SleepFut: std::future::Future, +{ + for attempt in 0..MAX_AGENT_REGISTRATION_ATTEMPTS { + match request().await { + Ok(token) => return Ok(token), + Err(error) + if registration_is_retryable(&error) + && attempt + 1 < MAX_AGENT_REGISTRATION_ATTEMPTS => + { + let delay = agent_registration_retry_delay(&error); + tracing::warn!( + attempt = attempt + 1, + retry_in_ms = delay.as_millis(), + error = %error, + "transient Relaycast agent registration failure; retrying" + ); + sleep(delay).await; + } + Err(error) if registration_is_retryable(&error) => { + return Err(RegRetryOutcome::RetryableExhausted(error)); + } + Err(error) => return Err(RegRetryOutcome::Fatal(error)), + } + } + unreachable!("the registration retry loop always returns on its final attempt") +} + +async fn retry_agent_registration_with_budget( + agent_name: &str, + budget: Duration, + request: F, + sleep: S, +) -> Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, + S: FnMut(Duration) -> SleepFut, + SleepFut: std::future::Future, +{ + match tokio::time::timeout(budget, retry_agent_registration_with(request, sleep)).await { + Ok(result) => result, + Err(_) => Err(RegRetryOutcome::RetryableExhausted( + RelaycastRegistrationError::Transport { + agent_name: agent_name.to_string(), + detail: format!( + "registration retry budget exhausted after {}s", + budget.as_secs() + ), + }, + )), + } +} + /// Attempt to register an agent token with up to 3 retries for transient errors. +/// +/// The Relaycast SDK's typed rate-limit errors carry its cooldown. Honor +/// that duration (under a hard one-minute cap) instead of immediately retrying +/// through the same admission window. Transport failures retain the short SDK +/// fallback because they do not carry a retry delay. The complete operation is +/// capped below Cloud's five-minute step-provisioning budget, so a hung request +/// or a second full cooldown cannot strand the caller indefinitely. pub async fn retry_agent_registration( http: &RelaycastHttpClient, name: &str, @@ -1454,7 +1532,13 @@ pub async fn retry_agent_registration( detail: "SDK relay client not initialized".to_string(), }) })?; - sdk_retry_agent_registration(registration, name, cli).await + retry_agent_registration_with_budget( + name, + MAX_AGENT_REGISTRATION_ELAPSED, + || registration.register_agent_token(name, cli), + |delay| tokio::time::sleep(delay), + ) + .await } /// The declared fields alone, trimmed, with blanks omitted. @@ -1522,9 +1606,11 @@ mod tests { use crate::{fleet_wire::AgentRegistrationMetadata, ids::ChannelName}; use super::{ - format_worker_preregistration_error, registration_is_retryable, - registration_retry_after_secs, ImpersonationAwareRegistrationError, MessageInjectionMode, - RecipientReachability, RegisterIntent, RelaycastHttpClient, + agent_registration_retry_delay, format_worker_preregistration_error, + registration_is_retryable, registration_retry_after_secs, retry_agent_registration_with, + retry_agent_registration_with_budget, ImpersonationAwareRegistrationError, + MessageInjectionMode, RecipientReachability, RegRetryOutcome, RegisterIntent, + RelaycastHttpClient, }; fn seeded_http_client(base_url: &str) -> RelaycastHttpClient { @@ -1549,6 +1635,172 @@ mod tests { assert_eq!(registration_retry_after_secs(&error), Some(60)); } + #[test] + fn registration_retry_delay_honors_typed_cooldown_with_a_hard_cap() { + let advertised = AgentRegistrationError::RateLimited { + agent_name: "worker-a".to_string(), + retry_after_secs: 17, + detail: "rate limited".to_string(), + }; + let excessive = AgentRegistrationError::Blocked { + agent_name: "worker-a".to_string(), + retry_after_secs: 600, + }; + let transport = AgentRegistrationError::Transport { + agent_name: "worker-a".to_string(), + detail: "connection reset".to_string(), + }; + + assert_eq!( + agent_registration_retry_delay(&advertised), + Duration::from_secs(17) + ); + assert_eq!( + agent_registration_retry_delay(&excessive), + Duration::from_secs(60) + ); + assert_eq!( + agent_registration_retry_delay(&transport), + Duration::from_secs(2) + ); + } + + #[tokio::test] + async fn registration_retry_waits_for_the_typed_rate_limit_delay() { + let mut outcomes = std::collections::VecDeque::from([ + Err(AgentRegistrationError::RateLimited { + agent_name: "worker-a".to_string(), + retry_after_secs: 23, + detail: "rate limited".to_string(), + }), + Ok("at_live_after_retry".to_string()), + ]); + let mut delays = Vec::new(); + + let token = retry_agent_registration_with( + || std::future::ready(outcomes.pop_front().expect("bounded test outcome")), + |delay| { + delays.push(delay); + std::future::ready(()) + }, + ) + .await + .expect("the retry after the advertised delay should succeed"); + + assert_eq!(token, "at_live_after_retry"); + assert_eq!(delays, vec![Duration::from_secs(23)]); + } + + #[tokio::test] + async fn registration_retry_preserves_terminal_classification_and_attempt_budget() { + let mut transient_outcomes = std::collections::VecDeque::from([ + Err(AgentRegistrationError::Transport { + agent_name: "worker-a".to_string(), + detail: "first".to_string(), + }), + Err(AgentRegistrationError::Transport { + agent_name: "worker-a".to_string(), + detail: "second".to_string(), + }), + Err(AgentRegistrationError::Transport { + agent_name: "worker-a".to_string(), + detail: "third".to_string(), + }), + ]); + let mut transient_delays = Vec::new(); + + let exhausted = retry_agent_registration_with( + || { + std::future::ready( + transient_outcomes + .pop_front() + .expect("three-attempt test outcome"), + ) + }, + |delay| { + transient_delays.push(delay); + std::future::ready(()) + }, + ) + .await; + + assert!(matches!( + exhausted, + Err(RegRetryOutcome::RetryableExhausted( + AgentRegistrationError::Transport { detail, .. } + )) if detail == "third" + )); + assert_eq!(transient_delays, vec![Duration::from_secs(2); 2]); + assert!(transient_outcomes.is_empty()); + + let mut fatal_delays = Vec::new(); + let fatal = retry_agent_registration_with( + || { + std::future::ready(Err(AgentRegistrationError::Api { + agent_name: "worker-a".to_string(), + status: 401, + detail: "unauthorized".to_string(), + })) + }, + |delay| { + fatal_delays.push(delay); + std::future::ready(()) + }, + ) + .await; + + assert!(matches!( + fatal, + Err(RegRetryOutcome::Fatal(AgentRegistrationError::Api { + status: 401, + .. + })) + )); + assert!(fatal_delays.is_empty()); + } + + #[tokio::test] + async fn registration_retry_budget_cancels_a_hung_request() { + let result = retry_agent_registration_with_budget( + "worker-a", + Duration::from_millis(10), + || std::future::pending::>(), + |_| std::future::ready(()), + ) + .await; + + assert!(matches!( + result, + Err(RegRetryOutcome::RetryableExhausted( + AgentRegistrationError::Transport { detail, .. } + )) if detail.contains("retry budget exhausted") + )); + } + + #[tokio::test] + async fn registration_retry_budget_cancels_a_cooldown_sleep() { + let result = retry_agent_registration_with_budget( + "worker-a", + Duration::from_millis(10), + || { + std::future::ready(Err(AgentRegistrationError::RateLimited { + agent_name: "worker-a".to_string(), + retry_after_secs: 60, + detail: "rate limited".to_string(), + })) + }, + |_| std::future::pending::<()>(), + ) + .await; + + assert!(matches!( + result, + Err(RegRetryOutcome::RetryableExhausted( + AgentRegistrationError::Transport { detail, .. } + )) if detail.contains("retry budget exhausted") + )); + } + #[test] fn format_registration_error_includes_worker_name() { let error = AgentRegistrationError::Transport { From 17875d5badcbb23259dc993b3b44aac963e40584 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 10 Sep 2026 09:26:20 +0200 Subject: [PATCH 50/50] fix(ci): parse retry-after portably --- packages/cli/src/cli/ci-standalone-smoke.test.ts | 4 +++- scripts/ci-standalone-smoke.sh | 12 +++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/cli/ci-standalone-smoke.test.ts b/packages/cli/src/cli/ci-standalone-smoke.test.ts index f92f3ba3ef..a4934f9663 100644 --- a/packages/cli/src/cli/ci-standalone-smoke.test.ts +++ b/packages/cli/src/cli/ci-standalone-smoke.test.ts @@ -69,7 +69,8 @@ elif [[ "$method" = GET && "$url" = */v1/workspace ]]; then printf '%s\n' "\$((verify_count + 1))" > "\$verify_count_file" fi if [ -n "\${FAKE_VERIFY_RETRY_AFTER:-}" ]; then - printf 'HTTP/2 %s\r\nRetry-After: %s\r\n\r\n' "\$verify_status" "\$FAKE_VERIFY_RETRY_AFTER" > "\$headers" + retry_after_header="\${FAKE_VERIFY_RETRY_AFTER_HEADER:-Retry-After}" + printf 'HTTP/2 %s\r\n%s: %s\r\n\r\n' "\$verify_status" "\$retry_after_header" "\$FAKE_VERIFY_RETRY_AFTER" > "\$headers" fi echo "\$verify_status" else @@ -328,6 +329,7 @@ describe('ci-standalone-smoke workspace reuse', () => { FAKE_DELETE_ERROR_CODE: 'internal_error', FAKE_VERIFY_STATUSES: '503,401', FAKE_VERIFY_RETRY_AFTER: '3', + FAKE_VERIFY_RETRY_AFTER_HEADER: 'retry-after', }, timeout: 10_000, }); diff --git a/scripts/ci-standalone-smoke.sh b/scripts/ci-standalone-smoke.sh index 6c9937112c..607e6d2c6c 100755 --- a/scripts/ci-standalone-smoke.sh +++ b/scripts/ci-standalone-smoke.sh @@ -147,11 +147,13 @@ cleanup() { # database overload. Accept only that bounded vocabulary; a # malformed, date-form, or excessive value falls back to 2s. verify_delay="$(awk ' - BEGIN { IGNORECASE = 1 } - /^Retry-After:[[:space:]]*/ { - gsub(/\r/, "") - sub(/^[^:]*:[[:space:]]*/, "") - value = $0 + { + line = $0 + gsub(/\r/, "", line) + } + tolower(line) ~ /^retry-after:[[:space:]]*/ { + sub(/^[^:]*:[[:space:]]*/, "", line) + value = line } END { print value } ' "$VERIFY_HEADERS" 2>/dev/null || true)"