From ea5328c6b86a8839783dadc47961eac0b354aed2 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Fri, 2 Oct 2026 08:59:30 -0700 Subject: [PATCH 1/9] feat(fleet): poll async sandbox preparation Co-Authored-By: Claude Opus 5.5 --- .../commands/fleet-sandbox-regression.test.ts | 11 +- packages/cli/src/cli/commands/fleet.test.ts | 38 +- packages/cli/src/cli/commands/fleet.ts | 17 +- packages/cloud/src/fleet-sandbox.test.ts | 726 ++++++++++++++++++ packages/cloud/src/fleet-sandbox.ts | 464 ++++++++++- packages/cloud/src/index.ts | 1 + .../cases/1763-zero-config-sandbox/run.mjs | 14 +- 7 files changed, 1231 insertions(+), 40 deletions(-) diff --git a/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts b/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts index 6a7aa5e23..85cc9a584 100644 --- a/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts +++ b/packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts @@ -174,7 +174,10 @@ describe('fleet sandbox command regressions', () => { expect(resolveRepository).toHaveBeenCalledWith(outsideGit, undefined); expect(materialize).not.toHaveBeenCalled(); - expect(ensure).toHaveBeenCalledWith(expect.objectContaining({ mountRelayfile: true })); + expect(ensure).toHaveBeenCalledWith( + expect.objectContaining({ mountRelayfile: true }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) + ); expect((ensure.mock.calls[0]?.[0] as Record).relayfilePaths).toBeUndefined(); expect(placement).toHaveBeenCalledWith( expect.objectContaining({ input: expect.objectContaining({ worker_cwd: '/workspace' }) }) @@ -257,7 +260,8 @@ describe('fleet sandbox command regressions', () => { sandboxId: REPLAY_SANDBOX_ID, name: REPLAY_SANDBOX_NAME, mountRelayfile: true, - }) + }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); const ensureInput = ensure.mock.calls[0]?.[0] as Record; expect(ensureInput.repos).toBeUndefined(); @@ -324,7 +328,8 @@ describe('fleet sandbox command regressions', () => { expect.objectContaining({ repos: [selection.repository], repoRevisions: { [selection.repository]: REVISION }, - }) + }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); }); }); diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 7236424cb..64de95ee9 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1184,6 +1184,7 @@ describe('fleet command support', () => { expect(ensureInput).toEqual({ workspaceId: 'rw_abc', requiredCapability: 'spawn:codex', + preparationMode: 'async-v1', maxAgents: 1, mountRelayfile: true, relayfilePaths: ['/live-review/run-123/**'], @@ -1257,6 +1258,7 @@ describe('fleet command support', () => { workerCwd: '/srv/agent-workforce/cloud/packages/web', }; const events: string[] = []; + const warnings: string[] = []; const materializeCloudRelayfileRepository = vi.fn(async () => { events.push('materialize'); return { @@ -1322,7 +1324,7 @@ describe('fleet command support', () => { deleteCloudFleetSandbox: vi.fn(async () => undefined), createFleetWorkspaceClient: vi.fn() as never, log: () => undefined, - warn: () => undefined, + warn: (message) => warnings.push(message), error: () => undefined, }); @@ -1361,9 +1363,20 @@ describe('fleet command support', () => { '/.skills/**', '/memory/**', ], - }) + }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0] as Record; + const ensureOptions = ensureCloudFleetSandbox.mock.calls[0]?.[1]; + ensureOptions?.onPreparationProgress?.({ + sandboxId: REPLAY_SANDBOX_ID, + state: 'pending', + phase: 'relayfile_mount_bootstrap', + generation: 2, + }); + expect(warnings).toEqual([ + 'Cloud sandbox preparation: relayfile_mount_bootstrap (pending, generation 2).', + ]); expect(ensureInput.repos).toBeUndefined(); expect(ensureInput.repoRevisions).toBeUndefined(); expect(placement.spawn).toHaveBeenCalledWith( @@ -1761,7 +1774,12 @@ describe('fleet command support', () => { expect(resolveWorkspaceByKey).toHaveBeenCalledWith('rk_live_pinned'); expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( - expect.objectContaining({ workspaceId: 'cloud-pinned', workloadProfile: 'long-running-agent' }) + expect.objectContaining({ + workspaceId: 'cloud-pinned', + workloadProfile: 'long-running-agent', + preparationMode: 'async-v1', + }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); expect(createWorkspaceRelay).toHaveBeenCalledWith( expect.objectContaining({ @@ -2461,7 +2479,9 @@ describe('fleet command support', () => { expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( expect.objectContaining({ workloadProfile: 'long-running-agent', - }) + preparationMode: 'async-v1', + }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); }); @@ -2670,7 +2690,8 @@ describe('fleet command support', () => { { from: 'user' } ); expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( - expect.objectContaining({ workspaceId: 'rw_captured' }) + expect.objectContaining({ workspaceId: 'rw_captured', preparationMode: 'async-v1' }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); }); @@ -2736,7 +2757,8 @@ describe('fleet command support', () => { ); expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( - expect.objectContaining({ workspaceId: cloudWorkspaceId }) + expect.objectContaining({ workspaceId: cloudWorkspaceId, preparationMode: 'async-v1' }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) ); expect(createWorkspaceRelay).toHaveBeenCalledWith({ projectRoot: process.cwd(), @@ -2903,6 +2925,10 @@ describe('fleet command support', () => { /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ ); expect(ensureInput?.name).toBe(`fleet-sandbox-${ensureInput?.sandboxId?.slice('sbx_'.length)}`); + expect(ensureInput?.preparationMode).toBe('async-v1'); + expect(ensureCloudFleetSandbox.mock.calls[0]?.[1]).toEqual({ + onPreparationProgress: expect.any(Function), + }); }); it('rejects a replay sandbox name that does not match its identity', async () => { diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index ba8a884e1..b4a23e916 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -11,6 +11,7 @@ import { resolveWorkspaceByKey, type CloudFleetSandboxProviderId, type CloudRelayfileRepositoryMaterialization, + type EnsureCloudFleetSandboxInput, type EnsureCloudFleetSandboxResult, } from '@agent-relay/cloud'; import { HarnessDriverClient } from '@agent-relay/harness-driver'; @@ -713,10 +714,13 @@ export function registerFleetCommands( sandboxProvider === undefined || sandboxProvider === 'agent37' ? 'long-running-agent' : 'standard-long-running-agent'; + const useAsyncPreparation = + sandboxId !== undefined && (sandboxProvider === undefined || sandboxProvider === 'agent37'); try { - sandbox = await deps.ensureCloudFleetSandbox({ + const ensureInput: EnsureCloudFleetSandboxInput = { workspaceId: relayWorkspaceId, requiredCapability: `spawn:${cli}`, + ...(useAsyncPreparation ? { preparationMode: 'async-v1' as const } : {}), maxAgents: 1, mountRelayfile: mountSandboxRelayfile, ...(liveRepository @@ -734,7 +738,16 @@ export function registerFleetCommands( ...(checkoutRepository && sandboxRepository ? { repoRevisions: { [sandboxRepository.repository]: sandboxRepository.revision } } : {}), - }); + }; + sandbox = useAsyncPreparation + ? await deps.ensureCloudFleetSandbox(ensureInput, { + onPreparationProgress: (progress) => { + deps.warn( + `Cloud sandbox preparation: ${progress.phase} (${progress.state}, generation ${progress.generation}).` + ); + }, + }) + : await deps.ensureCloudFleetSandbox(ensureInput); assertSandboxRepositoryRevision(sandbox, checkoutRepository ? sandboxRepository : undefined); } catch (error) { if ( diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index d90e1800f..e853d9d2a 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -19,6 +19,7 @@ import { materializeCloudRelayfileRepository, normalizeRelaycastTarget, } from './fleet-sandbox.js'; +import { CloudAuthError } from './types.js'; const auth = { accessToken: 'access', @@ -48,6 +49,23 @@ const PROVIDER_OUTCOME_MATRIX = NON_AGENT_PROVIDER_IDS.flatMap((providerId) => (['provisioned', 'reused'] as const).map((outcome) => ({ providerId, outcome })) ); +function preparationEnvelope( + state: 'pending' | 'ready' | 'terminal' | 'cleanup_pending', + phase: string, + generation: number, + extra: Record = {} +) { + return { + version: 1, + mode: 'async-v1', + sandboxId: SANDBOX_ID, + state, + phase, + generation, + ...extra, + }; +} + describe('Cloud fleet sandbox client', () => { beforeEach(() => { vi.resetAllMocks(); @@ -395,6 +413,714 @@ describe('Cloud fleet sandbox client', () => { }); }); + it('advances async preparation one durable phase per request and reports deduplicated progress', async () => { + const progress = vi.fn(); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth: refreshedAuth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'agent_relay_cli_bootstrap', 0), { + status: 202, + }), + auth: refreshedAuth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'agent_relay_cli_bootstrap', 0), { + status: 202, + }), + auth: refreshedAuth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'relayfile_mount_bootstrap', 1), { + status: 202, + }), + auth: refreshedAuth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 2, { + result: { + outcome: 'provisioned', + nodeId: 'node-async', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-async', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + relayfileMountPath: '/workspace', + providerId: 'agent37', + }, + }) + ), + auth: refreshedAuth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1, onPreparationProgress: progress } + ) + ).resolves.toMatchObject({ + outcome: 'provisioned', + sandboxId: SANDBOX_ID, + nodeId: 'node-async', + providerId: 'agent37', + }); + + const requests = mocks.authorizedApiFetch.mock.calls.slice(1); + expect(requests.map((call) => [call[1], call[2]?.method])).toEqual([ + ['/api/v1/fleet/nodes/sandbox/ensure', 'POST'], + [`/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + [`/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`, 'POST'], + [`/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`, 'POST'], + ]); + expect(JSON.parse(String(requests[0]?.[2]?.body))).toMatchObject({ + preparationMode: 'async-v1', + sandboxId: SANDBOX_ID, + providerId: 'agent37', + }); + expect(progress.mock.calls.map(([value]) => value)).toEqual([ + expect.objectContaining({ state: 'pending', phase: 'agent_relay_cli_bootstrap', generation: 0 }), + expect.objectContaining({ state: 'pending', phase: 'relayfile_mount_bootstrap', generation: 1 }), + expect.objectContaining({ state: 'ready', phase: 'broker_visible', generation: 2 }), + ]); + }); + + it('retries an accepted async preparation after rate limiting through durable status', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'rate limited' }, { status: 429, headers: { 'retry-after': '0' } }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'agent_relay_cli_bootstrap', 1), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 2, { + result: { + outcome: 'provisioned', + nodeId: 'node-after-rate-limit', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-rate-limit', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: false, + providerId: 'agent37', + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-after-rate-limit', sandboxId: SANDBOX_ID }); + + expect(mocks.authorizedApiFetch.mock.calls.slice(2).map((call) => call[2]?.method)).toEqual([ + 'GET', + 'GET', + 'POST', + ]); + }); + + it('recovers a dropped initial async ensure response by reading durable status without replaying ensure', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockRejectedValueOnce(new TypeError('connection reset')) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 1, { + result: { + outcome: 'provisioned', + nodeId: 'node-recovered', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-recovered', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: false, + providerId: 'agent37', + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-recovered', sandboxId: SANDBOX_ID }); + + expect( + mocks.authorizedApiFetch.mock.calls.filter((call) => call[1] === '/api/v1/fleet/nodes/sandbox/ensure') + ).toHaveLength(1); + expect(mocks.authorizedApiFetch.mock.calls[2]?.[2]?.method).toBe('GET'); + }); + + it('recovers an initial async 5xx by reading durable status without replaying ensure', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'gateway response lost' }, { status: 504 }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 1, { + result: { + outcome: 'provisioned', + nodeId: 'node-after-gateway', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-gateway', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: false, + providerId: 'agent37', + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-after-gateway', sandboxId: SANDBOX_ID }); + + expect( + mocks.authorizedApiFetch.mock.calls.filter((call) => call[1] === '/api/v1/fleet/nodes/sandbox/ensure') + ).toHaveLength(1); + expect(mocks.authorizedApiFetch.mock.calls[2]?.[2]?.method).toBe('GET'); + }); + + it('recovers a successful async ensure with a truncated envelope through durable status', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: new Response(null, { status: 202 }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 1, { + result: { + outcome: 'provisioned', + nodeId: 'node-after-truncated-ensure', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-truncated-ensure', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: false, + providerId: 'agent37', + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-after-truncated-ensure', sandboxId: SANDBOX_ID }); + + expect( + mocks.authorizedApiFetch.mock.calls.filter((call) => call[1] === '/api/v1/fleet/nodes/sandbox/ensure') + ).toHaveLength(1); + expect(mocks.authorizedApiFetch.mock.calls[2]?.[2]?.method).toBe('GET'); + }); + + it('preserves an initial async sandbox identity mismatch instead of polling the wrong record', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ + ...preparationEnvelope('pending', 'provider_allocation', 0), + sandboxId: 'sbx_00000000-0000-4000-8000-000000000099', + }), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).rejects.toThrow(/returned sandboxId .* instead of requested sandboxId/u); + + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + }); + + it('bounds status polling when the initial async request was never accepted', async () => { + vi.spyOn(Date, 'now').mockReturnValueOnce(0).mockReturnValue(110_001); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockRejectedValueOnce(new TypeError('connection refused')) + .mockResolvedValueOnce({ + response: Response.json({ error: 'not found' }, { status: 404 }), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + sandboxId: SANDBOX_ID, + providerId: 'agent37', + outcomeUnknown: true, + confirmedProvisioned: false, + }); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + }); + + it('wraps the internal async deadline with accepted cleanup identity', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { timeoutMs: 5, preparationPollIntervalMs: 20 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + sandboxId: SANDBOX_ID, + providerId: 'agent37', + confirmedProvisioned: true, + outcomeUnknown: false, + }); + }); + + it('preserves caller cancellation after async preparation was accepted', async () => { + const controller = new AbortController(); + const cancelled = new Error('caller cancelled'); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockImplementationOnce(async () => { + controller.abort(cancelled); + return { + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }; + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { signal: controller.signal, preparationPollIntervalMs: 1 } + ) + ).rejects.toBe(cancelled); + }); + + it('reads durable status after a dropped async advance before issuing another advance', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockRejectedValueOnce(new TypeError('advance response dropped')) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'agent_relay_cli_bootstrap', 1), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 2, { + result: { + outcome: 'provisioned', + nodeId: 'node-after-drop', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-drop', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: false, + providerId: 'agent37', + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-after-drop' }); + + expect(mocks.authorizedApiFetch.mock.calls.slice(2).map((call) => call[2]?.method)).toEqual([ + 'GET', + 'POST', + 'GET', + 'POST', + ]); + }); + + it('surfaces a durable terminal async failure without retrying it', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('terminal', 'cleanup', 1, { + failure: { + code: 'preparation_failed', + error: 'Sandbox preparation failed.', + phase: 'relayfile_mount_bootstrap', + }, + }), + { status: 502 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + confirmedProvisioned: true, + sandboxId: SANDBOX_ID, + }); + expect((error as Error).message).toContain('relayfile_mount_bootstrap'); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(3); + }); + + it('observes cleanup_pending without advancing until Cloud reports terminal', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('cleanup_pending', 'cleanup', 1), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('cleanup_pending', 'cleanup', 1), { + status: 202, + }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('terminal', 'cleanup', 2, { + failure: { + code: 'preparation_deadline_exceeded', + error: 'Sandbox preparation deadline exceeded.', + phase: 'cleanup', + }, + }), + { status: 502 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ confirmedProvisioned: true, sandboxId: SANDBOX_ID }); + expect(mocks.authorizedApiFetch.mock.calls.slice(2).map((call) => call[2]?.method)).toEqual([ + 'GET', + 'GET', + ]); + }); + + it('surfaces an initial async authentication failure without polling', async () => { + const authError = new CloudAuthError('AUTH_REFRESH_EXPIRED', 'Sign in again.'); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockRejectedValueOnce(authError); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + providerId: 'agent37', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).rejects.toBe(authError); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + }); + + it('surfaces an async polling authentication failure without retrying it', async () => { + const authError = new CloudAuthError('AUTH_REFRESH_EXPIRED', 'Sign in again.'); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0), { + status: 202, + }), + auth, + }) + .mockRejectedValueOnce(authError); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + providerId: 'agent37', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).rejects.toBe(authError); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(3); + }); + + it('marks an invalid ready result safe only after exact async cleanup identity is proven', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 1, { + result: { + outcome: 'provisioned', + nodeId: 'node-invalid-ready', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + providerId: 'agent37', + }, + }) + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + repos: ['AgentWorkforce/relay'], + repoRevisions: { 'AgentWorkforce/relay': '0123456789abcdef0123456789abcdef01234567' }, + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + sandboxId: SANDBOX_ID, + nodeName: SANDBOX_NAME, + providerId: 'agent37', + confirmedProvisioned: true, + outcomeUnknown: false, + }); + }); + it('rejects an invalid sandbox identity before contacting Cloud', async () => { await expect( ensureCloudFleetSandbox({ diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index a698a92a5..8d43b3dec 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -1,6 +1,6 @@ import { authorizedApiFetch, ensureCloudSession } from './auth.js'; import { redactCredentialValues } from './redact.js'; -import { defaultApiUrl } from './types.js'; +import { CloudAuthError, defaultApiUrl } from './types.js'; type JsonRecord = Record; @@ -24,6 +24,9 @@ const DEFAULT_RESOLUTION_TIMEOUT_MS = 120_000; // not abandon a successful server-side request without receiving its sandbox // identity (which prevents the CLI from cleaning it up safely). const DEFAULT_ENSURE_TIMEOUT_MS = 480_000; +const DEFAULT_ASYNC_PREPARATION_TIMEOUT_MS = 35 * 60_000; +const ASYNC_PREPARATION_REQUEST_TIMEOUT_MS = 110_000; +const DEFAULT_ASYNC_PREPARATION_POLL_INTERVAL_MS = 1_000; const DEFAULT_DELETE_TIMEOUT_MS = 30_000; const DEFAULT_RELAYFILE_REPOSITORY_MATERIALIZE_TIMEOUT_MS = 20 * 60_000; const DEFAULT_RELAYFILE_REPOSITORY_POLL_INTERVAL_MS = 2_000; @@ -34,6 +37,16 @@ export type CloudFleetSandboxRequestOptions = { apiUrl?: string; signal?: AbortSignal; timeoutMs?: number; + /** Receives safe durable phase changes while an async-v1 sandbox is prepared. */ + onPreparationProgress?: (progress: CloudFleetSandboxPreparationProgress) => void; + preparationPollIntervalMs?: number; +}; + +export type CloudFleetSandboxPreparationProgress = { + sandboxId: string; + state: 'pending' | 'ready' | 'terminal' | 'cleanup_pending'; + phase: string; + generation: number; }; export type MaterializeCloudRelayfileRepositoryInput = { @@ -108,6 +121,8 @@ class CloudFleetSandboxIdentityMismatchError extends Error {} export type EnsureCloudFleetSandboxInput = { /** Cloud UUID or unified rw_* workspace id. */ workspaceId: string; + /** Opt in to Cloud's durable, poll-driven Agent37 preparation contract. */ + preparationMode?: 'async-v1'; /** Caller-declared one-time Cloud identity used to resume a cut-off provision. */ sandboxId?: string; name?: string; @@ -379,6 +394,18 @@ function endpointError(action: string, response: Response, payload: unknown): Er ); } +function retryAfterDelayMs(response: Response): number | null { + const value = response.headers.get('retry-after')?.trim(); + if (!value) return null; + const seconds = Number(value); + if (Number.isFinite(seconds) && seconds >= 0) { + return Math.min(MAX_TIMER_MS, Math.max(0, Math.ceil(seconds * 1000))); + } + const retryAt = Date.parse(value); + if (!Number.isFinite(retryAt)) return null; + return Math.min(MAX_TIMER_MS, Math.max(0, retryAt - Date.now())); +} + function requiredString(payload: JsonRecord, key: string, context: string): string { const value = readString(payload, key); if (!value) throw new Error(`${context} response is missing ${key}.`); @@ -649,6 +676,19 @@ function confirmsProvisionedSandboxIdentity( return readString(payload, 'providerId') === requestedProviderId; } +function confirmsAsyncPreparedSandboxIdentity( + payload: unknown, + expectedSandboxId: string, + expectedNodeName: string | undefined, + requestedProviderId: CloudFleetSandboxProviderId | undefined +): boolean { + if (!isObject(payload) || requestedProviderId === undefined) return false; + if (readString(payload, 'outcome') !== 'provisioned') return false; + if (readString(payload, 'sandboxId') !== expectedSandboxId) return false; + if (expectedNodeName !== undefined && readString(payload, 'nodeName') !== expectedNodeName) return false; + return readString(payload, 'providerId') === requestedProviderId; +} + function normalizeEnsureResult( payload: unknown, cloudWorkspaceId: string, @@ -756,6 +796,85 @@ function normalizeEnsureResult( throw new Error('Cloud fleet sandbox response has an unknown outcome.'); } +const ASYNC_PREPARATION_STATES = new Set(['pending', 'ready', 'terminal', 'cleanup_pending']); +const ASYNC_PREPARATION_PHASES = new Set([ + 'provider_allocation', + 'agent_relay_cli_bootstrap', + 'relayfile_mount_bootstrap', + 'spawn_cli_bootstrap', + 'repo_clone', + 'finalize', + 'broker_visible', + 'cleanup', +]); + +type AsyncPreparationEnvelope = CloudFleetSandboxPreparationProgress & { + result?: JsonRecord; + failure?: { code: string; error: string; phase: string; causeDetail?: string }; +}; + +function readAsyncPreparationEnvelope( + payload: unknown, + expectedSandboxId: string +): AsyncPreparationEnvelope | null { + if (!isObject(payload) || payload.mode !== 'async-v1' || payload.version !== 1) return null; + const sandboxId = readString(payload, 'sandboxId'); + if (sandboxId !== expectedSandboxId) { + throw new CloudFleetSandboxIdentityMismatchError( + `Cloud returned sandboxId ${sandboxId ?? 'missing'} instead of requested sandboxId ${expectedSandboxId}.` + ); + } + const state = readString(payload, 'state'); + const phase = readString(payload, 'phase'); + const generation = readNumber(payload, 'generation'); + if ( + !state || + !ASYNC_PREPARATION_STATES.has(state) || + !phase || + !ASYNC_PREPARATION_PHASES.has(phase) || + generation === undefined || + !Number.isSafeInteger(generation) || + generation < 0 + ) { + throw new Error('Cloud fleet sandbox preparation response was invalid.'); + } + const envelope: AsyncPreparationEnvelope = { + sandboxId, + state: state as AsyncPreparationEnvelope['state'], + phase, + generation, + }; + if (payload.result !== undefined) { + if (!isObject(payload.result)) { + throw new Error('Cloud fleet sandbox preparation result was invalid.'); + } + envelope.result = payload.result; + } + if (payload.failure !== undefined) { + if (!isObject(payload.failure)) { + throw new Error('Cloud fleet sandbox preparation failure was invalid.'); + } + const code = readString(payload.failure, 'code'); + const error = readString(payload.failure, 'error'); + const failurePhase = readString(payload.failure, 'phase'); + const causeDetail = readString(payload.failure, 'causeDetail'); + if (!code || !error || !failurePhase) { + throw new Error('Cloud fleet sandbox preparation failure was invalid.'); + } + envelope.failure = { + code, + error, + phase: failurePhase, + ...(causeDetail === undefined ? {} : { causeDetail }), + }; + } + return envelope; +} + +function asyncPreparationRequestSignal(overall: AbortSignal): AbortSignal { + return AbortSignal.any([overall, AbortSignal.timeout(ASYNC_PREPARATION_REQUEST_TIMEOUT_MS)]); +} + /** * Materialize one exact GitHub revision into the selected workspace's live * Relayfile tree and wait until the decoded working-tree mount can consume it. @@ -908,6 +1027,19 @@ export async function ensureCloudFleetSandbox( } validateRequestedRepos(input.repos); const repoRevisions = validateRepoRevisions(input.repos, input.repoRevisions); + const asyncPreparation = input.preparationMode === 'async-v1'; + if (asyncPreparation && sandboxIdentity.sandboxId === undefined) { + throw new Error('Async Cloud fleet preparation requires a caller-declared sandboxId.'); + } + // Async-v1 is the replay-safe Agent37 contract. Make the implicit default + // explicit on the wire and in cleanup proof so a malformed ready result can + // never lose the provider identity merely because the CLI omitted --provider. + const requestedProviderId = asyncPreparation ? (input.providerId ?? 'agent37') : input.providerId; + const preparationPollIntervalMs = normalizeTimerMs( + options.preparationPollIntervalMs ?? DEFAULT_ASYNC_PREPARATION_POLL_INTERVAL_MS, + false, + 'Cloud fleet preparation poll interval' + ); const session = await ensureCloudSession({ apiUrl: options.apiUrl || defaultApiUrl(), @@ -915,34 +1047,246 @@ export async function ensureCloudFleetSandbox( }); const resolutionSignal = boundedSignal(options, DEFAULT_RESOLUTION_TIMEOUT_MS); const resolved = await resolveCloudWorkspaceId(workspaceId, session.auth, resolutionSignal); - const signal = boundedSignal(options, DEFAULT_ENSURE_TIMEOUT_MS); + const signal = boundedSignal( + options, + asyncPreparation ? DEFAULT_ASYNC_PREPARATION_TIMEOUT_MS : DEFAULT_ENSURE_TIMEOUT_MS + ); + let activeAuth = resolved.auth; + const ensureBody = JSON.stringify({ + workspaceId: resolved.cloudWorkspaceId, + requiredCapability, + ...(asyncPreparation ? { preparationMode: 'async-v1' } : {}), + ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), + ...(sandboxIdentity.name === undefined ? {} : { name: sandboxIdentity.name }), + ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), + ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), + ...(input.relayfilePaths === undefined ? {} : { relayfilePaths: [...input.relayfilePaths] }), + ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), + ...(requestedProviderId !== undefined ? { providerId: requestedProviderId } : {}), + ...(input.workloadProfile !== undefined ? { workloadProfile: input.workloadProfile } : {}), + ...(input.waitTimeoutMs !== undefined ? { waitTimeoutMs: input.waitTimeoutMs } : {}), + ...(input.repos !== undefined && input.repos.length > 0 ? { repos: [...input.repos] } : {}), + ...(repoRevisions === undefined ? {} : { repoRevisions }), + }); + + let sawAcceptedPreparation = false; + let lastProgressSignature: string | undefined; + const consumePreparation = (payload: unknown): EnsureCloudFleetSandboxResult | null => { + const envelope = readAsyncPreparationEnvelope(payload, sandboxIdentity.sandboxId!); + if (!envelope) return null; + sawAcceptedPreparation = true; + const signature = `${envelope.state}:${envelope.phase}:${envelope.generation}`; + if (signature !== lastProgressSignature) { + options.onPreparationProgress?.({ + sandboxId: envelope.sandboxId, + state: envelope.state, + phase: envelope.phase, + generation: envelope.generation, + }); + lastProgressSignature = signature; + } + if (envelope.state === 'ready') { + try { + if (!envelope.result) { + throw new Error('Cloud fleet sandbox preparation completed without a result.'); + } + return normalizeEnsureResult( + envelope.result, + resolved.cloudWorkspaceId, + sandboxIdentity.sandboxId!, + sandboxIdentity.name, + requestedProviderId, + repoRevisions + ); + } catch (error) { + const confirmedProvisioned = confirmsAsyncPreparedSandboxIdentity( + envelope.result, + sandboxIdentity.sandboxId!, + sandboxIdentity.name, + requestedProviderId + ); + throw new CloudFleetSandboxProvisionError( + error instanceof Error ? error.message : 'Cloud fleet sandbox preparation result was invalid.', + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: sandboxIdentity.sandboxId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...(confirmedProvisioned ? { confirmedProvisioned: true } : { outcomeUnknown: true }), + cause: error, + } + ); + } + } + if (envelope.state === 'terminal') { + const failure = envelope.failure; + throw new CloudFleetSandboxProvisionError( + redactCredentialValues( + failure + ? `Cloud fleet sandbox preparation failed during ${failure.phase}: ${failure.error}` + : 'Cloud fleet sandbox preparation failed.' + ), + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: sandboxIdentity.sandboxId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + confirmedProvisioned: true, + } + ); + } + return null; + }; + + const pollPreparation = async (initialPayload?: unknown): Promise => { + const unconfirmedDeadline = Date.now() + ASYNC_PREPARATION_REQUEST_TIMEOUT_MS; + let payload = initialPayload; + let method: 'GET' | 'POST' = initialPayload === undefined ? 'GET' : 'POST'; + let nextPollDelayMs: number | null = null; + for (;;) { + if (payload !== undefined) { + const ready = consumePreparation(payload); + if (ready) return ready; + } + if (!sawAcceptedPreparation && Date.now() >= unconfirmedDeadline) { + throw new Error('Cloud fleet sandbox preparation was not accepted before the status deadline.'); + } + const requestedDelayMs = nextPollDelayMs ?? preparationPollIntervalMs; + nextPollDelayMs = null; + const delayMs = sawAcceptedPreparation + ? requestedDelayMs + : Math.min(requestedDelayMs, Math.max(0, unconfirmedDeadline - Date.now())); + await waitForDelay(delayMs, signal); + // Do not start another full request after the short lost-ensure + // reconciliation window. A long configured poll interval must not turn + // the 110-second uncertainty bound into a 220-second network attempt. + if (!sawAcceptedPreparation && Date.now() >= unconfirmedDeadline) { + throw new Error('Cloud fleet sandbox preparation was not accepted before the status deadline.'); + } + const path = `/api/v1/fleet/nodes/sandbox/${encodeURIComponent( + sandboxIdentity.sandboxId! + )}/preparation${ + method === 'GET' ? `?workspaceId=${encodeURIComponent(resolved.cloudWorkspaceId)}` : '' + }`; + let pollResponse: Response; + try { + const polled = await authorizedApiFetch( + activeAuth, + path, + { + method, + signal: asyncPreparationRequestSignal(signal), + ...(method === 'POST' + ? { body: JSON.stringify({ workspaceId: resolved.cloudWorkspaceId }) } + : {}), + }, + { interactive: false } + ); + activeAuth = polled.auth; + pollResponse = polled.response; + } catch (fetchError) { + if (signal.aborted) throw signal.reason ?? fetchError; + const errorName = fetchError instanceof Error ? fetchError.name : ''; + if ( + !(fetchError instanceof TypeError) && + errorName !== 'TimeoutError' && + errorName !== 'AbortError' + ) { + throw fetchError; + } + if (!sawAcceptedPreparation && Date.now() >= unconfirmedDeadline) { + throw fetchError; + } + // A lost advance response is not replayed. Read durable status first; + // only a confirmed pending record permits a later advance. + method = 'GET'; + payload = undefined; + continue; + } + payload = await readJson(pollResponse); + const envelope = readAsyncPreparationEnvelope(payload, sandboxIdentity.sandboxId!); + if (envelope) { + const ready = consumePreparation(payload); + if (ready) return ready; + // Cleanup is owned by Cloud. Keep observing it until Cloud publishes a + // terminal result; advancing here could race the reaper or restart work + // on a sandbox that is already being destroyed. + method = envelope.state === 'cleanup_pending' ? 'GET' : 'POST'; + payload = undefined; + continue; + } + if ( + method === 'GET' && + pollResponse.status === 404 && + !sawAcceptedPreparation && + Date.now() < unconfirmedDeadline + ) { + payload = undefined; + continue; + } + if (!pollResponse.ok) { + if ( + (pollResponse.status >= 500 || pollResponse.status === 429) && + (sawAcceptedPreparation || Date.now() < unconfirmedDeadline) + ) { + if (pollResponse.status === 429) { + nextPollDelayMs = retryAfterDelayMs(pollResponse) ?? preparationPollIntervalMs; + } + method = 'GET'; + payload = undefined; + continue; + } + throw endpointError('read fleet sandbox preparation status', pollResponse, payload); + } + throw new Error('Cloud fleet sandbox preparation response was invalid.'); + } + }; + let response: Response; try { - ({ response } = await authorizedApiFetch( - resolved.auth, + const ensured = await authorizedApiFetch( + activeAuth, '/api/v1/fleet/nodes/sandbox/ensure', { method: 'POST', - signal, - body: JSON.stringify({ - workspaceId: resolved.cloudWorkspaceId, - requiredCapability, - ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), - ...(sandboxIdentity.name === undefined ? {} : { name: sandboxIdentity.name }), - ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), - ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), - ...(input.relayfilePaths === undefined ? {} : { relayfilePaths: [...input.relayfilePaths] }), - ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), - ...(input.providerId !== undefined ? { providerId: input.providerId } : {}), - ...(input.workloadProfile !== undefined ? { workloadProfile: input.workloadProfile } : {}), - ...(input.waitTimeoutMs !== undefined ? { waitTimeoutMs: input.waitTimeoutMs } : {}), - ...(input.repos !== undefined && input.repos.length > 0 ? { repos: [...input.repos] } : {}), - ...(repoRevisions === undefined ? {} : { repoRevisions }), - }), + signal: asyncPreparation ? asyncPreparationRequestSignal(signal) : signal, + body: ensureBody, }, { interactive: false } - )); + ); + activeAuth = ensured.auth; + response = ensured.response; } catch (error) { + if (asyncPreparation) { + if (options.signal?.aborted) throw options.signal.reason ?? error; + if (error instanceof CloudAuthError) throw error; + const errorName = error instanceof Error ? error.name : ''; + const retryableTransport = + error instanceof TypeError || errorName === 'TimeoutError' || errorName === 'AbortError'; + if (!retryableTransport && !signal.aborted) throw error; + try { + return await pollPreparation(); + } catch (pollError) { + if (pollError instanceof CloudFleetSandboxProvisionError) throw pollError; + if (pollError instanceof CloudAuthError) throw pollError; + if (options.signal?.aborted) throw options.signal.reason ?? pollError; + throw new CloudFleetSandboxProvisionError( + redactCredentialValues( + `Cloud fleet sandbox preparation status could not be confirmed: ${ + pollError instanceof Error ? pollError.message : String(pollError) + }` + ), + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: sandboxIdentity.sandboxId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...(sawAcceptedPreparation ? { confirmedProvisioned: true } : { outcomeUnknown: true }), + cause: pollError, + } + ); + } + } throw new CloudFleetSandboxProvisionError( redactCredentialValues( `Cloud fleet sandbox request ended without a complete response: ${ @@ -960,6 +1304,72 @@ export async function ensureCloudFleetSandbox( ); } const payload = await readJson(response); + if (asyncPreparation) { + let envelope: AsyncPreparationEnvelope | null; + try { + envelope = readAsyncPreparationEnvelope(payload, sandboxIdentity.sandboxId!); + } catch (error) { + // A mismatched public identity is neither a truncated response nor an + // ambiguous gateway failure. Polling the caller-declared identity would + // hide the diagnostic and could point cleanup at the wrong sandbox. + if (error instanceof CloudFleetSandboxIdentityMismatchError) throw error; + envelope = null; + } + if (envelope) { + const ready = consumePreparation(payload); + if (ready) return ready; + try { + return await pollPreparation(); + } catch (pollError) { + if (pollError instanceof CloudFleetSandboxProvisionError) throw pollError; + if (pollError instanceof CloudAuthError) throw pollError; + if (options.signal?.aborted) throw options.signal.reason ?? pollError; + throw new CloudFleetSandboxProvisionError( + redactCredentialValues( + `Cloud fleet sandbox preparation did not complete: ${ + pollError instanceof Error ? pollError.message : String(pollError) + }` + ), + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: sandboxIdentity.sandboxId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + confirmedProvisioned: true, + cause: pollError, + } + ); + } + } + // A successful/accepted response can still lose or truncate its body after + // Cloud durably recorded the preparation. Reconcile every non-envelope + // async response through the declared sandbox's status endpoint instead of + // falling through to legacy response parsing or replaying ensure. + if (response.ok || response.status >= 500) { + try { + return await pollPreparation(); + } catch (pollError) { + if (pollError instanceof CloudFleetSandboxProvisionError) throw pollError; + if (pollError instanceof CloudAuthError) throw pollError; + if (options.signal?.aborted) throw options.signal.reason ?? pollError; + throw new CloudFleetSandboxProvisionError( + redactCredentialValues( + `Cloud fleet sandbox preparation status could not be confirmed: ${ + pollError instanceof Error ? pollError.message : String(pollError) + }` + ), + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: sandboxIdentity.sandboxId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...(sawAcceptedPreparation ? { confirmedProvisioned: true } : { outcomeUnknown: true }), + cause: pollError, + } + ); + } + } + } if (!response.ok) { const returnedSandboxId = isObject(payload) ? readString(payload, 'sandboxId') : undefined; if ( @@ -973,7 +1383,7 @@ export async function ensureCloudFleetSandbox( throw new CloudFleetSandboxProvisionError(mismatch.message, { cloudWorkspaceId: resolved.cloudWorkspaceId, ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(input.providerId === undefined ? {} : { providerId: input.providerId }), + ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), outcomeUnknown: true, cause: mismatch, }); @@ -988,7 +1398,7 @@ export async function ensureCloudFleetSandbox( cloudWorkspaceId: resolved.cloudWorkspaceId, ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(input.providerId === undefined ? {} : { providerId: input.providerId }), + ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), outcomeUnknown: true, cause: error, }); @@ -998,7 +1408,7 @@ export async function ensureCloudFleetSandbox( cloudWorkspaceId: resolved.cloudWorkspaceId, ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(input.providerId === undefined ? {} : { providerId: input.providerId }), + ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), outcomeUnknown: true, cause: error, }); @@ -1011,7 +1421,7 @@ export async function ensureCloudFleetSandbox( resolved.cloudWorkspaceId, sandboxIdentity.sandboxId, sandboxIdentity.name, - input.providerId, + requestedProviderId, repoRevisions ); } catch (error) { @@ -1019,7 +1429,7 @@ export async function ensureCloudFleetSandbox( payload, sandboxIdentity.sandboxId, sandboxIdentity.name, - input.providerId + requestedProviderId ); throw new CloudFleetSandboxProvisionError( error instanceof Error ? error.message : 'Cloud fleet sandbox response was invalid.', @@ -1027,7 +1437,7 @@ export async function ensureCloudFleetSandbox( cloudWorkspaceId: resolved.cloudWorkspaceId, ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(input.providerId === undefined ? {} : { providerId: input.providerId }), + ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), ...(confirmedProvisioned ? { confirmedProvisioned: true } : { outcomeUnknown: true }), cause: error, } diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 678428388..75011f6e3 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -102,6 +102,7 @@ export { type CloudFleetSandboxWorkloadProfile, type DeleteCloudFleetSandboxInput, type CloudFleetSandboxRequestOptions, + type CloudFleetSandboxPreparationProgress, type CloudFleetRelaycastRoute, type CloudFleetRelaycastTarget, } from './fleet-sandbox.js'; diff --git a/tests/relayflows/cases/1763-zero-config-sandbox/run.mjs b/tests/relayflows/cases/1763-zero-config-sandbox/run.mjs index 505b511b3..e35e6bf8c 100644 --- a/tests/relayflows/cases/1763-zero-config-sandbox/run.mjs +++ b/tests/relayflows/cases/1763-zero-config-sandbox/run.mjs @@ -43,11 +43,21 @@ test('fleet sandbox CLI materializes exact source through a scoped live Relayfil if (!output) throw new Error('Missing observation path.'); const requests = []; mocks.ensureCloudSession.mockResolvedValue({ auth, client: {} }); - mocks.authorizedApiFetch.mockImplementation(async (_auth, _path, request) => { + mocks.authorizedApiFetch.mockImplementation(async (_auth, requestPath, request) => { const body = request?.body ? JSON.parse(request.body) : null; requests.push({ body }); if (requests.length === 1) return { response: Response.json({ cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99' }), auth }; - return { response: Response.json({ outcome: 'provisioned', cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', nodeId: 'node-proof', nodeName: body?.name ?? 'sandbox-proof', sandboxId: body?.sandboxId ?? 'sbx_123e4567-e89b-42d3-a456-426614174000', relayWorkspaceId: 'rw-proof', relayfileMounted: true, providerId: 'agent37', relaycastTarget: { route: 'agent37-isolated', baseUrl: 'https://agent37-cast.agentrelay.com', workspaceId: 'rw-proof', relaycastApiKey: 'rk_live_probe' }, repoRevisions: body?.repoRevisions ?? undefined }, { status: 201 }), auth }; + const ensureBody = requests[1]?.body ?? {}; + const sandboxId = ensureBody.sandboxId ?? 'sbx_123e4567-e89b-42d3-a456-426614174000'; + const result = { outcome: 'provisioned', cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', nodeId: 'node-proof', nodeName: ensureBody.name ?? 'sandbox-proof', sandboxId, providerSandboxId: 'provider-proof', relayWorkspaceId: 'rw-proof', relayfileMounted: true, providerId: 'agent37', relaycastTarget: { route: 'agent37-isolated', baseUrl: 'https://agent37-cast.agentrelay.com', workspaceId: 'rw-proof', relaycastApiKey: 'rk_live_probe' }, repoRevisions: ensureBody.repoRevisions ?? undefined }; + if (${JSON.stringify(arm)} === 'head') { + if (requestPath === '/api/v1/fleet/nodes/sandbox/ensure' && request?.method === 'POST') return { response: Response.json({ version: 1, mode: 'async-v1', sandboxId, state: 'pending', phase: 'provider_allocation', generation: 0 }, { status: 202 }), auth }; + const preparationPath = '/api/v1/fleet/nodes/sandbox/' + encodeURIComponent(sandboxId) + '/preparation'; + if (requestPath === preparationPath + '?workspaceId=50587328-441d-4acb-b8f3-dbe1b3c5de99' && request?.method === 'GET') return { response: Response.json({ version: 1, mode: 'async-v1', sandboxId, state: 'pending', phase: 'agent_relay_cli_bootstrap', generation: 1 }, { status: 202 }), auth }; + if (requestPath === preparationPath && request?.method === 'POST') return { response: Response.json({ version: 1, mode: 'async-v1', sandboxId, state: 'ready', phase: 'broker_visible', generation: 2, result }), auth }; + throw new Error('Unexpected async preparation request: ' + request?.method + ' ' + requestPath); + } + return { response: Response.json(result, { status: 201 }), auth }; }); const logs = [], warnings = [], deletes = [], releases = [], materializations = [], spawnInputs = []; const program = new Command(); program.exitOverride(); From aae39118c11977e1eace57a869d8290d6448a6d2 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Fri, 2 Oct 2026 12:06:03 -0700 Subject: [PATCH 2/9] fix(fleet): support legacy synchronous ensure responses Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/cloud/src/fleet-sandbox.test.ts | 121 +++++++++++++++++++++++ packages/cloud/src/fleet-sandbox.ts | 72 +++++++++++++- 2 files changed, 188 insertions(+), 5 deletions(-) diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index e853d9d2a..9b3a952a4 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -496,6 +496,127 @@ describe('Cloud fleet sandbox client', () => { ]); }); + it('accepts a synchronous 201 result when an older Cloud ignores async preparation mode', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeId: 'node-from-old-cloud', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-from-old-cloud', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + relayfileMountPath: '/workspace', + providerId: 'agent37', + }, + { status: 201 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }) + ).resolves.toMatchObject({ + outcome: 'provisioned', + nodeId: 'node-from-old-cloud', + sandboxId: SANDBOX_ID, + }); + + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + expect(JSON.parse(String(mocks.authorizedApiFetch.mock.calls[1]?.[2]?.body))).toMatchObject({ + preparationMode: 'async-v1', + sandboxId: SANDBOX_ID, + }); + }); + + it('retries synchronously when an older Cloud rejects the async preparation field', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'Unrecognized key: preparationMode' }, { status: 400 }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeId: 'node-after-compatibility-retry', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-compatibility-retry', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + providerId: 'agent37', + }, + { status: 201 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }) + ).resolves.toMatchObject({ + outcome: 'provisioned', + nodeId: 'node-after-compatibility-retry', + sandboxId: SANDBOX_ID, + }); + + const ensureRequests = mocks.authorizedApiFetch.mock.calls.slice(1); + expect(ensureRequests).toHaveLength(2); + expect(JSON.parse(String(ensureRequests[0]?.[2]?.body))).toHaveProperty('preparationMode', 'async-v1'); + expect(JSON.parse(String(ensureRequests[1]?.[2]?.body))).not.toHaveProperty('preparationMode'); + }); + + it('does not replay ensure for an unrelated 400 response', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'Repository revision is invalid' }, { status: 400 }), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }) + ).rejects.toThrow('Repository revision is invalid'); + + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + }); + it('retries an accepted async preparation after rate limiting through durable status', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 8d43b3dec..1ecb8511a 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -871,10 +871,36 @@ function readAsyncPreparationEnvelope( return envelope; } +function isLegacyEnsureOutcome(payload: unknown): boolean { + if (!isObject(payload)) return false; + return ['provisioned', 'reused', 'provisioning_timeout'].includes(readString(payload, 'outcome') ?? ''); +} + +function rejectsAsyncPreparationMode(response: Response, payload: unknown): boolean { + if (response.status !== 400 || !isObject(payload)) return false; + const diagnostic = [ + readString(payload, 'code'), + readString(payload, 'error'), + readString(payload, 'message'), + ] + .filter((value): value is string => value !== undefined) + .join(' ') + .toLowerCase(); + return diagnostic.includes('preparationmode') || diagnostic.includes('preparation_mode'); +} + function asyncPreparationRequestSignal(overall: AbortSignal): AbortSignal { return AbortSignal.any([overall, AbortSignal.timeout(ASYNC_PREPARATION_REQUEST_TIMEOUT_MS)]); } +function asyncPreparationInitialRequestSignal(overall: AbortSignal): AbortSignal { + // Until every Cloud deployment understands async-v1, the first request may + // be handled synchronously by an older server. Preserve the legacy client's + // 480-second response window for that one capability probe; every status or + // advance request remains capped at 110 seconds. + return AbortSignal.any([overall, AbortSignal.timeout(DEFAULT_ENSURE_TIMEOUT_MS)]); +} + /** * Materialize one exact GitHub revision into the selected workspace's live * Relayfile tree and wait until the decoded working-tree mount can consume it. @@ -1052,7 +1078,7 @@ export async function ensureCloudFleetSandbox( asyncPreparation ? DEFAULT_ASYNC_PREPARATION_TIMEOUT_MS : DEFAULT_ENSURE_TIMEOUT_MS ); let activeAuth = resolved.auth; - const ensureBody = JSON.stringify({ + const ensureRequest = { workspaceId: resolved.cloudWorkspaceId, requiredCapability, ...(asyncPreparation ? { preparationMode: 'async-v1' } : {}), @@ -1067,7 +1093,8 @@ export async function ensureCloudFleetSandbox( ...(input.waitTimeoutMs !== undefined ? { waitTimeoutMs: input.waitTimeoutMs } : {}), ...(input.repos !== undefined && input.repos.length > 0 ? { repos: [...input.repos] } : {}), ...(repoRevisions === undefined ? {} : { repoRevisions }), - }); + }; + const ensureBody = JSON.stringify(ensureRequest); let sawAcceptedPreparation = false; let lastProgressSignature: string | undefined; @@ -1249,7 +1276,7 @@ export async function ensureCloudFleetSandbox( '/api/v1/fleet/nodes/sandbox/ensure', { method: 'POST', - signal: asyncPreparation ? asyncPreparationRequestSignal(signal) : signal, + signal: asyncPreparation ? asyncPreparationInitialRequestSignal(signal) : signal, body: ensureBody, }, { interactive: false } @@ -1303,8 +1330,43 @@ export async function ensureCloudFleetSandbox( } ); } - const payload = await readJson(response); - if (asyncPreparation) { + let payload = await readJson(response); + let legacyCompatibilityResponse = asyncPreparation && response.ok && isLegacyEnsureOutcome(payload); + if (asyncPreparation && rejectsAsyncPreparationMode(response, payload)) { + try { + const retried = await authorizedApiFetch( + activeAuth, + '/api/v1/fleet/nodes/sandbox/ensure', + { + method: 'POST', + signal: asyncPreparationInitialRequestSignal(signal), + body: JSON.stringify({ ...ensureRequest, preparationMode: undefined }), + }, + { interactive: false } + ); + activeAuth = retried.auth; + response = retried.response; + payload = await readJson(response); + legacyCompatibilityResponse = true; + } catch (error) { + throw new CloudFleetSandboxProvisionError( + redactCredentialValues( + `Cloud fleet sandbox compatibility request ended without a complete response: ${ + error instanceof Error ? error.message : String(error) + }` + ), + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: sandboxIdentity.sandboxId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + outcomeUnknown: true, + cause: error, + } + ); + } + } + if (asyncPreparation && !legacyCompatibilityResponse) { let envelope: AsyncPreparationEnvelope | null; try { envelope = readAsyncPreparationEnvelope(payload, sandboxIdentity.sandboxId!); From 33a7d5040423a0d4c82977a5b5b74d929415d86a Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Wed, 7 Oct 2026 23:23:28 -0700 Subject: [PATCH 3/9] test(cloud): bind async-prepared Relaycast target to the answering Cloud Covers the semantic merge with #1894: the async ready record must be normalized against the auth that read it, so a DEV Cloud result keeps its DEV Relaycast target and relaycastCloudApiUrl. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 756e53c4-bcb0-4453-a820-50043b9d1357 --- packages/cloud/src/fleet-sandbox.test.ts | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index e038d029c..bfa5c9d68 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -449,6 +449,56 @@ describe('Cloud fleet sandbox client', () => { ).resolves.toMatchObject({ relaycastTarget: DEV_RELAYCAST_TARGET }); }); + it('validates an async-prepared Relaycast target against the auth that read the ready record', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'agent_relay_cli_bootstrap', 0), { + status: 202, + }), + auth: devAuth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 1, { + result: { + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-async-dev', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-async-dev', + relayWorkspaceId: 'rw_abc', + relaycastTarget: DEV_RELAYCAST_TARGET, + relayfileMounted: true, + }, + }) + ), + auth: devAuth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ + nodeId: 'node-async-dev', + relaycastTarget: DEV_RELAYCAST_TARGET, + relaycastCloudApiUrl: DEV_CLOUD_API_URL, + }); + }); + it('resolves the unified workspace and provisions a ready mounted sandbox', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ From 8a3776a28a5baab52b6a5171f1acad8092d0ac77 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Wed, 7 Oct 2026 23:27:20 -0700 Subject: [PATCH 4/9] fix(cloud): let Cloud route unpinned async sandbox preparation Unpinned `fleet spawn --sandbox` now uses async-v1 whenever it has a sandbox identity, and the client was writing the implied `providerId: agent37` onto the ensure request. Cloud treats an explicit provider as the strict canary path, and the relay#1656 regression case (Flows v2 shard 21) failed: "The CLI pinned a provider". The wire now carries only the caller's provider; Cloud #4137 routes an unpinned async-v1 request to Agent37 itself. Agent37 attribution is applied only once Cloud has confirmed a durable async record (ready normalization and cleanup identity). Unconfirmed failures and legacy synchronous answers keep the caller's provider, so an older Cloud that capability-routes the request elsewhere is accepted instead of failing as a provider mismatch. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 756e53c4-bcb0-4453-a820-50043b9d1357 --- packages/cloud/src/fleet-sandbox.test.ts | 120 ++++++++++++++++++++++- packages/cloud/src/fleet-sandbox.ts | 46 +++++---- 2 files changed, 144 insertions(+), 22 deletions(-) diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index bfa5c9d68..128800675 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -644,11 +644,14 @@ describe('Cloud fleet sandbox client', () => { [`/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`, 'POST'], [`/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`, 'POST'], ]); - expect(JSON.parse(String(requests[0]?.[2]?.body))).toMatchObject({ + const ensureBody = JSON.parse(String(requests[0]?.[2]?.body)); + expect(ensureBody).toMatchObject({ preparationMode: 'async-v1', sandboxId: SANDBOX_ID, - providerId: 'agent37', }); + // Cloud routes unpinned async-v1 to Agent37 itself; naming the provider + // would switch Cloud to the strict explicit-provider path (relay#1656). + expect(ensureBody).not.toHaveProperty('providerId'); expect(progress.mock.calls.map(([value]) => value)).toEqual([ expect.objectContaining({ state: 'pending', phase: 'agent_relay_cli_bootstrap', generation: 0 }), expect.objectContaining({ state: 'pending', phase: 'relayfile_mount_bootstrap', generation: 1 }), @@ -656,6 +659,48 @@ describe('Cloud fleet sandbox client', () => { ]); }); + it('accepts an older Cloud capability-routing an unpinned async request to another provider', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeId: 'node-routed', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: DAYTONA_PROVIDER_SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + providerId: 'daytona', + }, + { status: 201 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }) + ).resolves.toMatchObject({ + outcome: 'provisioned', + sandboxId: SANDBOX_ID, + providerId: 'daytona', + }); + expect(JSON.parse(String(mocks.authorizedApiFetch.mock.calls[1]?.[2]?.body))).not.toHaveProperty( + 'providerId' + ); + }); + it('accepts a synchronous 201 result when an older Cloud ignores async preparation mode', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -1060,7 +1105,7 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ sandboxId: SANDBOX_ID, - providerId: 'agent37', + providerId: undefined, outcomeUnknown: true, confirmedProvisioned: false, }); @@ -1101,6 +1146,75 @@ describe('Cloud fleet sandbox client', () => { }); }); + it('attributes Agent37 cleanup identity once status confirms a lost unpinned async ensure', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockRejectedValueOnce(new TypeError('fetch failed')) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'provider_allocation', 0)), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { timeoutMs: 40, preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + sandboxId: SANDBOX_ID, + providerId: 'agent37', + confirmedProvisioned: true, + outcomeUnknown: false, + }); + }); + + it('rejects a confirmed unpinned async ready result attributed to another provider', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 1, { + result: { + outcome: 'provisioned', + nodeId: 'node-wrong-provider', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: DAYTONA_PROVIDER_SANDBOX_ID, + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + providerId: 'daytona', + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }) + ).rejects.toThrow('Cloud returned provider daytona instead of requested provider agent37.'); + }); + it('preserves caller cancellation after async preparation was accepted', async () => { const controller = new AbortController(); const cancelled = new Error('caller cancelled'); diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 884e06319..2ab0c0525 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -1169,10 +1169,12 @@ export async function ensureCloudFleetSandbox( if (asyncPreparation && sandboxIdentity.sandboxId === undefined) { throw new Error('Async Cloud fleet preparation requires a caller-declared sandboxId.'); } - // Async-v1 is the replay-safe Agent37 contract. Make the implicit default - // explicit on the wire and in cleanup proof so a malformed ready result can - // never lose the provider identity merely because the CLI omitted --provider. - const requestedProviderId = asyncPreparation ? (input.providerId ?? 'agent37') : input.providerId; + // Cloud routes an unpinned async-v1 request to Agent37 itself, so the wire + // carries only the caller's provider: pinning the implied default would turn + // capability routing into the strict explicit-provider path. Once Cloud has + // confirmed a durable async record, attribute it to Agent37 so a malformed + // ready result can never lose the provider needed for exact cleanup. + const asyncProviderId = asyncPreparation ? (input.providerId ?? 'agent37') : input.providerId; const preparationPollIntervalMs = normalizeTimerMs( options.preparationPollIntervalMs ?? DEFAULT_ASYNC_PREPARATION_POLL_INTERVAL_MS, false, @@ -1200,7 +1202,7 @@ export async function ensureCloudFleetSandbox( ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), ...(input.relayfilePaths === undefined ? {} : { relayfilePaths: [...input.relayfilePaths] }), ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), - ...(requestedProviderId !== undefined ? { providerId: requestedProviderId } : {}), + ...(input.providerId !== undefined ? { providerId: input.providerId } : {}), ...(input.workloadProfile !== undefined ? { workloadProfile: input.workloadProfile } : {}), ...(input.waitTimeoutMs !== undefined ? { waitTimeoutMs: input.waitTimeoutMs } : {}), ...(input.repos !== undefined && input.repos.length > 0 ? { repos: [...input.repos] } : {}), @@ -1209,6 +1211,12 @@ export async function ensureCloudFleetSandbox( const ensureBody = JSON.stringify(ensureRequest); let sawAcceptedPreparation = false; + // Before Cloud confirms an async record, an older Cloud may have routed the + // request synchronously, so only the caller's own provider is attributable. + const attributedProvider = (): { providerId?: CloudFleetSandboxProviderId } => { + const providerId = sawAcceptedPreparation ? asyncProviderId : input.providerId; + return providerId === undefined ? {} : { providerId }; + }; let lastProgressSignature: string | undefined; const consumePreparation = (payload: unknown): EnsureCloudFleetSandboxResult | null => { const envelope = readAsyncPreparationEnvelope(payload, sandboxIdentity.sandboxId!); @@ -1234,7 +1242,7 @@ export async function ensureCloudFleetSandbox( resolved.cloudWorkspaceId, sandboxIdentity.sandboxId!, sandboxIdentity.name, - requestedProviderId, + asyncProviderId, repoRevisions, activeAuth.apiUrl ); @@ -1243,7 +1251,7 @@ export async function ensureCloudFleetSandbox( envelope.result, sandboxIdentity.sandboxId!, sandboxIdentity.name, - requestedProviderId + asyncProviderId ); throw new CloudFleetSandboxProvisionError( error instanceof Error ? error.message : 'Cloud fleet sandbox preparation result was invalid.', @@ -1251,7 +1259,7 @@ export async function ensureCloudFleetSandbox( cloudWorkspaceId: resolved.cloudWorkspaceId, sandboxId: sandboxIdentity.sandboxId, ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...(asyncProviderId === undefined ? {} : { providerId: asyncProviderId }), ...(confirmedProvisioned ? { confirmedProvisioned: true } : { outcomeUnknown: true }), cause: error, } @@ -1270,7 +1278,7 @@ export async function ensureCloudFleetSandbox( cloudWorkspaceId: resolved.cloudWorkspaceId, sandboxId: sandboxIdentity.sandboxId, ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...(asyncProviderId === undefined ? {} : { providerId: asyncProviderId }), confirmedProvisioned: true, } ); @@ -1420,7 +1428,7 @@ export async function ensureCloudFleetSandbox( cloudWorkspaceId: resolved.cloudWorkspaceId, sandboxId: sandboxIdentity.sandboxId, ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...attributedProvider(), ...(sawAcceptedPreparation ? { confirmedProvisioned: true } : { outcomeUnknown: true }), cause: pollError, } @@ -1472,7 +1480,7 @@ export async function ensureCloudFleetSandbox( cloudWorkspaceId: resolved.cloudWorkspaceId, sandboxId: sandboxIdentity.sandboxId, ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), outcomeUnknown: true, cause: error, } @@ -1509,7 +1517,7 @@ export async function ensureCloudFleetSandbox( cloudWorkspaceId: resolved.cloudWorkspaceId, sandboxId: sandboxIdentity.sandboxId, ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...(asyncProviderId === undefined ? {} : { providerId: asyncProviderId }), confirmedProvisioned: true, cause: pollError, } @@ -1537,7 +1545,7 @@ export async function ensureCloudFleetSandbox( cloudWorkspaceId: resolved.cloudWorkspaceId, sandboxId: sandboxIdentity.sandboxId, ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...attributedProvider(), ...(sawAcceptedPreparation ? { confirmedProvisioned: true } : { outcomeUnknown: true }), cause: pollError, } @@ -1558,7 +1566,7 @@ export async function ensureCloudFleetSandbox( throw new CloudFleetSandboxProvisionError(mismatch.message, { cloudWorkspaceId: resolved.cloudWorkspaceId, ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), outcomeUnknown: true, cause: mismatch, }); @@ -1586,7 +1594,7 @@ export async function ensureCloudFleetSandbox( cloudWorkspaceId: resolved.cloudWorkspaceId, ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), outcomeUnknown: true, cause: error, }); @@ -1596,7 +1604,7 @@ export async function ensureCloudFleetSandbox( cloudWorkspaceId: resolved.cloudWorkspaceId, ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), outcomeUnknown: true, cause: error, }); @@ -1609,7 +1617,7 @@ export async function ensureCloudFleetSandbox( resolved.cloudWorkspaceId, sandboxIdentity.sandboxId, sandboxIdentity.name, - requestedProviderId, + input.providerId, repoRevisions, activeAuth.apiUrl ); @@ -1618,7 +1626,7 @@ export async function ensureCloudFleetSandbox( payload, sandboxIdentity.sandboxId, sandboxIdentity.name, - requestedProviderId + input.providerId ); throw new CloudFleetSandboxProvisionError( error instanceof Error ? error.message : 'Cloud fleet sandbox response was invalid.', @@ -1626,7 +1634,7 @@ export async function ensureCloudFleetSandbox( cloudWorkspaceId: resolved.cloudWorkspaceId, ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), - ...(requestedProviderId === undefined ? {} : { providerId: requestedProviderId }), + ...(input.providerId === undefined ? {} : { providerId: input.providerId }), ...(confirmedProvisioned ? { confirmedProvisioned: true } : { outcomeUnknown: true }), cause: error, } From 51eb3303cae0f592222cea5ebb3be55f38bbd46f Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Wed, 7 Oct 2026 23:32:37 -0700 Subject: [PATCH 5/9] fix(fleet): report a terminal async preparation as proven absent with its typed cause Cloud #4137 writes an async-v1 record `terminal` only after the provider rejected allocation, dispatch was abandoned, or the reaper proved the provider sandbox absent. The client used to map that to confirmedProvisioned and the CLI then issued a redundant delete without ever saying whether anything was left running. - A terminal record now yields `sandboxAbsent` plus a typed `preparationFailure` (code, phase, causeStage). The CLI names the cause and states no sandbox was left running, with no cleanup call. - The production smoke B shape (typed 503 relayfile_mount_failed / initial_sync_deadline) is reconciled through the exact identity's read-only status, never by replaying ensure, and the 503's causeStage labels the terminal record when Cloud does not persist it. - Accept any bounded phase token: #4137 added cli_credential_setup, relayfile_mount, workspace_skills and fleet_enrollment, which the old allowlist rejected, turning a mount-phase record into outcome-unknown. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 756e53c4-bcb0-4453-a820-50043b9d1357 --- packages/cli/src/cli/commands/fleet.test.ts | 76 +++++++++++ packages/cli/src/cli/commands/fleet.ts | 10 +- packages/cloud/src/fleet-sandbox.test.ts | 142 +++++++++++++++++++- packages/cloud/src/fleet-sandbox.ts | 78 +++++++++-- packages/cloud/src/index.ts | 1 + 5 files changed, 290 insertions(+), 17 deletions(-) diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index c09e8ff70..56be3f1f0 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -3919,6 +3919,7 @@ describe('fleet command support', () => { expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); expect(warnings.join('\n')).toContain(`check Cloud Fleet for node '${REPLAY_SANDBOX_NAME}'`); + expect(warnings.join('\n')).not.toContain('no sandbox was left running'); expect(warnings.join('\n')).toContain(`--sandbox-id '${REPLAY_SANDBOX_ID}'`); }); @@ -3993,6 +3994,81 @@ describe('fleet command support', () => { expect(warnings.join('\n')).not.toContain('check Cloud Fleet'); }); + it('names a terminal relayfile_mount_failed cause and reports no sandbox left running without cleanup', async () => { + const warnings: string[] = []; + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const terminalMessage = + 'Cloud fleet sandbox preparation failed during relayfile_mount (relayfile_mount_failed, initial_sync_deadline): Relayfile mount failed during relayfile_mount. Retry provisioning the sandbox.'; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + resolveSandboxRepository: () => undefined, + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => { + throw new CloudFleetSandboxProvisionError(terminalMessage, { + cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + sandboxId: REPLAY_SANDBOX_ID, + nodeName: REPLAY_SANDBOX_NAME, + providerId: 'agent37', + sandboxAbsent: true, + preparationFailure: { + code: 'relayfile_mount_failed', + phase: 'relayfile_mount', + causeStage: 'initial_sync_deadline', + }, + }); + }), + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: (...args: unknown[]) => warnings.push(args.join(' ')), + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-id', + REPLAY_SANDBOX_ID, + '--sandbox-name', + REPLAY_SANDBOX_NAME, + '--workspace-id', + 'rw_abc', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); + expect(warnings).toEqual([ + `${terminalMessage} Cloud confirmed sandbox '${REPLAY_SANDBOX_ID}' is not running; no sandbox was left running.`, + ]); + expect(warnings.join('\n')).not.toContain('outcome is unknown'); + }); + it('preserves the caller Daytona ID after a matched malformed provisioned response', async () => { const warnings: string[] = []; const deleteCloudFleetSandbox = vi.fn(async () => undefined); diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index c452c0457..360d9fca1 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -823,7 +823,15 @@ export function registerFleetCommands( : await deps.ensureCloudFleetSandbox(ensureInput); assertSandboxRepositoryRevision(sandbox, checkoutRepository ? sandboxRepository : undefined); } catch (error) { - if ( + if (error instanceof CloudFleetSandboxProvisionError && error.sandboxAbsent) { + // Cloud's terminal record for this exact identity already proves the + // provider sandbox is gone; a delete here would only race its reaper. + deps.warn( + `${error.message} Cloud confirmed sandbox '${ + error.sandboxId ?? sandboxId ?? 'the requested sandbox' + }' is not running; no sandbox was left running.` + ); + } else if ( shouldCleanupSandbox && error instanceof CloudFleetSandboxProvisionError && error.confirmedProvisioned && diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 128800675..97e10d975 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -1354,13 +1354,151 @@ describe('Cloud fleet sandbox client', () => { expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); expect(error).toMatchObject({ - confirmedProvisioned: true, + confirmedProvisioned: false, + outcomeUnknown: false, + sandboxAbsent: true, sandboxId: SANDBOX_ID, + preparationFailure: { code: 'preparation_failed', phase: 'relayfile_mount_bootstrap' }, }); expect((error as Error).message).toContain('relayfile_mount_bootstrap'); expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(3); }); + it('reports the production initial_sync_deadline 503 as a typed proven-absent failure, not an unknown outcome', async () => { + const preparationPath = `/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`; + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + // Cloud build 6b8f1de8, smoke B: the mount's initial sync hit the 250 s + // deadline and the ensure request answered with this typed 503. + .mockResolvedValueOnce({ + response: Response.json( + { + error: + 'Relayfile mount failed during relayfile_mount: initial sync deadline exceeded. Retry provisioning the sandbox.', + code: 'relayfile_mount_failed', + phase: 'relayfile_mount', + causeName: 'FleetSandboxRelayfileMountError', + causeStage: 'initial_sync_deadline', + sandboxId: SANDBOX_ID, + }, + { status: 503 } + ), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('cleanup_pending', 'cleanup', 3), { status: 202 }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('terminal', 'cleanup', 4, { + failure: { + code: 'relayfile_mount_failed', + error: 'Relayfile mount failed during relayfile_mount. Retry provisioning the sandbox.', + phase: 'relayfile_mount', + }, + }), + { status: 502 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + sandboxId: SANDBOX_ID, + nodeName: SANDBOX_NAME, + sandboxAbsent: true, + confirmedProvisioned: false, + outcomeUnknown: false, + preparationFailure: { + code: 'relayfile_mount_failed', + phase: 'relayfile_mount', + causeStage: 'initial_sync_deadline', + }, + }); + expect((error as Error).message).toContain('relayfile_mount_failed, initial_sync_deadline'); + // The typed 503 is reconciled through the exact identity's read-only status; + // ensure is never replayed and Cloud-owned cleanup is never advanced. + expect(mocks.authorizedApiFetch.mock.calls.slice(1).map((call) => [call[1], call[2]?.method])).toEqual([ + ['/api/v1/fleet/nodes/sandbox/ensure', 'POST'], + [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + ]); + }); + + it('reconciles a lost final advance response by exact identity before reporting terminal', async () => { + const preparationPath = `/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`; + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'relayfile_mount', 3), { status: 202 }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'relayfile_mount', 3), { status: 202 }), + auth, + }) + // The advance that ran the failing mount tick loses its response. + .mockRejectedValueOnce(new TypeError('fetch failed')) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('terminal', 'cleanup', 5, { + failure: { + code: 'relayfile_mount_failed', + error: 'Relayfile mount failed during relayfile_mount. Retry provisioning the sandbox.', + phase: 'relayfile_mount', + causeStage: 'initial_sync_deadline', + }, + }), + { status: 502 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toMatchObject({ + sandboxId: SANDBOX_ID, + sandboxAbsent: true, + outcomeUnknown: false, + preparationFailure: { code: 'relayfile_mount_failed', causeStage: 'initial_sync_deadline' }, + }); + expect(mocks.authorizedApiFetch.mock.calls.slice(1).map((call) => [call[1], call[2]?.method])).toEqual([ + ['/api/v1/fleet/nodes/sandbox/ensure', 'POST'], + [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + [preparationPath, 'POST'], + [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + ]); + }); + it('observes cleanup_pending without advancing until Cloud reports terminal', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ @@ -1406,7 +1544,7 @@ describe('Cloud fleet sandbox client', () => { ).catch((caught: unknown) => caught); expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); - expect(error).toMatchObject({ confirmedProvisioned: true, sandboxId: SANDBOX_ID }); + expect(error).toMatchObject({ confirmedProvisioned: false, sandboxAbsent: true, sandboxId: SANDBOX_ID }); expect(mocks.authorizedApiFetch.mock.calls.slice(2).map((call) => call[2]?.method)).toEqual([ 'GET', 'GET', diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 2ab0c0525..95ee2d762 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -88,6 +88,14 @@ export type CloudFleetSandboxCapacityExhaustion = { readonly limit: number; }; +/** Credential-free typed failure from a terminal async-v1 preparation record. */ +export type CloudFleetSandboxPreparationFailure = { + readonly code: string; + readonly phase: string; + /** Mount command boundary, e.g. initial_sync_deadline, when Cloud reported one. */ + readonly causeStage?: string; +}; + /** Describes provisioning failure without granting unproven cleanup authority. */ export class CloudFleetSandboxProvisionError extends Error { readonly cloudWorkspaceId?: string; @@ -106,6 +114,15 @@ export class CloudFleetSandboxProvisionError extends Error { readonly retryable: boolean; /** Safe aggregate counts for each provider that blocked allocation. */ readonly capacity: readonly CloudFleetSandboxCapacityExhaustion[]; + /** + * Cloud's durable async-v1 record for this exact sandbox identity is + * terminal, which Cloud writes only after the provider rejected allocation, + * dispatch was abandoned, or the provider sandbox was proven absent. No + * sandbox for this identity is left running and no cleanup is owed. + */ + readonly sandboxAbsent: boolean; + /** Typed Cloud failure for a terminal preparation, e.g. relayfile_mount_failed. */ + readonly preparationFailure?: CloudFleetSandboxPreparationFailure; constructor( message: string, @@ -120,6 +137,8 @@ export class CloudFleetSandboxProvisionError extends Error { noSandboxCreated?: boolean; retryable?: boolean; capacity?: readonly CloudFleetSandboxCapacityExhaustion[]; + sandboxAbsent?: boolean; + preparationFailure?: CloudFleetSandboxPreparationFailure; cause?: unknown; } = {} ) { @@ -135,6 +154,9 @@ export class CloudFleetSandboxProvisionError extends Error { this.noSandboxCreated = !this.confirmedProvisioned && identity.noSandboxCreated === true; this.retryable = identity.retryable === true; this.capacity = identity.capacity?.map((entry) => ({ ...entry })) ?? []; + this.sandboxAbsent = !this.confirmedProvisioned && identity.sandboxAbsent === true; + this.preparationFailure = + identity.preparationFailure === undefined ? undefined : { ...identity.preparationFailure }; } } @@ -909,20 +931,17 @@ function normalizeEnsureResult( } const ASYNC_PREPARATION_STATES = new Set(['pending', 'ready', 'terminal', 'cleanup_pending']); -const ASYNC_PREPARATION_PHASES = new Set([ - 'provider_allocation', - 'agent_relay_cli_bootstrap', - 'relayfile_mount_bootstrap', - 'spawn_cli_bootstrap', - 'repo_clone', - 'finalize', - 'broker_visible', - 'cleanup', -]); +// Phase is progress metadata only: state decides the outcome. Cloud's async-v1 +// record currently reports provider_allocation, agent_relay_cli_bootstrap, +// relayfile_mount_bootstrap, spawn_cli_bootstrap, repo_clone, +// cli_credential_setup, relayfile_mount, workspace_skills, fleet_enrollment, +// finalize, broker_visible and cleanup. Accept any bounded phase token so a +// newer Cloud adding a phase is not misread as an invalid (unknown) outcome. +const ASYNC_PREPARATION_PHASE_TOKEN = /^[a-z][a-z0-9_]{0,63}$/; type AsyncPreparationEnvelope = CloudFleetSandboxPreparationProgress & { result?: JsonRecord; - failure?: { code: string; error: string; phase: string; causeDetail?: string }; + failure?: { code: string; error: string; phase: string; causeDetail?: string; causeStage?: string }; }; function readAsyncPreparationEnvelope( @@ -943,7 +962,7 @@ function readAsyncPreparationEnvelope( !state || !ASYNC_PREPARATION_STATES.has(state) || !phase || - !ASYNC_PREPARATION_PHASES.has(phase) || + !ASYNC_PREPARATION_PHASE_TOKEN.test(phase) || generation === undefined || !Number.isSafeInteger(generation) || generation < 0 @@ -970,6 +989,7 @@ function readAsyncPreparationEnvelope( const error = readString(payload.failure, 'error'); const failurePhase = readString(payload.failure, 'phase'); const causeDetail = readString(payload.failure, 'causeDetail'); + const causeStage = readString(payload.failure, 'causeStage'); if (!code || !error || !failurePhase) { throw new Error('Cloud fleet sandbox preparation failure was invalid.'); } @@ -978,6 +998,7 @@ function readAsyncPreparationEnvelope( error, phase: failurePhase, ...(causeDetail === undefined ? {} : { causeDetail }), + ...(causeStage === undefined ? {} : { causeStage }), }; } return envelope; @@ -1211,6 +1232,10 @@ export async function ensureCloudFleetSandbox( const ensureBody = JSON.stringify(ensureRequest); let sawAcceptedPreparation = false; + // A typed ensure failure (e.g. a 503 relayfile_mount_failed carrying + // causeStage initial_sync_deadline) is only a label; the durable status + // record decides the outcome. Keep it to name the cause of a terminal record. + let ensureFailureHint: { code: string; causeStage?: string } | undefined; // Before Cloud confirms an async record, an older Cloud may have routed the // request synchronously, so only the caller's own provider is attributable. const attributedProvider = (): { providerId?: CloudFleetSandboxProviderId } => { @@ -1267,11 +1292,19 @@ export async function ensureCloudFleetSandbox( } } if (envelope.state === 'terminal') { + // Cloud writes terminal only after it rejected allocation, abandoned + // dispatch, or proved the provider sandbox absent; it owns that cleanup. const failure = envelope.failure; + const causeStage = + failure?.causeStage ?? + (failure !== undefined && ensureFailureHint?.code === failure.code + ? ensureFailureHint.causeStage + : undefined); + const typed = failure ? [failure.code, ...(causeStage === undefined ? [] : [causeStage])].join(', ') : ''; throw new CloudFleetSandboxProvisionError( redactCredentialValues( failure - ? `Cloud fleet sandbox preparation failed during ${failure.phase}: ${failure.error}` + ? `Cloud fleet sandbox preparation failed during ${failure.phase} (${typed}): ${failure.error}` : 'Cloud fleet sandbox preparation failed.' ), { @@ -1279,7 +1312,16 @@ export async function ensureCloudFleetSandbox( sandboxId: sandboxIdentity.sandboxId, ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), ...(asyncProviderId === undefined ? {} : { providerId: asyncProviderId }), - confirmedProvisioned: true, + sandboxAbsent: true, + ...(failure === undefined + ? {} + : { + preparationFailure: { + code: failure.code, + phase: failure.phase, + ...(causeStage === undefined ? {} : { causeStage }), + }, + }), } ); } @@ -1452,6 +1494,14 @@ export async function ensureCloudFleetSandbox( ); } let payload = await readJson(response); + if (asyncPreparation && !response.ok && isObject(payload)) { + const code = readString(payload, 'code'); + const hintSandboxId = readString(payload, 'sandboxId'); + if (code && (hintSandboxId === undefined || hintSandboxId === sandboxIdentity.sandboxId)) { + const causeStage = readString(payload, 'causeStage'); + ensureFailureHint = { code, ...(causeStage === undefined ? {} : { causeStage }) }; + } + } let legacyCompatibilityResponse = asyncPreparation && response.ok && isLegacyEnsureOutcome(payload); if (asyncPreparation && rejectsAsyncPreparationMode(response, payload)) { try { diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 3b6c530af..2c5652db9 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -105,6 +105,7 @@ export { type CloudFleetSandboxWorkloadProfile, type DeleteCloudFleetSandboxInput, type CloudFleetSandboxRequestOptions, + type CloudFleetSandboxPreparationFailure, type CloudFleetSandboxPreparationProgress, type CloudFleetRelaycastRoute, type CloudFleetRelaycastTarget, From c116f625f41ab1a8ac1d64f496f449502d34d3f0 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Wed, 7 Oct 2026 23:35:29 -0700 Subject: [PATCH 6/9] test(fleet): pin async preparation for --checkout and --no-sandbox-relayfile; changelog The exact checkout revision and the mount choice are part of the durable async request, so assert both reach ensure with preparationMode async-v1. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 756e53c4-bcb0-4453-a820-50043b9d1357 --- CHANGELOG.md | 12 +++++++++++- packages/cli/src/cli/commands/fleet.test.ts | 9 +++++++++ packages/cloud/src/fleet-sandbox.ts | 4 +++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba9311050..ec7c2c10b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,17 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Minor] + +### Added + +- `agent-relay fleet spawn --sandbox` (unpinned or `--sandbox-provider agent37`) prepares the sandbox through Cloud's durable `async-v1` preparation: it prints each preparation phase and polls by the exact sandbox identity instead of holding one long request open, so a slow Relayfile initial sync no longer ends as an unknown outcome. Older Cloud deployments keep working through the synchronous response. +- `@agent-relay/cloud` `ensureCloudFleetSandbox` accepts `preparationMode: 'async-v1'` and an `onPreparationProgress` callback; `CloudFleetSandboxProvisionError` exposes `sandboxAbsent` and a typed `preparationFailure` (`code`, `phase`, `causeStage`). + +### Fixed + +- When Cloud ends a sandbox preparation (for example `relayfile_mount_failed` / `initial_sync_deadline`), `fleet spawn --sandbox` names the cause and confirms no sandbox was left running, without a redundant cleanup call or unknown-outcome guidance. +- A lost Cloud response during sandbox preparation is reconciled through read-only status for the same sandbox identity; the CLI never resubmits the spawn or a preparation step blindly. ## [13.1.5] - 2026-10-08 diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 56be3f1f0..ab02af381 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -2211,7 +2211,12 @@ describe('fleet command support', () => { expect(findProjectRoot).toHaveBeenCalledWith(path.resolve(process.cwd(), 'packages/web')); expect(selectionOptions[0]).toMatchObject({ projectRoot: "/local/cloud/packages/web team's" }); const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0] as Record; + // The exact checkout revision is part of the durable async request, so a + // resumed preparation clones the same commit it was accepted for. expect(ensureInput).toMatchObject({ + preparationMode: 'async-v1', + forceProvision: true, + providerId: 'agent37', repos: ['AgentWorkforce/cloud'], repoRevisions: { 'AgentWorkforce/cloud': revision }, }); @@ -2693,6 +2698,10 @@ describe('fleet command support', () => { { from: 'user' } ); + expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( + expect.objectContaining({ preparationMode: 'async-v1', providerId: 'agent37', mountRelayfile: false }), + expect.objectContaining({ onPreparationProgress: expect.any(Function) }) + ); expect(createWorkspaceRelay).toHaveBeenCalledWith({ projectRoot: process.cwd(), token: undefined, diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 95ee2d762..0eef92b1c 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -1300,7 +1300,9 @@ export async function ensureCloudFleetSandbox( (failure !== undefined && ensureFailureHint?.code === failure.code ? ensureFailureHint.causeStage : undefined); - const typed = failure ? [failure.code, ...(causeStage === undefined ? [] : [causeStage])].join(', ') : ''; + const typed = failure + ? [failure.code, ...(causeStage === undefined ? [] : [causeStage])].join(', ') + : ''; throw new CloudFleetSandboxProvisionError( redactCredentialValues( failure From 07378e4943c68c85c3c8d97282bc164fd6ff0fca Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Wed, 7 Oct 2026 23:37:08 -0700 Subject: [PATCH 7/9] fix(cloud): surface auth and cancellation from the async compatibility retry The one-time synchronous retry after an older Cloud rejects preparationMode wrapped CloudAuthError and caller cancellation as an outcome-unknown provisioning failure, unlike the initial and status paths (cubic, PR review). Tests: compatibility-retry auth and cancellation surface unchanged; an envelope-shaped invalid 5xx ensure body reconciles through status; the DEV Relaycast test now distinguishes ensure auth from the auth that read the ready record; the capability-routing fallback asserts async-v1 was sent. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 756e53c4-bcb0-4453-a820-50043b9d1357 --- packages/cloud/src/fleet-sandbox.test.ts | 116 ++++++++++++++++++++++- packages/cloud/src/fleet-sandbox.ts | 4 + 2 files changed, 116 insertions(+), 4 deletions(-) diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 97e10d975..8d6e3d256 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -459,7 +459,9 @@ describe('Cloud fleet sandbox client', () => { response: Response.json(preparationEnvelope('pending', 'agent_relay_cli_bootstrap', 0), { status: 202, }), - auth: devAuth, + // Only the read of the ready record carries DEV auth, so this proves + // normalization uses the auth that read the record, not the ensure's. + auth, }) .mockResolvedValueOnce({ response: Response.json( @@ -696,9 +698,9 @@ describe('Cloud fleet sandbox client', () => { sandboxId: SANDBOX_ID, providerId: 'daytona', }); - expect(JSON.parse(String(mocks.authorizedApiFetch.mock.calls[1]?.[2]?.body))).not.toHaveProperty( - 'providerId' - ); + const ensureBody = JSON.parse(String(mocks.authorizedApiFetch.mock.calls[1]?.[2]?.body)); + expect(ensureBody).toMatchObject({ preparationMode: 'async-v1', sandboxId: SANDBOX_ID }); + expect(ensureBody).not.toHaveProperty('providerId'); }); it('accepts a synchronous 201 result when an older Cloud ignores async preparation mode', async () => { @@ -1577,6 +1579,112 @@ describe('Cloud fleet sandbox client', () => { expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); }); + it('reconciles an envelope-shaped but invalid 5xx ensure body through durable status', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { version: 1, mode: 'async-v1', sandboxId: SANDBOX_ID, state: 'exploded', generation: -1 }, + { status: 502 } + ), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 2, { + result: { + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-after-invalid-5xx', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-invalid-5xx', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-after-invalid-5xx', sandboxId: SANDBOX_ID }); + expect(mocks.authorizedApiFetch.mock.calls[2]?.[2]?.method).toBe('GET'); + }); + + it('surfaces a compatibility-retry authentication failure unchanged', async () => { + const authError = new CloudAuthError('AUTH_REFRESH_EXPIRED', 'Sign in again.'); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'Unrecognized key: preparationMode' }, { status: 400 }), + auth, + }) + .mockRejectedValueOnce(authError); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }) + ).rejects.toBe(authError); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(3); + }); + + it('preserves caller cancellation during the compatibility retry', async () => { + const controller = new AbortController(); + const cancelled = new Error('caller cancelled'); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ error: 'Unrecognized key: preparationMode' }, { status: 400 }), + auth, + }) + .mockImplementationOnce(async () => { + controller.abort(cancelled); + throw new DOMException('This operation was aborted', 'AbortError'); + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { signal: controller.signal } + ) + ).rejects.toBe(cancelled); + }); + it('surfaces an async polling authentication failure without retrying it', async () => { const authError = new CloudAuthError('AUTH_REFRESH_EXPIRED', 'Sign in again.'); mocks.authorizedApiFetch diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 0eef92b1c..2b2b6606f 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -1522,6 +1522,10 @@ export async function ensureCloudFleetSandbox( payload = await readJson(response); legacyCompatibilityResponse = true; } catch (error) { + // Match the initial and status paths: authentication failures and caller + // cancellation are not lost responses and must surface unchanged. + if (error instanceof CloudAuthError) throw error; + if (options.signal?.aborted) throw options.signal.reason ?? error; throw new CloudFleetSandboxProvisionError( redactCredentialValues( `Cloud fleet sandbox compatibility request ended without a complete response: ${ From b43487424adadc9993eb3a10ea54e1a5ff8967b7 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Wed, 7 Oct 2026 23:38:36 -0700 Subject: [PATCH 8/9] fix(cloud): reconcile a truncated status or advance response instead of failing A 2xx GET/POST to /preparation with an empty or malformed body threw a plain error, which the caller wrapped as confirmedProvisioned, so the CLI deleted a sandbox whose tick Cloud may already have committed (Cursor Bugbot, PR review). Treat it as a lost response: read durable status by exact identity before any further advance. An identity mismatch still surfaces as a diagnostic. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 756e53c4-bcb0-4453-a820-50043b9d1357 --- packages/cloud/src/fleet-sandbox.test.ts | 57 ++++++++++++++++++++++++ packages/cloud/src/fleet-sandbox.ts | 14 +++++- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 8d6e3d256..4517186b9 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -1627,6 +1627,63 @@ describe('Cloud fleet sandbox client', () => { expect(mocks.authorizedApiFetch.mock.calls[2]?.[2]?.method).toBe('GET'); }); + it('reconciles a truncated successful advance through durable status instead of failing the sandbox', async () => { + const preparationPath = `/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`; + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'relayfile_mount', 3), { status: 202 }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json(preparationEnvelope('pending', 'relayfile_mount', 3), { status: 202 }), + auth, + }) + // The advance committed its tick, but the body was cut off in transit. + .mockResolvedValueOnce({ response: new Response('{"version":1,"mo', { status: 200 }), auth }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 4, { + result: { + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-after-truncated-advance', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-after-truncated-advance', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + }, + }) + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ) + ).resolves.toMatchObject({ nodeId: 'node-after-truncated-advance', sandboxId: SANDBOX_ID }); + expect(mocks.authorizedApiFetch.mock.calls.slice(1).map((call) => [call[1], call[2]?.method])).toEqual([ + ['/api/v1/fleet/nodes/sandbox/ensure', 'POST'], + [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + [preparationPath, 'POST'], + [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + ]); + }); + it('surfaces a compatibility-retry authentication failure unchanged', async () => { const authError = new CloudAuthError('AUTH_REFRESH_EXPIRED', 'Sign in again.'); mocks.authorizedApiFetch diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 2b2b6606f..dee723c7d 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -1396,7 +1396,14 @@ export async function ensureCloudFleetSandbox( continue; } payload = await readJson(pollResponse); - const envelope = readAsyncPreparationEnvelope(payload, sandboxIdentity.sandboxId!); + let envelope: AsyncPreparationEnvelope | null; + try { + envelope = readAsyncPreparationEnvelope(payload, sandboxIdentity.sandboxId!); + } catch (error) { + // A record for another identity is a diagnostic, not a lost response. + if (error instanceof CloudFleetSandboxIdentityMismatchError) throw error; + envelope = null; + } if (envelope) { const ready = consumePreparation(payload); if (ready) return ready; @@ -1430,7 +1437,10 @@ export async function ensureCloudFleetSandbox( } throw endpointError('read fleet sandbox preparation status', pollResponse, payload); } - throw new Error('Cloud fleet sandbox preparation response was invalid.'); + // A truncated or malformed success may hide a committed tick. Treat it as + // a lost response: read durable status before any further advance. + method = 'GET'; + payload = undefined; } }; From 5e51d9a95f7bf0361ee83669518cfc13af224c6b Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 02:25:41 -0700 Subject: [PATCH 9/9] fix(fleet): parse an older Cloud's typed 5xx and print the resumable sandbox identity Chief review of #1881 (GO-WITH-CONDITIONS): - Every async 5xx was routed to status polling. An older Cloud that ignores preparationMode answers a typed 503 (relayfile_mount_failed, sandbox_capacity_exhausted) synchronously and has no status route, so the CLI polled 404s for 110 s and reported outcome-unknown: the smoke-B shape again. A typed error body (`code`) now goes to the legacy parser, which surfaces capacity as a definitive rejection and names code + causeStage; only envelopes and untyped gateway 5xx are reconciled through status. The 503 cause hint is gone: Cloud persists causeStage in the terminal record (cloud#4137 839d71e1b). - The production test now models the real async shape (mount tick -> cleanup_pending -> terminal with causeStage). - `fleet spawn --sandbox` prints the generated sbx_ identity before Cloud work and how to resume it with --sandbox-id, since the identity is not persisted across a killed process. - Retry-After on 202 progress (initial ensure and each tick) paces the next status/advance request. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 756e53c4-bcb0-4453-a820-50043b9d1357 --- CHANGELOG.md | 2 + packages/cli/src/cli/commands/fleet.test.ts | 4 + packages/cli/src/cli/commands/fleet.ts | 12 +- packages/cloud/src/fleet-sandbox.test.ts | 173 +++++++++++++++++--- packages/cloud/src/fleet-sandbox.ts | 61 ++++--- 5 files changed, 206 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec7c2c10b..04458ce84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - When Cloud ends a sandbox preparation (for example `relayfile_mount_failed` / `initial_sync_deadline`), `fleet spawn --sandbox` names the cause and confirms no sandbox was left running, without a redundant cleanup call or unknown-outcome guidance. - A lost Cloud response during sandbox preparation is reconciled through read-only status for the same sandbox identity; the CLI never resubmits the spawn or a preparation step blindly. +- `fleet spawn --sandbox` prints the generated sandbox identity before provisioning; re-running an interrupted spawn with `--sandbox-id ` resumes that sandbox instead of creating a second one. +- Against a Cloud that does not support async preparation, a typed failure such as `relayfile_mount_failed` or a capacity rejection is reported at once with its cause instead of after a status-polling wait. ## [13.1.5] - 2026-10-08 diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index ab02af381..e12fc0df1 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -1378,9 +1378,13 @@ describe('fleet command support', () => { phase: 'relayfile_mount_bootstrap', generation: 2, }); + // The generated identity is printed before Cloud work so an interrupted + // run can be resumed with --sandbox-id instead of creating a second sandbox. expect(warnings).toEqual([ + `Cloud sandbox identity: ${String(ensureInput.sandboxId)}. If this command is interrupted, re-run it with --sandbox-id ${String(ensureInput.sandboxId)} to resume the same sandbox instead of creating another.`, 'Cloud sandbox preparation: relayfile_mount_bootstrap (pending, generation 2).', ]); + expect(ensureInput.sandboxId).toMatch(/^sbx_[0-9a-f-]{36}$/); expect(ensureInput.repos).toBeUndefined(); expect(ensureInput.repoRevisions).toBeUndefined(); expect(placement.spawn).toHaveBeenCalledWith( diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 360d9fca1..fd1ff0464 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -458,7 +458,10 @@ export function registerFleetCommands( '--sandbox-name ', 'Explicit sandbox node name (custom unless --sandbox-id requires matching fleet-sandbox-)' ) - .option('--sandbox-id ', 'Reuse a caller-declared sbx_ identity for an exact replay') + .option( + '--sandbox-id ', + 'Reuse a caller-declared sbx_ identity for an exact replay; re-run an interrupted --sandbox spawn with the identity it printed to resume it' + ) .option('--workspace-id ', 'Explicit Relay workspace identity required for sandbox provisioning') .option('--sandbox-provider ', 'Sandbox provider: daytona, e2b, or agent37') .option( @@ -793,6 +796,13 @@ export function registerFleetCommands( : 'standard-long-running-agent'; const useAsyncPreparation = sandboxId !== undefined && (sandboxProvider === undefined || sandboxProvider === 'agent37'); + if (useAsyncPreparation && sandboxIdOption === undefined) { + // The generated identity is not persisted. Print it before Cloud work + // starts so an interrupted run can be resumed instead of duplicated. + deps.warn( + `Cloud sandbox identity: ${sandboxId}. If this command is interrupted, re-run it with --sandbox-id ${sandboxId} to resume the same sandbox instead of creating another.` + ); + } try { const ensureInput: EnsureCloudFleetSandboxInput = { workspaceId: relayWorkspaceId, diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 4517186b9..c32e710ce 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -1366,41 +1366,45 @@ describe('Cloud fleet sandbox client', () => { expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(3); }); - it('reports the production initial_sync_deadline 503 as a typed proven-absent failure, not an unknown outcome', async () => { + it('reports the production initial_sync_deadline mount failure as a typed proven-absent failure, not an unknown outcome', async () => { const preparationPath = `/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`; mocks.authorizedApiFetch .mockResolvedValueOnce({ response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), auth, }) - // Cloud build 6b8f1de8, smoke B: the mount's initial sync hit the 250 s - // deadline and the ensure request answered with this typed 503. .mockResolvedValueOnce({ - response: Response.json( - { - error: - 'Relayfile mount failed during relayfile_mount: initial sync deadline exceeded. Retry provisioning the sandbox.', - code: 'relayfile_mount_failed', - phase: 'relayfile_mount', - causeName: 'FleetSandboxRelayfileMountError', - causeStage: 'initial_sync_deadline', - sandboxId: SANDBOX_ID, - }, - { status: 503 } - ), + response: Response.json(preparationEnvelope('pending', 'relayfile_mount', 3), { status: 202 }), auth, }) .mockResolvedValueOnce({ - response: Response.json(preparationEnvelope('cleanup_pending', 'cleanup', 3), { status: 202 }), + response: Response.json(preparationEnvelope('pending', 'relayfile_mount', 3), { status: 202 }), + auth, + }) + // Cloud build 6b8f1de8, smoke B: the mount's initial sync hit the 250 s + // deadline. Under async-v1 that tick moves the record to cleanup. + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('cleanup_pending', 'cleanup', 4, { + failure: { + code: 'relayfile_mount_failed', + error: 'Relayfile mount failed during relayfile_mount. Retry provisioning the sandbox.', + phase: 'relayfile_mount', + causeStage: 'initial_sync_deadline', + }, + }), + { status: 202 } + ), auth, }) .mockResolvedValueOnce({ response: Response.json( - preparationEnvelope('terminal', 'cleanup', 4, { + preparationEnvelope('terminal', 'cleanup', 5, { failure: { code: 'relayfile_mount_failed', error: 'Relayfile mount failed during relayfile_mount. Retry provisioning the sandbox.', phase: 'relayfile_mount', + causeStage: 'initial_sync_deadline', }, }), { status: 502 } @@ -1434,15 +1438,146 @@ describe('Cloud fleet sandbox client', () => { }, }); expect((error as Error).message).toContain('relayfile_mount_failed, initial_sync_deadline'); - // The typed 503 is reconciled through the exact identity's read-only status; - // ensure is never replayed and Cloud-owned cleanup is never advanced. + // One ensure, never replayed; Cloud-owned cleanup is observed, never advanced. expect(mocks.authorizedApiFetch.mock.calls.slice(1).map((call) => [call[1], call[2]?.method])).toEqual([ ['/api/v1/fleet/nodes/sandbox/ensure', 'POST'], [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], + [preparationPath, 'POST'], [`${preparationPath}?workspaceId=${CLOUD_WORKSPACE_ID}`, 'GET'], ]); }); + it('paces preparation by the Retry-After that Cloud sends with pending progress', async () => { + const pending = () => + new Response(JSON.stringify(preparationEnvelope('pending', 'relayfile_mount', 3)), { + status: 202, + headers: { 'content-type': 'application/json', 'retry-after': '0' }, + }); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ response: pending(), auth }) + .mockResolvedValueOnce({ response: pending(), auth }) + .mockResolvedValueOnce({ + response: Response.json( + preparationEnvelope('ready', 'broker_visible', 4, { + result: { + outcome: 'provisioned', + providerId: 'agent37', + nodeId: 'node-paced', + nodeName: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + providerSandboxId: 'provider-paced', + relayWorkspaceId: 'rw_abc', + relaycastTarget: RELAYCAST_TARGET, + relayfileMounted: true, + }, + }) + ), + auth, + }); + + // A 60 s default interval would time this test out if Retry-After were ignored. + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 60_000 } + ) + ).resolves.toMatchObject({ nodeId: 'node-paced' }); + }); + + it('surfaces an older Cloud typed relayfile_mount_failed 503 immediately instead of polling a missing status route', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + // An older Cloud ignores preparationMode and answers synchronously with + // the production smoke B failure. + .mockResolvedValueOnce({ + response: Response.json( + { + error: + 'Relayfile mount failed during relayfile_mount: initial sync deadline exceeded. Retry provisioning the sandbox.', + code: 'relayfile_mount_failed', + phase: 'relayfile_mount', + causeStage: 'initial_sync_deadline', + sandboxId: SANDBOX_ID, + }, + { status: 503 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ sandboxId: SANDBOX_ID, sandboxAbsent: false }); + expect((error as Error).message).toContain('(relayfile_mount_failed, initial_sync_deadline)'); + // No status reads: the typed body is a complete synchronous answer. + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + }); + + it('surfaces an older Cloud typed capacity 503 as a definitive pre-allocation rejection without polling', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + error: 'Sandbox capacity is exhausted before allocation; no sandbox was created', + code: 'sandbox_capacity_exhausted', + capacity: [{ provider: 'agent37', current: 14, limit: 10 }], + retryable: true, + no_sandbox_created: true, + }, + { status: 503 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + name: SANDBOX_NAME, + sandboxId: SANDBOX_ID, + requiredCapability: 'spawn:codex', + forceProvision: true, + preparationMode: 'async-v1', + }, + { preparationPollIntervalMs: 1 } + ).catch((caught: unknown) => caught); + + expect(error).toMatchObject({ + code: 'sandbox_capacity_exhausted', + noSandboxCreated: true, + retryable: true, + capacity: [{ provider: 'agent37', current: 14, limit: 10 }], + }); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + }); + it('reconciles a lost final advance response by exact identity before reporting terminal', async () => { const preparationPath = `/api/v1/fleet/nodes/sandbox/${SANDBOX_ID}/preparation`; mocks.authorizedApiFetch diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index dee723c7d..b71bb4810 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -469,6 +469,19 @@ function endpointError(action: string, response: Response, payload: unknown): Er ); } +/** A Cloud error body that names its failure, as opposed to a gateway page. */ +function isTypedErrorBody(payload: unknown): boolean { + return isObject(payload) && readString(payload, 'code') !== undefined; +} + +function typedCauseSuffix(payload: unknown): string { + if (!isObject(payload)) return ''; + const code = readString(payload, 'code'); + if (!code) return ''; + const causeStage = readString(payload, 'causeStage'); + return ` (${code}${causeStage === undefined ? '' : `, ${causeStage}`})`; +} + function retryAfterDelayMs(response: Response): number | null { const value = response.headers.get('retry-after')?.trim(); if (!value) return null; @@ -1232,10 +1245,6 @@ export async function ensureCloudFleetSandbox( const ensureBody = JSON.stringify(ensureRequest); let sawAcceptedPreparation = false; - // A typed ensure failure (e.g. a 503 relayfile_mount_failed carrying - // causeStage initial_sync_deadline) is only a label; the durable status - // record decides the outcome. Keep it to name the cause of a terminal record. - let ensureFailureHint: { code: string; causeStage?: string } | undefined; // Before Cloud confirms an async record, an older Cloud may have routed the // request synchronously, so only the caller's own provider is attributable. const attributedProvider = (): { providerId?: CloudFleetSandboxProviderId } => { @@ -1295,11 +1304,9 @@ export async function ensureCloudFleetSandbox( // Cloud writes terminal only after it rejected allocation, abandoned // dispatch, or proved the provider sandbox absent; it owns that cleanup. const failure = envelope.failure; - const causeStage = - failure?.causeStage ?? - (failure !== undefined && ensureFailureHint?.code === failure.code - ? ensureFailureHint.causeStage - : undefined); + // Cloud persists the safe mount cause stage (e.g. initial_sync_deadline) + // in the durable failure record (cloud#4137 839d71e1b). + const causeStage = failure?.causeStage; const typed = failure ? [failure.code, ...(causeStage === undefined ? [] : [causeStage])].join(', ') : ''; @@ -1330,11 +1337,14 @@ export async function ensureCloudFleetSandbox( return null; }; - const pollPreparation = async (initialPayload?: unknown): Promise => { + const pollPreparation = async ( + initialPayload?: unknown, + initialDelayMs: number | null = null + ): Promise => { const unconfirmedDeadline = Date.now() + ASYNC_PREPARATION_REQUEST_TIMEOUT_MS; let payload = initialPayload; let method: 'GET' | 'POST' = initialPayload === undefined ? 'GET' : 'POST'; - let nextPollDelayMs: number | null = null; + let nextPollDelayMs: number | null = initialDelayMs; for (;;) { if (payload !== undefined) { const ready = consumePreparation(payload); @@ -1411,6 +1421,8 @@ export async function ensureCloudFleetSandbox( // terminal result; advancing here could race the reaper or restart work // on a sandbox that is already being destroyed. method = envelope.state === 'cleanup_pending' ? 'GET' : 'POST'; + // Each advance goes through Cloud's ensure limiter; pace by its hint. + nextPollDelayMs = retryAfterDelayMs(pollResponse); payload = undefined; continue; } @@ -1438,7 +1450,9 @@ export async function ensureCloudFleetSandbox( throw endpointError('read fleet sandbox preparation status', pollResponse, payload); } // A truncated or malformed success may hide a committed tick. Treat it as - // a lost response: read durable status before any further advance. + // a lost response: read durable status before any further advance. This + // also covers Cloud's bare `{ state }` fallback body (no `mode`/`version`), + // returned when it cannot re-read the record after a failed tick. method = 'GET'; payload = undefined; } @@ -1506,14 +1520,6 @@ export async function ensureCloudFleetSandbox( ); } let payload = await readJson(response); - if (asyncPreparation && !response.ok && isObject(payload)) { - const code = readString(payload, 'code'); - const hintSandboxId = readString(payload, 'sandboxId'); - if (code && (hintSandboxId === undefined || hintSandboxId === sandboxIdentity.sandboxId)) { - const causeStage = readString(payload, 'causeStage'); - ensureFailureHint = { code, ...(causeStage === undefined ? {} : { causeStage }) }; - } - } let legacyCompatibilityResponse = asyncPreparation && response.ok && isLegacyEnsureOutcome(payload); if (asyncPreparation && rejectsAsyncPreparationMode(response, payload)) { try { @@ -1568,7 +1574,7 @@ export async function ensureCloudFleetSandbox( const ready = consumePreparation(payload); if (ready) return ready; try { - return await pollPreparation(); + return await pollPreparation(undefined, retryAfterDelayMs(response)); } catch (pollError) { if (pollError instanceof CloudFleetSandboxProvisionError) throw pollError; if (pollError instanceof CloudAuthError) throw pollError; @@ -1591,10 +1597,13 @@ export async function ensureCloudFleetSandbox( } } // A successful/accepted response can still lose or truncate its body after - // Cloud durably recorded the preparation. Reconcile every non-envelope - // async response through the declared sandbox's status endpoint instead of - // falling through to legacy response parsing or replaying ensure. - if (response.ok || response.status >= 500) { + // Cloud durably recorded the preparation, and a gateway 5xx can hide an + // accepted request. Reconcile those through the declared sandbox's status + // endpoint instead of replaying ensure. A typed 5xx error body (`code`) is + // a complete synchronous answer, e.g. from an older Cloud that ignored + // preparationMode: parse it below so its cause and capacity detail + // surface immediately instead of polling a status route it lacks. + if (response.ok || (response.status >= 500 && !isTypedErrorBody(payload))) { try { return await pollPreparation(); } catch (pollError) { @@ -1656,7 +1665,7 @@ export async function ensureCloudFleetSandbox( // replayable as an unknown outcome, even for legacy custom-name callers; // never copy an unverified response ID into cleanup authority. if (response.status >= 500) { - throw new CloudFleetSandboxProvisionError(error.message, { + throw new CloudFleetSandboxProvisionError(`${error.message}${typedCauseSuffix(payload)}`, { cloudWorkspaceId: resolved.cloudWorkspaceId, ...(sandboxIdentity.sandboxId === undefined ? {} : { sandboxId: sandboxIdentity.sandboxId }), ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }),