Repository navigation
feat(fleet): poll async sandbox preparation #1881
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
ea5328c
aae3911
faeb9da
33a7d50
8a3776a
51eb330
c116f62
07378e4
b434874
5e51d9a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1191,6 +1191,7 @@ describe('fleet command support', () => { | |
| expect(ensureInput).toEqual({ | ||
| workspaceId: 'rw_abc', | ||
| requiredCapability: 'spawn:codex', | ||
| preparationMode: 'async-v1', | ||
| maxAgents: 1, | ||
| mountRelayfile: true, | ||
| relayfilePaths: ['/live-review/run-123/**'], | ||
|
|
@@ -1264,6 +1265,7 @@ describe('fleet command support', () => { | |
| workerCwd: '/srv/agent-workforce/cloud/packages/web', | ||
| }; | ||
| const events: string[] = []; | ||
| const warnings: string[] = []; | ||
| const materializeCloudRelayfileRepository = vi.fn(async () => { | ||
| events.push('materialize'); | ||
| return { | ||
|
|
@@ -1329,7 +1331,7 @@ describe('fleet command support', () => { | |
| deleteCloudFleetSandbox: vi.fn(async () => undefined), | ||
| createFleetWorkspaceClient: vi.fn() as never, | ||
| log: () => undefined, | ||
| warn: () => undefined, | ||
| warn: (message) => warnings.push(message), | ||
| error: () => undefined, | ||
| }); | ||
|
|
||
|
|
@@ -1365,9 +1367,24 @@ describe('fleet command support', () => { | |
| '/github/repos/AgentWorkforce/cloud/.relayfile/**', | ||
| '/.skills/**', | ||
| ], | ||
| }) | ||
| }), | ||
| expect.objectContaining({ onPreparationProgress: expect.any(Function) }) | ||
| ); | ||
| const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0] as Record<string, unknown>; | ||
| const ensureOptions = ensureCloudFleetSandbox.mock.calls[0]?.[1]; | ||
| ensureOptions?.onPreparationProgress?.({ | ||
| sandboxId: REPLAY_SANDBOX_ID, | ||
| state: 'pending', | ||
| phase: 'relayfile_mount_bootstrap', | ||
| generation: 2, | ||
| }); | ||
| // The generated identity is printed before Cloud work so an interrupted | ||
| // run can be resumed with --sandbox-id instead of creating a second sandbox. | ||
| expect(warnings).toEqual([ | ||
| `Cloud sandbox identity: ${String(ensureInput.sandboxId)}. If this command is interrupted, re-run it with --sandbox-id ${String(ensureInput.sandboxId)} to resume the same sandbox instead of creating another.`, | ||
| 'Cloud sandbox preparation: relayfile_mount_bootstrap (pending, generation 2).', | ||
| ]); | ||
| expect(ensureInput.sandboxId).toMatch(/^sbx_[0-9a-f-]{36}$/); | ||
| expect(ensureInput.repos).toBeUndefined(); | ||
| expect(ensureInput.repoRevisions).toBeUndefined(); | ||
| expect(placement.spawn).toHaveBeenCalledWith( | ||
|
|
@@ -1482,7 +1499,8 @@ describe('fleet command support', () => { | |
| expect.objectContaining({ | ||
| mountRelayfile: true, | ||
| relayfilePaths: ['/github/repos/AgentWorkforce/agent-assistant/contents/**', '/memory/**'], | ||
| }) | ||
| }), | ||
| expect.objectContaining({ onPreparationProgress: expect.any(Function) }) | ||
| ); | ||
| const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0] as Record<string, unknown>; | ||
| expect(ensureInput.repos).toBeUndefined(); | ||
|
|
@@ -1602,7 +1620,8 @@ describe('fleet command support', () => { | |
| expect.objectContaining({ | ||
| mountRelayfile: true, | ||
| relayfilePaths: ['/github/repos/AgentWorkforce/cloud/contents/**', '/memory/**'], | ||
| }) | ||
| }), | ||
| expect.objectContaining({ onPreparationProgress: expect.any(Function) }) | ||
| ); | ||
| const spawnInput = placement.spawn.mock.calls[0]?.[0]?.input as { task?: string }; | ||
| expect(placement.spawn).toHaveBeenCalledWith( | ||
|
|
@@ -1970,7 +1989,8 @@ describe('fleet command support', () => { | |
| { from: 'user' } | ||
| ); | ||
| expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( | ||
| expect.objectContaining({ relayfilePaths: paths16 }) | ||
| expect.objectContaining({ relayfilePaths: paths16 }), | ||
| expect.objectContaining({ onPreparationProgress: expect.any(Function) }) | ||
| ); | ||
|
|
||
| const tooMany = new Command(); | ||
|
|
@@ -2195,7 +2215,12 @@ describe('fleet command support', () => { | |
| expect(findProjectRoot).toHaveBeenCalledWith(path.resolve(process.cwd(), 'packages/web')); | ||
| expect(selectionOptions[0]).toMatchObject({ projectRoot: "/local/cloud/packages/web team's" }); | ||
| const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0] as Record<string, unknown>; | ||
| // The exact checkout revision is part of the durable async request, so a | ||
| // resumed preparation clones the same commit it was accepted for. | ||
| expect(ensureInput).toMatchObject({ | ||
| preparationMode: 'async-v1', | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Prompt for AI agents
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Answered with evidence, no change. |
||
| forceProvision: true, | ||
| providerId: 'agent37', | ||
| repos: ['AgentWorkforce/cloud'], | ||
| repoRevisions: { 'AgentWorkforce/cloud': revision }, | ||
| }); | ||
|
|
@@ -2412,7 +2437,12 @@ describe('fleet command support', () => { | |
|
|
||
| expect(resolveWorkspaceByKey).toHaveBeenCalledWith('rk_live_pinned'); | ||
| expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( | ||
| expect.objectContaining({ workspaceId: 'cloud-pinned', workloadProfile: 'long-running-agent' }) | ||
| expect.objectContaining({ | ||
| workspaceId: 'cloud-pinned', | ||
| workloadProfile: 'long-running-agent', | ||
| preparationMode: 'async-v1', | ||
| }), | ||
| expect.objectContaining({ onPreparationProgress: expect.any(Function) }) | ||
| ); | ||
| expect(createWorkspaceRelay).toHaveBeenCalledWith( | ||
| expect.objectContaining({ | ||
|
|
@@ -2672,6 +2702,10 @@ describe('fleet command support', () => { | |
| { from: 'user' } | ||
| ); | ||
|
|
||
| expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: This call can never match: the test passes Prompt for AI agents
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Answered with evidence, no change. |
||
| expect.objectContaining({ preparationMode: 'async-v1', providerId: 'agent37', mountRelayfile: false }), | ||
| expect.objectContaining({ onPreparationProgress: expect.any(Function) }) | ||
| ); | ||
| expect(createWorkspaceRelay).toHaveBeenCalledWith({ | ||
| projectRoot: process.cwd(), | ||
| token: undefined, | ||
|
|
@@ -3273,7 +3307,9 @@ describe('fleet command support', () => { | |
| expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( | ||
| expect.objectContaining({ | ||
| workloadProfile: 'long-running-agent', | ||
| }) | ||
| preparationMode: 'async-v1', | ||
| }), | ||
| expect.objectContaining({ onPreparationProgress: expect.any(Function) }) | ||
| ); | ||
| expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); | ||
| }); | ||
|
|
@@ -3482,7 +3518,8 @@ describe('fleet command support', () => { | |
| { from: 'user' } | ||
| ); | ||
| expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( | ||
| expect.objectContaining({ workspaceId: 'rw_captured' }) | ||
| expect.objectContaining({ workspaceId: 'rw_captured', preparationMode: 'async-v1' }), | ||
| expect.objectContaining({ onPreparationProgress: expect.any(Function) }) | ||
| ); | ||
| }); | ||
|
|
||
|
|
@@ -3548,7 +3585,8 @@ describe('fleet command support', () => { | |
| ); | ||
|
|
||
| expect(ensureCloudFleetSandbox).toHaveBeenCalledWith( | ||
| expect.objectContaining({ workspaceId: cloudWorkspaceId }) | ||
| expect.objectContaining({ workspaceId: cloudWorkspaceId, preparationMode: 'async-v1' }), | ||
| expect.objectContaining({ onPreparationProgress: expect.any(Function) }) | ||
| ); | ||
| expect(createWorkspaceRelay).toHaveBeenCalledWith({ | ||
| projectRoot: process.cwd(), | ||
|
|
@@ -3715,6 +3753,10 @@ describe('fleet command support', () => { | |
| /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ | ||
| ); | ||
| expect(ensureInput?.name).toBe(`fleet-sandbox-${ensureInput?.sandboxId?.slice('sbx_'.length)}`); | ||
| expect(ensureInput?.preparationMode).toBe('async-v1'); | ||
| expect(ensureCloudFleetSandbox.mock.calls[0]?.[1]).toEqual({ | ||
| onPreparationProgress: expect.any(Function), | ||
| }); | ||
| }); | ||
|
|
||
| it('rejects a replay sandbox name that does not match its identity', async () => { | ||
|
|
@@ -3890,6 +3932,7 @@ describe('fleet command support', () => { | |
|
|
||
| expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); | ||
| expect(warnings.join('\n')).toContain(`check Cloud Fleet for node '${REPLAY_SANDBOX_NAME}'`); | ||
| expect(warnings.join('\n')).not.toContain('no sandbox was left running'); | ||
| expect(warnings.join('\n')).toContain(`--sandbox-id '${REPLAY_SANDBOX_ID}'`); | ||
| }); | ||
|
|
||
|
|
@@ -3964,6 +4007,81 @@ describe('fleet command support', () => { | |
| expect(warnings.join('\n')).not.toContain('check Cloud Fleet'); | ||
| }); | ||
|
|
||
| it('names a terminal relayfile_mount_failed cause and reports no sandbox left running without cleanup', async () => { | ||
| const warnings: string[] = []; | ||
| const deleteCloudFleetSandbox = vi.fn(async () => undefined); | ||
| const terminalMessage = | ||
| 'Cloud fleet sandbox preparation failed during relayfile_mount (relayfile_mount_failed, initial_sync_deadline): Relayfile mount failed during relayfile_mount. Retry provisioning the sandbox.'; | ||
| const program = new Command(); | ||
| program.exitOverride(); | ||
| registerFleetCommands(program, { | ||
| resolveSandboxRepository: () => undefined, | ||
| sdk: { | ||
| createAgentRelay: vi.fn() as never, | ||
| createWorkspaceRelay: vi.fn(() => ({ | ||
| workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, | ||
| })) as never, | ||
| createWorkspace: vi.fn() as never, | ||
| log: vi.fn(), | ||
| error: vi.fn(), | ||
| exit: (() => { | ||
| throw new Error('__exit__'); | ||
| }) as never, | ||
| }, | ||
| ensureCloudFleetSandbox: vi.fn(async () => { | ||
| throw new CloudFleetSandboxProvisionError(terminalMessage, { | ||
| cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', | ||
| sandboxId: REPLAY_SANDBOX_ID, | ||
| nodeName: REPLAY_SANDBOX_NAME, | ||
| providerId: 'agent37', | ||
| sandboxAbsent: true, | ||
| preparationFailure: { | ||
| code: 'relayfile_mount_failed', | ||
| phase: 'relayfile_mount', | ||
| causeStage: 'initial_sync_deadline', | ||
| }, | ||
| }); | ||
| }), | ||
| deleteCloudFleetSandbox, | ||
| createFleetWorkspaceClient: vi.fn() as never, | ||
| log: () => undefined, | ||
| warn: (...args: unknown[]) => warnings.push(args.join(' ')), | ||
| error: () => undefined, | ||
| }); | ||
|
|
||
| await expect( | ||
| program.parseAsync( | ||
| [ | ||
| 'fleet', | ||
| 'spawn', | ||
| 'codex', | ||
| '--sandbox', | ||
| '--sandbox-id', | ||
| REPLAY_SANDBOX_ID, | ||
| '--sandbox-name', | ||
| REPLAY_SANDBOX_NAME, | ||
| '--workspace-id', | ||
| 'rw_abc', | ||
| '--name', | ||
| 'sandbox-worker', | ||
| '--task', | ||
| 'Work', | ||
| '--workspace-key', | ||
| 'rk_live_test', | ||
| '--token', | ||
| 'at_live_lead', | ||
| ], | ||
| { from: 'user' } | ||
| ) | ||
| ).rejects.toThrow('__exit__'); | ||
|
|
||
| expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); | ||
| expect(warnings).toEqual([ | ||
| `${terminalMessage} Cloud confirmed sandbox '${REPLAY_SANDBOX_ID}' is not running; no sandbox was left running.`, | ||
| ]); | ||
| expect(warnings.join('\n')).not.toContain('outcome is unknown'); | ||
| }); | ||
|
|
||
| it('preserves the caller Daytona ID after a matched malformed provisioned response', async () => { | ||
| const warnings: string[] = []; | ||
| const deleteCloudFleetSandbox = vi.fn(async () => undefined); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P3: This assertion doesn't actually pin the sbx_<UUID> contract the command enforces elsewhere.
/^sbx_[0-9a-f-]{36}$/accepts any 36-char lowercase hex/hyphen string, so it can't catch a regression where the generated ID stops being an RFC 4122 UUID. Use the same strict pattern as CLOUD_SANDBOX_ID_PATTERN in fleet.ts:99 so the test asserts the documented identity contract.Prompt for AI agents
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Valid, no code change in this round:
/^sbx_[0-9a-f-]{36}$/only checks shape. The generator issbx_${randomUUID()}(fleet.ts:772, RFC 4122 v4) and--sandbox-idinput is already validated with the strictCLOUD_SANDBOX_ID_PATTERN(fleet.ts:99, testrejects an invalid replay sandbox ID before provisioning), so no current behaviour escapes; but the assertion should use the same strict pattern to pin the contract. Per the CI batching rule (reviewer asked for no push now), that tightening is queued for the next push on this PR rather than pushed alone.