diff --git a/CHANGELOG.md b/CHANGELOG.md index 6056d3697..6704ebda2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Patch] + +### Fixed + +- `agent-relay fleet spawn --sandbox` reports provider capacity and confirms no sandbox was created when Cloud rejects before allocation, without misleading leak-check guidance. ## [13.1.3] - 2026-10-07 diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 2fc26d89e..93cf4a155 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -3893,6 +3893,77 @@ describe('fleet command support', () => { expect(warnings.join('\n')).toContain(`--sandbox-id '${REPLAY_SANDBOX_ID}'`); }); + it('prints definitive capacity detail without cleanup or unknown-outcome guidance', async () => { + const warnings: string[] = []; + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const capacityMessage = + 'Sandbox capacity is exhausted before allocation (agent37: 14 current / 10 limit). No sandbox was created; retry when capacity is available.'; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + resolveSandboxRepository: () => undefined, + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => { + throw new CloudFleetSandboxProvisionError(capacityMessage, { + cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + nodeName: REPLAY_SANDBOX_NAME, + providerId: 'agent37', + code: 'sandbox_capacity_exhausted', + noSandboxCreated: true, + retryable: true, + capacity: [{ provider: 'agent37', current: 14, limit: 10 }], + }); + }), + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: (...args: unknown[]) => warnings.push(args.join(' ')), + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-id', + REPLAY_SANDBOX_ID, + '--sandbox-name', + REPLAY_SANDBOX_NAME, + '--workspace-id', + 'rw_abc', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); + expect(warnings).toEqual([capacityMessage]); + expect(warnings.join('\n')).not.toContain('outcome is unknown'); + expect(warnings.join('\n')).not.toContain('check Cloud Fleet'); + }); + it('preserves the caller Daytona ID after a matched malformed provisioned response', async () => { const warnings: string[] = []; const deleteCloudFleetSandbox = vi.fn(async () => undefined); diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 31252d9c6..c89649075 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -385,6 +385,7 @@ function mergeFleetNodeListOptions( return merged; } +/** Register fleet lifecycle and sandbox commands on the root CLI program. */ export function registerFleetCommands( program: Command, overrides: Partial = {} @@ -829,6 +830,8 @@ export function registerFleetCommands( }` ); }); + } else if (error instanceof CloudFleetSandboxProvisionError && error.noSandboxCreated) { + deps.warn(error.message); } else if (error instanceof CloudFleetSandboxProvisionError && error.outcomeUnknown) { deps.warn( `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index bb4e2df07..dd4a24cb7 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -626,6 +626,99 @@ describe('Cloud fleet sandbox client', () => { } ); + it('classifies a capacity 503 as a definitive pre-allocation rejection', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + error: 'Sandbox capacity is exhausted before allocation; no sandbox was created', + code: 'sandbox_capacity_exhausted', + capacity: [{ provider: 'agent37', current: 14, limit: 10 }], + retryable: true, + no_sandbox_created: true, + }, + { status: 503 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + providerId: 'agent37', + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + code: 'sandbox_capacity_exhausted', + providerId: 'agent37', + sandboxId: undefined, + confirmedProvisioned: false, + outcomeUnknown: false, + noSandboxCreated: true, + retryable: true, + capacity: [{ provider: 'agent37', current: 14, limit: 10 }], + }); + expect(String(error)).toContain('agent37: 14 current / 10 limit'); + expect(String(error)).toContain('No sandbox was created'); + }); + + it.each([ + ['current', 14.5, 10], + ['limit', 14, 10.5], + ])( + 'keeps a capacity 503 with fractional %s on the conservative unknown path', + async (_field, current, limit) => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + error: 'Sandbox capacity is exhausted before allocation; no sandbox was created', + code: 'sandbox_capacity_exhausted', + capacity: [{ provider: 'agent37', current, limit }], + retryable: true, + no_sandbox_created: true, + }, + { status: 503 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + sandboxId: SANDBOX_ID, + name: SANDBOX_NAME, + forceProvision: true, + providerId: 'agent37', + workloadProfile: 'long-running-agent', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + code: undefined, + sandboxId: SANDBOX_ID, + confirmedProvisioned: false, + outcomeUnknown: true, + noSandboxCreated: false, + retryable: false, + capacity: [], + }); + } + ); + it.each(['provisioned', 'provisioning_timeout'] as const)( 'requires a valid Daytona providerSandboxId for %s responses', async (outcome) => { diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 168a17519..612825a5d 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -68,10 +68,14 @@ export type CloudFleetSandboxProviderId = | 'agent37' | 'microsandbox'; -/** - * Carries every safe identifier Cloud returned when provisioning failed after - * the request may have created a billable sandbox. - */ +/** Safe aggregate provider-capacity detail returned before allocation. */ +export type CloudFleetSandboxCapacityExhaustion = { + readonly provider: CloudFleetSandboxProviderId; + readonly current: number; + readonly limit: number; +}; + +/** Describes provisioning failure without granting unproven cleanup authority. */ export class CloudFleetSandboxProvisionError extends Error { readonly cloudWorkspaceId?: string; readonly sandboxId?: string; @@ -79,7 +83,16 @@ export class CloudFleetSandboxProvisionError extends Error { readonly providerId?: CloudFleetSandboxProviderId; /** A 2xx response proved this exact caller-owned sandbox was provisioned. */ readonly confirmedProvisioned: boolean; + /** Cloud may have accepted the request without returning a complete outcome. */ readonly outcomeUnknown: boolean; + /** Stable Cloud error code for a validated pre-allocation capacity rejection. */ + readonly code?: 'sandbox_capacity_exhausted'; + /** Cloud affirmatively proved the rejected request allocated no sandbox. */ + readonly noSandboxCreated: boolean; + /** The same request can be retried after provider capacity becomes available. */ + readonly retryable: boolean; + /** Safe aggregate counts for each provider that blocked allocation. */ + readonly capacity: readonly CloudFleetSandboxCapacityExhaustion[]; constructor( message: string, @@ -90,6 +103,10 @@ export class CloudFleetSandboxProvisionError extends Error { providerId?: CloudFleetSandboxProviderId; confirmedProvisioned?: boolean; outcomeUnknown?: boolean; + code?: 'sandbox_capacity_exhausted'; + noSandboxCreated?: boolean; + retryable?: boolean; + capacity?: readonly CloudFleetSandboxCapacityExhaustion[]; cause?: unknown; } = {} ) { @@ -101,6 +118,10 @@ export class CloudFleetSandboxProvisionError extends Error { this.providerId = identity.providerId; this.confirmedProvisioned = identity.confirmedProvisioned === true; this.outcomeUnknown = !this.confirmedProvisioned && identity.outcomeUnknown === true; + this.code = identity.code; + this.noSandboxCreated = !this.confirmedProvisioned && identity.noSandboxCreated === true; + this.retryable = identity.retryable === true; + this.capacity = identity.capacity?.map((entry) => ({ ...entry })) ?? []; } } @@ -636,6 +657,55 @@ function readProviderId( return undefined; } +/** Parse only the complete, affirmative pre-allocation capacity contract. */ +function readCapacityExhaustion( + payload: unknown +): readonly CloudFleetSandboxCapacityExhaustion[] | undefined { + if ( + !isObject(payload) || + readString(payload, 'code') !== 'sandbox_capacity_exhausted' || + payload.no_sandbox_created !== true || + payload.retryable !== true || + !Array.isArray(payload.capacity) || + payload.capacity.length === 0 + ) { + return undefined; + } + const capacity: CloudFleetSandboxCapacityExhaustion[] = []; + for (const value of payload.capacity) { + if (!isObject(value)) return undefined; + const provider = readString(value, 'provider'); + const current = readNumber(value, 'current'); + const limit = readNumber(value, 'limit'); + if ( + provider === undefined || + !CLOUD_FLEET_SANDBOX_PROVIDER_IDS.includes(provider as CloudFleetSandboxProviderId) || + current === undefined || + current < 0 || + !Number.isInteger(current) || + limit === undefined || + limit < 0 || + !Number.isInteger(limit) + ) { + return undefined; + } + capacity.push({ + provider: provider as CloudFleetSandboxProviderId, + current, + limit, + }); + } + return capacity; +} + +/** Render safe aggregate provider counts for a definitive capacity rejection. */ +function capacityExhaustionMessage(capacity: readonly CloudFleetSandboxCapacityExhaustion[]): string { + const detail = capacity + .map(({ provider, current, limit }) => `${provider}: ${current} current / ${limit} limit`) + .join('; '); + return `Sandbox capacity is exhausted before allocation (${detail}). No sandbox was created; retry when capacity is available.`; +} + function assertExpectedSandboxIdentity(payload: JsonRecord, expectedSandboxId: string): void { const sandboxId = requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'); if (sandboxId !== expectedSandboxId) { @@ -1022,6 +1092,19 @@ export async function ensureCloudFleetSandbox( }); } const error = endpointError('provision the fleet sandbox', response, payload); + const capacity = response.status === 503 ? readCapacityExhaustion(payload) : undefined; + if (capacity !== undefined) { + throw new CloudFleetSandboxProvisionError(capacityExhaustionMessage(capacity), { + cloudWorkspaceId: resolved.cloudWorkspaceId, + ...(sandboxIdentity.name === undefined ? {} : { nodeName: sandboxIdentity.name }), + providerId: capacity.length === 1 ? capacity[0].provider : input.providerId, + code: 'sandbox_capacity_exhausted', + noSandboxCreated: true, + retryable: true, + capacity, + cause: error, + }); + } // Gateway/server failures can arrive after Cloud accepted the ensure // request but before it could return an identity. Keep every 5xx failure // replayable as an unknown outcome, even for legacy custom-name callers; diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 41621459f..598a5af25 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -101,6 +101,7 @@ export { type CloudFleetSandboxReused, type CloudFleetSandboxProvisioningTimeout, type CloudFleetSandboxProviderId, + type CloudFleetSandboxCapacityExhaustion, type CloudFleetSandboxWorkloadProfile, type DeleteCloudFleetSandboxInput, type CloudFleetSandboxRequestOptions, diff --git a/tests/relayflows/cases/1916-sandbox-capacity-rejection/case.json b/tests/relayflows/cases/1916-sandbox-capacity-rejection/case.json new file mode 100644 index 000000000..c6de5c2f7 --- /dev/null +++ b/tests/relayflows/cases/1916-sandbox-capacity-rejection/case.json @@ -0,0 +1,21 @@ +{ + "version": 1, + "id": "1916-sandbox-capacity-rejection", + "kind": "bugfix", + "title": "Classify a pre-allocation sandbox capacity rejection as definitive", + "runner": { + "command": ["node", "tests/relayflows/cases/1916-sandbox-capacity-rejection/run.mjs"] + }, + "requirements": [], + "timeoutSeconds": 900, + "expected": { + "base": { + "outcome": "bug", + "signature": "capacity_503_reported_as_unknown" + }, + "head": { + "outcome": "fixed", + "signature": "capacity_503_definitive_no_sandbox_created" + } + } +} diff --git a/tests/relayflows/cases/1916-sandbox-capacity-rejection/run.mjs b/tests/relayflows/cases/1916-sandbox-capacity-rejection/run.mjs new file mode 100644 index 000000000..424373b34 --- /dev/null +++ b/tests/relayflows/cases/1916-sandbox-capacity-rejection/run.mjs @@ -0,0 +1,241 @@ +/** + * RelayFlow proof for definitive pre-allocation capacity rejection handling. + * + * The probe presents the production Cloud client with the stable capacity 503 + * contract. The base conservatively reports an unknown outcome and retains the + * caller's replay identity; the head proves no sandbox was created, exposes + * safe aggregate counts, and withholds cleanup authority. + */ + +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const CASE_ID = '1916-sandbox-capacity-rejection'; +const targetDir = requiredDirectory('RELAY_PR_PROOF_TARGET_DIR'); +const harnessDir = requiredDirectory('RELAY_PR_PROOF_HARNESS_DIR'); +const resultPath = requiredValue('RELAY_PR_PROOF_RESULT_PATH'); +const arm = requiredValue('RELAY_PR_PROOF_ARM'); + +if (arm !== 'base' && arm !== 'head') { + throw new Error(`RELAY_PR_PROOF_ARM must be base or head, received ${JSON.stringify(arm)}.`); +} + +const expectedSha = + arm === 'base' ? process.env.RELAY_PR_PROOF_BASE_SHA : process.env.RELAY_PR_PROOF_HEAD_SHA; +if (!expectedSha) throw new Error(`Missing expected ${arm} SHA.`); +const targetSha = execFileSync('git', ['-C', targetDir, 'rev-parse', 'HEAD'], { + encoding: 'utf8', +}).trim(); +if (targetSha !== expectedSha) { + throw new Error(`Target checkout ${targetSha} does not match exact ${arm} SHA ${expectedSha}.`); +} + +const runnerPath = fileURLToPath(import.meta.url); +if (!isWithin(harnessDir, runnerPath)) { + throw new Error('The RelayFlow runner must execute from the exact-head harness checkout.'); +} + +const probePath = path.join( + targetDir, + 'packages/cloud/src/.relayflow-1916-sandbox-capacity-rejection.test.ts' +); +const observationPath = path.join(targetDir, '.relayflow-1916-sandbox-capacity-rejection-observation.json'); +const configPath = path.join(targetDir, '.relayflow-1916-sandbox-capacity-rejection.vitest.config.mjs'); + +const probeSource = String.raw`import { writeFile } from 'node:fs/promises'; +import { test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + ensureCloudSession: vi.fn(), + authorizedApiFetch: vi.fn(), +})); + +vi.mock('./auth.js', () => ({ + ensureCloudSession: mocks.ensureCloudSession, + authorizedApiFetch: mocks.authorizedApiFetch, +})); + +import { + CloudFleetSandboxProvisionError, + ensureCloudFleetSandbox, +} from './fleet-sandbox.js'; + +const auth = { + accessToken: 'relayflow-probe-access', + refreshToken: 'relayflow-probe-refresh', + accessTokenExpiresAt: '2099-01-01T00:00:00Z', + apiUrl: 'https://relayflow.invalid', +}; + +test('observes capacity rejection classification', async () => { + const observationPath = process.env.RELAY_PR1916_OBSERVATION_PATH; + if (!observationPath) throw new Error('Missing RELAY_PR1916_OBSERVATION_PATH.'); + + mocks.ensureCloudSession.mockResolvedValue({ auth, client: {} }); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99' }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + error: 'Sandbox capacity is exhausted before allocation; no sandbox was created', + code: 'sandbox_capacity_exhausted', + capacity: [{ provider: 'agent37', current: 14, limit: 10 }], + retryable: true, + no_sandbox_created: true, + }, + { status: 503 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_relayflow', + requiredCapability: 'spawn:codex', + sandboxId: 'sbx_123e4567-e89b-42d3-a456-426614174000', + name: 'fleet-sandbox-123e4567-e89b-42d3-a456-426614174000', + providerId: 'agent37', + forceProvision: true, + workloadProfile: 'long-running-agent', + }).catch((caught) => caught); + + await writeFile( + observationPath, + JSON.stringify({ + provisionError: error instanceof CloudFleetSandboxProvisionError, + message: String(error), + code: error?.code ?? null, + sandboxId: error?.sandboxId ?? null, + outcomeUnknown: error?.outcomeUnknown ?? null, + noSandboxCreated: error?.noSandboxCreated ?? null, + retryable: error?.retryable ?? null, + capacity: error?.capacity ?? null, + }), + 'utf8' + ); +}); +`; + +const configSource = `export default { + test: { + environment: 'node', + include: ['packages/cloud/src/.relayflow-1916-sandbox-capacity-rejection.test.ts'], + setupFiles: [], + }, +};\n`; + +try { + run( + 'npm', + ['ci', '--ignore-scripts', '--workspace', 'packages/cloud', '--include-workspace-root=false'], + targetDir, + 'Cloud workspace dependency installation' + ); + run('npm', ['run', 'build:config'], targetDir, 'configuration package build'); + run('npm', ['run', 'build:cloud'], targetDir, 'Cloud package build'); + + await rm(probePath, { force: true }); + await rm(configPath, { force: true }); + await rm(observationPath, { force: true }); + await writeFile(probePath, probeSource, { encoding: 'utf8', flag: 'wx' }); + await writeFile(configPath, configSource, { encoding: 'utf8', flag: 'wx' }); + run( + 'npm', + ['exec', '--', 'vitest', 'run', '--config', path.relative(targetDir, configPath)], + targetDir, + 'capacity rejection probe', + { RELAY_PR1916_OBSERVATION_PATH: observationPath } + ); + + const observation = JSON.parse(await readFile(observationPath, 'utf8')); + const baseObserved = + observation.provisionError === true && + observation.code === null && + observation.sandboxId === 'sbx_123e4567-e89b-42d3-a456-426614174000' && + observation.outcomeUnknown === true && + observation.noSandboxCreated === null && + observation.retryable === null && + observation.capacity === null; + const headObserved = + observation.provisionError === true && + observation.code === 'sandbox_capacity_exhausted' && + observation.sandboxId === null && + observation.outcomeUnknown === false && + observation.noSandboxCreated === true && + observation.retryable === true && + JSON.stringify(observation.capacity) === + JSON.stringify([{ provider: 'agent37', current: 14, limit: 10 }]) && + observation.message.includes('agent37: 14 current / 10 limit') && + observation.message.includes('No sandbox was created'); + + let outcome; + let signature; + let details; + if (baseObserved) { + outcome = 'bug'; + signature = 'capacity_503_reported_as_unknown'; + details = + 'The base treats a complete pre-allocation capacity response as unknown and retains the caller replay identity.'; + } else if (headObserved) { + outcome = 'fixed'; + signature = 'capacity_503_definitive_no_sandbox_created'; + details = + 'The head reports provider counts, proves no sandbox was created, and retains no cleanup identity.'; + } else { + throw new Error(`Unexpected capacity rejection observation: ${JSON.stringify(observation)}.`); + } + + await mkdir(path.dirname(resultPath), { recursive: true }); + await writeFile( + resultPath, + `${JSON.stringify({ version: 1, caseId: CASE_ID, arm, outcome, signature, details })}\n`, + 'utf8' + ); +} finally { + await rm(probePath, { force: true }); + await rm(configPath, { force: true }); + await rm(observationPath, { force: true }); +} + +/** Read a required non-empty RelayFlow environment value. */ +function requiredValue(name) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`Missing required environment variable ${name}.`); + return value; +} + +/** Resolve a required RelayFlow directory to an absolute path. */ +function requiredDirectory(name) { + return path.resolve(requiredValue(name)); +} + +/** Return true when a candidate path stays inside the expected checkout. */ +function isWithin(directory, candidate) { + const relative = path.relative(directory, candidate); + return ( + relative === '' || + (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) + ); +} + +/** Run a proof subprocess synchronously and surface a stable labeled failure. */ +function run(command, args, cwd, label, extraEnv = {}) { + const completed = spawnSync(command, args, { + cwd, + env: { ...process.env, ...extraEnv }, + stdio: ['ignore', 'inherit', 'inherit'], + }); + if (completed.error) throw new Error(`${label} could not start: ${completed.error.message}`); + if (completed.status !== 0) { + throw new Error( + `${label} failed with ${ + completed.signal ? `signal ${completed.signal}` : `exit code ${completed.status ?? 'unknown'}` + }.` + ); + } +}