From 2141d31c7905e7ab2604b561c5c91b5fef36c2a9 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Thu, 8 Oct 2026 23:50:33 +0000 Subject: [PATCH 1/5] fix(fleet): wait for late spawns and expose dispatch status --- CHANGELOG.md | 6 +- packages/cli/src/cli/commands/fleet.test.ts | 69 ++++++++++++++++++- packages/cli/src/cli/commands/fleet.ts | 59 +++++++++++++++- .../cli/lib/fleet-spawn-confirmation.test.ts | 36 ++++++++++ packages/sdk/src/messaging/relaycast.ts | 4 +- packages/sdk/src/messaging/types.ts | 2 +- summary.md | 19 +++++ 7 files changed, 187 insertions(+), 8 deletions(-) create mode 100644 summary.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 21f3330a1..4ec421ee8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Patch] + +### Fixed + +- `fleet spawn` waits up to six minutes for late launch confirmation by default; `fleet spawn-status ` reads the original dispatch to distinguish readiness, reported failure, and an outcome still awaiting confirmation without retrying the spawn. ## [13.2.0] - 2026-10-08 diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index e12fc0df1..51614202a 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -891,7 +891,7 @@ describe('fleet command support', () => { // #1430: acceptance by the node is not evidence of a launch, so a // targeted spawn asks the node to confirm unless told not to. confirm: true, - confirmTimeoutMs: 120_000, + confirmTimeoutMs: 360_000, input: { name: 'api-worker', cli: 'codex', @@ -5428,3 +5428,70 @@ describe('fleet command support', () => { expect(errors.join('\n')).toMatch(/relay cloud enroll/); }); }); + +describe('fleet spawn-status', () => { + it.each([ + ['invoked', undefined, 'unconfirmed_may_be_running'], + ['completed', { spawned: true, ready: true }, 'ready'], + ['failed', undefined, 'failed'], + ['completed', { spawned: true, ready: false }, 'accepted'], + ])('reports %s without redispatching or exposing raw output', async (status, output, state) => { + const getInvocation = vi.fn(async () => ({ + invocationId: 'inv_late', + status, + output: { ...output, secret: 'raw-secret' }, + error: 'raw-secret', + input: { task: 'private-task' }, + })); + const spawn = vi.fn(); + const logs: string[] = []; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn(() => ({ + messaging: { commands: { getInvocation }, placement: { spawn } }, + })) as never, + log: (message: unknown) => logs.push(String(message)), + error: vi.fn(), + exit: vi.fn() as never, + }, + }); + await program.parseAsync(['fleet', 'spawn-status', 'inv_late', '--token', 'at_test'], { from: 'user' }); + expect(getInvocation).toHaveBeenCalledWith('spawn', 'inv_late'); + expect(spawn).not.toHaveBeenCalled(); + expect(JSON.parse(logs[0]!).invocation.placement).toMatchObject({ invocationId: 'inv_late', state }); + expect(logs.join('')).not.toContain('raw-secret'); + expect(logs.join('')).not.toContain('private-task'); + }); +}); + +it('mints and releases a temporary reader for spawn-status with workspace credentials', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', ''); + const register = vi.fn(async () => ({ token: 'at_reader' })); + const release = vi.fn(async () => undefined); + const getInvocation = vi.fn(async () => ({ invocationId: 'inv_late', status: 'failed' })); + const createAgentRelay = vi.fn(() => ({ messaging: { commands: { getInvocation } } })); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createWorkspaceRelay: vi.fn(() => ({ workspace: { register, release } })) as never, + createAgentRelay: createAgentRelay as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + }); + await program.parseAsync(['fleet', 'spawn-status', 'inv_late', '--workspace-key', 'rk_test'], { + from: 'user', + }); + expect(register).toHaveBeenCalledWith( + expect.objectContaining({ metadata: { purpose: 'fleet-spawn-status' } }), + { strict: true } + ); + expect(createAgentRelay).toHaveBeenCalledWith(expect.objectContaining({ token: 'at_reader' })); + expect(release).toHaveBeenCalledWith( + expect.objectContaining({ name: register.mock.calls[0]![0].name, deleteAgent: true }) + ); +}); diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index fd1ff0464..f75465307 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -44,7 +44,7 @@ import { redactSecrets } from '../lib/redact.js'; import { attributableReleaseReason } from '../lib/release-reason.js'; import { retireOwnedIntegrationBindings } from './integration.js'; import { resolveSandboxRepository, type SandboxRepositorySelection } from '../lib/sandbox-repo.js'; -import { spawnPlacementReceipt } from '../lib/spawn-lifecycle.js'; +import { sanitizedSpawnReceipt, spawnPlacementReceipt } from '../lib/spawn-lifecycle.js'; import { resolveAgentToken, resolveWorkspaceSelection, @@ -436,6 +436,59 @@ export function registerFleetCommands( await runFleetAgentList(deps, options); }); + addSdkOptions( + group + .command('spawn-status') + .description('Read an existing spawn dispatch without launching another agent') + .argument('', 'Invocation ID returned by fleet spawn') + ).action(async (invocationId: string, options: Record) => { + await runSdk(deps.sdk, async () => { + const clientOptions = sdkOptionsFromOpts(options); + let token = resolveAgentToken(clientOptions); + let launcherName: string | undefined; + let workspace: ReturnType | undefined; + try { + if (!token) { + workspace = deps.sdk.createWorkspaceRelay(clientOptions); + const name = `fleet-status-launcher-${randomUUID().slice(0, 8)}`; + const launcher = await workspace.workspace.register( + { name, metadata: { purpose: 'fleet-spawn-status' } }, + { strict: true } + ); + launcherName = name; + token = launcher.token; + if (!token) throw new Error('The temporary fleet status launcher did not receive an agent token.'); + } + const relay = deps.sdk.createAgentRelay({ ...clientOptions, token }); + const invocation = await relay.messaging.commands.getInvocation('spawn', invocationId); + // Keep this read-only surface on the existing safe receipt projection. + const receipt = sanitizedSpawnReceipt(invocation as unknown as Record); + const result = spawnInvocationWithPlacement(receipt); + const output = receipt.output as Record | undefined; + // --no-confirm dispatches may legitimately report launch without readiness. + // Preserve that evidence instead of claiming the launched worker failed. + if ( + ['completed', 'success', 'succeeded'].includes(String(receipt.status).toLowerCase()) && + output?.spawned === true && + output.ready === false + ) { + result.placement = { ...spawnPlacementReceipt(receipt), state: 'accepted' }; + } + printJson(deps.sdk, { invocation: result }); + } finally { + if (launcherName && workspace) { + await workspace.workspace + .release({ + name: launcherName, + reason: 'Temporary fleet spawn status launcher completed', + deleteAgent: true, + }) + .catch((error) => deps.warn(redactSecrets(String(error)))); + } + } + }); + }); + addSdkOptions( group .command('spawn') @@ -489,7 +542,7 @@ export function registerFleetCommands( .option( '--confirm-timeout ', `How long a targeted spawn waits for harness readiness (minimum ${MIN_VERIFIED_CONFIRM_TIMEOUT_MS}ms)`, - '120000' + '360000' ) ).action(async (cli: string, options: Record) => { await runSdk(deps.sdk, async () => { @@ -576,7 +629,7 @@ export function registerFleetCommands( { organization, project, workstream, role, objective }, task ); - const confirmTimeoutText = optionalText(options.confirmTimeout, 'Confirm timeout') ?? '120000'; + const confirmTimeoutText = optionalText(options.confirmTimeout, 'Confirm timeout') ?? '360000'; const confirmTimeoutMs = Number(confirmTimeoutText); if (!Number.isFinite(confirmTimeoutMs) || confirmTimeoutMs <= 0) { throw new Error('--confirm-timeout must be a positive number of milliseconds.'); diff --git a/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts b/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts index 498134efb..f741b7b91 100644 --- a/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts +++ b/packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts @@ -519,3 +519,39 @@ describe('targeted spawn readiness contract', () => { } }); }); + +it('confirms a five-minute launch using the default budget without dispatching again', async () => { + vi.useFakeTimers(); + try { + const started = Date.now(); + const { client, invoke } = createClient(async (name, invocationId) => ({ + invocation_id: invocationId, + action_name: name, + status: Date.now() - started >= 300_000 ? 'completed' : 'invoked', + output: { spawned: true, ready: true }, + })); + const pending = client.placement.spawn(spawnInput({ confirm: true })); + await vi.advanceTimersByTimeAsync(300_500); + expect((await pending).placement.confirmed).toBe(true); + expect(invoke).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } +}); + +it('returns a pollable invocation ID when the extended default budget expires', async () => { + vi.useFakeTimers(); + try { + const { client, invoke } = createClient(async () => ({ status: 'invoked' })); + const pending = client.placement.spawn(spawnInput({ confirm: true })).catch((error) => error); + await vi.advanceTimersByTimeAsync(360_500); + const error = await pending; + expect(error.code).toBe('spawn_unconfirmed'); + expect(error.state).toBe('unconfirmed_may_be_running'); + expect(error.invocationId).toBe('inv-1430'); + expect(error.message).toContain('fleet spawn-status inv-1430'); + expect(invoke).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } +}); diff --git a/packages/sdk/src/messaging/relaycast.ts b/packages/sdk/src/messaging/relaycast.ts index 3497b2385..8acbbe947 100644 --- a/packages/sdk/src/messaging/relaycast.ts +++ b/packages/sdk/src/messaging/relaycast.ts @@ -125,7 +125,7 @@ export { RelayPlacementError } from './relaycast-placement.js'; export type { RelaySpawnDispatchState, RelaySpawnPlacementState } from './relaycast-placement.js'; export type { RelaycastMessagingOptions } from './relaycast-client.js'; -const DEFAULT_CONFIRM_TIMEOUT_MS = 120_000; +const DEFAULT_CONFIRM_TIMEOUT_MS = 360_000; const DEFAULT_CONFIRM_POLL_MS = 500; /** `setTimeout` clamps anything larger, firing immediately instead of waiting. */ const MAX_CONFIRM_TIMEOUT_MS = 2_147_483_647; @@ -1102,7 +1102,7 @@ export class RelaycastMessagingClient implements RelayMessagingClient { `node '${context.node}' accepted ${actionName} (invocation ${invocationId}) but never reported a result within ${budgetMs}ms. ` + `The node advertised capacity and acknowledged the dispatch; nothing confirmed that it launched. ` + `The invocation may still be running, so do not retry blindly. ` + - `Check that node's broker version, or re-run without confirmation to accept an unconfirmed dispatch.` + + `Poll this dispatch with agent-relay fleet spawn-status ${invocationId}; an unknown outcome is not proof that no process started.` + (lastReadError ? ` Last read error: ${lastReadError}` : ''), { ...errorContext, diff --git a/packages/sdk/src/messaging/types.ts b/packages/sdk/src/messaging/types.ts index 683c36296..a5f644d02 100644 --- a/packages/sdk/src/messaging/types.ts +++ b/packages/sdk/src/messaging/types.ts @@ -674,7 +674,7 @@ export interface RelaySpawnPlacementInput { * How long to wait for that terminal result. Must exceed the node's own * readiness window (the broker's `verify_ready` mode holds the action open * for up to 90s, and starts that clock only once the launch completes). - * Defaults to 120000; `fleet spawn` enforces a 95000 floor for verified + * Defaults to 360000; `fleet spawn` enforces a 95000 floor for verified * targeted spawns so the two windows nest. */ confirmTimeoutMs?: number; diff --git a/summary.md b/summary.md new file mode 100644 index 000000000..d39bb49ba --- /dev/null +++ b/summary.md @@ -0,0 +1,19 @@ +# Fix late fleet spawn confirmation and provide dispatch status reads + +`fleet spawn` previously exhausted its two-minute confirmation budget while agents were still launching, then suggested redispatching without confirmation. Launches taking 1–5 minutes now have a six-minute default budget in the CLI and SDK. Explicit timeout overrides and the existing verified-spawn minimum remain supported. + +Add `agent-relay fleet spawn-status ` to read the original spawn invocation without dispatching another worker. Timeout errors retain their structured invocation ID and direct callers to this command. It uses existing lifecycle receipts to report confirmed readiness, terminal failure, or an outcome still awaiting confirmation. A successful unverified launch retains `spawned: true, ready: false` with state `accepted`. Workspace-only callers mint and clean up a temporary reader using the established launcher pattern. Output excludes task payloads and raw handler errors. + +Silence cannot establish that a process never started. A node that never returns a result remains explicitly unconfirmed; a terminal node failure is reported as failed. No roster-only inference of readiness is introduced. These tests use deterministic fixtures, not the affected physical fleet nodes. + +Validation: + +- `npx vitest run packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts packages/cli/src/cli/commands/fleet.test.ts packages/cli/src/cli/lib/spawn-lifecycle.test.ts` — 124 tests passed. +- `npm --prefix packages/sdk run check` — passed. +- `npx tsc -p packages/cli/tsconfig.json --noEmit` — passed. +- `npx eslint packages/cli/src/cli/commands/fleet.ts packages/cli/src/cli/commands/fleet.test.ts packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts` — no errors; four existing warnings in fleet.ts. +- `git diff --check` — passed. + +Regression coverage includes a five-minute launch with exactly one dispatch, expiration of the extended budget with a pollable ID, pending/ready/failed/unverified status reads, safe receipt output, and temporary reader cleanup. + +No workflow files changed. The existing unrelated active Trail trajectory prevented starting a new one; it was left intact. From 95bf06914ea444fc05822a201f8c7b283afc566a Mon Sep 17 00:00:00 2001 From: Relayflow Date: Fri, 9 Oct 2026 00:06:31 +0000 Subject: [PATCH 2/5] fix(fleet): carry spawn-status onto the enumerated command surfaces `fleet spawn-status` was registered without updating the surfaces that enumerate commands, so the bootstrap leaf inventory and the Fleet CLI inventory test failed. - Add the leaf to `expectedLeafCommands` and to the current-main Fleet inventory counts, with an assertion that it is a public leaf: the spawn timeout error names the command, so a hidden or missing one makes that guidance unactionable. - Refresh the trusted Fleet CLI inventory snapshot and the matrix pin for the new leaf and the 360000ms `--confirm-timeout` default, and declare its option coverage. The Daytona board defers the leaf rather than mapping an operation: a read-only poll of a recorded invocation id proves nothing about placement that the spawn it reads has not already proven. Six `node` records with pre-existing snapshot drift (`--state-dir`, `--broker-url`, `--api-key`) are restored verbatim to keep that drift out of this change. - Cover the evidence the fix exists for: launch proof and the answering node id survive the sanitized read, and a dispatch that has not reported acquires no launch claim. - Document the command in the CLI README and the feature manifest, and split the changelog entry (new command is Added, hence Minor). Co-Authored-By: Claude Opus 5 --- .agentworkforce/features/manifest.yaml | 7 ++ CHANGELOG.md | 8 +- packages/cli/README.md | 17 ++++ packages/cli/src/cli/bootstrap.test.ts | 1 + packages/cli/src/cli/commands/fleet.test.ts | 97 +++++++++++++------ summary.md | 13 +-- tests/fixtures/verify-fleet-daytona.test.ts | 15 ++- .../cleanroom/fleet-cli-inventory.json | 87 ++++++++++++++++- .../cleanroom/fleet-daytona.matrix.json | 26 ++++- 9 files changed, 230 insertions(+), 41 deletions(-) diff --git a/.agentworkforce/features/manifest.yaml b/.agentworkforce/features/manifest.yaml index f95f9a189..190c3aed0 100644 --- a/.agentworkforce/features/manifest.yaml +++ b/.agentworkforce/features/manifest.yaml @@ -724,6 +724,13 @@ categories: location: packages/cli/src/cli/commands/fleet.ts verify_tier: 5 + - id: fleet-spawn-status + name: Read Fleet Spawn Dispatch Status + cli: relay fleet spawn-status + description: Read a recorded fleet spawn dispatch without spawning again, separating confirmed readiness, terminal failure, and an outcome the node has not reported yet + location: packages/cli/src/cli/commands/fleet.ts + verify_tier: 5 + - id: fleet-release name: Release Fleet Agent cli: relay fleet release [--reason ] [--delete-agent] diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ec421ee8..ff053631b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,15 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased - Patch] +## [Unreleased - Minor] + +### Added + +- `agent-relay fleet spawn-status ` reads a recorded spawn dispatch without spawning again, separating confirmed readiness, a node-reported failure, and an outcome the node has not reported yet. ### Fixed -- `fleet spawn` waits up to six minutes for late launch confirmation by default; `fleet spawn-status ` reads the original dispatch to distinguish readiness, reported failure, and an outcome still awaiting confirmation without retrying the spawn. +- `fleet spawn` (and SDK placement confirmation) now waits six minutes for launch confirmation instead of two, so agents that register minutes after dispatch are confirmed rather than reported as unconfirmed; the timeout error names the `fleet spawn-status` poll for its invocation id instead of suggesting a redispatch. ## [13.2.0] - 2026-10-08 diff --git a/packages/cli/README.md b/packages/cli/README.md index c6cf79265..701b1b242 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -312,6 +312,23 @@ set `RELAY_AGENT_TOKEN` to the token returned by login (`agent-relay cloud login`) but does not need an agent token: when one is absent, it creates and removes a short-lived launcher identity automatically. +A targeted spawn waits for the node to confirm the launch, for six minutes by +default (`--confirm-timeout `, minimum 95000). Agents that take minutes to +register are confirmed inside that window rather than reported as unconfirmed. +If the window does expire, the error carries the invocation id: read that +dispatch instead of spawning again, because a retry under the same name +collides with a worker that may still be starting. + +```bash +agent-relay fleet spawn-status inv_01J... +``` + +The reply's `placement.state` separates the outcomes: `ready` (launched and +confirmed), `failed` (the node reported a terminal failure), `accepted` +(launched, readiness unverified), and `unconfirmed_may_be_running` (the node +still has not reported — not evidence that nothing started). Polling never +dispatches another agent. + Without placement options, `fleet spawn` connects to the local project's broker and passes the caller's exact directory, including a nested package, to the worker. Start the local broker with `agent-relay node up` if it is not running; diff --git a/packages/cli/src/cli/bootstrap.test.ts b/packages/cli/src/cli/bootstrap.test.ts index f07483fdb..7943a8742 100644 --- a/packages/cli/src/cli/bootstrap.test.ts +++ b/packages/cli/src/cli/bootstrap.test.ts @@ -46,6 +46,7 @@ const expectedLeafCommands = [ 'fleet nodes list', 'fleet release', 'fleet spawn', + 'fleet spawn-status', 'fleet status', // cloud 'cloud login', diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 51614202a..d5e66d2d6 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -5464,34 +5464,77 @@ describe('fleet spawn-status', () => { expect(logs.join('')).not.toContain('raw-secret'); expect(logs.join('')).not.toContain('private-task'); }); -}); -it('mints and releases a temporary reader for spawn-status with workspace credentials', async () => { - vi.stubEnv('RELAY_AGENT_TOKEN', ''); - const register = vi.fn(async () => ({ token: 'at_reader' })); - const release = vi.fn(async () => undefined); - const getInvocation = vi.fn(async () => ({ invocationId: 'inv_late', status: 'failed' })); - const createAgentRelay = vi.fn(() => ({ messaging: { commands: { getInvocation } } })); - const program = new Command(); - program.exitOverride(); - registerFleetCommands(program, { - sdk: { - createWorkspaceRelay: vi.fn(() => ({ workspace: { register, release } })) as never, - createAgentRelay: createAgentRelay as never, - log: vi.fn(), - error: vi.fn(), - exit: vi.fn() as never, - }, + // #1430 follow-up: the reported failure was a caller unable to tell a worker + // that launched late from one that never started. The state alone is not that + // evidence — the launch proof and the node that answered have to survive the + // read, and a silent dispatch must not acquire a launch claim it never made. + it.each([ + [ + 'a node that has not reported yet', + { invocationId: 'inv_late', status: 'invoked', dispatched_node_id: 'node_sf_mini' }, + { state: 'unconfirmed_may_be_running', dispatchState: 'dispatched', dispatchedNodeId: 'node_sf_mini' }, + undefined, + ], + [ + 'a launch confirmed after the spawn gave up', + { + invocationId: 'inv_late', + status: 'completed', + handler_node_id: 'node_sf_mini', + output: { spawned: true, ready: true }, + }, + { state: 'ready', dispatchState: 'dispatched', handlerNodeId: 'node_sf_mini' }, + { spawned: true, ready: true }, + ], + ])('keeps the launch evidence for %s', async (_label, invocation, placement, output) => { + const logs: string[] = []; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn(() => ({ + messaging: { commands: { getInvocation: vi.fn(async () => invocation) } }, + })) as never, + log: (message: unknown) => logs.push(String(message)), + error: vi.fn(), + exit: vi.fn() as never, + }, + }); + await program.parseAsync(['fleet', 'spawn-status', 'inv_late', '--token', 'at_test'], { from: 'user' }); + + const reported = JSON.parse(logs[0]!).invocation; + expect(reported.placement).toMatchObject(placement); + expect(reported.output).toEqual(output); }); - await program.parseAsync(['fleet', 'spawn-status', 'inv_late', '--workspace-key', 'rk_test'], { - from: 'user', + + it('mints and releases a temporary reader for spawn-status with workspace credentials', async () => { + vi.stubEnv('RELAY_AGENT_TOKEN', ''); + const register = vi.fn(async () => ({ token: 'at_reader' })); + const release = vi.fn(async () => undefined); + const getInvocation = vi.fn(async () => ({ invocationId: 'inv_late', status: 'failed' })); + const createAgentRelay = vi.fn(() => ({ messaging: { commands: { getInvocation } } })); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createWorkspaceRelay: vi.fn(() => ({ workspace: { register, release } })) as never, + createAgentRelay: createAgentRelay as never, + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as never, + }, + }); + await program.parseAsync(['fleet', 'spawn-status', 'inv_late', '--workspace-key', 'rk_test'], { + from: 'user', + }); + expect(register).toHaveBeenCalledWith( + expect.objectContaining({ metadata: { purpose: 'fleet-spawn-status' } }), + { strict: true } + ); + expect(createAgentRelay).toHaveBeenCalledWith(expect.objectContaining({ token: 'at_reader' })); + expect(release).toHaveBeenCalledWith( + expect.objectContaining({ name: register.mock.calls[0]![0].name, deleteAgent: true }) + ); }); - expect(register).toHaveBeenCalledWith( - expect.objectContaining({ metadata: { purpose: 'fleet-spawn-status' } }), - { strict: true } - ); - expect(createAgentRelay).toHaveBeenCalledWith(expect.objectContaining({ token: 'at_reader' })); - expect(release).toHaveBeenCalledWith( - expect.objectContaining({ name: register.mock.calls[0]![0].name, deleteAgent: true }) - ); }); diff --git a/summary.md b/summary.md index d39bb49ba..af947ec94 100644 --- a/summary.md +++ b/summary.md @@ -6,14 +6,15 @@ Add `agent-relay fleet spawn-status ` to read the original spawn Silence cannot establish that a process never started. A node that never returns a result remains explicitly unconfirmed; a terminal node failure is reported as failed. No roster-only inference of readiness is introduced. These tests use deterministic fixtures, not the affected physical fleet nodes. +The new leaf is carried onto the surfaces that enumerate commands: the bootstrap leaf-command inventory, the trusted Fleet CLI inventory snapshot (`tests/relayflows/cleanroom/fleet-cli-inventory.json`) and its matrix pin, the `fleet` README, and the feature manifest. The Daytona board defers `fleet spawn-status` rather than mapping it to an operation: a read-only poll of a recorded invocation id proves nothing about placement that the spawn it reads has not already proven. + Validation: -- `npx vitest run packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts packages/cli/src/cli/commands/fleet.test.ts packages/cli/src/cli/lib/spawn-lifecycle.test.ts` — 124 tests passed. -- `npm --prefix packages/sdk run check` — passed. -- `npx tsc -p packages/cli/tsconfig.json --noEmit` — passed. -- `npx eslint packages/cli/src/cli/commands/fleet.ts packages/cli/src/cli/commands/fleet.test.ts packages/cli/src/cli/lib/fleet-spawn-confirmation.test.ts` — no errors; four existing warnings in fleet.ts. -- `git diff --check` — passed. +- `sh .relayflow/check.sh` — all checks passed (npm ci, codegen check, build, typecheck, lint, format, PR-proof and subscription guards, `vitest run`: 200 test files, 3754 tests). +- Regression coverage: a five-minute launch confirmed with exactly one dispatch; expiration of the extended budget returning a pollable ID whose message names the poll command; pending/ready/failed/unverified status reads; launch evidence (`output.spawned`/`ready`, dispatch and handler node ids) surviving the sanitized read; safe receipt output; temporary reader cleanup; `fleet spawn-status` present as a public leaf in both command inventories. + +Two check failures were environmental, not code: `node-claim.test.ts` needs `lsof`/`ps` and `broker-process-identity.test.ts` compiles a C fixture with `cc`. GitHub's ubuntu runners ship these (`node-compat.yml` installs `lsof procps`; `test-install.yml` installs `build-essential`), so the install step was added to the uncommitted `.relayflow/check.sh` rather than changing any test. -Regression coverage includes a five-minute launch with exactly one dispatch, expiration of the extended budget with a pollable ID, pending/ready/failed/unverified status reads, safe receipt output, and temporary reader cleanup. +Pre-existing, left alone: the committed Fleet CLI inventory snapshot is stale for six `node` commands (`--state-dir`, `--broker-url`, `--api-key` were never snapshotted after `addBrokerOptions` was applied to them), so the qualification job's inventory comparison fails on main independently of this change. The snapshot regenerated here restores those six records verbatim to keep that drift out of this change. No workflow files changed. The existing unrelated active Trail trajectory prevented starting a new one; it was left intact. diff --git a/tests/fixtures/verify-fleet-daytona.test.ts b/tests/fixtures/verify-fleet-daytona.test.ts index 015572b5e..004588934 100644 --- a/tests/fixtures/verify-fleet-daytona.test.ts +++ b/tests/fixtures/verify-fleet-daytona.test.ts @@ -2059,10 +2059,10 @@ describe('complete Daytona Fleet board', () => { const actual = await collectFleetCliInventory('packages/cli/dist/cli/index.js'); // The trusted verifier runs from current main while the candidate CLI is // hydrated separately. Keep this assertion pinned to main's known - // 30-executable-leaf/36-record surface; candidate inventory equality is checked in + // 31-executable-leaf/37-record surface; candidate inventory equality is checked in // the qualification job against the hydrated artifact. - expect(actual.commands).toHaveLength(36); - expect(actual.commands.filter(({ leaf }: { leaf: boolean }) => leaf)).toHaveLength(30); + expect(actual.commands).toHaveLength(37); + expect(actual.commands.filter(({ leaf }: { leaf: boolean }) => leaf)).toHaveLength(31); expect(inventorySha256(actual)).toMatch(/^[a-f0-9]{64}$/); for (const verb of ['config', 'enable', 'disable', 'inherit']) { expect(actual.commands.find(({ path }: { path: string }) => path === `fleet ${verb}`)).toMatchObject({ @@ -2082,6 +2082,15 @@ describe('complete Daytona Fleet board', () => { hidden: false, leaf: true, }); + // The read-only poll for a spawn whose confirmation window expired. It has + // to be on the published surface for the timeout error that names it to be + // actionable. + expect(actual.commands.find(({ path }: { path: string }) => path === 'fleet spawn-status')).toMatchObject( + { + hidden: false, + leaf: true, + } + ); expect( actual.commands .find(({ path }: { path: string }) => path === 'node up') diff --git a/tests/relayflows/cleanroom/fleet-cli-inventory.json b/tests/relayflows/cleanroom/fleet-cli-inventory.json index 498bdbd84..0bf5b75ed 100644 --- a/tests/relayflows/cleanroom/fleet-cli-inventory.json +++ b/tests/relayflows/cleanroom/fleet-cli-inventory.json @@ -1044,7 +1044,7 @@ "conflictsWith": [], "implied": null, "envVar": null, - "defaultValue": "120000", + "defaultValue": "360000", "presetArg": null }, { @@ -1457,6 +1457,91 @@ } ] }, + { + "path": "fleet spawn-status", + "aliases": [], + "hidden": false, + "leaf": true, + "arguments": [ + { + "name": "invocation-id", + "required": true, + "variadic": false, + "choices": null, + "defaultValue": null + } + ], + "options": [ + { + "flags": "--base-url ", + "short": null, + "long": "--base-url", + "mandatory": false, + "valueRequired": true, + "valueOptional": false, + "variadic": false, + "negate": false, + "hidden": false, + "choices": null, + "conflictsWith": [], + "implied": null, + "envVar": null, + "defaultValue": null, + "presetArg": null + }, + { + "flags": "--token ", + "short": null, + "long": "--token", + "mandatory": false, + "valueRequired": true, + "valueOptional": false, + "variadic": false, + "negate": false, + "hidden": false, + "choices": null, + "conflictsWith": [], + "implied": null, + "envVar": null, + "defaultValue": null, + "presetArg": null + }, + { + "flags": "--wk ", + "short": null, + "long": "--wk", + "mandatory": false, + "valueRequired": true, + "valueOptional": false, + "variadic": false, + "negate": false, + "hidden": false, + "choices": null, + "conflictsWith": [], + "implied": null, + "envVar": null, + "defaultValue": null, + "presetArg": null + }, + { + "flags": "--workspace-key ", + "short": null, + "long": "--workspace-key", + "mandatory": false, + "valueRequired": true, + "valueOptional": false, + "variadic": false, + "negate": false, + "hidden": false, + "choices": null, + "conflictsWith": [], + "implied": null, + "envVar": null, + "defaultValue": null, + "presetArg": null + } + ] + }, { "path": "fleet status", "aliases": [], diff --git a/tests/relayflows/cleanroom/fleet-daytona.matrix.json b/tests/relayflows/cleanroom/fleet-daytona.matrix.json index 15ab20607..b31bebbca 100644 --- a/tests/relayflows/cleanroom/fleet-daytona.matrix.json +++ b/tests/relayflows/cleanroom/fleet-daytona.matrix.json @@ -8,7 +8,7 @@ "requiredSnapshotRelayVersion": "11.10.3", "artifactRoot": ".workflow-artifacts/verify-fleet-daytona", "inventoryFile": "fleet-cli-inventory.json", - "inventorySha256": "8bdc4be9341d4efb5a5e04ae8fe601fd30a9d664681b82ab2c4c729f46b4ac16", + "inventorySha256": "88839ab72869ab2cfc6d0757700dd7f0253581b921c5c3d9097dd08ff4ba23df", "commandSurface": { "fleet agent list": [ "fleet-agent-list-json", @@ -118,7 +118,7 @@ "node-agent-set-model-app-server-a": ["node agent spawn", "node agent set-model"], "node-agent-set-model-app-server-b": ["node agent spawn", "node agent set-model"] }, - "deferredCommandSurface": [], + "deferredCommandSurface": ["fleet spawn-status"], "optionCoverage": { "fleet agent list": [ { @@ -572,6 +572,28 @@ "operationId": "fleet-spawn-metadata-channel-model-cwd" } ], + "fleet spawn-status": [ + { + "option": "--base-url", + "status": "skipped", + "reason": "The proof uses the credential-bound workspace endpoint; alternate endpoint selection is covered by CLI connection tests without changing the Daytona target." + }, + { + "option": "--token", + "status": "skipped", + "reason": "Token argv would duplicate reveal-once credentials; the board proves redaction and uses the bound workspace key." + }, + { + "option": "--wk", + "status": "skipped", + "reason": "The legacy --wk credential alias is covered by CLI connection tests; the board must not mint a second credential source." + }, + { + "option": "--workspace-key", + "status": "skipped", + "reason": "The bound workspace key is injected only where the operation needs it; repeating it on every Fleet read would multiply secret-bearing argv." + } + ], "fleet status": [ { "option": "--base-url", From 99c51b39a59474aef7e658ce1175df6194a7c207 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Fri, 9 Oct 2026 00:09:35 +0000 Subject: [PATCH 3/5] Keep relayflow working files out of the change --- summary.md | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 summary.md diff --git a/summary.md b/summary.md deleted file mode 100644 index af947ec94..000000000 --- a/summary.md +++ /dev/null @@ -1,20 +0,0 @@ -# Fix late fleet spawn confirmation and provide dispatch status reads - -`fleet spawn` previously exhausted its two-minute confirmation budget while agents were still launching, then suggested redispatching without confirmation. Launches taking 1–5 minutes now have a six-minute default budget in the CLI and SDK. Explicit timeout overrides and the existing verified-spawn minimum remain supported. - -Add `agent-relay fleet spawn-status ` to read the original spawn invocation without dispatching another worker. Timeout errors retain their structured invocation ID and direct callers to this command. It uses existing lifecycle receipts to report confirmed readiness, terminal failure, or an outcome still awaiting confirmation. A successful unverified launch retains `spawned: true, ready: false` with state `accepted`. Workspace-only callers mint and clean up a temporary reader using the established launcher pattern. Output excludes task payloads and raw handler errors. - -Silence cannot establish that a process never started. A node that never returns a result remains explicitly unconfirmed; a terminal node failure is reported as failed. No roster-only inference of readiness is introduced. These tests use deterministic fixtures, not the affected physical fleet nodes. - -The new leaf is carried onto the surfaces that enumerate commands: the bootstrap leaf-command inventory, the trusted Fleet CLI inventory snapshot (`tests/relayflows/cleanroom/fleet-cli-inventory.json`) and its matrix pin, the `fleet` README, and the feature manifest. The Daytona board defers `fleet spawn-status` rather than mapping it to an operation: a read-only poll of a recorded invocation id proves nothing about placement that the spawn it reads has not already proven. - -Validation: - -- `sh .relayflow/check.sh` — all checks passed (npm ci, codegen check, build, typecheck, lint, format, PR-proof and subscription guards, `vitest run`: 200 test files, 3754 tests). -- Regression coverage: a five-minute launch confirmed with exactly one dispatch; expiration of the extended budget returning a pollable ID whose message names the poll command; pending/ready/failed/unverified status reads; launch evidence (`output.spawned`/`ready`, dispatch and handler node ids) surviving the sanitized read; safe receipt output; temporary reader cleanup; `fleet spawn-status` present as a public leaf in both command inventories. - -Two check failures were environmental, not code: `node-claim.test.ts` needs `lsof`/`ps` and `broker-process-identity.test.ts` compiles a C fixture with `cc`. GitHub's ubuntu runners ship these (`node-compat.yml` installs `lsof procps`; `test-install.yml` installs `build-essential`), so the install step was added to the uncommitted `.relayflow/check.sh` rather than changing any test. - -Pre-existing, left alone: the committed Fleet CLI inventory snapshot is stale for six `node` commands (`--state-dir`, `--broker-url`, `--api-key` were never snapshotted after `addBrokerOptions` was applied to them), so the qualification job's inventory comparison fails on main independently of this change. The snapshot regenerated here restores those six records verbatim to keep that drift out of this change. - -No workflow files changed. The existing unrelated active Trail trajectory prevented starting a new one; it was left intact. From e4bdb248903fd2ba97c15b0f65245ee3e072ebe0 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 10 Oct 2026 00:51:12 -0700 Subject: [PATCH 4/5] fix(fleet): spawn-status reads with the minted token alone; keep invocation node ids spawn-status passed the workspace key alongside its minted launcher token, which createAgentRelay rejects, and sent that token to the default gateway rather than the one that minted it. Read with the token and the resolved workspace origin only, as the sandbox spawn path does. normalizeActionInvocation dropped handler_node_id and dispatched_node_id, so spawn-status could never report the node that answered a dispatch. Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/cli/src/cli/commands/fleet.test.ts | 5 ++++- packages/cli/src/cli/commands/fleet.ts | 9 +++++++- .../src/__tests__/relaycast-translate.test.ts | 21 ++++++++++++++++++- .../sdk/src/messaging/relaycast-translate.ts | 6 ++++++ packages/sdk/src/messaging/types.ts | 2 ++ 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index d5e66d2d6..3e805082a 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -5510,6 +5510,7 @@ describe('fleet spawn-status', () => { it('mints and releases a temporary reader for spawn-status with workspace credentials', async () => { vi.stubEnv('RELAY_AGENT_TOKEN', ''); + vi.stubEnv('RELAY_BASE_URL', 'https://relay.isolated.test'); const register = vi.fn(async () => ({ token: 'at_reader' })); const release = vi.fn(async () => undefined); const getInvocation = vi.fn(async () => ({ invocationId: 'inv_late', status: 'failed' })); @@ -5532,7 +5533,9 @@ describe('fleet spawn-status', () => { expect.objectContaining({ metadata: { purpose: 'fleet-spawn-status' } }), { strict: true } ); - expect(createAgentRelay).toHaveBeenCalledWith(expect.objectContaining({ token: 'at_reader' })); + // The minted token reads alone, from the gateway that minted it: passing + // the workspace key too is rejected by createAgentRelay. + expect(createAgentRelay).toHaveBeenCalledWith({ token: 'at_reader', baseUrl: 'https://relay.isolated.test' }); expect(release).toHaveBeenCalledWith( expect.objectContaining({ name: register.mock.calls[0]![0].name, deleteAgent: true }) ); diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index f75465307..398b3ee0c 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -459,7 +459,14 @@ export function registerFleetCommands( token = launcher.token; if (!token) throw new Error('The temporary fleet status launcher did not receive an agent token.'); } - const relay = deps.sdk.createAgentRelay({ ...clientOptions, token }); + // A minted launcher token must read from the gateway that minted it, and + // never alongside the workspace key that minted it (createAgentRelay + // rejects that pair), matching the spawn path's minted-token client. + const relay = deps.sdk.createAgentRelay( + launcherName + ? { token, baseUrl: resolveWorkspaceTransport(clientOptions).baseUrl } + : { ...clientOptions, token } + ); const invocation = await relay.messaging.commands.getInvocation('spawn', invocationId); // Keep this read-only surface on the existing safe receipt projection. const receipt = sanitizedSpawnReceipt(invocation as unknown as Record); diff --git a/packages/sdk/src/__tests__/relaycast-translate.test.ts b/packages/sdk/src/__tests__/relaycast-translate.test.ts index 6e3d5ce05..8ecc30a1c 100644 --- a/packages/sdk/src/__tests__/relaycast-translate.test.ts +++ b/packages/sdk/src/__tests__/relaycast-translate.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { toRelayNode } from '../messaging/relaycast-translate.js'; +import { normalizeActionInvocation, toRelayNode } from '../messaging/relaycast-translate.js'; describe('toRelayNode fleet liveness', () => { it('omits stale active-agent load for an offline node', () => { @@ -18,3 +18,22 @@ describe('toRelayNode fleet liveness', () => { expect(node.activeAgents).toBeUndefined(); }); }); + +describe('normalizeActionInvocation dispatch evidence', () => { + it('keeps the node that handled or was dispatched the invocation', () => { + const invocation = normalizeActionInvocation({ + invocation_id: 'inv_late', + action_name: 'spawn', + status: 'failed', + handler_node_id: 'node_a', + dispatched_node_id: 'node_b', + }); + expect(invocation).toMatchObject({ handlerNodeId: 'node_a', dispatchedNodeId: 'node_b' }); + }); + + it('omits node ids the server did not report', () => { + const invocation = normalizeActionInvocation({ invocation_id: 'inv', status: 'invoked' }); + expect(invocation).not.toHaveProperty('handlerNodeId'); + expect(invocation).not.toHaveProperty('dispatchedNodeId'); + }); +}); diff --git a/packages/sdk/src/messaging/relaycast-translate.ts b/packages/sdk/src/messaging/relaycast-translate.ts index ecab4a16d..b69f51de7 100644 --- a/packages/sdk/src/messaging/relaycast-translate.ts +++ b/packages/sdk/src/messaging/relaycast-translate.ts @@ -232,6 +232,12 @@ export function normalizeActionInvocation(raw: unknown): RelayActionInvocation { actionName: readStr(record, 'actionName', 'action_name') ?? '', callerId: (readStr(record, 'callerId', 'caller_id') ?? null) as string | null, callerName: (readStr(record, 'callerName', 'caller_name') ?? null) as string | null, + ...(readStr(record, 'handlerNodeId', 'handler_node_id') + ? { handlerNodeId: readStr(record, 'handlerNodeId', 'handler_node_id') } + : {}), + ...(readStr(record, 'dispatchedNodeId', 'dispatched_node_id') + ? { dispatchedNodeId: readStr(record, 'dispatchedNodeId', 'dispatched_node_id') } + : {}), input: readRecord(record, 'input') ?? {}, output: readRecord(record, 'output') ?? null, status: readStr(record, 'status') ?? 'invoked', diff --git a/packages/sdk/src/messaging/types.ts b/packages/sdk/src/messaging/types.ts index 74d1dc8ea..8151ffa9a 100644 --- a/packages/sdk/src/messaging/types.ts +++ b/packages/sdk/src/messaging/types.ts @@ -598,6 +598,8 @@ export interface RelayActionInvocation { actionName: string; callerId?: string | null; callerName?: string | null; + handlerNodeId?: string | null; + dispatchedNodeId?: string | null; input?: Record; output?: Record | null; status: string; From 705190a353a42779c9ffe80af2607b702f72aa7b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 10 Oct 2026 08:12:29 +0000 Subject: [PATCH 5/5] style: auto-format with Prettier --- packages/cli/src/cli/commands/fleet.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 3e805082a..4baf9f832 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -5535,7 +5535,10 @@ describe('fleet spawn-status', () => { ); // The minted token reads alone, from the gateway that minted it: passing // the workspace key too is rejected by createAgentRelay. - expect(createAgentRelay).toHaveBeenCalledWith({ token: 'at_reader', baseUrl: 'https://relay.isolated.test' }); + expect(createAgentRelay).toHaveBeenCalledWith({ + token: 'at_reader', + baseUrl: 'https://relay.isolated.test', + }); expect(release).toHaveBeenCalledWith( expect.objectContaining({ name: register.mock.calls[0]![0].name, deleteAgent: true }) );