chore: combine relay #1917, #1923, #1881 (merge commit, do not squash)

CHANGELOG.md

@@ -5,7 +5,23 @@ All notable changes to Agent Relay will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [Unreleased - Minor]
+
+### Added
+
+- `agent-relay fleet spawn --sandbox` (unpinned or `--sandbox-provider agent37`) now uses Cloud's durable `async-v1` preparation and prints each preparation phase, so a slow Relayfile initial sync no longer ends as an unknown outcome. Older Cloud deployments keep working through the synchronous response.
+- `@agent-relay/cloud` `ensureCloudFleetSandbox` accepts `preparationMode: 'async-v1'` and an `onPreparationProgress` callback; `CloudFleetSandboxProvisionError` exposes `sandboxAbsent` and a typed `preparationFailure` (`code`, `phase`, `causeStage`).
+
+### Fixed
+
+- When Cloud ends a sandbox preparation (for example `relayfile_mount_failed` / `initial_sync_deadline`), `fleet spawn --sandbox` names the cause and confirms no sandbox was left running, without a redundant cleanup call or unknown-outcome guidance.
+- A lost Cloud response during `fleet spawn --sandbox` preparation no longer causes the spawn or a preparation step to be resubmitted.
+- `fleet spawn --sandbox` prints the generated sandbox identity before provisioning; re-running an interrupted spawn with `--sandbox-id <id>` resumes that sandbox instead of creating a second one.
+- Against a Cloud that does not support async preparation, a typed failure such as `relayfile_mount_failed` or a capacity rejection is reported at once with its cause instead of after a status-polling wait.
+- Workspace-scoped commands such as `agent-relay fleet release` continue using the workspace's Relaycast server when `--base-url` is omitted; explicit conflicting `--base-url` values are still rejected.
+- Fleet spawns now recover from transient Relaycast `d1_pressure`; persistent registration pressure returns a named `d1_pressure` error within 25 seconds, and any success arriving after retry-count or wall-clock exhaustion is explicitly deregistered.
+- Acknowledged `agent.deregister`, cumulative `delivery.ack`, and non-task action results now recover from transient Relaycast `d1_pressure`; exhausted non-task action results are replayed after reconnect.
+- `agent-relay fleet spawn` reports registration `d1_pressure` as retryable only when the registration is proven to have failed before any change was made.
 
 ## [13.1.5] - 2026-10-08
 

packages/cli/src/cli/commands/fleet-lifecycle-integration.test.ts

@@ -41,6 +41,9 @@ describe('fleet CLI lifecycle routing', () => {
     relayHome = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-fleet-lifecycle-home-'));
     vi.stubEnv('AGENT_RELAY_PROJECT', projectRoot);
     vi.stubEnv('AGENT_RELAY_HOME', relayHome);
+    vi.stubEnv('RELAY_WORKSPACE_KEY', 'rk_live_workspace');
+    vi.stubEnv('RELAY_API_KEY', '');
+    vi.stubEnv('RELAY_BASE_URL', 'https://cast.agentrelay.com');
     const dataDir = path.join(projectRoot, '.agentworkforce', 'relay');
     writeProjectWorkspaceKey(dataDir, 'rk_live_workspace', { workspaceId: TARGET.workspaceId });
 

packages/cli/src/cli/commands/fleet-sandbox-regression.test.ts

@@ -174,7 +174,10 @@ describe('fleet sandbox command regressions', () => {
 
     expect(resolveRepository).toHaveBeenCalledWith(outsideGit, undefined, undefined, 'strict');
     expect(materialize).not.toHaveBeenCalled();
-    expect(ensure).toHaveBeenCalledWith(expect.objectContaining({ mountRelayfile: true }));
+    expect(ensure).toHaveBeenCalledWith(
+      expect.objectContaining({ mountRelayfile: true }),
+      expect.objectContaining({ onPreparationProgress: expect.any(Function) })
+    );
     expect((ensure.mock.calls[0]?.[0] as Record<string, unknown>).relayfilePaths).toBeUndefined();
     expect(placement).toHaveBeenCalledWith(
       expect.objectContaining({ input: expect.objectContaining({ worker_cwd: '/workspace' }) })
@@ -257,7 +260,8 @@ describe('fleet sandbox command regressions', () => {
         sandboxId: REPLAY_SANDBOX_ID,
         name: REPLAY_SANDBOX_NAME,
         mountRelayfile: true,
-      })
+      }),
+      expect.objectContaining({ onPreparationProgress: expect.any(Function) })
     );
     const ensureInput = ensure.mock.calls[0]?.[0] as Record<string, unknown>;
     expect(ensureInput.repos).toBeUndefined();
@@ -324,7 +328,8 @@ describe('fleet sandbox command regressions', () => {
       expect.objectContaining({
         repos: [selection.repository],
         repoRevisions: { [selection.repository]: REVISION },
-      })
+      }),
+      expect.objectContaining({ onPreparationProgress: expect.any(Function) })
     );
   });
 });

packages/cli/src/cli/commands/fleet.test.ts

@@ -1191,6 +1191,7 @@ describe('fleet command support', () => {
     expect(ensureInput).toEqual({
       workspaceId: 'rw_abc',
       requiredCapability: 'spawn:codex',
+      preparationMode: 'async-v1',
       maxAgents: 1,
       mountRelayfile: true,
       relayfilePaths: ['/live-review/run-123/**'],
@@ -1264,6 +1265,7 @@ describe('fleet command support', () => {
       workerCwd: '/srv/agent-workforce/cloud/packages/web',
     };
     const events: string[] = [];
+    const warnings: string[] = [];
     const materializeCloudRelayfileRepository = vi.fn(async () => {
       events.push('materialize');
       return {
@@ -1329,7 +1331,7 @@ describe('fleet command support', () => {
       deleteCloudFleetSandbox: vi.fn(async () => undefined),
       createFleetWorkspaceClient: vi.fn() as never,
       log: () => undefined,
-      warn: () => undefined,
+      warn: (message) => warnings.push(message),
       error: () => undefined,
     });
 
@@ -1365,9 +1367,24 @@ describe('fleet command support', () => {
           '/github/repos/AgentWorkforce/cloud/.relayfile/**',
           '/.skills/**',
         ],
-      })
+      }),
+      expect.objectContaining({ onPreparationProgress: expect.any(Function) })
     );
     const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0] as Record<string, unknown>;
+    const ensureOptions = ensureCloudFleetSandbox.mock.calls[0]?.[1];
+    ensureOptions?.onPreparationProgress?.({
+      sandboxId: REPLAY_SANDBOX_ID,
+      state: 'pending',
+      phase: 'relayfile_mount_bootstrap',
+      generation: 2,
+    });
+    // The generated identity is printed before Cloud work so an interrupted
+    // run can be resumed with --sandbox-id instead of creating a second sandbox.
+    expect(warnings).toEqual([
+      `Cloud sandbox identity: ${String(ensureInput.sandboxId)}. If this command is interrupted, re-run it with --sandbox-id ${String(ensureInput.sandboxId)} to resume the same sandbox instead of creating another.`,
+      'Cloud sandbox preparation: relayfile_mount_bootstrap (pending, generation 2).',
+    ]);
+    expect(ensureInput.sandboxId).toMatch(/^sbx_[0-9a-f-]{36}$/);
     expect(ensureInput.repos).toBeUndefined();
     expect(ensureInput.repoRevisions).toBeUndefined();
     expect(placement.spawn).toHaveBeenCalledWith(
@@ -1482,7 +1499,8 @@ describe('fleet command support', () => {
       expect.objectContaining({
         mountRelayfile: true,
         relayfilePaths: ['/github/repos/AgentWorkforce/agent-assistant/contents/**', '/memory/**'],
-      })
+      }),
+      expect.objectContaining({ onPreparationProgress: expect.any(Function) })
     );
     const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0] as Record<string, unknown>;
     expect(ensureInput.repos).toBeUndefined();
@@ -1602,7 +1620,8 @@ describe('fleet command support', () => {
       expect.objectContaining({
         mountRelayfile: true,
         relayfilePaths: ['/github/repos/AgentWorkforce/cloud/contents/**', '/memory/**'],
-      })
+      }),
+      expect.objectContaining({ onPreparationProgress: expect.any(Function) })
     );
     const spawnInput = placement.spawn.mock.calls[0]?.[0]?.input as { task?: string };
     expect(placement.spawn).toHaveBeenCalledWith(
@@ -1970,7 +1989,8 @@ describe('fleet command support', () => {
       { from: 'user' }
     );
     expect(ensureCloudFleetSandbox).toHaveBeenCalledWith(
-      expect.objectContaining({ relayfilePaths: paths16 })
+      expect.objectContaining({ relayfilePaths: paths16 }),
+      expect.objectContaining({ onPreparationProgress: expect.any(Function) })
     );
 
     const tooMany = new Command();
@@ -2195,7 +2215,12 @@ describe('fleet command support', () => {
     expect(findProjectRoot).toHaveBeenCalledWith(path.resolve(process.cwd(), 'packages/web'));
     expect(selectionOptions[0]).toMatchObject({ projectRoot: "/local/cloud/packages/web team's" });
     const ensureInput = ensureCloudFleetSandbox.mock.calls[0]?.[0] as Record<string, unknown>;
+    // The exact checkout revision is part of the durable async request, so a
+    // resumed preparation clones the same commit it was accepted for.
     expect(ensureInput).toMatchObject({
+      preparationMode: 'async-v1',
+      forceProvision: true,
+      providerId: 'agent37',
       repos: ['AgentWorkforce/cloud'],
       repoRevisions: { 'AgentWorkforce/cloud': revision },
     });
@@ -2412,7 +2437,12 @@ describe('fleet command support', () => {
 
     expect(resolveWorkspaceByKey).toHaveBeenCalledWith('rk_live_pinned');
     expect(ensureCloudFleetSandbox).toHaveBeenCalledWith(
-      expect.objectContaining({ workspaceId: 'cloud-pinned', workloadProfile: 'long-running-agent' })
+      expect.objectContaining({
+        workspaceId: 'cloud-pinned',
+        workloadProfile: 'long-running-agent',
+        preparationMode: 'async-v1',
+      }),
+      expect.objectContaining({ onPreparationProgress: expect.any(Function) })
     );
     expect(createWorkspaceRelay).toHaveBeenCalledWith(
       expect.objectContaining({
@@ -2672,6 +2702,10 @@ describe('fleet command support', () => {
       { from: 'user' }
     );
 
+    expect(ensureCloudFleetSandbox).toHaveBeenCalledWith(
+      expect.objectContaining({ preparationMode: 'async-v1', providerId: 'agent37', mountRelayfile: false }),
+      expect.objectContaining({ onPreparationProgress: expect.any(Function) })
+    );
     expect(createWorkspaceRelay).toHaveBeenCalledWith({
       projectRoot: process.cwd(),
       token: undefined,
@@ -3273,7 +3307,9 @@ describe('fleet command support', () => {
     expect(ensureCloudFleetSandbox).toHaveBeenCalledWith(
       expect.objectContaining({
         workloadProfile: 'long-running-agent',
-      })
+        preparationMode: 'async-v1',
+      }),
+      expect.objectContaining({ onPreparationProgress: expect.any(Function) })
     );
     expect(deleteCloudFleetSandbox).not.toHaveBeenCalled();
   });
@@ -3482,7 +3518,8 @@ describe('fleet command support', () => {
       { from: 'user' }
     );
     expect(ensureCloudFleetSandbox).toHaveBeenCalledWith(
-      expect.objectContaining({ workspaceId: 'rw_captured' })
+      expect.objectContaining({ workspaceId: 'rw_captured', preparationMode: 'async-v1' }),
+      expect.objectContaining({ onPreparationProgress: expect.any(Function) })
     );
   });
 
@@ -3548,7 +3585,8 @@ describe('fleet command support', () => {
     );
 
     expect(ensureCloudFleetSandbox).toHaveBeenCalledWith(
-      expect.objectContaining({ workspaceId: cloudWorkspaceId })
+      expect.objectContaining({ workspaceId: cloudWorkspaceId, preparationMode: 'async-v1' }),
+      expect.objectContaining({ onPreparationProgress: expect.any(Function) })
     );
     expect(createWorkspaceRelay).toHaveBeenCalledWith({
       projectRoot: process.cwd(),
@@ -3715,6 +3753,10 @@ describe('fleet command support', () => {
       /^sbx_[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/
     );
     expect(ensureInput?.name).toBe(`fleet-sandbox-${ensureInput?.sandboxId?.slice('sbx_'.length)}`);
+    expect(ensureInput?.preparationMode).toBe('async-v1');
+    expect(ensureCloudFleetSandbox.mock.calls[0]?.[1]).toEqual({
+      onPreparationProgress: expect.any(Function),
+    });
   });
 
   it('rejects a replay sandbox name that does not match its identity', async () => {
@@ -3890,6 +3932,7 @@ describe('fleet command support', () => {
 
     expect(deleteCloudFleetSandbox).not.toHaveBeenCalled();
     expect(warnings.join('\n')).toContain(`check Cloud Fleet for node '${REPLAY_SANDBOX_NAME}'`);
+    expect(warnings.join('\n')).not.toContain('no sandbox was left running');
     expect(warnings.join('\n')).toContain(`--sandbox-id '${REPLAY_SANDBOX_ID}'`);
   });
 
@@ -3964,6 +4007,81 @@ describe('fleet command support', () => {
     expect(warnings.join('\n')).not.toContain('check Cloud Fleet');
   });
 
+  it('names a terminal relayfile_mount_failed cause and reports no sandbox left running without cleanup', async () => {
+    const warnings: string[] = [];
+    const deleteCloudFleetSandbox = vi.fn(async () => undefined);
+    const terminalMessage =
+      'Cloud fleet sandbox preparation failed during relayfile_mount (relayfile_mount_failed, initial_sync_deadline): Relayfile mount failed during relayfile_mount. Retry provisioning the sandbox.';
+    const program = new Command();
+    program.exitOverride();
+    registerFleetCommands(program, {
+      resolveSandboxRepository: () => undefined,
+      sdk: {
+        createAgentRelay: vi.fn() as never,
+        createWorkspaceRelay: vi.fn(() => ({
+          workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) },
+        })) as never,
+        createWorkspace: vi.fn() as never,
+        log: vi.fn(),
+        error: vi.fn(),
+        exit: (() => {
+          throw new Error('__exit__');
+        }) as never,
+      },
+      ensureCloudFleetSandbox: vi.fn(async () => {
+        throw new CloudFleetSandboxProvisionError(terminalMessage, {
+          cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99',
+          sandboxId: REPLAY_SANDBOX_ID,
+          nodeName: REPLAY_SANDBOX_NAME,
+          providerId: 'agent37',
+          sandboxAbsent: true,
+          preparationFailure: {
+            code: 'relayfile_mount_failed',
+            phase: 'relayfile_mount',
+            causeStage: 'initial_sync_deadline',
+          },
+        });
+      }),
+      deleteCloudFleetSandbox,
+      createFleetWorkspaceClient: vi.fn() as never,
+      log: () => undefined,
+      warn: (...args: unknown[]) => warnings.push(args.join(' ')),
+      error: () => undefined,
+    });
+
+    await expect(
+      program.parseAsync(
+        [
+          'fleet',
+          'spawn',
+          'codex',
+          '--sandbox',
+          '--sandbox-id',
+          REPLAY_SANDBOX_ID,
+          '--sandbox-name',
+          REPLAY_SANDBOX_NAME,
+          '--workspace-id',
+          'rw_abc',
+          '--name',
+          'sandbox-worker',
+          '--task',
+          'Work',
+          '--workspace-key',
+          'rk_live_test',
+          '--token',
+          'at_live_lead',
+        ],
+        { from: 'user' }
+      )
+    ).rejects.toThrow('__exit__');
+
+    expect(deleteCloudFleetSandbox).not.toHaveBeenCalled();
+    expect(warnings).toEqual([
+      `${terminalMessage} Cloud confirmed sandbox '${REPLAY_SANDBOX_ID}' is not running; no sandbox was left running.`,
+    ]);
+    expect(warnings.join('\n')).not.toContain('outcome is unknown');
+  });
+
   it('preserves the caller Daytona ID after a matched malformed provisioned response', async () => {
     const warnings: string[] = [];
     const deleteCloudFleetSandbox = vi.fn(async () => undefined);

packages/cli/src/cli/commands/fleet.ts

@@ -13,6 +13,7 @@ import {
   resolveWorkspaceByKey,
   type CloudFleetSandboxProviderId,
   type CloudRelayfileRepositoryMaterialization,
+  type EnsureCloudFleetSandboxInput,
   type EnsureCloudFleetSandboxResult,
 } from '@agent-relay/cloud';
 import { HarnessDriverClient } from '@agent-relay/harness-driver';
@@ -457,7 +458,10 @@ export function registerFleetCommands(
         '--sandbox-name <name>',
         'Explicit sandbox node name (custom unless --sandbox-id requires matching fleet-sandbox-<UUID>)'
       )
-      .option('--sandbox-id <id>', 'Reuse a caller-declared sbx_<UUID> identity for an exact replay')
+      .option(
+        '--sandbox-id <id>',
+        'Reuse a caller-declared sbx_<UUID> identity for an exact replay; re-run an interrupted --sandbox spawn with the identity it printed to resume it'
+      )
       .option('--workspace-id <id>', 'Explicit Relay workspace identity required for sandbox provisioning')
       .option('--sandbox-provider <provider>', 'Sandbox provider: daytona, e2b, or agent37')
       .option(
@@ -790,10 +794,20 @@ export function registerFleetCommands(
           sandboxProvider === undefined || sandboxProvider === 'agent37'
             ? 'long-running-agent'
             : 'standard-long-running-agent';
+        const useAsyncPreparation =
+          sandboxId !== undefined && (sandboxProvider === undefined || sandboxProvider === 'agent37');
+        if (useAsyncPreparation && sandboxIdOption === undefined) {
+          // The generated identity is not persisted. Print it before Cloud work
+          // starts so an interrupted run can be resumed instead of duplicated.
+          deps.warn(
+            `Cloud sandbox identity: ${sandboxId}. If this command is interrupted, re-run it with --sandbox-id ${sandboxId} to resume the same sandbox instead of creating another.`
+          );
+        }
         try {
-          sandbox = await deps.ensureCloudFleetSandbox({
+          const ensureInput: EnsureCloudFleetSandboxInput = {
             workspaceId: relayWorkspaceId,
             requiredCapability: `spawn:${cli}`,
+            ...(useAsyncPreparation ? { preparationMode: 'async-v1' as const } : {}),
             maxAgents: 1,
             mountRelayfile: mountSandboxRelayfile,
             ...(sandboxMountPaths === undefined ? {} : { relayfilePaths: sandboxMountPaths }),
@@ -807,10 +821,27 @@ export function registerFleetCommands(
             ...(checkoutRepository && sandboxRepository
               ? { repoRevisions: { [sandboxRepository.repository]: sandboxRepository.revision } }
               : {}),
-          });
+          };
+          sandbox = useAsyncPreparation
+            ? await deps.ensureCloudFleetSandbox(ensureInput, {
+                onPreparationProgress: (progress) => {
+                  deps.warn(
+                    `Cloud sandbox preparation: ${progress.phase} (${progress.state}, generation ${progress.generation}).`
+                  );
+                },
+              })
+            : await deps.ensureCloudFleetSandbox(ensureInput);
           assertSandboxRepositoryRevision(sandbox, checkoutRepository ? sandboxRepository : undefined);
         } catch (error) {
-          if (
+          if (error instanceof CloudFleetSandboxProvisionError && error.sandboxAbsent) {
+            // Cloud's terminal record for this exact identity already proves the
+            // provider sandbox is gone; a delete here would only race its reaper.
+            deps.warn(
+              `${error.message} Cloud confirmed sandbox '${
+                error.sandboxId ?? sandboxId ?? 'the requested sandbox'
+              }' is not running; no sandbox was left running.`
+            );
+          } else if (
             shouldCleanupSandbox &&
             error instanceof CloudFleetSandboxProvisionError &&
             error.confirmedProvisioned &&