[build-tools] Upload the last agent-device artifacts before the daemon stops

.changeset/agent-device-final-artifact-drain.md

@@ -0,0 +1,5 @@
+---
+'@expo/build-tools': patch
+---
+
+Upload the final agent-device session artifacts before stopping the daemon, including sessions that reuse a warm Simulator.

packages/build-tools/src/steps/functions/__tests__/deviceSessionHost-orchestration.test.ts

@@ -6,6 +6,7 @@ import type { CustomBuildContext } from '../../../customBuildContext';
 import { Sentry } from '../../../sentry';
 import { AndroidEmulatorUtils } from '../../../utils/AndroidEmulatorUtils';
 import { turtleFetch } from '../../../utils/turtleFetch';
+import { pollAgentDeviceArtifactsForUploadAsync } from '../../utils/agentDeviceArtifacts';
 import { startAgentDeviceEventCollectionAsync } from '../../utils/agentDeviceEvents';
 import { startAppiumEventCollectionAsync } from '../../utils/appiumEvents';
 import { startDeviceSessionHostAsync } from '../../utils/deviceSessionHost';
@@ -54,6 +55,7 @@ const logger = { info: jest.fn(), warn: jest.fn(), child: jest.fn().mockReturnTh
 
 beforeEach(() => {
   jest.clearAllMocks();
+  jest.mocked(pollAgentDeviceArtifactsForUploadAsync).mockResolvedValue();
   // The daemon package is external. No installation or daemon process runs in these tests.
   jest.spyOn(fs, 'existsSync').mockReturnValue(true);
   // Only the emulator logcat folder: the Appium step writes into its own temp folder.

packages/build-tools/src/steps/functions/__tests__/remoteSessionLocalEgress.test.ts

@@ -7,6 +7,7 @@ import { type CustomBuildContext } from '../../../customBuildContext';
 import { IosSimulatorUtils } from '../../../utils/IosSimulatorUtils';
 import { isProcessDescendantOfAsync } from '../../../utils/processes';
 import { turtleFetch } from '../../../utils/turtleFetch';
+import { pollAgentDeviceArtifactsForUploadAsync } from '../../utils/agentDeviceArtifacts';
 import { startDeviceSessionHostAsync } from '../../utils/deviceSessionHost';
 import {
   monitorLocalEgressAsync,
@@ -124,6 +125,7 @@ describe.each(controllers)('%s local egress', (_name, createFunction, controller
   }
   beforeEach(() => {
     jest.clearAllMocks();
+    jest.mocked(pollAgentDeviceArtifactsForUploadAsync).mockResolvedValue();
     lifetime = registerLocalEgressResources(stopEgress);
     stopEgress.mockResolvedValue(undefined);
     stopPreview.mockResolvedValue(undefined);

packages/build-tools/src/steps/functions/__tests__/startAgentDeviceRemoteSession-orchestration.test.ts

@@ -1,12 +1,15 @@
 import { BuildRuntimePlatform } from '@expo/steps';
 import spawn from '@expo/turtle-spawn';
+import fetch from 'node-fetch';
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import { Readable } from 'node:stream';
 
 import { type CustomBuildContext } from '../../../customBuildContext';
 import { pollAgentDeviceArtifactsForUploadAsync } from '../../utils/agentDeviceArtifacts';
 import { startAgentDeviceEventCollectionAsync } from '../../utils/agentDeviceEvents';
+import { uploadDeviceRunSessionArtifactAsync } from '../../utils/deviceRunSessionArtifacts';
 import { startDeviceSessionHostAsync } from '../../utils/deviceSessionHost';
 import {
   getDeviceRunSessionIdOrThrow,
@@ -40,6 +43,9 @@ jest.mock('../../../sentry');
 jest.mock('../../utils/agentDeviceArtifacts', () => ({
   pollAgentDeviceArtifactsForUploadAsync: jest.fn(),
 }));
+jest.mock('../../utils/deviceRunSessionArtifacts', () => ({
+  uploadDeviceRunSessionArtifactAsync: jest.fn(),
+}));
 jest.mock('../../utils/agentDeviceEvents', () => ({
   startAgentDeviceEventCollectionAsync: jest.fn(),
 }));
@@ -77,10 +83,7 @@ const NO_CAPTURE: CaptureInputs = { networkCapture: false, networkCaptureFields:
 async function runAsync(
   logger: { info: jest.Mock; warn: jest.Mock; child: jest.Mock },
   runtimePlatform: BuildRuntimePlatform,
-  capture: CaptureInputs = NO_CAPTURE,
-  disableShare = false,
-  privatePreviewUrl?: string,
-  maxIdleTimeMinutes?: number
+  options: Partial<Parameters<typeof runAgentDeviceRemoteSessionAsync>[1]> = {}
 ): Promise<void> {
   await runAgentDeviceRemoteSessionAsync(ctx, {
     env: {},
@@ -92,16 +95,15 @@ async function runAsync(
       ngrokAuthtoken: 'ngrok-token',
     },
     packageVersion: undefined,
-    maxIdleTimeMinutes,
+    maxIdleTimeMinutes: undefined,
     maxDurationSeconds: undefined,
-    capture,
-    disableShare,
-    privatePreviewUrl,
-    tasks: createStartupTasks(logger as never),
+    capture: NO_CAPTURE,
+    tasks: createStartupTasks(logger as never, options.signal),
     device:
       runtimePlatform === BuildRuntimePlatform.DARWIN
         ? { iosSimulatorUdid: 'selected-udid', application: Promise.resolve() }
         : { booted: Promise.resolve(), ready: Promise.resolve() },
+    ...options,
   });
 }
 
@@ -238,6 +240,173 @@ describe('runAgentDeviceRemoteSessionAsync orchestration', () => {
     expect(mockPreviewStopAsync).toHaveBeenCalledTimes(1);
   });
 
+  it.each([false, true])(
+    'finishes the final artifact upload before stopping its owner (warm=%s)',
+    async warm => {
+      const logger = { info: jest.fn(), warn: jest.fn(), child: jest.fn().mockReturnThis() };
+      const { pollAgentDeviceArtifactsForUploadAsync: pollActual } = jest.requireActual<
+        typeof import('../../utils/agentDeviceArtifacts')
+      >('../../utils/agentDeviceArtifacts');
+      jest.mocked(pollAgentDeviceArtifactsForUploadAsync).mockImplementation(pollActual);
+      const { Response } = jest.requireActual<typeof import('node-fetch')>('node-fetch');
+      const artifacts: { id: string; filename: string }[] = [];
+      let releaseUpload = () => {};
+      const pendingUpload = new Promise<void>(resolve => {
+        releaseUpload = resolve;
+      });
+      let markUploadStarted = () => {};
+      const uploadStarted = new Promise<void>(resolve => {
+        markUploadStarted = resolve;
+      });
+      jest.mocked(uploadDeviceRunSessionArtifactAsync).mockImplementationOnce(async () => {
+        markUploadStarted();
+        await pendingUpload;
+      });
+      let inventories = 0;
+      jest.mocked(fetch).mockImplementation(async url => {
+        if (String(url).endsWith('/artifacts')) {
+          inventories += 1;
+          return new Response(JSON.stringify({ artifacts }));
+        }
+        return new Response(Readable.from([Buffer.from('screenshot')]));
+      });
+      jest.mocked(waitForDeviceRunSessionStoppedAsync).mockImplementation(async () => {
+        while (inventories === 0) {
+          await new Promise(resolve => setTimeout(resolve, 5));
+        }
+        // The next periodic inventory is 5 seconds away, so only the final one can see this file.
+        artifacts.push({ id: 'late', filename: 'late.png' });
+      });
+
+      const running = runAsync(
+        logger,
+        warm ? BuildRuntimePlatform.DARWIN : BuildRuntimePlatform.LINUX,
+        {
+          warm: warm
+            ? ({
+                agentDeviceStateDir: '/warm/agent',
+                tools: { port: 4310, token: 'warm-token' },
+              } as WarmIosSession)
+            : undefined,
+        }
+      );
+      try {
+        await uploadStarted;
+        expect(mockDaemonStopAsync).not.toHaveBeenCalled();
+        if (warm) {
+          expect(mockPreviewStopAsync).not.toHaveBeenCalled();
+        }
+      } finally {
+        releaseUpload();
+        await running;
+      }
+
+      expect(uploadDeviceRunSessionArtifactAsync).toHaveBeenCalledWith(
+        ctx,
+        expect.objectContaining({ artifactId: 'late', filename: 'late.png' })
+      );
+      expect(inventories).toBe(2);
+      expect(mockPreviewStopAsync).toHaveBeenCalledTimes(1);
+      expect(mockDaemonStopAsync).toHaveBeenCalledTimes(warm ? 0 : 1);
+    }
+  );
+
+  it('stops the daemon when the final artifact drain fails', async () => {
+    const logger = { info: jest.fn(), warn: jest.fn(), child: jest.fn().mockReturnThis() };
+    const failure = new Error('drain failed');
+    jest.mocked(pollAgentDeviceArtifactsForUploadAsync).mockRejectedValue(failure);
+
+    await expect(runAsync(logger, BuildRuntimePlatform.LINUX)).rejects.toBe(failure);
+
+    expect(pollAgentDeviceArtifactsForUploadAsync).toHaveBeenCalledWith(
+      ctx,
+      expect.objectContaining({ stopPollingSignal: expect.any(AbortSignal) })
+    );
+    expect(mockDaemonStopAsync).toHaveBeenCalledTimes(1);
+  });
+
+  it.each([false, true])(
+    'stops the warm host after a failed drain (collector startup fails=%s)',
+    async startupFails => {
+      const logger = { info: jest.fn(), warn: jest.fn(), child: jest.fn().mockReturnThis() };
+      const drainFailure = new Error('drain failed');
+      const startupFailure = new Error('event collector failed');
+      jest.mocked(pollAgentDeviceArtifactsForUploadAsync).mockRejectedValueOnce(drainFailure);
+      if (startupFails) {
+        jest.mocked(startAgentDeviceEventCollectionAsync).mockRejectedValueOnce(startupFailure);
+      }
+      await expect(
+        runAsync(logger, BuildRuntimePlatform.DARWIN, {
+          warm: {
+            agentDeviceStateDir: '/warm/agent',
+            tools: { port: 4310, token: 'warm-token' },
+          } as WarmIosSession,
+        })
+      ).rejects.toBe(startupFails ? startupFailure : drainFailure);
+      expect(mockPreviewStopAsync).toHaveBeenCalledTimes(1);
+      expect(mockTunnelStopAsync).toHaveBeenCalledTimes(1);
+      expect(mockDaemonStopAsync).not.toHaveBeenCalled();
+      expect(logger.warn).toHaveBeenCalledWith(
+        { err: drainFailure },
+        'Could not stop the agent-device daemon during remote session teardown.'
+      );
+    }
+  );
+
+  it('cancels a warm upload and drains its poller before stopping the host', async () => {
+    const logger = { info: jest.fn(), warn: jest.fn(), child: jest.fn().mockReturnThis() };
+    const controller = new AbortController();
+    const cancellation = new Error('job cancelled');
+    const { pollAgentDeviceArtifactsForUploadAsync: pollActual } = jest.requireActual<
+      typeof import('../../utils/agentDeviceArtifacts')
+    >('../../utils/agentDeviceArtifacts');
+    const pollingFinished = jest.fn();
+    jest
+      .mocked(pollAgentDeviceArtifactsForUploadAsync)
+      .mockImplementation((...args) => pollActual(...args).finally(pollingFinished));
+    const { Response } = jest.requireActual<typeof import('node-fetch')>('node-fetch');
+    jest
+      .mocked(fetch)
+      .mockImplementation(async url =>
+        String(url).endsWith('/artifacts')
+          ? new Response(JSON.stringify({ artifacts: [{ id: 'active', filename: 'active.png' }] }))
+          : new Response(Readable.from(['png']))
+      );
+    let markUploadStarted = () => {};
+    const uploadStarted = new Promise<void>(resolve => {
+      markUploadStarted = resolve;
+    });
+    jest
+      .mocked(uploadDeviceRunSessionArtifactAsync)
+      .mockImplementationOnce(async (_ctx, { signal }) => {
+        markUploadStarted();
+        await new Promise<void>((_resolve, reject) => {
+          signal?.addEventListener('abort', () => reject(signal.reason), { once: true });
+        });
+      });
+    jest.mocked(waitForDeviceRunSessionStoppedAsync).mockImplementationOnce(async () => {
+      await uploadStarted;
+      controller.abort(cancellation);
+      throw cancellation;
+    });
+    await expect(
+      runAsync(logger, BuildRuntimePlatform.DARWIN, {
+        signal: controller.signal,
+        warm: {
+          agentDeviceStateDir: '/warm/agent',
+          tools: { port: 4310, token: 'warm-token' },
+        } as WarmIosSession,
+      })
+    ).rejects.toBe(cancellation);
+    expect(pollingFinished).toHaveBeenCalledTimes(1);
+    expect(pollingFinished.mock.invocationCallOrder[0]).toBeLessThan(
+      mockPreviewStopAsync.mock.invocationCallOrder[0]
+    );
+    expect(mockPreviewStopAsync).toHaveBeenCalledTimes(1);
+    expect(mockTunnelStopAsync).toHaveBeenCalledTimes(1);
+    expect(mockDaemonStopAsync).not.toHaveBeenCalled();
+  });
+
   it('redacts learned daemon credentials in arbitrary output and retained diagnostics', async () => {
     const logger = { info: jest.fn(), warn: jest.fn(), child: jest.fn().mockReturnThis() };
     const token = 'daemon secret/+';
@@ -267,7 +436,7 @@ describe('runAgentDeviceRemoteSessionAsync orchestration', () => {
     } as never);
     jest.mocked(host.getLastPreviewActivityAt).mockReturnValue(previewInputAt);
 
-    await runAsync(logger, BuildRuntimePlatform.LINUX, NO_CAPTURE, false, undefined, 5);
+    await runAsync(logger, BuildRuntimePlatform.LINUX, { maxIdleTimeMinutes: 5 });
 
     const { idleTimeout } = jest.mocked(waitForDeviceRunSessionStoppedAsync).mock.calls[0][0];
     expect(idleTimeout?.maxIdleTimeMinutes).toBe(5);
@@ -278,8 +447,7 @@ describe('runAgentDeviceRemoteSessionAsync orchestration', () => {
     const logger = { info: jest.fn(), warn: jest.fn(), child: jest.fn().mockReturnThis() };
 
     await runAsync(logger, BuildRuntimePlatform.DARWIN, {
-      networkCapture: true,
-      networkCaptureFields: ['header'],
+      capture: { networkCapture: true, networkCaptureFields: ['header'] },
     });
 
     expect(startDeviceSessionHostAsync).toHaveBeenCalledWith(
@@ -290,13 +458,10 @@ describe('runAgentDeviceRemoteSessionAsync orchestration', () => {
 
   it('hands a private preview choice to the session host', async () => {
     const logger = { info: jest.fn(), warn: jest.fn(), child: jest.fn().mockReturnThis() };
-    await runAsync(
-      logger,
-      BuildRuntimePlatform.DARWIN,
-      NO_CAPTURE,
-      true,
-      'https://expo.dev/private-preview'
-    );
+    await runAsync(logger, BuildRuntimePlatform.DARWIN, {
+      disableShare: true,
+      privatePreviewUrl: 'https://expo.dev/private-preview',
+    });
     expect(startDeviceSessionHostAsync).toHaveBeenCalledWith(
       ctx,
       expect.objectContaining({

packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts

@@ -142,6 +142,9 @@ export async function runAgentDeviceRemoteSessionAsync(
   let agentDeviceTunnel: Awaited<ReturnType<typeof startNgrokTunnelAsync>> | undefined;
   let sessionHost: DeviceSessionHost | undefined;
   let eventCollection: Awaited<ReturnType<typeof startAgentDeviceEventCollectionAsync>> | undefined;
+  let artifactUploads: Promise<void> | undefined;
+  // Ends periodic polling; the poller then runs one bounded final inventory and upload drain.
+  const stopArtifactPolling = new AbortController();
   let sessionFailed = false;
   const daemonSecrets = [ngrokAuthtoken];
   const hostBootedDevice = 'iosSimulatorUdid' in device ? device : undefined;
@@ -264,12 +267,16 @@ export async function runAgentDeviceRemoteSessionAsync(
       return;
     }
     tasks.signal.throwIfAborted();
-    void pollAgentDeviceArtifactsForUploadAsync(ctx, {
+    artifactUploads = pollAgentDeviceArtifactsForUploadAsync(ctx, {
       deviceRunSessionId,
       daemonUrl: `http://127.0.0.1:${daemonPort}`,
       daemonToken,
       logger,
+      signal,
+      stopPollingSignal: stopArtifactPolling.signal,
     });
+    // Attach a handler at once; the teardown still awaits and reports a rejection.
+    void artifactUploads.catch(() => {});
 
     eventCollection = await startAgentDeviceEventCollectionAsync({
       ctx,
@@ -302,29 +309,41 @@ export async function runAgentDeviceRemoteSessionAsync(
     // everything that was started.
     await Promise.allSettled([agentDeviceStartup, sessionHostStartup, deviceReady]);
     const startedDaemon = daemonProcess;
+    const toolTeardown = (async () => {
+      try {
+        if (eventCollection) {
+          await stopAgentDeviceEventCollectionSafelyAsync({
+            eventCollection,
+            deviceRunSessionId,
+            logger,
+          });
+        }
+      } finally {
+        try {
+          // The final inventory reads from the daemon, so it finishes before the daemon stops.
+          stopArtifactPolling.abort();
+          await artifactUploads;
+        } finally {
+          await startedDaemon?.stopAsync();
+        }
+      }
+    })();
     await finishRemoteSessionAsync({
       logger,
       sessionFailed,
       teardown: [
         ['agent-device tunnel', agentDeviceTunnel?.stopAsync()],
+        ['agent-device daemon', toolTeardown],
         [
-          'agent-device daemon',
-          (startedDaemon || eventCollection) &&
-            (async () => {
-              try {
-                if (eventCollection) {
-                  await stopAgentDeviceEventCollectionSafelyAsync({
-                    eventCollection,
-                    deviceRunSessionId,
-                    logger,
-                  });
-                }
-              } finally {
-                await startedDaemon?.stopAsync();
-              }
-            })(),
+          'session host',
+          (async () => {
+            if (warm) {
+              // The warm host owns the daemon too; keep it alive through the final downloads.
+              await Promise.allSettled([toolTeardown]);
+            }
+            await sessionHost?.finishAsync();
+          })(),
         ],
-        ['session host', sessionHost?.finishAsync()],
       ],
     });
   }