fix(apple-runner): restart without resending a command written then lost

contracts/fixtures/dispatch-disclosure.json

@@ -256,7 +256,13 @@
   {
     "id": "ios-runner.transport.written-then-lost",
     "producer": "ios-runner",
-    "trigger": "a connect attempt posted the command and then timed out (fetch deadline, simctl curl exit 28); the failure keeps the runner_connect_refused restart verdict, and the restart that replays it failed",
+    "trigger": "a mutating command was posted and the runner process died before replying; the status probe fails, the session is invalidated, and the command is not sent again (details.reason runner_reply_lost)",
+    "dispatched": "unknown"
+  },
+  {
+    "id": "ios-runner.transport.read-only-written-then-lost",
+    "producer": "ios-runner",
+    "trigger": "a connect attempt posted a read-only command and then timed out (fetch deadline, simctl curl exit 28); the runner is restarted and the read is sent again, and the resend failed; the first send may have run, so the runner reports unknown, and the daemon's read-only rule (daemon.read-only-command) reports no to the caller",
     "dispatched": "unknown"
   },
   {
@@ -289,6 +295,12 @@
     "trigger": "transport lost; status probe reports lifecycleState completed without a readable retained reply",
     "dispatched": "unknown"
   },
+  {
+    "id": "ios-runner.status.read-only-completed-without-retained-reply",
+    "producer": "ios-runner",
+    "trigger": "transport lost on a read-only command; status probe reports lifecycleState completed without a readable retained reply; the runner reports unknown, and the daemon's read-only rule (daemon.read-only-command) reports no to the caller",
+    "dispatched": "unknown"
+  },
   {
     "id": "ios-runner.status.accepted",
     "producer": "ios-runner",

docs/adr/0011-interaction-guarantee-contract.md

@@ -192,6 +192,18 @@ repeat. A producer that runs several device inputs inside one bound operation
 Each row names its driver file by id prefix, and that file drives the real
 producer.
 
+The Apple runner does not resend a mutating command whose first send may have
+run: a restart resends only a command the first attempt provably did not write,
+or a read-only one. A mutation whose reply stays lost (the runner dies
+mid-command, or status recovery finds neither a retained result nor a runner
+answer) fails with `reason: runner_reply_lost` and `dispatched: unknown` (rows
+`ios-runner.transport.written-then-lost`,
+`ios-runner.status.completed-without-retained-reply`,
+`ios-runner.status.accepted`, `ios-runner.status.started`,
+`ios-runner.status.notAccepted`, `ios-runner.status.probe-failed`, and
+`ios-runner.status.unavailable`). The `ios-runner.status.failed*` rows carry the
+runner's own answer instead. A read keeps its transport error and is resent.
+
 Remaining gaps: a failure before the router's locked scope (session
 resolution, lock acquisition, lease and daemon-policy admission) never reaches
 the disclosure and carries no `dispatched`. It sends nothing, but a consumer

packages/platform-apple/src/runner/__tests__/fake-runner-server.ts

@@ -13,7 +13,11 @@ import type { AddressInfo } from 'node:net';
 export type FakeRunnerResponse =
   | { kind: 'ok'; data: Record<string, unknown> }
   | { kind: 'runnerError'; code: string; message: string }
-  | { kind: 'hangUp' };
+  | { kind: 'hangUp' }
+  /** Hangs up on this request and every later one for the command: the entry is never consumed. */
+  | { kind: 'hangUpAlways' }
+  /** Hangs up and stops listening, as a runner process that died mid-command. */
+  | { kind: 'exit' };
 
 export type FakeRunnerRequest = {
   command: string;
@@ -44,6 +48,7 @@ export async function startFakeRunnerServer(
     : Object.fromEntries(Object.entries(script).map(([key, list]) => [key, [...list]]));
   const remaining = sequential ?? [];
   const requests: FakeRunnerRequest[] = [];
+  let stopped: Promise<void> | undefined;
   const server = http.createServer((req, res) => {
     let raw = '';
     req.on('data', (chunk) => {
@@ -53,24 +58,18 @@ export async function startFakeRunnerServer(
       const body = parseBody(raw);
       requests.push({ command: String(body.command ?? ''), body });
       const next = byCommand
-        ? (byCommand[String(body.command ?? '')]?.shift() ?? { kind: 'ok' as const, data: {} })
-        : remaining.shift();
-      if (!next) {
-        res.statusCode = 500;
-        res.end(JSON.stringify({ ok: false, error: { message: 'fake runner script exhausted' } }));
-        return;
-      }
-      if (next.kind === 'hangUp') {
+        ? (takeScriptedResponse(byCommand[String(body.command ?? '')]) ?? {
+            kind: 'ok' as const,
+            data: {},
+          })
+        : takeScriptedResponse(remaining);
+      if (next?.kind === 'exit') {
         res.destroy();
+        stopped ??= new Promise<void>((resolve) => server.close(() => resolve()));
+        server.closeAllConnections();
         return;
       }
-      if (next.kind === 'runnerError') {
-        res.setHeader('content-type', 'application/json');
-        res.end(JSON.stringify({ ok: false, error: { code: next.code, message: next.message } }));
-        return;
-      }
-      res.setHeader('content-type', 'application/json');
-      res.end(JSON.stringify({ ok: true, data: next.data }));
+      writeFakeRunnerResponse(res, next);
     });
   });
   await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
@@ -79,12 +78,41 @@ export async function startFakeRunnerServer(
     port,
     requests,
     close: () =>
+      stopped ??
       new Promise<void>((resolve, reject) =>
         server.close((error) => (error ? reject(error) : resolve())),
       ),
   };
 }
 
+/** The next scripted reply; a `hangUpAlways` entry stays at the head of its queue. */
+function takeScriptedResponse(
+  queue: FakeRunnerResponse[] | undefined,
+): FakeRunnerResponse | undefined {
+  return queue?.[0]?.kind === 'hangUpAlways' ? queue[0] : queue?.shift();
+}
+
+function writeFakeRunnerResponse(
+  res: http.ServerResponse,
+  next: Exclude<FakeRunnerResponse, { kind: 'exit' }> | undefined,
+): void {
+  if (!next) {
+    res.statusCode = 500;
+    res.end(JSON.stringify({ ok: false, error: { message: 'fake runner script exhausted' } }));
+    return;
+  }
+  if (next.kind === 'hangUp' || next.kind === 'hangUpAlways') {
+    res.destroy();
+    return;
+  }
+  res.setHeader('content-type', 'application/json');
+  if (next.kind === 'runnerError') {
+    res.end(JSON.stringify({ ok: false, error: { code: next.code, message: next.message } }));
+    return;
+  }
+  res.end(JSON.stringify({ ok: true, data: next.data }));
+}
+
 function parseBody(raw: string): Record<string, unknown> {
   try {
     const parsed: unknown = JSON.parse(raw);

packages/platform-apple/src/runner/__tests__/runner-command-recovery.test.ts

@@ -6,6 +6,7 @@ import { IOS_SIMULATOR } from './device-fixtures.ts';
 import type { ExecResult } from '@agent-device/host-kit/command';
 import { handleRunnerTransportErrorAfterCommandSend } from '../runner-command-recovery.ts';
 import type { RunnerCommand } from '../runner-contract.ts';
+import { RUNNER_REPLY_LOST_REASON } from '../runner-error-classification.ts';
 import type { RunnerSession } from '../runner-session.ts';
 import {
   startFakeRunnerServer,
@@ -122,22 +123,53 @@ test('an unknown lifecycle state invalidates the session and says so', async ()
   assert.equal(invalidate.mock.calls[0]?.[1], 'transport_error_after_command_send');
 });
 
-test('a failing status probe retains the invalidation and rethrows the transport error', async () => {
+test('notAccepted from a restarted runner fails the lost command as unknown, naming the lost reply', async () => {
+  const { result, invalidate } = await runRecovery({
+    script: [{ kind: 'ok', data: { lifecycleState: 'notAccepted' } }],
+  });
+
+  await assert.rejects(result, (error: unknown) => {
+    assert.ok(error instanceof AppError);
+    assert.equal(error.details?.reason, RUNNER_REPLY_LOST_REASON);
+    assert.equal(error.details?.dispatched, 'unknown');
+    return true;
+  });
+  assert.equal(invalidate.mock.calls.length, 1);
+});
+
+test('a failing status probe retains the invalidation and names the lost reply', async () => {
   const { result, invalidate, transportError } = await runRecovery({
     script: [{ kind: 'runnerError', code: 'COMMAND_FAILED', message: 'status probe exploded' }],
+    transportError: new AppError('COMMAND_FAILED', 'socket hang up', { reason: 'socket_reset' }),
   });
 
-  await assert.rejects(result, (error: unknown) => error === transportError);
+  await assert.rejects(result, (error: unknown) => {
+    assert.ok(error instanceof AppError);
+    assert.notEqual(error.message, transportError.message);
+    assert.equal(error.details?.transportError, transportError.message);
+    assert.equal(error.cause, transportError);
+    assert.equal(error.details?.reason, RUNNER_REPLY_LOST_REASON);
+    assert.equal(error.details?.transportReason, 'socket_reset');
+    assert.equal(error.details?.recovery, 'status_probe_failed');
+    assert.equal(error.details?.dispatched, 'unknown');
+    return true;
+  });
   assert.equal(invalidate.mock.calls.length, 1);
 });
 
-test('a command without an id cannot be probed: invalidate and rethrow', async () => {
+test('a command without an id cannot be probed: invalidate and name the lost reply', async () => {
   const { result, invalidate, transportError } = await runRecovery({
     script: [],
     command: { command: 'tap', x: 10, y: 10 } as RunnerCommand,
   });
 
-  await assert.rejects(result, (error: unknown) => error === transportError);
+  await assert.rejects(result, (error: unknown) => {
+    assert.ok(error instanceof AppError);
+    assert.equal(error.cause, transportError);
+    assert.equal(error.details?.reason, RUNNER_REPLY_LOST_REASON);
+    assert.equal(error.details?.recovery, 'status_recovery_unavailable');
+    return true;
+  });
   assert.equal(invalidate.mock.calls.length, 1);
   assert.equal(server?.requests.length, 0);
 });

packages/platform-apple/src/runner/__tests__/runner-command-retry.test.ts

@@ -5,6 +5,7 @@ import {
   createTestRequestCancellation,
   makeRunnerSession,
   runnerConnectFailure,
+  unwrittenConnectRefusal,
 } from './runner-session-fixtures.ts';
 import { AppError } from '@agent-device/kernel/errors';
 import { Deadline } from '../host.ts';
@@ -49,6 +50,7 @@ vi.mock('../runner-xctestrun.ts', async () => {
 
 import { prepareIosRunner, runAppleRunnerCommand } from '../runner-client.ts';
 import { resetRunnerRecycleLedgerForTests } from '../runner-recycle-ledger.ts';
+import { RUNNER_REPLY_LOST_REASON } from '../runner-error-classification.ts';
 import type { RunnerXctestrunArtifact } from '../runner-xctestrun.ts';
 
 const requestCancellation = createTestRequestCancellation();
@@ -306,7 +308,7 @@ test('mutating commands restart stale ready sessions when the preflight probe ne
 
   mockEnsureRunnerSession.mockResolvedValueOnce(staleSession).mockResolvedValueOnce(freshSession);
   mockExecuteRunnerCommandWithSession
-    .mockRejectedValueOnce(runnerConnectFailure('runner_connect_refused'))
+    .mockRejectedValueOnce(unwrittenConnectRefusal())
     .mockResolvedValueOnce({ message: 'tapped' });
 
   const result = await runAppleRunnerCommand(IOS_SIMULATOR, { command: 'tap', x: 120, y: 240 });
@@ -329,7 +331,7 @@ test('mutating commands retry startup sessions with stale bundle cleanup', async
 
   mockEnsureRunnerSession.mockResolvedValueOnce(startupSession).mockResolvedValueOnce(freshSession);
   mockExecuteRunnerCommandWithSession
-    .mockRejectedValueOnce(runnerConnectFailure('runner_connect_refused'))
+    .mockRejectedValueOnce(unwrittenConnectRefusal())
     .mockResolvedValueOnce({ message: 'tapped' });
 
   const result = await runAppleRunnerCommand(IOS_SIMULATOR, { command: 'tap', x: 120, y: 240 });
@@ -474,10 +476,9 @@ test('mutating commands keep invalidating when status recovery probe fails', asy
   await assert.rejects(
     () => runAppleRunnerCommand(IOS_SIMULATOR, { command: 'tap', x: 120, y: 240 }),
     (error: unknown) => {
-      // A failed status probe re-throws the original transport error, not the probe's own.
       assert.ok(error instanceof AppError);
-      assert.equal(error.code, 'COMMAND_FAILED');
-      assert.equal(error.message, 'fetch failed');
+      assert.equal(error.details?.reason, RUNNER_REPLY_LOST_REASON);
+      assert.equal(error.details?.transportError, 'fetch failed');
       return true;
     },
   );
@@ -814,7 +815,7 @@ test('mutating commands invalidate the retry session without replaying again', a
 
   mockEnsureRunnerSession.mockResolvedValueOnce(staleSession).mockResolvedValueOnce(freshSession);
   mockExecuteRunnerCommandWithSession
-    .mockRejectedValueOnce(runnerConnectFailure('runner_connect_refused'))
+    .mockRejectedValueOnce(unwrittenConnectRefusal())
     .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
     .mockResolvedValueOnce({ lifecycleState: 'notAccepted' });
 
@@ -970,10 +971,9 @@ test('sequence invalidates the session when the status probe fails', async () =>
         steps: [{ kind: 'tap', x: 1, y: 2 }],
       }),
     (error: unknown) => {
-      // A failed status probe re-throws the original transport error, not the probe's own.
       assert.ok(error instanceof AppError);
-      assert.equal(error.code, 'COMMAND_FAILED');
-      assert.equal(error.message, 'fetch failed');
+      assert.equal(error.details?.reason, RUNNER_REPLY_LOST_REASON);
+      assert.equal(error.details?.transportError, 'fetch failed');
       return true;
     },
   );
@@ -1189,10 +1189,9 @@ test('a later command in the same request cannot pay for a second recycle boot',
   const requestId = 'req-restart-cap';
   const staleSession = makeRunnerSession({ port: 8100, state: 'ready' });
   const freshSession = makeRunnerSession({ port: 8101, state: 'starting' });
-
   mockEnsureRunnerSession.mockResolvedValueOnce(staleSession).mockResolvedValueOnce(freshSession);
   mockExecuteRunnerCommandWithSession
-    .mockRejectedValueOnce(runnerConnectFailure('runner_connect_refused'))
+    .mockRejectedValueOnce(unwrittenConnectRefusal())
     .mockResolvedValueOnce({ message: 'tapped' });
 
   // First command consumes the request's only recycle via restart-and-replay.