fix(test): make the iOS simulator smoke lane's red/green signal meaningful (#2491)

test/integration/ios-simulator-e2e-snapshot-acquisition.test.ts

@@ -0,0 +1,113 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { assertSimulatorSnapshotAcquisition } from './ios-simulator-e2e/live-snapshot-depth-frontier.ts';
+
+// #3328, pinned without a simulator: the depth-frontier scenario accepts either acquisition it
+// can present depth facts from — the AX bridge (which publishes no verdict) or the XCTest runner
+// the route sent the capture to (which always stamps its strategy) — and rejects only a capture
+// whose own typed verdict says it degraded or served nothing. The positives are the exact shapes
+// of the two failing main runs (37145696803, 37459193672), whose healthy `tree` verdict the old
+// assertion failed on the word `tree`.
+
+const HITTABLE = 'viewport-derived hittability evidence';
+
+function capture(quality: unknown): { json?: any } {
+  return {
+    json: {
+      success: true,
+      data: {
+        nodes: [{ index: 0, depth: 0, hittable: true }],
+        ...(quality === undefined ? {} : { snapshotQuality: quality }),
+        warnings: [HITTABLE],
+      },
+    },
+  };
+}
+
+test('an AX-bridge capture carries no verdict and is accepted', () => {
+  assertSimulatorSnapshotAcquisition(capture(undefined), 'regular depth-1 snapshot');
+});
+
+for (const backend of ['tree', 'queries', 'private-ax'] as const) {
+  test(`a runner-served capture disclosing the ${backend} strategy is accepted`, () => {
+    assertSimulatorSnapshotAcquisition(
+      capture({ state: 'healthy', backend, timing: { acquisitionMs: 64.7, presentationMs: 0.09 } }),
+      'regular depth-1 snapshot',
+    );
+  });
+}
+
+for (const reasonCode of ['deferred', 'requested-backend'] as const) {
+  test(`a pre-selected-backend recovered verdict (${reasonCode}) is accepted`, () => {
+    assertSimulatorSnapshotAcquisition(
+      capture({ state: 'recovered', backend: 'private-ax', reasonCode }),
+      'full raw visible-depth snapshot',
+    );
+  });
+}
+
+test('a capture that recovered to another strategy mid-capture is rejected', () => {
+  // The closest negative to the pre-selected pair: the same `recovered` state with a degradation
+  // code means the strategy the presented depth belongs to failed mid-capture, so the tree the
+  // frontier assertions would read is not a comparable view of the screen (#1569).
+  for (const reasonCode of ['capture-failed', 'presentation-failed', undefined]) {
+    assert.throws(
+      () =>
+        assertSimulatorSnapshotAcquisition(
+          capture({ state: 'recovered', backend: 'tree', ...(reasonCode ? { reasonCode } : {}) }),
+          'regular depth-1 snapshot',
+        ),
+      /fell back to another capture strategy mid-capture/,
+    );
+  }
+});
+
+test('a sparse capture is rejected whatever strategy it names', () => {
+  assert.throws(
+    () =>
+      assertSimulatorSnapshotAcquisition(
+        capture({ state: 'sparse', backend: 'tree', reasonCode: 'sparse-tree' }),
+        'regular depth-1 snapshot',
+      ),
+    /no backend served this screen/,
+  );
+});
+
+test('a verdict with a state outside the declared vocabulary is rejected', () => {
+  // The kernel declares ['healthy', 'recovered', 'sparse'] and the runner's allCases is pinned to
+  // the same fixture; a verdict outside that set means the two sides diverged, and certifying
+  // depth facts off a tree the lane cannot classify is the same meaningless-green defect as
+  // ruling a sparse capture healthy.
+  for (const state of ['unknown', 'degraded', undefined]) {
+    assert.throws(
+      () =>
+        assertSimulatorSnapshotAcquisition(
+          capture({ ...(state === undefined ? {} : { state }), backend: 'tree' }),
+          'regular depth-1 snapshot',
+        ),
+      /state outside the declared vocabulary/,
+    );
+  }
+});
+
+test('a capture that declares its hittability evidence missing is rejected', () => {
+  assert.throws(
+    () =>
+      assertSimulatorSnapshotAcquisition(
+        {
+          json: {
+            success: true,
+            data: {
+              nodes: [{ index: 0, depth: 0 }],
+              warnings: [
+                'iOS snapshot acquisition does not provide hittability evidence; regular snapshots omit unverified hittability while raw snapshots preserve supplied facts.',
+              ],
+            },
+          },
+        },
+        'regular depth-1 snapshot',
+      ),
+    /must derive hittability/,
+  );
+});

test/integration/ios-simulator-e2e-step-retry-policy.test.ts

@@ -0,0 +1,74 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { WAIT_REASONS } from '@agent-device/contracts/wait';
+import type { CliJsonResult } from './cli-json.ts';
+import { isObservationPreventedStepMiss } from './ios-simulator-e2e/live-step-retry-policy.ts';
+
+// The #2491 retry policy for the iOS live lane, pinned without a simulator: which typed wire
+// verdicts the lane may re-issue one step for, and — the closest negatives — which look close but
+// must fail at once. The pairing of `retriable: true` with an observation-prevented
+// `details.reason` is the whole predicate; neither signal alone activates a re-issue.
+
+function stepResult(json: unknown, status = 1): CliJsonResult {
+  return { json, status, stdout: '', stderr: '' };
+}
+
+function waitFailure(reason: string, retriable: boolean | undefined): CliJsonResult {
+  return stepResult({
+    error: {
+      code: 'COMMAND_FAILED',
+      details: { reason },
+      ...(retriable === undefined ? {} : { retriable }),
+    },
+  });
+}
+
+for (const reason of [
+  WAIT_REASONS.captureStalled,
+  WAIT_REASONS.runnerRestartExhausted,
+  WAIT_REASONS.readinessExhausted,
+]) {
+  test(`${reason} with the product's retriable verdict re-issues the step`, () => {
+    assert.equal(isObservationPreventedStepMiss(waitFailure(reason, true)), true);
+  });
+}
+
+test('an observation-prevented reason without the product retriable verdict does not re-issue', () => {
+  // The negative that keeps the predicate keyed on the conjunction: a response carrying the same
+  // reason but marked non-retriable (or carrying no verdict) must not buy a re-issue.
+  assert.equal(
+    isObservationPreventedStepMiss(waitFailure(WAIT_REASONS.captureStalled, false)),
+    false,
+  );
+  assert.equal(
+    isObservationPreventedStepMiss(waitFailure(WAIT_REASONS.captureStalled, undefined)),
+    false,
+  );
+});
+
+test('a retriable target-absent or deadline-exceeded wait does not re-issue', () => {
+  // The screen was readable: the target was not there, or a readable capture consumed the
+  // remaining budget. Re-issuing would hide a real absence, so these stay hard failures.
+  assert.equal(isObservationPreventedStepMiss(waitFailure(WAIT_REASONS.targetAbsent, true)), false);
+  assert.equal(
+    isObservationPreventedStepMiss(waitFailure(WAIT_REASONS.deadlineExceeded, true)),
+    false,
+  );
+});
+
+test('a non-wait retriable failure does not re-issue', () => {
+  // A retriable verdict on a response with no wait taxonomy reason (e.g. a leased-busy device on
+  // a click) is not evidence that observation of THIS step was prevented.
+  assert.equal(
+    isObservationPreventedStepMiss(
+      stepResult({ error: { code: 'DEVICE_IN_USE', retriable: true } }),
+    ),
+    false,
+  );
+});
+
+test('a successful step result is never a re-attributable miss', () => {
+  assert.equal(isObservationPreventedStepMiss(stepResult(undefined, 0)), false);
+  assert.equal(isObservationPreventedStepMiss(stepResult(undefined)), false);
+});

test/integration/ios-simulator-e2e/live-harness.ts

@@ -13,6 +13,7 @@ import {
 } from '../live-device-e2e/runtime.ts';
 import type { IosSimulatorBehaviorId } from './behavior-coverage.ts';
 import { liveCommandsForScenario } from './coverage.ts';
+import { isObservationPreventedStepMiss } from './live-step-retry-policy.ts';
 import { liveBehaviorsForScenario, writeCoverageReport } from './live-coverage-report.ts';
 
 export { assertCoverageComplete, writeCoverageReport } from './live-coverage-report.ts';
@@ -67,7 +68,26 @@ const harness = createLiveDeviceHarness<LiveContext, IosSimulatorBehaviorId>({
   writeCoverageReport,
 });
 
-export const { runScenario, runStep, sessionExists, verifyBehavior, verifyCommand } = harness;
+/**
+ * Every iOS live step carries the observation-prevented re-issue policy (#2491): one runner
+ * restart or bridge rebuild re-issues its own wait instead of failing the job, while a readable
+ * miss (target absent, budget exhausted after a readable capture, wrong asserted value) fails at
+ * once exactly as before. Steps that already own a miss policy (`allowFailure` / `expectFailure`,
+ * like the destination-wait and cleanup retry loops) are exempt inside the harness itself.
+ */
+export function runStep(
+  context: LiveContext,
+  step: string,
+  args: string[],
+  options: Parameters<typeof harness.runStep>[3] = {},
+): Promise<CliJsonResult> {
+  return harness.runStep(context, step, args, {
+    reattemptInfrastructureMiss: isObservationPreventedStepMiss,
+    ...options,
+  });
+}
+
+export const { runScenario, sessionExists, verifyBehavior, verifyCommand } = harness;
 
 export function verifyNestedReplayCommand(
   context: LiveContext,

test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts

@@ -1,5 +1,7 @@
 import assert from 'node:assert/strict';
 
+import { SNAPSHOT_QUALITY_STATES } from '@agent-device/kernel/snapshot';
+
 import {
   assertWaitText,
   type LiveSnapshotNode as SnapshotNode,
@@ -31,7 +33,7 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P
     '1',
     '--debug',
   ]);
-  assertSimulatorBridgeSnapshot(regular, 'regular depth-1 snapshot');
+  assertSimulatorSnapshotAcquisition(regular, 'regular depth-1 snapshot');
   const regularNodes = snapshotNodes(regular);
   const regularRoot = requireRoot(regularNodes, 'regular depth-1 snapshot');
   const projectedChild = requireIdentifier(regularNodes, CHILD_ID, 'regular depth-1 snapshot');
@@ -59,7 +61,7 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P
     'snapshot',
     '--raw',
   ]);
-  assertSimulatorBridgeSnapshot(rawFull, 'full raw visible-depth snapshot');
+  assertSimulatorSnapshotAcquisition(rawFull, 'full raw visible-depth snapshot');
   const rawFullNodes = snapshotNodes(rawFull);
   const rawChild = requireIdentifier(rawFullNodes, CHILD_ID, 'full raw visible-depth snapshot');
   assert.ok(
@@ -73,7 +75,7 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P
     '--depth',
     '1',
   ]);
-  assertSimulatorBridgeSnapshot(rawDepthOne, 'raw depth-1 visible-depth snapshot');
+  assertSimulatorSnapshotAcquisition(rawDepthOne, 'raw depth-1 visible-depth snapshot');
   const rawDepthOneNodes = snapshotNodes(rawDepthOne);
   assert.equal(
     rawDepthOneNodes.some((node) => node.identifier === CHILD_ID),
@@ -122,12 +124,60 @@ function numericDepth(node: SnapshotNode): number {
   return node.depth as number;
 }
 
-function assertSimulatorBridgeSnapshot(result: { json?: any }, description: string): void {
-  assert.equal(
-    result.json?.data?.snapshotQuality?.backend,
-    undefined,
-    `${description} must not carry XCTest tree quality metadata: ${JSON.stringify(result)}`,
-  );
+/**
+ * Reason codes that name a PRE-SELECTED backend rather than a capture that degraded
+ * (`deferred`: the runner's penalty circuit; `requested-backend`: the caller asked for that
+ * strategy). They are the two codes the product's own quality-warning renderer exempts from any
+ * degradation sentence for the same reason: nothing on THIS capture went wrong.
+ */
+const PRE_SELECTED_REASON_CODES: ReadonlySet<unknown> = new Set(['deferred', 'requested-backend']);
+
+/**
+ * The acquisition disclosure this scenario accepts before reading depth facts off the capture,
+ * keyed on the typed `snapshotQuality` verdict and never on the fallback warning's wording.
+ *
+ * A regular or raw capture of the fixture is served one of two ways, and each discloses itself
+ * here (#3328):
+ *
+ * - the host AX bridge served it. The bridge publishes no quality verdict at all
+ *   (`presentIosSnapshotAcquisition` reports the tree it read without one), so an absent verdict IS
+ *   the bridge's disclosure.
+ * - the route sent the capture to the XCTest runner instead — the bridge probe circuit is open for
+ *   this app generation, the bridge is still being prepared, a system surface is presented — and
+ *   the runner always stamps which of its own strategies served the payload. That disclosure is
+ *   legitimate on a regular snapshot: the bridge is a fast path, its circuit is per app generation,
+ *   and a runner-served capture keeps the hittability evidence this scenario asserts below.
+ *
+ * What the scenario may NOT be read from is a capture whose own verdict says it degraded or
+ * served nothing: `sparse` means no backend served the screen, and `recovered` with a code other
+ * than the two pre-selected ones means the strategy the presented depth semantics belong to failed
+ * mid-capture and another answered (#1569 — two strategies are not comparable views of one screen).
+ * Either one fails here with the response that proved it. A verdict whose state is outside the
+ * kernel-declared `SNAPSHOT_QUALITY_STATES` exhausts nothing and fails too — the lane must not
+ * certify a tree whose acquisition it does not classify.
+ */
+export function assertSimulatorSnapshotAcquisition(
+  result: { json?: any },
+  description: string,
+): void {
+  const quality = result.json?.data?.snapshotQuality;
+  if (quality !== undefined) {
+    assert.ok(
+      (SNAPSHOT_QUALITY_STATES as readonly unknown[]).includes(quality.state),
+      `${description} disclosed a quality state outside the declared vocabulary: ${JSON.stringify(result)}`,
+    );
+    assert.notEqual(
+      quality.state,
+      'sparse',
+      `${description} reports no backend served this screen: ${JSON.stringify(result)}`,
+    );
+    if (quality.state === 'recovered') {
+      assert.ok(
+        PRE_SELECTED_REASON_CODES.has(quality.reasonCode),
+        `${description} fell back to another capture strategy mid-capture, so its presented depth is not comparable: ${JSON.stringify(result)}`,
+      );
+    }
+  }
   assert.equal(
     result.json?.data?.warnings?.includes(MISSING_HITTABILITY_WARNING) ?? false,
     false,

test/integration/ios-simulator-e2e/live-step-retry-policy.ts

@@ -0,0 +1,38 @@
+import { WAIT_REASONS } from '@agent-device/contracts/wait';
+import type { CliJsonResult } from '../cli-json.ts';
+
+/**
+ * The wait verdicts that say the observation itself was prevented — the runner restarted,
+ * readiness work (runner start, bridge target discovery) ate the budget, or no capture was ever
+ * readable. Each is a lane condition on this host, not an answer about the screen: the product
+ * itself publishes all three with `retriable: true` and a "then retry" hint on the wire
+ * (`wait-polling.ts`), and they are the observed reds of the smoke lane (run 37356199982's
+ * `wait_runner_restart_exhausted` with `readableCaptures: 0`).
+ */
+const OBSERVATION_PREVENTED_WAIT_REASONS: ReadonlySet<string> = new Set([
+  WAIT_REASONS.captureStalled,
+  WAIT_REASONS.runnerRestartExhausted,
+  WAIT_REASONS.readinessExhausted,
+]);
+
+/**
+ * Whether one failed E2E step's result says the DEVICE prevented the observation, so re-issuing
+ * the same step is a measurement of the same question rather than a second bite at a real defect.
+ *
+ * Keyed on the conjunction of two typed signals — the wire `retriable` verdict the daemon hoists
+ * from its throw sites AND `error.details.reason` from the wait taxonomy — never on error text.
+ * Either signal alone is too wide: `retriable` also rides failures the lane must not re-issue
+ * blind (e.g. an app the lane itself is expected to have launched via another route), and a reason
+ * set copied onto a response the product marks non-retriable would retry a verdict the product
+ * says cannot succeed unchanged. The pairing is pinned in
+ * `test/integration/ios-simulator-e2e-step-retry-policy.test.ts`.
+ *
+ * `wait_target_absent` (the screen was readable and simply did not hold the target),
+ * `wait_deadline_exceeded` (a readable capture consumed the budget), a wrong assertion value, a
+ * runner crash, and any non-wait failure are all non-retriable by this predicate.
+ */
+export function isObservationPreventedStepMiss(result: CliJsonResult): boolean {
+  const error = result.json?.error;
+  if (!error || result.status === 0) return false;
+  return error.retriable === true && OBSERVATION_PREVENTED_WAIT_REASONS.has(error.details?.reason);
+}