test(ios): speed up fixtures and run native regressions nightly (#3824)

* test(ios): reuse fixtures across native regression assertions

* test(ios): observe interruption opt-out during native dispatch

* docs: record native fixture reuse measurements and mutation proof

* perf(ios): avoid polling fixtures that already exist

* ci(ios): keep native fault regressions nightly and check shared branches on PRs

* test(ios): count real focus taps and retain dropped-tap recovery

* test(ios): use immediate fixture checks in the upstream latency regression

* test(ios): remove the duplicate drag input from the preflight table

* chore(gates): describe the generated iOS branch checks

* docs: record full XCTest profiling and nightly coverage proof

Author: thymikee

.github/actions/smoke-ios-run/action.yml

@@ -47,7 +47,7 @@ runs:
       with:
         runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
         preferred-device-name: iPhone 17 Pro
-    - name: Run targeted iOS runner XCTest regressions
+    - name: Run iOS runner platform checks
       id: ios-xctest
       if: steps.xctest-impact.outputs.run != 'false'
       run: |
@@ -66,13 +66,11 @@ runs:
           -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \
           -resultBundlePath "$GITHUB_WORKSPACE/test/artifacts/ios-runner-regressions.xcresult" \
           "${IOS_PR_TEST_ARGS[@]}" 2>&1 | tee /tmp/agent-device-runner-regressions.log
-        node --input-type=module -e '
-          import { readFileSync } from "node:fs";
-          const log = readFileSync("/tmp/agent-device-runner-regressions.log", "utf8");
-          if (!/\] AGENT_DEVICE_RUNNER_OPTIONAL_PROBE_WAIT_COMPLETED$/m.test(log)) {
-            throw new Error("Optional observation ended the runner test before its wait completed");
-          }
-        '
+        xcrun xcresulttool get test-results summary \
+          --path "$GITHUB_WORKSPACE/test/artifacts/ios-runner-regressions.xcresult" \
+          --compact > "$GITHUB_WORKSPACE/test/artifacts/ios-runner-regressions-summary.json"
+        RESULT_SUMMARY_PATH="$GITHUB_WORKSPACE/test/artifacts/ios-runner-regressions-summary.json" \
+          XCTEST_LANE=pr node --experimental-strip-types scripts/xctest-run-summary.ts
       shell: bash
     - name: Preflight iOS runner through public CLI
       run: |

.github/workflows/xctest-nightly.yml

@@ -1,50 +1,9 @@
 name: XCTest Nightly
 
-# The full iOS runner XCTest suite (#1781 A7). The iOS PR phase in apple.yml selects a source-derived
-# subset of the target's methods in an `-only-testing:` list; everything outside that list ran
-# nowhere at all. This lane drops the filter and runs the test plan whole, skipping only
-# `testCommand` — which is the runner's server entry point rather than a test (see the step
-# below).
-#
-# No count is quoted here on purpose. `pnpm check:xctest-selection` prints the live split
-# (declared / PR-selected / skipped / nightly-only) and is the only place those numbers are
-# derived rather than remembered. A hand-copied figure in a comment is the same class of
-# defect this lane exists to close: between this file being written and first review, the
-# suite gained three methods and the PR list gained five entries, so every number originally
-# written here was already wrong.
-#
-# Under the lane rule in #1781:
-#
-#   - Catches: regressions in the simulator-only tests (guard `… && os(iOS)`: they launch
-#     the host app, route through SpringBoard, or assert an iOS-only branch) that apple.yml's
-#     PR list leaves out. Since the A7 classification the pure-decision tests also run on
-#     the macOS phase on every PR (apple.yml), so this lane's
-#     unique reach is the simulator-only remainder — `pnpm check:xctest-selection` prints
-#     both numbers. `testCommand` is skipped, and the tests guarded to macOS/tvOS only
-#     cannot execute on this lane's iOS Simulator destination (the host lane runs them).
-#   - Evidence: the first execution found #1812 in the dark set (2 of 155 red, both real);
-#     the other ~114 never-run tests all passed.
-#   - Cost: this is a public repository, so nothing here is billed. The real cost is one macOS
-#     concurrency slot per night. The Apple runner build shares apple.yml's cache (identical
-#     inputs and build variant), but scheduled runs only see DEFAULT-BRANCH caches, so the hit
-#     depends on apple.yml's `push: main` run having populated the cache at the same source
-#     hash — on a miss this lane also pays a full runner build. Adds nothing to any PR.
-#   - Kill criterion: what ONLY this lane reaches is 11 tests / ~42 s of simulator time
-#     (measured from the first night's xcresult) — the app-launching, SpringBoard-probing
-#     class. Once apple.yml runs the whole iOS-compiled bundle instead of its selected
-#     list — the follow-up stacked on the classification — every test reaches a PR lane and
-#     this lane goes. That trade is not free: those 11 are the flakiest tests in the suite,
-#     and they would land on a blocking 15-minute gate that is already cancelled ~19% of the
-#     time, so measure their flake rate here before moving them.
-#
-# Scheduled + manual only. It is deliberately NOT a PR gate: apple.yml is already over half of
-# this repo's CI spend (#1781 A9), and moving the unrun set onto it would make that worse for
-# tests whose flakiness on a shared runner is unknown.
-#
-# Note on `concurrency` below: it is keyed on `github.ref`, so a manual dispatch on the same
-# ref cancels an in-flight scheduled run. That is the intended trade — an operator asking for
-# a run wants the fresh one — but it means "dispatch while the nightly is running" loses the
-# nightly's result rather than queueing behind it.
+# Run all native iOS regressions nightly and on manual dispatch. PRs keep the host
+# decision suite, shared platform-branch checks, and public CLI E2E smoke.
+# Native fault injection and identity checks remain here; healthy E2E does not
+# replace those contracts. The source-derived count guards against a zero-test run.
 
 on:
   schedule:
@@ -63,13 +22,6 @@ jobs:
   full-suite:
     name: iOS Runner Full XCTest Suite
     runs-on: macos-26
-    # Measured, not guessed. A full local run of this exact command on matching toolchain
-    # (Xcode 26.2, iOS 26.2 Simulator) executed 154 tests in 120s wall — the suite itself is
-    # ~2 minutes, not the bulk of the job. What dominates the budget is the runner build on a
-    # derived-data cache MISS, plus checkout and simulator boot. the iOS phase in apple.yml has an 80-minute limit
-    # while doing strictly more than this lane (same build and boot, plus replays, prepare,
-    # and the fixture E2E), so half of that is generous headroom for a cache-miss night and
-    # still caps a wedged run at 45 minutes instead of two hours of held macOS slot.
     timeout-minutes: 45
     env:
       # Must match apple.yml. The runtime version rides in the derived-data cache key, and
@@ -141,7 +93,14 @@ jobs:
             -xctestrun "$XCTESTRUN_PATH" \
             -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \
             -skip-testing:AgentDeviceRunnerUITests/RunnerTests/testCommand \
-            -resultBundlePath "$RESULT_BUNDLE_PATH"
+            -resultBundlePath "$RESULT_BUNDLE_PATH" 2>&1 | tee .tmp/xctest-nightly/runner.log
+          node --input-type=module -e '
+            import { readFileSync } from "node:fs";
+            const log = readFileSync(".tmp/xctest-nightly/runner.log", "utf8");
+            if (!/\] AGENT_DEVICE_RUNNER_OPTIONAL_PROBE_WAIT_COMPLETED$/m.test(log)) {
+              throw new Error("Optional observation ended the runner test before its wait completed");
+            }
+          '
 
       # Best-effort and never the job's verdict on its own; the step below is what asserts.
       # `--compact` first because a red night's summary is the large one, and the job summary
@@ -166,7 +125,7 @@ jobs:
       # guards (XCTEST_LANE above), so a guard that compiles a file of tests out of iOS reads
       # as red too. The failure list is capped so a badly red night cannot blow the 1 MiB
       # job-summary limit and lose the headline with it.
-      - name: Report the run and assert it executed the source-derived set
+      - name: Report the run and assert the source-derived count
         if: always()
         run: |
           set -euo pipefail

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -156,10 +156,11 @@ final class RunnerTests: XCTestCase {
   // `-D AGENT_DEVICE_RUNNER_UNIT_TESTS` and are classified by their `#if` guard (#1781 A7):
   //   - `#if AGENT_DEVICE_RUNNER_UNIT_TESTS` alone: a pure runner decision (rule table,
   //     geometry, parser, policy, journal, dispatch bookkeeping) that needs no launched app.
-  //     Runs on the macOS host lane on every PR (ci.yml, no simulator) and on the iOS lanes.
+  //     Runs on the macOS host lane on every PR and on the full iOS nightly lane.
   //   - `… && os(iOS)` (or a nested `#if os(iOS)`): runner/XCTest semantics — launches the
   //     host app, routes through SpringBoard, swizzles XCUIApplication, or asserts an
-  //     iOS-only branch. Simulator lanes only (ios.yml PR list, xctest-nightly.yml).
+  //     iOS-only branch. Full simulator coverage runs nightly (xctest-nightly.yml).
+  //   - Shared methods with platform-dependent bodies also run on the PR simulator lane.
   // `pnpm check:xctest-selection` derives each lane's reachable set from these guards and
   // fails when a declared test is reachable by no lane, so a test gated to a platform nothing
   // runs (the old tvOS-only pair) cannot go dark silently.

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/FixtureExistence.swift

@@ -0,0 +1,9 @@
+#if AGENT_DEVICE_RUNNER_UNIT_TESTS
+import XCTest
+
+extension XCUIElement {
+  func waitForFixtureExistence(timeout: TimeInterval) -> Bool {
+    exists || waitForExistence(timeout: timeout)
+  }
+}
+#endif

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift

@@ -131,7 +131,7 @@ extension RunnerTests {
       invalidateCachedTarget(reason: "unit_test_cleanup")
       app.terminate()
     }
-    XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout))
+    XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout))
     // Resolution is a pure read; a contended hosted simulator can spend far more than the 10 s
     // command timeout on it, so it runs under the sibling fixtures' resolution allowance.
     let alert = try XCTUnwrap(
@@ -160,7 +160,7 @@ extension RunnerTests {
       invalidateCachedTarget(reason: "unit_test_cleanup")
       app.terminate()
     }
-    XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout))
+    XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout))
     let alert = try XCTUnwrap(
       resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.alertResolutionAllowance))
     )

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift

@@ -5,6 +5,7 @@ import ObjectiveC.runtime
 
 private final class AlertTapDeadlineStub: NSObject {
   static var status: RunnerTapSynthesisStatus?
+  static var interruptionHandlingDisabled: Bool?
 
   @objc(synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:)
   class func synthesizeTap(
@@ -15,7 +16,8 @@ private final class AlertTapDeadlineStub: NSObject {
     deadline: NSDate?,
     errorMessage: AutoreleasingUnsafeMutablePointer<NSString?>?
   ) -> RunnerTapSynthesisStatus {
-    status ?? (deadline == nil ? .succeeded : .deadlineExceeded)
+    interruptionHandlingDisabled = (application.value(forKey: "doesNotHandleUIInterruptions") as? NSNumber)?.boolValue
+    return status ?? (deadline == nil ? .succeeded : .deadlineExceeded)
   }
 }
 #endif
@@ -49,7 +51,7 @@ extension RunnerTests {
       invalidateCachedTarget(reason: "unit_test_cleanup")
       app.terminate()
     }
-    XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout))
+    XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout))
     let alert = try resolveAlertBeforeTheCommand()
     let response = handleAlert(alert, action: "accept", deadline: .distantPast)
     XCTAssertFalse(response.ok)
@@ -67,7 +69,7 @@ extension RunnerTests {
       invalidateCachedTarget(reason: "unit_test_cleanup")
       app.terminate()
     }
-    XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout))
+    XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout))
     let alert = try resolveAlertBeforeTheCommand()
     alertButtonHittabilityProbeOverrideForTesting = { probeDeadline in
       while Date() < probeDeadline {
@@ -90,7 +92,7 @@ extension RunnerTests {
       invalidateCachedTarget(reason: "unit_test_cleanup")
       app.terminate()
     }
-    XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout))
+    XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout))
     let alert = try resolveAlertBeforeTheCommand()
     let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" })
     let frame = button.frame
@@ -110,7 +112,7 @@ extension RunnerTests {
       invalidateCachedTarget(reason: "unit_test_cleanup")
       app.terminate()
     }
-    XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout))
+    XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout))
     let alert = try resolveAlertBeforeTheCommand()
     let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" })
     let frame = button.frame
@@ -121,6 +123,7 @@ extension RunnerTests {
     method_setImplementation(method, method_getImplementation(stub))
     defer {
       AlertTapDeadlineStub.status = nil
+      AlertTapDeadlineStub.interruptionHandlingDisabled = nil
       method_setImplementation(method, original)
     }
 
@@ -150,7 +153,7 @@ extension RunnerTests {
       invalidateCachedTarget(reason: "unit_test_cleanup")
       app.terminate()
     }
-    XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout))
+    XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout))
     let alert = try resolveAlertBeforeTheCommand()
 
     let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline))
@@ -165,7 +168,7 @@ extension RunnerTests {
   }
 
   @MainActor
-  func testAlertActivationDoesNotConsultInterruptionHandlingWithANotificationBanner() throws {
+  func testAlertActivationConfirmsBeforeDeadlineWithoutHandlingTheNotificationBanner() throws {
     try withNotificationBannerAlert { alert in
       var consultedInterruptions: [String] = []
       let monitor = addUIInterruptionMonitor(withDescription: "alert activation banner") { element in
@@ -174,24 +177,41 @@ extension RunnerTests {
       }
       defer { removeUIInterruptionMonitor(monitor) }
 
-      _ = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline))
-
-      XCTAssertEqual(consultedInterruptions, [], "alert activation consulted XCTest's interruption handling")
-    }
-  }
-
-  @MainActor
-  func testAlertActivationConfirmsBeforeDeadlineWithANotificationBanner() throws {
-    try withNotificationBannerAlert { alert in
+      try assertAlertActivationDisablesInterruptionHandling(alert)
       let deadline = Date().addingTimeInterval(RunnerTests.alertActivationDeadline)
       let response = handleAlert(alert, action: "accept", deadline: deadline)
 
+      XCTAssertEqual(consultedInterruptions, [], "alert activation consulted XCTest's interruption handling")
       XCTAssertTrue(response.ok, String(describing: response.error))
       XCTAssertLessThan(Date(), deadline)
       XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0")
     }
   }
 
+  @MainActor
+  private func assertAlertActivationDisablesInterruptionHandling(_ alert: RunnerAlert) throws {
+    let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" })
+    let frame = button.frame
+    let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:")
+    let method = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector))
+    let stub = try XCTUnwrap(class_getClassMethod(AlertTapDeadlineStub.self, selector))
+    let original = method_getImplementation(method)
+    AlertTapDeadlineStub.status = .deadlineExceeded
+    AlertTapDeadlineStub.interruptionHandlingDisabled = nil
+    method_setImplementation(method, method_getImplementation(stub))
+    defer {
+      AlertTapDeadlineStub.status = nil
+      AlertTapDeadlineStub.interruptionHandlingDisabled = nil
+      method_setImplementation(method, original)
+    }
+
+    let outcome = activateAlertButton(
+      alert, button: button, action: "accept", frame: frame, deadline: .distantFuture
+    )
+    XCTAssertNil(outcome)
+    XCTAssertEqual(AlertTapDeadlineStub.interruptionHandlingDisabled, true)
+  }
+
   @MainActor
   private func withNotificationBannerAlert(_ assertions: (RunnerAlert) throws -> Void) throws {
     app.launchArguments = ["--agent-device-alert-replacement-regression", "--agent-device-alert-banner"]
@@ -204,7 +224,7 @@ extension RunnerTests {
       _ = banner.waitForNonExistence(timeout: 15)
     }
     try acceptNotificationAuthorizationUntilAlertAppears()
-    XCTAssertTrue(banner.waitForExistence(timeout: appExistenceTimeout), "the fixture keeps a banner up")
+    XCTAssertTrue(banner.waitForFixtureExistence(timeout: appExistenceTimeout), "the fixture keeps a banner up")
     let alert = try resolveAlertBeforeTheCommand()
 
     try assertions(alert)
@@ -255,7 +275,7 @@ extension RunnerTests {
       invalidateCachedTarget(reason: "unit_test_cleanup")
       app.terminate()
     }
-    XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout))
+    XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout))
     let original = try resolveAlertBeforeTheCommand()
     alertResolutionOverrideForTesting = { _ in original }
     let timeoutMs = Int(RunnerTests.alertActivationDeadline * 1000)

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift

@@ -87,7 +87,7 @@ extension RunnerTests {
   private func launchCrowdedScreen(extraArguments: [String]) {
     app.launchArguments = ["--agent-device-crowded-screen"] + extraArguments
     app.launch()
-    XCTAssertTrue(app.staticTexts["agent-device-crowded-row-499"].waitForExistence(timeout: appExistenceTimeout))
+    XCTAssertTrue(app.staticTexts["agent-device-crowded-row-499"].waitForFixtureExistence(timeout: appExistenceTimeout))
   }
 
   @MainActor