test(ios): resolve the alert before the deadline the replacement fixtures assert on

Nine iOS CI failures of the alert replacement fixtures answered ALERT_DEADLINE_EXCEEDED with "No
alert button was activated": the command deadline also covered resolution, about two dozen
accessibility reads before any tap, and a contended hosted simulator spent it all there (job
106268275388 sampled a 0.823 s read, sized a 39.5 s deadline, then spent 39 s resolving). The
latency-sampled deadline from #2708 cannot predict a host that slows down mid-command.

The fixtures now resolve the alert first under a 60 s allowance and hand it to the command through
`alertResolutionOverrideForTesting`, so `timeoutMs` bounds only activation and verification, which
is what they prove: the replacement is never activated. That window gets a fixed 30 s. A confirmed
answer returns as soon as verification sees the replacement; only the unprovable-replacement
fixture pays it in full. The latency sampler and its decomposition test are removed.

The busy-app fixture asserted wall-clock success under a fixed 6 s deadline. It now records whether
the button was answered while its animation was still in flight, and the test asserts that instead.
Reverting the seam to a plain `activateElement` fails it with "Answered after the app went idle".

Author: thymikee

.github/workflows/ios.yml

@@ -189,7 +189,6 @@ jobs:
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertAcceptDoesNotActivateAReplacementWithASharedButton \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDismissDoesNotActivateAReplacementWithTheSameTitle \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertCannotProveAnIdenticalReplacementAndDoesNotActivateIt \
-            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertReplacementCommandDeadlineScalesWithMeasuredLatency \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDeadlineBeforeActivationLeavesTheOriginalUntouched \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertHittableProbeCompletingAfterDeadlineLeavesTheOriginalUntouched \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertActivationIgnoresAnAppThatNeverSettlesBeforeTheDeadline \

apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m

@@ -62,6 +62,7 @@ int main(int argc, const char *argv[]) {
 
 @interface AgentDeviceRunnerViewController : UIViewController
 @property(nonatomic, strong) UILabel *alertActionStatus;
+@property(nonatomic, strong) UILabel *alertActivationBusyAnswer;
 @property(nonatomic, assign) NSUInteger firstAlertActions;
 @property(nonatomic, assign) NSUInteger replacementAlertActions;
 @property(nonatomic, assign) BOOL alertFixtureStarted;
@@ -183,6 +184,10 @@ - (void)presentAlertFixtureReplacement:(BOOL)replacement {
         ? UIAlertActionStyleCancel : UIAlertActionStyleDefault;
     [alert addAction:[UIAlertAction actionWithTitle:buttonTitle style:style handler:^(UIAlertAction *action) {
       (void)action;
+      if (!replacement) {
+        self.alertActivationBusyAnswer.text = self.alertActivationBusyBackstop != nil
+            ? @"Answered while busy" : @"Answered after the app went idle";
+      }
       [self stopAlertActivationBusy];
       [self stopAlertBanner];
       if (replacement) {
@@ -253,6 +258,19 @@ - (void)viewDidLoad {
     [self updateAlertActionStatus];
   }
 
+  if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-activation-busy"]) {
+    UILabel *busyAnswer = [[UILabel alloc] init];
+    busyAnswer.text = @"Unanswered";
+    busyAnswer.accessibilityIdentifier = @"agent-device-alert-busy-answer";
+    busyAnswer.translatesAutoresizingMaskIntoConstraints = NO;
+    [self.view addSubview:busyAnswer];
+    [NSLayoutConstraint activateConstraints:@[
+      [busyAnswer.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
+      [busyAnswer.topAnchor constraintEqualToAnchor:label.bottomAnchor constant:24],
+    ]];
+    self.alertActivationBusyAnswer = busyAnswer;
+  }
+
   if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) {
     UITextField *textField = [[UITextField alloc] init];
     textField.accessibilityIdentifier = @"agent-device-hardware-keyboard-input";

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift

@@ -18,23 +18,6 @@ extension RunnerTests {
     )
   }
 
-  func testAlertReplacementCommandDeadlineScalesWithMeasuredLatency() {
-    // A quiet host still asks for the production default: 24 reads at 10 ms is a quarter of a second.
-    XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 0.001), 10_000)
-    XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 0.01), 10_000)
-    // The linear region is where a contended host lands: one second per read buys the 24 reads
-    // twice over, which is what the red nightly nights needed and 10 s did not cover.
-    XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 1), 48_000)
-    // Past the cap the fixture gives up rather than dominate the lane, whatever the host costs.
-    XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 2.5), 60_000)
-    XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 300), 60_000)
-    XCTAssertEqual(
-      RunnerTests.alertResolutionReadsBeforeActivation,
-      24,
-      "the decomposition below is the traced command path; re-trace it before changing a count"
-    )
-  }
-
   func testAlertDeadlineBeforeActivationLeavesTheOriginalUntouched() throws {
     app.launchArguments = ["--agent-device-alert-replacement-regression"]
     app.launch()
@@ -43,7 +26,7 @@ extension RunnerTests {
       app.terminate()
     }
     XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout))
-    let alert = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10)))
+    let alert = try resolveAlertBeforeTheCommand()
     let response = handleAlert(alert, action: "accept", deadline: .distantPast)
     XCTAssertFalse(response.ok)
     XCTAssertEqual(response.error?.code, "ALERT_DEADLINE_EXCEEDED")
@@ -60,7 +43,7 @@ extension RunnerTests {
       app.terminate()
     }
     XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout))
-    let alert = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10)))
+    let alert = try resolveAlertBeforeTheCommand()
     alertButtonHittabilityProbeOverrideForTesting = { probeDeadline in
       while Date() < probeDeadline {
         Thread.sleep(forTimeInterval: min(0.02, max(0, probeDeadline.timeIntervalSinceNow)))
@@ -85,28 +68,16 @@ extension RunnerTests {
       app.terminate()
     }
     XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout))
-    let alert = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10)))
+    let alert = try resolveAlertBeforeTheCommand()
 
-    let deadline = Date().addingTimeInterval(6)
-    let startedAt = Date()
-    let response = handleAlert(alert, action: "accept", deadline: deadline)
-    let elapsed = Date().timeIntervalSince(startedAt)
+    let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline))
 
-    // The fixture keeps an animation in flight, which is what XCTest waits out before it synthesises
-    // an event, so this comes back early only because activation refused to wait for an app that has
-    // no intention of settling (#2546).
-    XCTAssertLessThan(elapsed, 9, "activation waited \(elapsed)s for a busy app to idle")
+    // The fixture keeps an animation in flight until a button is answered or its 20 s backstop stops
+    // it, and an in-flight animation is what XCTest waits out before it synthesises an event. An
+    // answer that arrives while the app is still busy is one activation did not wait to idle (#2546).
     XCTAssertTrue(response.ok, String(describing: response.error))
-    let recordedActions = app.staticTexts["agent-device-alert-actions"].label
-    if response.ok {
-      XCTAssertEqual(recordedActions, "First actions: 1; replacement actions: 0")
-    } else {
-      XCTAssertEqual(
-        recordedActions,
-        "First actions: 0; replacement actions: 0",
-        "a caller told about an expired deadline must not have a button activated behind it"
-      )
-    }
+    XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0")
+    XCTAssertEqual(app.staticTexts["agent-device-alert-busy-answer"].label, "Answered while busy")
   }
 
   func testAlertActivationDoesNotWaitOutANotificationBanner() throws {
@@ -127,9 +98,9 @@ extension RunnerTests {
     }
     acceptNotificationAuthorizationUntilAlertAppears()
     XCTAssertTrue(banner.waitForExistence(timeout: appExistenceTimeout), "the fixture keeps a banner up")
-    let alert = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(30)))
+    let alert = try resolveAlertBeforeTheCommand()
 
-    let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(30))
+    let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline))
 
     XCTAssertEqual(consultedInterruptions, [], "alert activation waited on XCTest's interruption handling")
     XCTAssertTrue(response.ok, String(describing: response.error))
@@ -152,81 +123,45 @@ extension RunnerTests {
     XCTAssertTrue(fixtureAlert.exists, "the banner fixture needs notification authorization before it presents its alert")
   }
 
-  /// The accessibility round trips `resolveAlert` spends before a button is chosen, decomposed from
-  /// the implementation rather than counted off one trace: the blocking-modal probe scans
-  /// SpringBoard's alert and sheet lists and re-reads the candidate it settles on; the app's own
-  /// alert list is resolved and then read for existence and frame; `actionableElements` issues one
-  /// fetch per member of `actionableTypes`; and every candidate then pays `exists`, `isHittable`,
-  /// `elementType`, `frame`, `label` and `isEnabled` of its own.
-  static let alertResolutionModalProbeReads = 3
-  static let alertResolutionAlertRootReads = 3
-  static let alertResolutionActionableTypeReads = 6
-  static let alertResolutionButtonCandidates = 2
-  static let alertResolutionCandidateReads = 6
-
-  static let alertResolutionReadsBeforeActivation =
-    alertResolutionModalProbeReads + alertResolutionAlertRootReads + alertResolutionActionableTypeReads
-    + alertResolutionButtonCandidates * alertResolutionCandidateReads
-
-  /// The reads that follow resolution — the hittability wait and the first verification observation
-  /// run on the same channel — plus the allowance for a host that gets slower mid-command.
-  private static let alertCommandSafetyFactor: TimeInterval = 2
-
-  /// No higher than the slowest test this lane already runs (61 s on a green nightly), so a badly
-  /// starved host cannot make these fixtures the lane's worst contributor. The cap covers the worst
-  /// night traced so far, which needed 40.4 s of deadline.
-  private static let alertCommandTimeoutCap: TimeInterval = 60
-
-  /// The alert command's deadline for the replacement fixtures.
-  ///
-  /// `timeoutMs` bounds the whole command, so it is only meaningful in units of what one
-  /// accessibility read costs this host right now: `RunnerTests+Alert.swift` starts the deadline at
-  /// dispatch and the resolution above spends all of it reading before anything is activated. The
-  /// hosted lane runs a read at about 10 ms on a healthy night and about 1.7 s on a red one, which
-  /// is why the fixed 10 s these fixtures asked for passed on green nights and expired mid-resolution
-  /// on red ones (runs 35, 36 and 37 needed 31.2 s, 30.6 s and 40.4 s). Deriving the budget from a
-  /// measured read is `docs/agents/testing.md`'s preferred answer for a timeout that only fails on a
-  /// contended host; production still defaults to `defaultAlertCommandTimeout` for real callers.
-  static func alertReplacementCommandTimeoutMs(measuredRoundTrip: TimeInterval) -> Int {
-    let derived = measuredRoundTrip * Double(alertResolutionReadsBeforeActivation) * alertCommandSafetyFactor
-    return Int((min(max(derived, defaultAlertCommandTimeout), alertCommandTimeoutCap) * 1000).rounded())
-  }
+  /// Resolution reads the alert, its owner and every candidate button before anything is activated,
+  /// and a contended hosted simulator has spent 40 s of reads there. These fixtures prove what
+  /// activation and verification do, so they resolve first under this allowance, which only a failed
+  /// resolution ever spends.
+  static let alertResolutionAllowance: TimeInterval = 60
 
-  /// Samples the read the command is about to pay 24 times, once the alert is up, so the sample comes
-  /// from the window the command runs in. Two samples and the slower one wins: a single lucky read
-  /// must not size the budget low, and this is a worst-of measurement, not a mean.
-  private func measuredAccessibilityRoundTrip() -> TimeInterval {
-    let recordedActions = app.staticTexts["agent-device-alert-actions"]
-    var roundTrip = TimeInterval(0.001)
-    for _ in 0..<2 {
-      let startedAt = Date()
-      _ = recordedActions.label
-      roundTrip = max(roundTrip, Date().timeIntervalSince(startedAt))
-    }
-    return roundTrip
+  /// The deadline activation and verification run under once the alert is resolved: a confirmed
+  /// answer returns as soon as verification sees the replacement, so only a fixture whose answer is
+  /// the deadline itself pays it in full. It buys the dozen reads after resolution 2.5 s each, above the
+  /// 1.7 s a read cost on the worst hosted nights traced (#2708).
+  static let alertActivationDeadline: TimeInterval = 30
+
+  private func resolveAlertBeforeTheCommand() throws -> RunnerAlert {
+    try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.alertResolutionAllowance)))
   }
 
   private func assertReplacementAlertUntouched(action: String, arguments: [String], confirmed: Bool) throws {
     app.launchArguments = ["--agent-device-alert-replacement-regression"] + arguments
     app.launch()
     defer {
+      alertResolutionOverrideForTesting = nil
       invalidateCachedTarget(reason: "unit_test_cleanup")
       app.terminate()
     }
     XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout))
-    let measuredRoundTrip = measuredAccessibilityRoundTrip()
-    let timeoutMs = RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: measuredRoundTrip)
+    let original = try resolveAlertBeforeTheCommand()
+    alertResolutionOverrideForTesting = { _ in original }
+    let timeoutMs = Int(RunnerTests.alertActivationDeadline * 1000)
     let command = try runnerCommandFixture(
       #"{"command":"alert","commandId":"alert-replacement","action":"\#(action)","timeoutMs":\#(timeoutMs)}"#
     )
     let response = try executeOnMainPrepared(command: command, activeApp: app)
-    let budget = "timeoutMs \(timeoutMs) from a \(String(format: "%.3f", measuredRoundTrip))s read"
-    XCTAssertEqual(response.ok, confirmed, "\(budget): \(String(describing: response.error))")
+    alertResolutionOverrideForTesting = nil
+    XCTAssertEqual(response.ok, confirmed, String(describing: response.error))
     if !confirmed { XCTAssertEqual(response.error?.code, "ALERT_DEADLINE_EXCEEDED") }
     XCTAssertTrue(app.alerts.firstMatch.exists, "the replacement must remain visible")
     XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0")
-    let current = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10)))
-    let inspection = handleAlert(current, action: "get", deadline: Date().addingTimeInterval(10))
+    let current = try resolveAlertBeforeTheCommand()
+    let inspection = handleAlert(current, action: "get", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline))
     XCTAssertTrue(inspection.ok)
     XCTAssertEqual(inspection.data?.items?.sorted(), ["Cancel", "OK"])
     XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0")