diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index dc1cda0ec1..81434bc9a0 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -189,10 +189,10 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertAcceptDoesNotActivateAReplacementWithASharedButton \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDismissDoesNotActivateAReplacementWithTheSameTitle \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertCannotProveAnIdenticalReplacementAndDoesNotActivateIt \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertReplacementCommandDeadlineScalesWithMeasuredLatency \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDeadlineBeforeActivationLeavesTheOriginalUntouched \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertHittableProbeCompletingAfterDeadlineLeavesTheOriginalUntouched \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertActivationIgnoresAnAppThatNeverSettlesBeforeTheDeadline \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertActivationDoesNotWaitOutANotificationBanner \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSystemModalProbeSliceSharesAndClampsToPlanDeadline \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bf19bbaba..688e453617 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ CI it spent its whole 10 s budget there (`wait_capture_stalled` with `captures: 1`). The probe now joins the running discovery, bounded by the discovery's own deadline, and then observes the launch as before. A resolution failure other than a pending discovery still ends the probe at once. +- Fixed (ios): `alert accept` and `alert dismiss` no longer land their tap after the command's + deadline when a notification banner is on screen. XCTest checks for SpringBoard banners and alerts + before every event and its default handler waited up to 15 s for a banner to leave, so the command + answered `ALERT_DEADLINE_EXCEEDED` ("The button was activated once") for a tap that arrived after + the caller was told it had failed (#2546's late tap, from the banner side). Alert activation now + opts out of XCTest's interruption handling, which also stops that handler from pressing a button of + its own choosing on an unrelated system alert while the command answers the alert it resolved. - Fixed (ios): a local Simulator snapshot taken through the host AX bridge once again publishes the geometric `hittable` fact, so `is hittable` and a `hittable:` selector resolve the same controls on the bridge and the XCTest runner. The snapshot capability table has declared `hittable = diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 7ed0af0bb4..937f0f3d59 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -56,15 +56,25 @@ int main(int argc, const char *argv[]) { #else #import +#if TARGET_OS_IOS +#import +#endif @interface AgentDeviceRunnerViewController : UIViewController @property(nonatomic, strong) UILabel *alertActionStatus; +@property(nonatomic, strong) UILabel *alertActivationBusyAnswer; @property(nonatomic, assign) NSUInteger firstAlertActions; @property(nonatomic, assign) NSUInteger replacementAlertActions; @property(nonatomic, assign) BOOL alertFixtureStarted; @property(nonatomic, strong) NSTimer *alertActivationBusyBackstop; +@property(nonatomic, strong) NSTimer *alertBannerRepost; @end +#if TARGET_OS_IOS +@interface AgentDeviceRunnerViewController () +@end +#endif + @implementation AgentDeviceRunnerViewController #if TARGET_OS_IOS @@ -72,15 +82,21 @@ @implementation AgentDeviceRunnerViewController // may receive an event: the app keeps reporting work in flight, which is the state that cost an alert // command its whole deadline in #2546. It stops the moment an alert button is answered, since that // answer is the event the runner is trying to land, and the backstop stops it even when no answer -// arrives so a regressed run finishes rather than waiting out XCTest's own timeout. A layer +// arrives so a regressed run finishes rather than waiting out XCTest's own timeout. The test passes +// the backstop after `--agent-device-alert-activation-busy`, sized to outlast its whole resolution +// and activation budget, so a slow host cannot end the busy state before the answer lands. A layer // animation on its own is not enough; only a UIView animation counts as in-flight work here. -static NSTimeInterval const kAgentDeviceAlertActivationBusyWindow = 20.0; +static NSTimeInterval AgentDeviceAlertActivationBusyWindow(void) { + NSArray *arguments = NSProcessInfo.processInfo.arguments; + NSUInteger flag = [arguments indexOfObject:@"--agent-device-alert-activation-busy"]; + return flag + 1 < arguments.count ? arguments[flag + 1].doubleValue : 0; +} - (void)startAlertActivationBusy { if (self.alertActivationBusyBackstop != nil) { return; } - self.alertActivationBusyBackstop = [NSTimer scheduledTimerWithTimeInterval:kAgentDeviceAlertActivationBusyWindow + self.alertActivationBusyBackstop = [NSTimer scheduledTimerWithTimeInterval:AgentDeviceAlertActivationBusyWindow() target:self selector:@selector(stopAlertActivationBusy) userInfo:nil @@ -101,6 +117,57 @@ - (void)stopAlertActivationBusy { self.alertActivationBusyBackstop = nil; } +// A banner from this app, shown over its own alert and re-posted before the previous one expires so +// one is on screen for as long as the first alert is unanswered. XCTest treats such a banner as an +// interruption of every event aimed at the app (#2546's late tap, from the banner side). +- (void)startAlertBannerThen:(dispatch_block_t)presentAlert { + UNUserNotificationCenter *center = UNUserNotificationCenter.currentNotificationCenter; + center.delegate = self; + [center requestAuthorizationWithOptions:UNAuthorizationOptionAlert + completionHandler:^(BOOL granted, NSError *error) { + (void)error; + if (!granted) { + return; + } + dispatch_async(dispatch_get_main_queue(), ^{ + presentAlert(); + [self postAlertBanner]; + self.alertBannerRepost = [NSTimer scheduledTimerWithTimeInterval:2.0 + target:self + selector:@selector(postAlertBanner) + userInfo:nil + repeats:YES]; + }); + }]; +} + +- (void)postAlertBanner { + UNMutableNotificationContent *content = [[UNMutableNotificationContent alloc] init]; + content.title = @"Agent Device banner"; + content.body = @"Shown over the alert fixture"; + UNNotificationRequest *request = [UNNotificationRequest requestWithIdentifier:NSUUID.UUID.UUIDString + content:content + trigger:nil]; + [UNUserNotificationCenter.currentNotificationCenter addNotificationRequest:request withCompletionHandler:nil]; +} + +- (void)stopAlertBanner { + if (self.alertBannerRepost == nil) { + return; + } + [self.alertBannerRepost invalidate]; + self.alertBannerRepost = nil; + [UNUserNotificationCenter.currentNotificationCenter removeAllDeliveredNotifications]; +} + +- (void)userNotificationCenter:(UNUserNotificationCenter *)center + willPresentNotification:(UNNotification *)notification + withCompletionHandler:(void (^)(UNNotificationPresentationOptions options))completionHandler { + (void)center; + (void)notification; + completionHandler(UNNotificationPresentationOptionBanner); +} + - (void)updateAlertActionStatus { self.alertActionStatus.text = [NSString stringWithFormat:@"First actions: %lu; replacement actions: %lu", @@ -123,7 +190,12 @@ - (void)presentAlertFixtureReplacement:(BOOL)replacement { ? UIAlertActionStyleCancel : UIAlertActionStyleDefault; [alert addAction:[UIAlertAction actionWithTitle:buttonTitle style:style handler:^(UIAlertAction *action) { (void)action; + if (!replacement) { + self.alertActivationBusyAnswer.text = self.alertActivationBusyBackstop != nil + ? @"Answered while busy" : @"Answered after the app went idle"; + } [self stopAlertActivationBusy]; + [self stopAlertBanner]; if (replacement) { self.replacementAlertActions += 1; } else { @@ -145,9 +217,16 @@ - (void)viewDidAppear:(BOOL)animated { if (!self.alertFixtureStarted && [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-replacement-regression"]) { self.alertFixtureStarted = YES; - [self presentAlertFixtureReplacement:NO]; - if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-activation-busy"]) { - [self startAlertActivationBusy]; + dispatch_block_t presentAlert = ^{ + [self presentAlertFixtureReplacement:NO]; + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-activation-busy"]) { + [self startAlertActivationBusy]; + } + }; + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-banner"]) { + [self startAlertBannerThen:presentAlert]; + } else { + presentAlert(); } } } @@ -185,6 +264,19 @@ - (void)viewDidLoad { [self updateAlertActionStatus]; } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-activation-busy"]) { + UILabel *busyAnswer = [[UILabel alloc] init]; + busyAnswer.text = @"Unanswered"; + busyAnswer.accessibilityIdentifier = @"agent-device-alert-busy-answer"; + busyAnswer.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:busyAnswer]; + [NSLayoutConstraint activateConstraints:@[ + [busyAnswer.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [busyAnswer.topAnchor constraintEqualToAnchor:label.bottomAnchor constant:24], + ]]; + self.alertActivationBusyAnswer = busyAnswer; + } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) { UITextField *textField = [[UITextField alloc] init]; textField.accessibilityIdentifier = @"agent-device-hardware-keyboard-input"; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index 4503003d31..483862a3af 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -76,15 +76,17 @@ extension RunnerTests { buttonFrame.origin.x, buttonFrame.origin.y, buttonFrame.size.width, buttonFrame.size.height, buttonFrame.midX, buttonFrame.midY ) - // The hittable read above is this activation's readiness gate, so XCTest's pre-synthesis wait - // adds nothing and can cost more than the command has: the tap would land after the deadline + // The hittable read above is this activation's readiness gate, so XCTest's pre-synthesis waits + // add nothing and can cost more than the command has: the tap would land after the deadline // expired and the alert would be answered by a button the caller was told nothing about // (#2546). The post-tap settle stays, because the verification below reads the alert this tap // replaces; an alert that dismisses and presents an identical replacement passes through a // window with no alert, and a first read landing there reports a dismissal nothing proved. var outcome = RunnerInteractionOutcome.performed - withBoundedInteractionIdleTimeoutIfSupported(alert.ownerApp, waits: .preEventSkipped) { - outcome = activateElement(app: alert.ownerApp, element: button, action: "alert \(action)") + withUIInterruptionHandlingDisabledIfSupported(alert.ownerApp) { + withBoundedInteractionIdleTimeoutIfSupported(alert.ownerApp, waits: .preEventSkipped) { + outcome = activateElement(app: alert.ownerApp, element: button, action: "alert \(action)") + } } if let response = unsupportedResponse(for: outcome) { return response @@ -114,6 +116,22 @@ extension RunnerTests { ) } + /// Before each event XCTest looks for SpringBoard elements over the target and hands them to its + /// interruption handler, which waits up to 15 s for a notification banner to leave and taps a + /// button of its own choosing on any other alert. An alert command answers exactly the alert it + /// resolved, with the button it chose, before its deadline, so it opts out of both. + private func withUIInterruptionHandlingDisabledIfSupported(_ target: XCUIApplication, operation: () -> Void) { + let key = "doesNotHandleUIInterruptions" + guard target.responds(to: NSSelectorFromString("setDoesNotHandleUIInterruptions:")) else { + operation() + return + } + let previous = target.value(forKey: key) as? NSNumber + target.setValue(true, forKey: key) + defer { target.setValue(previous?.boolValue ?? false, forKey: key) } + operation() + } + private func runnerAlert(_ modal: ResolvedBlockingSystemModal) -> RunnerAlert? { let buttons = modal.actions.filter { isEnabledElement($0) } guard !buttons.isEmpty else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift index 508ed18691..b57b0ed27c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift @@ -18,23 +18,6 @@ extension RunnerTests { ) } - func testAlertReplacementCommandDeadlineScalesWithMeasuredLatency() { - // A quiet host still asks for the production default: 24 reads at 10 ms is a quarter of a second. - XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 0.001), 10_000) - XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 0.01), 10_000) - // The linear region is where a contended host lands: one second per read buys the 24 reads - // twice over, which is what the red nightly nights needed and 10 s did not cover. - XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 1), 48_000) - // Past the cap the fixture gives up rather than dominate the lane, whatever the host costs. - XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 2.5), 60_000) - XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 300), 60_000) - XCTAssertEqual( - RunnerTests.alertResolutionReadsBeforeActivation, - 24, - "the decomposition below is the traced command path; re-trace it before changing a count" - ) - } - func testAlertDeadlineBeforeActivationLeavesTheOriginalUntouched() throws { app.launchArguments = ["--agent-device-alert-replacement-regression"] app.launch() @@ -43,7 +26,7 @@ extension RunnerTests { app.terminate() } XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) - let alert = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10))) + let alert = try resolveAlertBeforeTheCommand() let response = handleAlert(alert, action: "accept", deadline: .distantPast) XCTAssertFalse(response.ok) XCTAssertEqual(response.error?.code, "ALERT_DEADLINE_EXCEEDED") @@ -60,7 +43,7 @@ extension RunnerTests { app.terminate() } XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) - let alert = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10))) + let alert = try resolveAlertBeforeTheCommand() alertButtonHittabilityProbeOverrideForTesting = { probeDeadline in while Date() < probeDeadline { Thread.sleep(forTimeInterval: min(0.02, max(0, probeDeadline.timeIntervalSinceNow))) @@ -77,7 +60,8 @@ extension RunnerTests { func testAlertActivationIgnoresAnAppThatNeverSettlesBeforeTheDeadline() throws { app.launchArguments = [ "--agent-device-alert-replacement-regression", - "--agent-device-alert-activation-busy" + "--agent-device-alert-activation-busy", + String(RunnerTests.alertResolutionAllowance + RunnerTests.alertActivationDeadline) ] app.launch() defer { @@ -85,105 +69,101 @@ extension RunnerTests { app.terminate() } XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) - let alert = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10))) + let alert = try resolveAlertBeforeTheCommand() - let deadline = Date().addingTimeInterval(6) - let startedAt = Date() - let response = handleAlert(alert, action: "accept", deadline: deadline) - let elapsed = Date().timeIntervalSince(startedAt) + let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) - // The fixture keeps an animation in flight, which is what XCTest waits out before it synthesises - // an event, so this comes back early only because activation refused to wait for an app that has - // no intention of settling (#2546). - XCTAssertLessThan(elapsed, 9, "activation waited \(elapsed)s for a busy app to idle") + // The fixture keeps an animation in flight until a button is answered or its backstop, which + // outlasts resolution plus activation, stops it, and an in-flight animation is what XCTest waits + // out before it synthesises an event. An answer that arrives while the app is still busy is one + // activation did not wait to idle (#2546). XCTAssertTrue(response.ok, String(describing: response.error)) - let recordedActions = app.staticTexts["agent-device-alert-actions"].label - if response.ok { - XCTAssertEqual(recordedActions, "First actions: 1; replacement actions: 0") - } else { - XCTAssertEqual( - recordedActions, - "First actions: 0; replacement actions: 0", - "a caller told about an expired deadline must not have a button activated behind it" - ) - } + XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") + XCTAssertEqual(app.staticTexts["agent-device-alert-busy-answer"].label, "Answered while busy") } - /// The accessibility round trips `resolveAlert` spends before a button is chosen, decomposed from - /// the implementation rather than counted off one trace: the blocking-modal probe scans - /// SpringBoard's alert and sheet lists and re-reads the candidate it settles on; the app's own - /// alert list is resolved and then read for existence and frame; `actionableElements` issues one - /// fetch per member of `actionableTypes`; and every candidate then pays `exists`, `isHittable`, - /// `elementType`, `frame`, `label` and `isEnabled` of its own. - static let alertResolutionModalProbeReads = 3 - static let alertResolutionAlertRootReads = 3 - static let alertResolutionActionableTypeReads = 6 - static let alertResolutionButtonCandidates = 2 - static let alertResolutionCandidateReads = 6 - - static let alertResolutionReadsBeforeActivation = - alertResolutionModalProbeReads + alertResolutionAlertRootReads + alertResolutionActionableTypeReads - + alertResolutionButtonCandidates * alertResolutionCandidateReads - - /// The reads that follow resolution — the hittability wait and the first verification observation - /// run on the same channel — plus the allowance for a host that gets slower mid-command. - private static let alertCommandSafetyFactor: TimeInterval = 2 - - /// No higher than the slowest test this lane already runs (61 s on a green nightly), so a badly - /// starved host cannot make these fixtures the lane's worst contributor. The cap covers the worst - /// night traced so far, which needed 40.4 s of deadline. - private static let alertCommandTimeoutCap: TimeInterval = 60 - - /// The alert command's deadline for the replacement fixtures. - /// - /// `timeoutMs` bounds the whole command, so it is only meaningful in units of what one - /// accessibility read costs this host right now: `RunnerTests+Alert.swift` starts the deadline at - /// dispatch and the resolution above spends all of it reading before anything is activated. The - /// hosted lane runs a read at about 10 ms on a healthy night and about 1.7 s on a red one, which - /// is why the fixed 10 s these fixtures asked for passed on green nights and expired mid-resolution - /// on red ones (runs 35, 36 and 37 needed 31.2 s, 30.6 s and 40.4 s). Deriving the budget from a - /// measured read is `docs/agents/testing.md`'s preferred answer for a timeout that only fails on a - /// contended host; production still defaults to `defaultAlertCommandTimeout` for real callers. - static func alertReplacementCommandTimeoutMs(measuredRoundTrip: TimeInterval) -> Int { - let derived = measuredRoundTrip * Double(alertResolutionReadsBeforeActivation) * alertCommandSafetyFactor - return Int((min(max(derived, defaultAlertCommandTimeout), alertCommandTimeoutCap) * 1000).rounded()) + func testAlertActivationDoesNotWaitOutANotificationBanner() throws { + app.launchArguments = ["--agent-device-alert-replacement-regression", "--agent-device-alert-banner"] + app.launch() + let banner = XCUIApplication(bundleIdentifier: "com.apple.springboard") + .descendants(matching: .any)["NotificationShortLookView"] + var consultedInterruptions: [String] = [] + let monitor = addUIInterruptionMonitor(withDescription: "alert activation banner") { element in + consultedInterruptions.append(element.identifier) + return false + } + defer { + removeUIInterruptionMonitor(monitor) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + _ = banner.waitForNonExistence(timeout: 15) + } + acceptNotificationAuthorizationUntilAlertAppears() + XCTAssertTrue(banner.waitForExistence(timeout: appExistenceTimeout), "the fixture keeps a banner up") + let alert = try resolveAlertBeforeTheCommand() + + let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) + + XCTAssertEqual(consultedInterruptions, [], "alert activation waited on XCTest's interruption handling") + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") } - /// Samples the read the command is about to pay 24 times, once the alert is up, so the sample comes - /// from the window the command runs in. Two samples and the slower one wins: a single lucky read - /// must not size the budget low, and this is a worst-of measurement, not a mean. - private func measuredAccessibilityRoundTrip() -> TimeInterval { - let recordedActions = app.staticTexts["agent-device-alert-actions"] - var roundTrip = TimeInterval(0.001) - for _ in 0..<2 { - let startedAt = Date() - _ = recordedActions.label - roundTrip = max(roundTrip, Date().timeIntervalSince(startedAt)) + /// The banner fixture presents its alert only once this app may post notifications; a fresh + /// simulator asks first, through SpringBoard. + private func acceptNotificationAuthorizationUntilAlertAppears() { + let allow = XCUIApplication(bundleIdentifier: "com.apple.springboard").alerts.buttons["Allow"] + let fixtureAlert = app.alerts.firstMatch + let deadline = Date().addingTimeInterval(appExistenceTimeout) + while Date() < deadline, !fixtureAlert.exists { + if allow.exists { + allow.tap() + } else { + Thread.sleep(forTimeInterval: 0.25) + } } - return roundTrip + XCTAssertTrue(fixtureAlert.exists, "the banner fixture needs notification authorization before it presents its alert") + } + + /// Resolution reads the alert, its owner and every candidate button before anything is activated, + /// and a contended hosted simulator has spent 40 s of reads there. These fixtures prove what + /// activation and verification do, so they resolve first under this allowance, which only a failed + /// resolution ever spends. + static let alertResolutionAllowance: TimeInterval = 60 + + /// The deadline activation and verification run under once the alert is resolved: a confirmed + /// answer returns as soon as verification sees the replacement, so only a fixture whose answer is + /// the deadline itself pays it in full. It buys the dozen reads after resolution 2.5 s each, above the + /// 1.7 s a read cost on the worst hosted nights traced (#2708). + static let alertActivationDeadline: TimeInterval = 30 + + private func resolveAlertBeforeTheCommand() throws -> RunnerAlert { + try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.alertResolutionAllowance))) } private func assertReplacementAlertUntouched(action: String, arguments: [String], confirmed: Bool) throws { app.launchArguments = ["--agent-device-alert-replacement-regression"] + arguments app.launch() defer { + alertResolutionOverrideForTesting = nil invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) - let measuredRoundTrip = measuredAccessibilityRoundTrip() - let timeoutMs = RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: measuredRoundTrip) + let original = try resolveAlertBeforeTheCommand() + alertResolutionOverrideForTesting = { _ in original } + let timeoutMs = Int(RunnerTests.alertActivationDeadline * 1000) let command = try runnerCommandFixture( #"{"command":"alert","commandId":"alert-replacement","action":"\#(action)","timeoutMs":\#(timeoutMs)}"# ) let response = try executeOnMainPrepared(command: command, activeApp: app) - let budget = "timeoutMs \(timeoutMs) from a \(String(format: "%.3f", measuredRoundTrip))s read" - XCTAssertEqual(response.ok, confirmed, "\(budget): \(String(describing: response.error))") + alertResolutionOverrideForTesting = nil + XCTAssertEqual(response.ok, confirmed, String(describing: response.error)) if !confirmed { XCTAssertEqual(response.error?.code, "ALERT_DEADLINE_EXCEEDED") } XCTAssertTrue(app.alerts.firstMatch.exists, "the replacement must remain visible") XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") - let current = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10))) - let inspection = handleAlert(current, action: "get", deadline: Date().addingTimeInterval(10)) + let current = try resolveAlertBeforeTheCommand() + let inspection = handleAlert(current, action: "get", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) XCTAssertTrue(inspection.ok) XCTAssertEqual(inspection.data?.items?.sorted(), ["Cancel", "OK"]) XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0")