diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 8ef6bc8ca7..0562d2fd14 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -205,6 +205,19 @@ extension RunnerTests { } } + /// The reason a capture's verdict carries. Once an XCTest-backed tier ran behind a deferred plan, + /// it ran on the bounded probe's short slice, so the capture reports that constraint ('budget') + /// whether the tier recovered it or the plan ended sparse, rather than the pre-selection + /// ('deferred') the plan was seeded with. + static func planVerdictReason( + xCTestTierRan: Bool, + state: SnapshotXCTestChannelPlanState, + firstFailure: (reason: String, code: String)? + ) -> (reason: String, code: String)? { + guard xCTestTierRan, state == .deferredToIndependentBackend else { return firstFailure } + return xcTestChannelStateFirstFailure(.boundedXCTestProbe) + } + /// Pure gate: a capture is planned as penalized when the channel penalty is /// active OR the daemon pinned the private-AX backend (same-backend evidence /// probe) — both mean "do not enter XCTest tree work first, and stamp the @@ -251,10 +264,14 @@ extension RunnerTests { let availablePlan = plan.filter { availableBackends.contains($0) } let recoveryPlan = availablePlan.filter { !$0.usesXCTestAccessibilityChannel } if !recoveryPlan.isEmpty { + // The independent backend reads only the app it can match as the active AX application, so + // an out-of-process surface over the app (the Save Password sheet) leaves it empty for the + // whole penalty. The tree, on the bounded probe's short slice, stays behind it; the query + // sweep does not, since its grind is what the penalty keeps off the main thread. return EffectiveSnapshotCapturePlan( - plan: recoveryPlan, + plan: recoveryPlan + availablePlan.filter { $0 == .recursiveTree }, xCTestChannelState: .deferredToIndependentBackend, - treeCaptureSliceBudgetOverride: nil, + treeCaptureSliceBudgetOverride: Self.penalizedXCTestProbeTreeSliceBudget, preferredBackend: nil ) } @@ -285,6 +302,7 @@ extension RunnerTests { // A caller may share the pre-plan system-modal probe's deadline; otherwise own the full budget (#1244). let deadline = deadline ?? Date().addingTimeInterval(Self.snapshotPlanBudget) let suppressXCTestPenalty = snapshotXCTestPenaltyWarmupExemption.consume() + var xCTestTierRan = false // Reorder is iOS-only because hostile screens can make XCTest tree/query work grind while // the app remains visually responsive. Simulators can avoid that channel through private AX; @@ -344,6 +362,7 @@ extension RunnerTests { } continue } + if kind.usesXCTestAccessibilityChannel { xCTestTierRan = true } let attempt = try captureWithBackend( kind, target: target, @@ -391,18 +410,23 @@ extension RunnerTests { } let recovered = kind != effectivePlan.first || effective.xCTestChannelState != .normal + let verdictReason = Self.planVerdictReason( + xCTestTierRan: kind.usesXCTestAccessibilityChannel, + state: effective.xCTestChannelState, + firstFailure: firstFailure + ) if recovered { NSLog( "AGENT_DEVICE_RUNNER_SNAPSHOT_RECOVERED backend=%@ reason=%@", kind.rawValue, - firstFailure?.reason ?? "sparse tree" + verdictReason?.reason ?? "sparse tree" ) } return stampedSnapshotPayload( capture, backend: kind, state: recovered ? .recovered : .healthy, - reason: recovered || firstFailure?.code == "requested-backend" ? firstFailure : nil + reason: recovered || firstFailure?.code == "requested-backend" ? verdictReason : nil ) } @@ -425,13 +449,18 @@ extension RunnerTests { } } + let terminalReason = Self.planVerdictReason( + xCTestTierRan: xCTestTierRan, + state: effective.xCTestChannelState, + firstFailure: firstFailure + ) let fallbackPayload = - best.map { stampedSnapshotPayload($0.capture, backend: $0.kind, state: .sparse, reason: firstFailure) } + best.map { stampedSnapshotPayload($0.capture, backend: $0.kind, state: .sparse, reason: terminalReason) } ?? stampedSnapshotPayload( SnapshotBackendCapture(payload: sparseTruncatedSnapshotPayload(), effectiveDepth: nil), backend: effectivePlan.last ?? plan.last ?? .recursiveTree, state: .sparse, - reason: firstFailure + reason: terminalReason ) return fallbackPayload } @@ -491,6 +520,11 @@ extension RunnerTests { } return (sweep.acquisition, sweep.outcome) case .privateAX: + #if AGENT_DEVICE_RUNNER_UNIT_TESTS + if let override = self.privateAXAcquisitionOverrideForTesting { + return (override(), .completed) + } + #endif return ( self.privateAXSnapshotAcquisition( target: target, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 3140827424..ae1b41951e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -5,6 +5,7 @@ // Created by Michał Pierzchała on 30/01/2026. // +import AgentDeviceSnapshotPresentation import XCTest import Network #if canImport(UIKit) @@ -199,6 +200,9 @@ final class RunnerTests: XCTestCase { // live SpringBoard alert. Production never compiles this property. Stored here (rather than in // the extension that reads it) because Swift extensions cannot hold stored properties. var systemModalProbeOverrideForTesting: (@MainActor (Date) -> DataPayload?)? + /// Stands in for the private AX tier's acquisition in unit-test builds, so a plan can meet the + /// out-of-process sheet the tier cannot read. Production builds compile none of this. + var privateAXAcquisitionOverrideForTesting: (() -> SnapshotAcquisition?)? var blockingSystemModalPresenceOverrideForTesting: Bool? var alertResolutionOverrideForTesting: (@MainActor (Date) -> RunnerAlert?)? var alertButtonHittabilityProbeOverrideForTesting: (@MainActor (Date) -> Bool)? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift index 285e1fd461..f3e9c5c234 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift @@ -380,5 +380,127 @@ extension RunnerTests { ) XCTAssertFalse(hasAbandonedMainThreadWork()) } + + /// The Save Password sheet shape: the channel is penalized and private AX cannot match the app + /// behind the out-of-process sheet. The deferred plan recovers through the bounded tree and + /// reports the bounded slice as its reason. That the sweep stays off the plan is the plan's own + /// contract (`effectiveSnapshotCapturePlan`). + func testPenalizedPlanRecoversThroughTheBoundedTreeWhenPrivateAXReadsNothing() throws { + let captureTarget = try launchPenalizedPrivateAXBlindTarget( + bundleId: "com.callstack.agentdevice.runner.penalized-tree-recovery-test" + ) + defer { tearDownPenalizedPrivateAXBlindTarget() } + + let payload = try runDeferredPlanOffMain(target: captureTarget) + + let quality = try XCTUnwrap(payload.snapshotQuality) + XCTAssertEqual(quality.reasonCode, "budget") + if quality.state == .sparse { + throw XCTSkip("the bounded tree did not answer within its slice on this host") + } + XCTAssertEqual(quality.state, .recovered) + XCTAssertEqual(quality.backend, SnapshotBackendKind.recursiveTree.rawValue) + XCTAssertGreaterThan(payload.nodes?.count ?? 0, 1, "the bounded tree answers with a real tree") + } + + /// When the bounded tree also grinds past its slice, the plan ends sparse, and the verdict still + /// names the bounded slice the capture ran on rather than the seeded pre-selection. + func testPenalizedPlanThatEndsSparseAfterTheBoundedTreeReportsTheBudget() throws { + guard + let snapshotMethod = class_getInstanceMethod( + XCUIApplication.self, + #selector(XCUIElement.snapshot) + ), + let stubMethod = class_getInstanceMethod( + RunnerBlockingSnapshotStub.self, + #selector(RunnerBlockingSnapshotStub.snapshot) + ) + else { + XCTFail("unable to install the blocking snapshot stub") + return + } + let captureTarget = try launchPenalizedPrivateAXBlindTarget( + bundleId: "com.callstack.agentdevice.runner.penalized-tree-timeout-test" + ) + RunnerBlockingSnapshotGate.release = DispatchSemaphore(value: 0) + RunnerBlockingSnapshotGate.entered = DispatchSemaphore(value: 0) + let originalImplementation = method_getImplementation(snapshotMethod) + method_setImplementation(snapshotMethod, method_getImplementation(stubMethod)) + defer { + RunnerBlockingSnapshotGate.release.signal() + method_setImplementation(snapshotMethod, originalImplementation) + let drainDeadline = Date().addingTimeInterval(3) + while hasAbandonedMainThreadWork(), Date() < drainDeadline { + RunLoop.current.run(until: Date().addingTimeInterval(0.05)) + } + tearDownPenalizedPrivateAXBlindTarget() + } + + let payload = try runDeferredPlanOffMain(target: captureTarget) { + RunnerBlockingSnapshotGate.release.signal() + } + + let quality = try XCTUnwrap(payload.snapshotQuality) + XCTAssertEqual(quality.state, .sparse) + XCTAssertEqual(quality.reasonCode, "budget") + } + + private func launchPenalizedPrivateAXBlindTarget(bundleId: String) throws -> SnapshotCaptureTarget { + app.launchArguments = ["--agent-device-selector-read-regression"] + app.launch() + XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) + XCTAssertFalse(app.frame.isEmpty) + MainActor.assumeIsolated { + mainOwned.app = app + mainOwned.bundleId = bundleId + } + snapshotXCTestPenaltyWarmupExemption.isPending = false + let captureTarget = MainActor.assumeIsolated { takeSnapshotCaptureTarget(app: app) } + penalizeSnapshotXCTestChannel(bundleId: captureTarget.bundleId, reason: "test-setup") + privateAXAcquisitionOverrideForTesting = { nil } + return captureTarget + } + + private func tearDownPenalizedPrivateAXBlindTarget() { + privateAXAcquisitionOverrideForTesting = nil + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + clearPrivateAXAcceptedDepth(reason: "test-cleanup") + MainActor.assumeIsolated { + invalidateCachedTarget(reason: "unit_test_cleanup") + } + app.terminate() + } + + /// Runs the regular plan for `target` off the main thread, as the command queue does, and + /// returns its payload; `afterPlan` runs on that queue once the plan has answered. + private func runDeferredPlanOffMain( + target: SnapshotCaptureTarget, + afterPlan: @escaping () -> Void = {} + ) throws -> DataPayload { + final class ResultBox { + var payload: DataPayload? + var error: Error? + } + let box = ResultBox() + let planned = expectation(description: "deferred plan answered") + DispatchQueue(label: "agent-device.runner.tests.deferred-plan").async { + do { + box.payload = try self.runSnapshotCapturePlan( + Self.regularVisiblePlan, + target: target, + options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), + terminal: .sparseWithFatalOnAXFailure, + deadline: Date().addingTimeInterval(20) + ) + } catch { + box.error = error + } + afterPlan() + planned.fulfill() + } + wait(for: [planned], timeout: 60) + if let error = box.error { throw error } + return try XCTUnwrap(box.payload) + } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift index 96d5eb1a3f..806db679f1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift @@ -403,14 +403,14 @@ extension RunnerTests { XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) } - func testEffectiveSnapshotCapturePlanDefersXCTestBackedTiersOnlyWhenPenalizedRegularPlan() { + func testEffectiveSnapshotCapturePlanPutsXCTestBackedTiersBehindPrivateAXOnlyWhenPenalizedRegularPlan() { let regular = Self.effectiveSnapshotCapturePlan( Self.regularVisiblePlan, xCTestChannelPenalized: true ) - XCTAssertEqual(regular.plan, [.privateAX]) + XCTAssertEqual(regular.plan, [.privateAX, .recursiveTree]) XCTAssertEqual(regular.xCTestChannelState, .deferredToIndependentBackend) - XCTAssertNil(regular.treeCaptureSliceBudgetOverride) + XCTAssertEqual(regular.treeCaptureSliceBudgetOverride, Self.penalizedXCTestProbeTreeSliceBudget) let unpenalized = Self.effectiveSnapshotCapturePlan( Self.regularVisiblePlan, @@ -430,6 +430,23 @@ extension RunnerTests { XCTAssertNil(raw.treeCaptureSliceBudgetOverride) } + func testPlanVerdictReasonNamesTheBoundedProbeOnceAnXCTestTierRanBehindPrivateAX() { + let deferred = Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend) + XCTAssertEqual( + Self.planVerdictReason( + xCTestTierRan: true, state: .deferredToIndependentBackend, firstFailure: deferred + )?.code, + "budget" + ) + XCTAssertEqual( + Self.planVerdictReason( + xCTestTierRan: false, state: .deferredToIndependentBackend, firstFailure: deferred + )?.code, + "deferred" + ) + XCTAssertNil(Self.planVerdictReason(xCTestTierRan: true, state: .normal, firstFailure: nil)) + } + func testEffectiveSnapshotCapturePlanUsesBoundedXCTestProbeWhenNoIndependentBackendRuns() { let physicalDevicePlan = Self.effectiveSnapshotCapturePlan( Self.regularVisiblePlan,