From 6cfe6a5f0e62da64f7922689e8b4d55c54ab37fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 18 Sep 2026 10:08:55 +0200 Subject: [PATCH 1/4] refactor(ios): declare bridge failure scope at the throw site, disclose unplaced surface hosts, move the coordinate-space rule into the presentation package Three follow-ups to #2653 (fixes #2612), each inside that PR's own new code. Failure scope. Whether a Simulator AX bridge failure retires the app generation from the bridge was decided at the route by matching the failure's kind and code. SnapshotSourceFailure now carries scope: 'capture' | 'generation', set where the failure is thrown (snapshotSourceCaptureError for bridge-preparation-pending and window-coordinate-space-unresolved), and the route reads the field. Disclosure. A runner capture that could not turn a rotated surface host back (orientation unreadable, or the private-AX tier anchored on the bridge's root box) published two coordinate spaces silently. SnapshotGeometrySpace.unplacedSurfaceHostCount runs after every tier's acquisition; a non-zero count rides the quality verdict as unresolvedCoordinateSpaceWindows and the daemon renders a warning. Package. The rotation table and the coordinate-space rule are pure CoreGraphics and now live in AgentDeviceSnapshotPresentation (SnapshotCoordinateSpace.swift) with their tests in the package suite, so swift test replays the golden table without a simulator. isSurfaceHost takes booleans instead of an XCUIElement.ElementType. --- CHANGELOG.md | 9 + .../RunnerScrollViewportPolicy.swift | 1 + .../RunnerTests+PrivateAXPresentation.swift | 2 +- .../RunnerTests+Snapshot.swift | 2 +- .../RunnerTests+SnapshotCapturePlan.swift | 33 ++ .../SnapshotCoordinateSpace.swift | 250 +++++++++ .../CoordinateSpaceTests.swift} | 474 ++++++------------ .../fixtures/window-coordinate-space.json | 2 +- .../adr/0004-ios-snapshot-backend-strategy.md | 19 +- .../src/snapshot-quality-verdict.test.ts | 17 + .../src/snapshot-quality-verdict.ts | 6 + .../snapshot-presentation/quality-warnings.ts | 15 + .../snapshot-presentation/warnings.test.ts | 17 + packages/kernel/src/snapshot.ts | 6 + .../src/snapshot-observability.test.ts | 2 +- .../platform-apple/src/snapshot-route.test.ts | 25 +- packages/platform-apple/src/snapshot-route.ts | 28 +- .../src/snapshot-source-facade.ts | 2 +- .../src/snapshot-source/adapter.test.ts | 1 + .../src/snapshot-source/adapter.ts | 9 +- .../src/snapshot-source/errors.ts | 19 +- .../src/snapshot-source/preparation.ts | 4 +- .../src/snapshot-source/types.ts | 10 + .../window-coordinate-space.test.ts | 6 +- .../window-coordinate-space.ts | 2 +- 25 files changed, 598 insertions(+), 363 deletions(-) create mode 100644 apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift rename apple/{runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift => snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift} (51%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 827ec0abea..88e60ccab3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,15 @@ in the app's own orientation space.` — and the route sends that one capture to the runner instead of retiring the app generation, so the next capture of a healthy app still uses the bridge. One table proves both languages apply one rule: `contracts/fixtures/window-coordinate-space.json` (#2612). +- Changed (ios): a runner capture that could not turn a rotated system surface back into the app's + orientation space — the interface orientation was unreadable, or the tier had only the bridge's + root box to anchor on — now says so instead of publishing two coordinate spaces silently: the + quality verdict carries `unresolvedCoordinateSpaceWindows` and the snapshot warns that rects under + those windows are not in the same space as the rest of the tree. Whether a Simulator AX bridge + failure retires the app generation from the bridge is now declared where the failure is thrown + (`scope: capture | generation`) rather than matched on its code at the route. The rotation table + and the coordinate-space rule moved from the XCTest bundle into the `AgentDeviceSnapshotPresentation` + package, where `swift test` replays the golden table without a simulator. - Added (limrun): `longpress` on Limrun iOS direct sessions. The interactor refused it as unsupported although the SDK exposes the HID primitives; it now holds one touch as a `performActions` batch of `touchDown`, `wait`, `touchUp`, defaulting to the 800 ms the Android diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift index 914cad4cde..18c8638222 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift @@ -1,4 +1,5 @@ import XCTest +import AgentDeviceSnapshotPresentation // The scroll viewport rule the runner shares with the TS runtime (#2500). // diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift index bfdf0db779..b2d8c8ae12 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift @@ -44,7 +44,7 @@ extension RunnerTests { let fields = privateAXFields(raw) let nodeSpace = SnapshotGeometrySpace.space( reportedBySurfaceHost: SnapshotGeometrySpace.isSurfaceHost( - elementType: fields.elementType, + isWindow: isWindowElement(fields.elementType), parentIsWindow: parentIsWindow ), reportedFrame: fields.rect, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 60e2e25e21..ef4eedd522 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -1069,7 +1069,7 @@ extension RunnerTests { ) -> SnapshotGeometrySpace { SnapshotGeometrySpace.space( reportedBySurfaceHost: SnapshotGeometrySpace.isSurfaceHost( - elementType: snapshot.elementType, + isWindow: isWindowElement(snapshot.elementType), parentIsWindow: parentIsWindow ), reportedFrame: snapshot.frame, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 7c3141c7a0..5ff3994583 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -27,6 +27,10 @@ struct SnapshotQuality: Codable { let collapsedLeafIndexes: [Int]? /// Coverage of the bounded custom-action pass, when the capture asked for one. let customActions: SnapshotCustomActionCoverage? + /// Surface hosts the published tree still carries in the device's native space: the capture could + /// not name the app's interface orientation to turn them back, so rects under them are not in the + /// space the rest of the tree publishes (#2612). Absent when every host was placed. + var unresolvedCoordinateSpaceWindows: Int? = nil /// Response-level timing for the accepted backend attempt, never repeated per node. var timing: SnapshotCaptureTiming? = nil } @@ -61,6 +65,8 @@ struct SnapshotBackendCapture { let effectiveDepth: Int? var customActions: SnapshotCustomActionCoverage? = nil var qualityPayload: DataPayload? = nil + /// Turned surface hosts the acquisition left as reported; nil when the tree holds none. + var unresolvedCoordinateSpaceWindows: Int? = nil var timing: SnapshotCaptureTiming? = nil } @@ -503,6 +509,13 @@ extension RunnerTests { var capture = presented capture.timing = timer.timing + // A host the walk could not place is disclosed, never silently published as an address: the + // orientation read failed, or the tier had only the bridge's root box to anchor on (#2612). + let unplaced = SnapshotGeometrySpace.unplacedSurfaceHostCount( + in: acquisition.nodes, + viewport: acquisition.viewport + ) + capture.unresolvedCoordinateSpaceWindows = unplaced > 0 ? unplaced : nil return SnapshotBackendAttempt( outcome: .captured(capture), timing: timer.timing @@ -625,6 +638,7 @@ extension RunnerTests { effectiveDepth: capture.effectiveDepth, collapsedLeafIndexes: Self.collapsedLeafIndexes(payload.nodes ?? []), customActions: capture.customActions, + unresolvedCoordinateSpaceWindows: capture.unresolvedCoordinateSpaceWindows, timing: capture.timing ) return DataPayload( @@ -876,6 +890,25 @@ extension RunnerTests { XCTAssertEqual(payload.nodes?.count, 1) } + /// A tree that still carries a turned host says so on the verdict, and one that holds none says + /// nothing: the daemon's warning keys off presence, so a zero must not ride the wire as a fact. + func testStampedPayloadDisclosesUnplacedSurfaceHosts() { + let payload = DataPayload( + nodes: [planTestNode(index: 0, type: "Application", label: "App")], + truncated: false + ) + let unplaced = stampedSnapshotPayload( + SnapshotBackendCapture( + payload: payload, effectiveDepth: nil, unresolvedCoordinateSpaceWindows: 1), + backend: .recursiveTree, state: "healthy", reason: nil) + XCTAssertEqual(unplaced.snapshotQuality?.unresolvedCoordinateSpaceWindows, 1) + + let placed = stampedSnapshotPayload( + SnapshotBackendCapture(payload: payload, effectiveDepth: nil), + backend: .recursiveTree, state: "healthy", reason: nil) + XCTAssertNil(placed.snapshotQuality?.unresolvedCoordinateSpaceWindows) + } + func testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance() { let complete = SnapshotBackendCapture( payload: DataPayload( diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift new file mode 100644 index 0000000000..51b5797920 --- /dev/null +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -0,0 +1,250 @@ +import Foundation +import CoreGraphics + +// Where a piece of geometry lives, and how it gets where the caller means it. +// +// DISPATCH: synthesized events skip XCTest's orientation handling, so a point planned in an app's +// interface space has to be rotated into the device's native (portrait-up) space before it is +// performed. Without that, a landscape tap lands somewhere else. +// +// CAPTURE: iOS hosts some system surfaces in that native space even while the app is rotated, so +// their whole subtree arrives quarter-turned. Measured on iPhone 17 Pro (iOS 26.2) with the system +// keyboard up in landscape: the app viewport is 874 x 402, `UIRemoteKeyboardWindow` reports its own +// box as 402 x 874, a key arrives 45 x 72 at x 154, and the key plane as 202 x 724 — a strip down +// the left edge — while the screenshot draws a 724 x 204 band docked at y 198, which is exactly +// what the live `app.keyboards` query measures. Rules that read the reported numbers refused app +// content the keyboard was nowhere near and let a tap through into a key (#2612, #2589). +// +// Both directions live in ONE table on purpose: capture rotates back with the exact inverse of what +// dispatch rotates forward, so a performed tap and a reported rect can never disagree about which +// pixel is which. The round-trip test in the package suite is that proof, not a comment asking for +// care. This file is pure geometry, which is why it lives here rather than in the XCTest bundle: +// nothing in it needs a simulator to run. + +/// The UIInterfaceOrientation raw values the rotation table switches on. XCTest exposes the app's +/// interface orientation as an integer, and 0 (unknown) deliberately rotates nothing. +public enum RunnerInterfaceOrientation { + public static let unknown = 0 + public static let portrait = 1 + public static let portraitUpsideDown = 2 + public static let landscapeRight = 3 + public static let landscapeLeft = 4 +} + +/// The quarter-turn between an app's interface space and the device's native (portrait-up) space. +public enum CoordinateSpaceRotation { + /// An interface-space point, in the native space synthesized events are performed in. + public static func native( + point: CGPoint, + in frame: CGRect, + interfaceOrientation: Int + ) -> CGPoint { + let localX = Double(point.x) - Double(frame.minX) + let localY = Double(point.y) - Double(frame.minY) + let width = Double(frame.width) + let height = Double(frame.height) + switch interfaceOrientation { + case RunnerInterfaceOrientation.landscapeRight: + return CGPoint(x: height - localY, y: localX) + case RunnerInterfaceOrientation.landscapeLeft: + return CGPoint(x: localY, y: width - localX) + case RunnerInterfaceOrientation.portraitUpsideDown: + return CGPoint(x: width - localX, y: height - localY) + default: // portrait, or an orientation the platform did not name + return CGPoint(x: localX, y: localY) + } + } + + /// An interface-space translation vector, in the same native space as `native(point:)`. + public static func native(vector: CGVector, interfaceOrientation: Int) -> CGVector { + switch interfaceOrientation { + case RunnerInterfaceOrientation.landscapeRight: + return CGVector(dx: -vector.dy, dy: vector.dx) + case RunnerInterfaceOrientation.landscapeLeft: + return CGVector(dx: vector.dy, dy: -vector.dx) + case RunnerInterfaceOrientation.portraitUpsideDown: + return CGVector(dx: -vector.dx, dy: -vector.dy) + default: // portrait, or an orientation the platform did not name + return vector + } + } + + /// The inverse of `native(point:)`: a native-space point, in the app's interface space. + public static func oriented( + point: CGPoint, + in frame: CGRect, + interfaceOrientation: Int + ) -> CGPoint { + let width = Double(frame.width) + let height = Double(frame.height) + let localX: Double + let localY: Double + switch interfaceOrientation { + case RunnerInterfaceOrientation.landscapeRight: + localX = Double(point.y) + localY = height - Double(point.x) + case RunnerInterfaceOrientation.landscapeLeft: + localX = width - Double(point.y) + localY = Double(point.x) + case RunnerInterfaceOrientation.portraitUpsideDown: + localX = width - Double(point.x) + localY = height - Double(point.y) + default: // portrait, or an orientation the platform did not name + localX = Double(point.x) + localY = Double(point.y) + } + return CGPoint(x: localX + Double(frame.minX), y: localY + Double(frame.minY)) + } + + /// The inverse of `native(point:)` for a rect. A quarter turn swaps the axes, so opposite corners + /// of the reported rect become opposite corners of the result; ordering them keeps the origin + /// top-left, which is what every consumer of a published rect reads. + public static func oriented( + rect: CGRect, + in frame: CGRect, + interfaceOrientation: Int + ) -> CGRect { + let leading = oriented( + point: CGPoint(x: rect.minX, y: rect.minY), + in: frame, + interfaceOrientation: interfaceOrientation + ) + let trailing = oriented( + point: CGPoint(x: rect.maxX, y: rect.maxY), + in: frame, + interfaceOrientation: interfaceOrientation + ) + return CGRect( + x: min(leading.x, trailing.x), + y: min(leading.y, trailing.y), + width: abs(trailing.x - leading.x), + height: abs(trailing.y - leading.y) + ) + } +} + +/// The space one captured subtree reports its rects in. +/// +/// RULE: the capture publishes every rect in the app's orientation space. A surface host declares the +/// space of its own subtree — its box is either the app's box, or the app's box turned through a +/// quarter, which is how a surface hosted in the device's native space announces itself — and anything +/// below it inherits that space. Nothing is declared, and every rect stays exactly as the platform +/// reported it, when the app frame is unusable, when the interface orientation names no quarter turn +/// (portrait, or an orientation the platform did not name), or when the app frame is square. Geometry +/// the capture cannot place is not a claim about where it is, and consumers already fail open on it the +/// way they do on any missing platform fact — but the capture says so: `unplacedSurfaceHostCount` +/// counts the hosts a published tree still carries turned, so the daemon can warn instead of +/// publishing two spaces silently. +/// +/// A half turn is a different claim: an upside-down app's own box is already the native box, so no +/// reported window distinguishes the two spaces and nothing is turned back. That is the same limit the +/// box-only rule accepts for a square app frame, and it is why the consumer rules in +/// `tap-keyboard-occlusion.ts` stay rather than becoming an assertion here. +public enum SnapshotGeometrySpace: Equatable { + /** The app's own orientation space: the one every consumer of a published tree reads. */ + case appOrientation + /** The device's native (portrait-up) space, with the way back to the app's space attached. */ + case deviceNative(appFrame: CGRect, interfaceOrientation: Int) + + /// How far a window's side lengths may miss the app's swapped side lengths and still be that + /// quarter turn. Measured captures match to the point; this absorbs float representation only. + public static let quarterTurnTolerance: Double = 1 + + /// The reported rect, in the space the capture publishes. + public func orientedFrame(of reportedFrame: CGRect) -> CGRect { + switch self { + case .appOrientation: + return reportedFrame + case .deviceNative(let appFrame, let interfaceOrientation): + return CoordinateSpaceRotation.oriented( + rect: reportedFrame, + in: appFrame, + interfaceOrientation: interfaceOrientation + ) + } + } + + /// The space one node's subtree reports in. + /// + /// Only a surface host may declare a space: the window itself, or the surface the window hands its + /// content to. XCTest reports `UIRemoteKeyboardWindow` with the app's own box and its child with + /// the turned one, so the window alone is not where the turn shows up, while a deep node claiming a + /// turned box is content whose reported bounds happen to be large and is left in the inherited + /// space. A host that reports anything other than a turned box declares the app's space for its + /// subtree, which is how a tree with no rotated surface stays exactly as reported. + public static func space( + reportedBySurfaceHost isSurfaceHost: Bool, + reportedFrame: CGRect, + inheritedFrom inherited: SnapshotGeometrySpace, + appFrame: CGRect, + interfaceOrientation: Int + ) -> SnapshotGeometrySpace { + guard isSurfaceHost else { return inherited } + guard namesQuarterTurn(interfaceOrientation) else { return .appOrientation } + guard isQuarterTurned(reportedFrame, relativeTo: appFrame) else { return .appOrientation } + return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) + } + + /// Whether this orientation is a turn the reported box can be measured against. Portrait and an + /// unnamed orientation leave the device's native space upright, so nothing under them is turned. + public static func namesQuarterTurn(_ interfaceOrientation: Int) -> Bool { + interfaceOrientation == RunnerInterfaceOrientation.landscapeLeft + || interfaceOrientation == RunnerInterfaceOrientation.landscapeRight + } + + /// Whether this node, or the window above it, is where a hosted surface's box appears. + public static func isSurfaceHost(isWindow: Bool, parentIsWindow: Bool) -> Bool { + isWindow || parentIsWindow + } + + /// The element type names under which a producer publishes the app's windows: the app's own + /// window, `UITextEffectsWindow`, `UIRemoteKeyboardWindow`. XCTest models each as its own element + /// typed `.application` or `.window`, and every producer names them this way in a published tree. + public static func isWindowType(_ type: String) -> Bool { + type == "Application" || type == "Window" + } + + /// Surface hosts a published tree still carries in the device's native space. + /// + /// A capture that could name the app's interface orientation turned every such host back, so its + /// published tree holds none. One that could not — the orientation read failed, or the tier had + /// only the bridge's own root box to anchor on — publishes the host as the platform reported it, + /// and this is where that shows: a host whose box is the viewport quarter-turned. The count is a + /// disclosure, not a repair; it rides the quality verdict so the daemon can say which captures hold + /// two spaces instead of letting rects under those hosts read as addresses (#2612). + public static func unplacedSurfaceHostCount(in nodes: [RawAXNode], viewport: CGRect) -> Int { + guard isPlottable(viewport) else { return 0 } + var count = 0 + for node in nodes { + let parentIsWindow = node.parentIndex.map { index in + nodes.indices.contains(index) && isWindowType(nodes[index].type) + } ?? false + guard isSurfaceHost(isWindow: isWindowType(node.type), parentIsWindow: parentIsWindow) + else { continue } + let frame = CGRect( + x: node.rect.x, y: node.rect.y, width: node.rect.width, height: node.rect.height + ) + if isQuarterTurned(frame, relativeTo: viewport) { count += 1 } + } + return count + } + + /// A window that reports the app's two side lengths on the other axes is hosted in the device's + /// native space. A square app frame cannot be told from its own quarter turn, so it is left alone + /// rather than guessed at. + private static func isQuarterTurned(_ frame: CGRect, relativeTo appFrame: CGRect) -> Bool { + guard isPlottable(frame), isPlottable(appFrame), + abs(appFrame.width - appFrame.height) > quarterTurnTolerance + else { + return false + } + return abs(frame.width - appFrame.height) <= quarterTurnTolerance + && abs(frame.height - appFrame.width) <= quarterTurnTolerance + } + + private static func isPlottable(_ frame: CGRect) -> Bool { + frame.origin.x.isFinite && frame.origin.y.isFinite + && frame.width.isFinite && frame.height.isFinite + && frame.width > 0 && frame.height > 0 + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift similarity index 51% rename from apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift rename to apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index f2a06c0415..9664943501 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -1,272 +1,61 @@ +import AgentDeviceSnapshotPresentation +import CoreGraphics +import Foundation import XCTest -// Where a piece of geometry lives, and how it gets where the caller means it. -// -// DISPATCH: synthesized events skip XCTest's orientation handling, so a point planned in an app's -// interface space has to be rotated into the device's native (portrait-up) space before it is -// performed. Without that, a landscape tap lands somewhere else. -// -// CAPTURE: iOS hosts some system surfaces in that native space even while the app is rotated, so -// their whole subtree arrives quarter-turned. Measured on iPhone 17 Pro (iOS 26.2) with the system -// keyboard up in landscape: the app viewport is 874 x 402, `UIRemoteKeyboardWindow` reports its own -// box as 402 x 874, a key arrives 45 x 72 at x 154, and the key plane as 202 x 724 — a strip down -// the left edge — while the screenshot draws a 724 x 204 band docked at y 198, which is exactly -// what the live `app.keyboards` query measures. Rules that read the reported numbers refused app -// content the keyboard was nowhere near and let a tap through into a key (#2612, #2589). -// -// Both directions live in ONE table on purpose: capture rotates back with the exact inverse of what -// dispatch rotates forward, so a performed tap and a reported rect can never disagree about which -// pixel is which. The round-trip test below is that proof, not a comment asking for care. - -/// The UIInterfaceOrientation raw values the rotation table switches on. XCTest exposes the app's -/// interface orientation as an integer, and 0 (unknown) deliberately rotates nothing. -enum RunnerInterfaceOrientation { - static let unknown = 0 - static let portrait = 1 - static let portraitUpsideDown = 2 - static let landscapeRight = 3 - static let landscapeLeft = 4 -} - -/// The quarter-turn between an app's interface space and the device's native (portrait-up) space. -enum CoordinateSpaceRotation { - /// An interface-space point, in the native space synthesized events are performed in. - static func native( - point: CGPoint, - in frame: CGRect, - interfaceOrientation: Int - ) -> CGPoint { - let localX = Double(point.x) - Double(frame.minX) - let localY = Double(point.y) - Double(frame.minY) - let width = Double(frame.width) - let height = Double(frame.height) - switch interfaceOrientation { - case RunnerInterfaceOrientation.landscapeRight: - return CGPoint(x: height - localY, y: localX) - case RunnerInterfaceOrientation.landscapeLeft: - return CGPoint(x: localY, y: width - localX) - case RunnerInterfaceOrientation.portraitUpsideDown: - return CGPoint(x: width - localX, y: height - localY) - default: // portrait, or an orientation the platform did not name - return CGPoint(x: localX, y: localY) - } - } - - /// An interface-space translation vector, in the same native space as `native(point:)`. - static func native(vector: CGVector, interfaceOrientation: Int) -> CGVector { - switch interfaceOrientation { - case RunnerInterfaceOrientation.landscapeRight: - return CGVector(dx: -vector.dy, dy: vector.dx) - case RunnerInterfaceOrientation.landscapeLeft: - return CGVector(dx: vector.dy, dy: -vector.dx) - case RunnerInterfaceOrientation.portraitUpsideDown: - return CGVector(dx: -vector.dx, dy: -vector.dy) - default: // portrait, or an orientation the platform did not name - return vector - } - } - - /// The inverse of `native(point:)`: a native-space point, in the app's interface space. - static func oriented( - point: CGPoint, - in frame: CGRect, - interfaceOrientation: Int - ) -> CGPoint { - let width = Double(frame.width) - let height = Double(frame.height) - let localX: Double - let localY: Double - switch interfaceOrientation { - case RunnerInterfaceOrientation.landscapeRight: - localX = Double(point.y) - localY = height - Double(point.x) - case RunnerInterfaceOrientation.landscapeLeft: - localX = width - Double(point.y) - localY = Double(point.x) - case RunnerInterfaceOrientation.portraitUpsideDown: - localX = width - Double(point.x) - localY = height - Double(point.y) - default: // portrait, or an orientation the platform did not name - localX = Double(point.x) - localY = Double(point.y) - } - return CGPoint(x: localX + Double(frame.minX), y: localY + Double(frame.minY)) - } - - /// The inverse of `native(point:)` for a rect. A quarter turn swaps the axes, so opposite corners - /// of the reported rect become opposite corners of the result; ordering them keeps the origin - /// top-left, which is what every consumer of a published rect reads. - static func oriented( - rect: CGRect, - in frame: CGRect, - interfaceOrientation: Int - ) -> CGRect { - let leading = oriented( - point: CGPoint(x: rect.minX, y: rect.minY), - in: frame, - interfaceOrientation: interfaceOrientation - ) - let trailing = oriented( - point: CGPoint(x: rect.maxX, y: rect.maxY), - in: frame, - interfaceOrientation: interfaceOrientation - ) - return CGRect( - x: min(leading.x, trailing.x), - y: min(leading.y, trailing.y), - width: abs(trailing.x - leading.x), - height: abs(trailing.y - leading.y) - ) - } -} - -/// The space one captured subtree reports its rects in. -/// -/// RULE: the capture publishes every rect in the app's orientation space. A surface host declares the -/// space of its own subtree — its box is either the app's box, or the app's box turned through a -/// quarter, which is how a surface hosted in the device's native space announces itself — and anything -/// below it inherits that space. Nothing is declared, and every rect stays exactly as the platform -/// reported it, when the app frame is unusable, when the interface orientation names no quarter turn -/// (portrait, or an orientation the platform did not name), or when the app frame is square. Geometry -/// the capture cannot place is not a claim about where it is, and consumers already fail open on it the -/// way they do on any missing platform fact. -/// -/// A half turn is a different claim: an upside-down app's own box is already the native box, so no -/// reported window distinguishes the two spaces and nothing is turned back. That is the same limit the -/// box-only rule accepts for a square app frame, and it is why the consumer rules in -/// `tap-keyboard-occlusion.ts` stay rather than becoming an assertion here. -enum SnapshotGeometrySpace: Equatable { - /** The app's own orientation space: the one every consumer of a published tree reads. */ - case appOrientation - /** The device's native (portrait-up) space, with the way back to the app's space attached. */ - case deviceNative(appFrame: CGRect, interfaceOrientation: Int) +/// The rotation table and the space a captured window declares (#2612). Pure geometry: it runs here, +/// without a simulator, and the runner's walkers are tested separately for threading it through. +final class CoordinateSpaceTests: XCTestCase { + private struct WindowCoordinateSpaceFixture: Decodable { + /// A frame as JSON can carry one. Infinity has no JSON spelling, so the unusable box a platform + /// hands back is named `{"infinite": true}`: `CGRect.infinite` here, and an infinite rect in the + /// vitest twin. + struct Frame: Decodable { + private enum CodingKeys: String, CodingKey { + case x, y, width, height, infinite + } - /// How far a window's side lengths may miss the app's swapped side lengths and still be that - /// quarter turn. Measured captures match to the point; this absorbs float representation only. - static let quarterTurnTolerance: Double = 1 + let cgRect: CGRect - /// The reported rect, in the space the capture publishes. - func orientedFrame(of reportedFrame: CGRect) -> CGRect { - switch self { - case .appOrientation: - return reportedFrame - case .deviceNative(let appFrame, let interfaceOrientation): - return CoordinateSpaceRotation.oriented( - rect: reportedFrame, - in: appFrame, - interfaceOrientation: interfaceOrientation - ) + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + guard try container.decodeIfPresent(Bool.self, forKey: .infinite) != true else { + self.cgRect = .infinite + return + } + self.cgRect = CGRect( + x: try container.decode(Double.self, forKey: .x), + y: try container.decode(Double.self, forKey: .y), + width: try container.decode(Double.self, forKey: .width), + height: try container.decode(Double.self, forKey: .height) + ) + } } - } - - /// The space one node's subtree reports in. - /// - /// Only a surface host may declare a space: the window itself, or the surface the window hands its - /// content to. XCTest reports `UIRemoteKeyboardWindow` with the app's own box and its child with - /// the turned one, so the window alone is not where the turn shows up, while a deep node claiming a - /// turned box is content whose reported bounds happen to be large and is left in the inherited - /// space. A host that reports anything other than a turned box declares the app's space for its - /// subtree, which is how a tree with no rotated surface stays exactly as reported. - static func space( - reportedBySurfaceHost isSurfaceHost: Bool, - reportedFrame: CGRect, - inheritedFrom inherited: SnapshotGeometrySpace, - appFrame: CGRect, - interfaceOrientation: Int - ) -> SnapshotGeometrySpace { - guard isSurfaceHost else { return inherited } - guard namesQuarterTurn(interfaceOrientation) else { return .appOrientation } - guard isQuarterTurned(reportedFrame, relativeTo: appFrame) else { return .appOrientation } - return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) - } - /// Whether this orientation is a turn the reported box can be measured against. Portrait and an - /// unnamed orientation leave the device's native space upright, so nothing under them is turned. - static func namesQuarterTurn(_ interfaceOrientation: Int) -> Bool { - interfaceOrientation == RunnerInterfaceOrientation.landscapeLeft - || interfaceOrientation == RunnerInterfaceOrientation.landscapeRight - } - - /// Whether this node, or the window above it, is where a hosted surface's box appears. - static func isSurfaceHost( - elementType: XCUIElement.ElementType?, - parentIsWindow: Bool - ) -> Bool { - elementType == .application || elementType == .window || parentIsWindow - } - - /// A window that reports the app's two side lengths on the other axes is hosted in the device's - /// native space. A square app frame cannot be told from its own quarter turn, so it is left alone - /// rather than guessed at. - private static func isQuarterTurned(_ frame: CGRect, relativeTo appFrame: CGRect) -> Bool { - guard isPlottable(frame), isPlottable(appFrame), - abs(appFrame.width - appFrame.height) > quarterTurnTolerance - else { - return false + struct Constants: Decodable { + let quarterTurnTolerance: Double } - return abs(frame.width - appFrame.height) <= quarterTurnTolerance - && abs(frame.height - appFrame.width) <= quarterTurnTolerance - } - - private static func isPlottable(_ frame: CGRect) -> Bool { - frame.origin.x.isFinite && frame.origin.y.isFinite - && frame.width.isFinite && frame.height.isFinite - && frame.width > 0 && frame.height > 0 - } -} -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct WindowCoordinateSpaceFixture: Decodable { - /// A frame as JSON can carry one. Infinity has no JSON spelling, so the unusable box a platform - /// hands back is named `{"infinite": true}`: `CGRect.infinite` here, and an infinite rect in the - /// vitest twin. - struct Frame: Decodable { - private enum CodingKeys: String, CodingKey { - case x, y, width, height, infinite + struct QuarterTurnCase: Decodable { + let name: String + let window: Frame + let app: Frame + let quarterTurned: Bool } - let cgRect: CGRect - - init(from decoder: Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - guard try container.decodeIfPresent(Bool.self, forKey: .infinite) != true else { - self.cgRect = .infinite - return - } - self.cgRect = CGRect( - x: try container.decode(Double.self, forKey: .x), - y: try container.decode(Double.self, forKey: .y), - width: try container.decode(Double.self, forKey: .width), - height: try container.decode(Double.self, forKey: .height) - ) + struct RotationCase: Decodable { + let name: String + let interfaceOrientation: Int + let app: Frame + let native: Frame + let oriented: Frame } - } - - struct Constants: Decodable { - let quarterTurnTolerance: Double - } - - struct QuarterTurnCase: Decodable { - let name: String - let window: Frame - let app: Frame - let quarterTurned: Bool - } - struct RotationCase: Decodable { - let name: String - let interfaceOrientation: Int - let app: Frame - let native: Frame - let oriented: Frame + let constants: Constants + let quarterTurnCases: [QuarterTurnCase] + let rotationCases: [RotationCase] } - let constants: Constants - let quarterTurnCases: [QuarterTurnCase] - let rotationCases: [RotationCase] -} - -extension RunnerTests { /// Identity in portrait/unknown, 90° per landscape, 180° upside-down. func testNativeSynthesizedPointRotatesByInterfaceOrientation() { let portrait = CGRect(x: 0, y: 0, width: 834, height: 1210) @@ -429,7 +218,7 @@ extension RunnerTests { XCTAssertEqual( SnapshotGeometrySpace.space( reportedBySurfaceHost: SnapshotGeometrySpace.isSurfaceHost( - elementType: XCUIElement.ElementType(rawValue: 20), + isWindow: false, parentIsWindow: false ), reportedFrame: CGRect(x: 154, y: 77, width: 45, height: 72), @@ -489,6 +278,85 @@ extension RunnerTests { ) } + /// Where a turned box has to appear before it declares a space. + func testOnlyASurfaceHostDeclaresTheCoordinateSpaceOfItsSubtree() { + let appFrame = CGRect(x: 0, y: 0, width: 874, height: 402) + let turned = CGRect(x: 0, y: 0, width: 402, height: 874) + let landscape = RunnerInterfaceOrientation.landscapeRight + + // XCTest gives `UIRemoteKeyboardWindow` the app's own box and its child the turned one, so the + // surface below a window is a host too. + XCTAssertTrue(SnapshotGeometrySpace.isSurfaceHost(isWindow: false, parentIsWindow: true)) + XCTAssertFalse(SnapshotGeometrySpace.isSurfaceHost(isWindow: false, parentIsWindow: false)) + XCTAssertTrue(SnapshotGeometrySpace.isWindowType("Application")) + XCTAssertTrue(SnapshotGeometrySpace.isWindowType("Window")) + XCTAssertFalse(SnapshotGeometrySpace.isWindowType("Other")) + + let windowSpace = SnapshotGeometrySpace.space( + reportedBySurfaceHost: true, + reportedFrame: appFrame, + inheritedFrom: .appOrientation, + appFrame: appFrame, + interfaceOrientation: landscape + ) + XCTAssertEqual(windowSpace, .appOrientation) + let surfaceSpace = SnapshotGeometrySpace.space( + reportedBySurfaceHost: true, + reportedFrame: turned, + inheritedFrom: windowSpace, + appFrame: appFrame, + interfaceOrientation: landscape + ) + XCTAssertEqual(surfaceSpace, .deviceNative(appFrame: appFrame, interfaceOrientation: landscape)) + // Deep in the tree a turned box is content reporting large bounds, not a hosted surface: it keeps + // the space it inherited rather than rewriting the space below it. + XCTAssertEqual( + SnapshotGeometrySpace.space( + reportedBySurfaceHost: false, + reportedFrame: turned, + inheritedFrom: windowSpace, + appFrame: appFrame, + interfaceOrientation: landscape + ), + .appOrientation + ) + } + + /// A published tree that still carries a turned host is one the capture could not place, and the + /// count is what lets the daemon say so. A tree the capture did turn back holds none. + func testUnplacedSurfaceHostCountNamesTheHostsATreeStillCarriesTurned() { + let viewport = CGRect(x: 0, y: 0, width: 874, height: 402) + // Application > [Window(app box) > Button, Window(app box) > Other(turned) > Key] + let unplaced = [ + node(0, type: "Application", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: nil), + node(1, type: "Window", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: 0), + node(2, type: "Button", rect: SnapshotRect(x: 204, y: 323, width: 91, height: 55), parent: 1), + node(3, type: "Window", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: 0), + node(4, type: "Other", rect: SnapshotRect(x: 0, y: 0, width: 402, height: 874), parent: 3), + node(5, type: "Key", rect: SnapshotRect(x: 154, y: 77, width: 45, height: 72), parent: 4), + ] + XCTAssertEqual(SnapshotGeometrySpace.unplacedSurfaceHostCount(in: unplaced, viewport: viewport), 1) + + // The same screen once the capture turned the host back: nothing is left to disclose. + let placed = unplaced.map { entry in + entry.index == 4 + ? node(4, type: "Other", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: 3) + : entry + } + XCTAssertEqual(SnapshotGeometrySpace.unplacedSurfaceHostCount(in: placed, viewport: viewport), 0) + + // A turned box deep in the tree is content, not a host; a viewport the capture could not resolve + // anchors nothing. + let deep = [ + node(0, type: "Application", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: nil), + node(1, type: "Window", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: 0), + node(2, type: "Other", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: 1), + node(3, type: "Other", rect: SnapshotRect(x: 0, y: 0, width: 402, height: 874), parent: 2), + ] + XCTAssertEqual(SnapshotGeometrySpace.unplacedSurfaceHostCount(in: deep, viewport: viewport), 0) + XCTAssertEqual(SnapshotGeometrySpace.unplacedSurfaceHostCount(in: unplaced, viewport: .infinite), 0) + } + /// Golden parity table (#2612): every case in contracts/fixtures/window-coordinate-space.json /// must agree with the vitest twin /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). Add @@ -497,9 +365,9 @@ extension RunnerTests { /// rotation rows carry their own orientation. func testWindowCoordinateSpaceMatchesGoldenParityTable() throws { let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner + .deletingLastPathComponent() // AgentDeviceSnapshotPresentationTests + .deletingLastPathComponent() // Tests + .deletingLastPathComponent() // snapshot-presentation .deletingLastPathComponent() // apple .deletingLastPathComponent() // repo root .appendingPathComponent("contracts") @@ -536,6 +404,14 @@ extension RunnerTests { "\(testCase.name) (interfaceOrientation \(interfaceOrientation))" ) } + // The disclosure reads the same rule off a published node: a host the table calls turned is + // the host the count names, whichever way the app is turned. + let host = node(0, type: "Window", rect: snapshotRect(testCase.window.cgRect), parent: nil) + XCTAssertEqual( + SnapshotGeometrySpace.unplacedSurfaceHostCount(in: [host], viewport: appFrame), + testCase.quarterTurned ? 1 : 0, + "\(testCase.name) (unplaced host count)" + ) } for testCase in fixture.rotationCases { XCTAssertEqual( @@ -549,60 +425,34 @@ extension RunnerTests { ) } } -} -#endif -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - /// Where a turned box has to appear before it declares a space. - func testOnlyASurfaceHostDeclaresTheCoordinateSpaceOfItsSubtree() { - let appFrame = CGRect(x: 0, y: 0, width: 874, height: 402) - let turned = CGRect(x: 0, y: 0, width: 402, height: 874) - let landscape = RunnerInterfaceOrientation.landscapeRight - - // XCTest gives `UIRemoteKeyboardWindow` the app's own box and its child the turned one, so the - // surface below a window is a host too. - XCTAssertTrue( - SnapshotGeometrySpace.isSurfaceHost( - elementType: XCUIElement.ElementType(rawValue: 21), - parentIsWindow: true - ) - ); - XCTAssertFalse( - SnapshotGeometrySpace.isSurfaceHost( - elementType: XCUIElement.ElementType(rawValue: 21), - parentIsWindow: false - ) - ); - - let windowSpace = SnapshotGeometrySpace.space( - reportedBySurfaceHost: true, - reportedFrame: appFrame, - inheritedFrom: .appOrientation, - appFrame: appFrame, - interfaceOrientation: landscape - ) - XCTAssertEqual(windowSpace, .appOrientation) - let surfaceSpace = SnapshotGeometrySpace.space( - reportedBySurfaceHost: true, - reportedFrame: turned, - inheritedFrom: windowSpace, - appFrame: appFrame, - interfaceOrientation: landscape - ) - XCTAssertEqual(surfaceSpace, .deviceNative(appFrame: appFrame, interfaceOrientation: landscape)) - // Deep in the tree a turned box is content reporting large bounds, not a hosted surface: it keeps - // the space it inherited rather than rewriting the space below it. - XCTAssertEqual( - SnapshotGeometrySpace.space( - reportedBySurfaceHost: false, - reportedFrame: turned, - inheritedFrom: windowSpace, - appFrame: appFrame, - interfaceOrientation: landscape - ), - .appOrientation + private func node(_ index: Int, type: String, rect: SnapshotRect, parent: Int?) -> RawAXNode { + RawAXNode( + index: index, + type: type, + label: nil, + identifier: nil, + value: nil, + rect: rect, + enabled: true, + focused: nil, + selected: nil, + hittable: false, + depth: parent == nil ? 0 : 1, + parentIndex: parent, + hiddenContentAbove: nil, + hiddenContentBelow: nil ) } + + private func snapshotRect(_ frame: CGRect) -> SnapshotRect { + guard frame.origin.x.isFinite, frame.origin.y.isFinite, frame.width.isFinite, + frame.height.isFinite + else { + return SnapshotRect(x: 0, y: 0, width: 0, height: 0) + } + return SnapshotRect( + x: Double(frame.minX), y: Double(frame.minY), + width: Double(frame.width), height: Double(frame.height)) + } } -#endif diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index 1424410655..a8497d9b07 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -1,5 +1,5 @@ { - "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side.", + "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side.", "constants": { "quarterTurnTolerance": 1 }, diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index e566b71489..f18fea84bf 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -405,11 +405,15 @@ oriented frame, so an address and the permission to tap it cannot disagree. Wher named — no usable app frame, an interface orientation the platform did not report, or an app frame square enough to be indistinguishable from its own quarter turn — the capture publishes what the platform reported rather than guessing at a rotation, and consumers fail open on that geometry the -way they do on any missing platform fact. Detection and the way back share one rotation table with -synthesized dispatch, and both languages replay it against -`contracts/fixtures/window-coordinate-space.json` in the ADR 0011 parity-table shape, which is what -stops capture from turning back with anything other than the exact inverse of what dispatch turns -forward. +way they do on any missing platform fact. It does not do so silently: a published tree that still +carries a surface host reporting the viewport quarter-turned is counted after every tier's +acquisition and stamped on the quality verdict as `unresolvedCoordinateSpaceWindows`, which the +daemon renders as a warning, so a tree holding two spaces never looks like one. Detection, the way +back and that count share one rotation table with synthesized dispatch — the pure geometry lives in +the `AgentDeviceSnapshotPresentation` package (`SnapshotCoordinateSpace.swift`), not in the XCTest +bundle — and both languages replay it against `contracts/fixtures/window-coordinate-space.json` in +the ADR 0011 parity-table shape, which is what stops capture from turning back with anything other +than the exact inverse of what dispatch turns forward. Decision. A producer that cannot name the app's interface orientation refuses the screen rather than publishing two spaces in one tree. The Simulator AX bridge's attribute set carries no @@ -427,8 +431,9 @@ hybrid app, so its screens keep the runner until it relaunches. A rotated surfac the screen in front of the reader and about nothing else — it is up now and gone after the next keystroke — so retiring the generation would move every later portrait capture of a healthy app onto the runner to work around one landscape keyboard, the cost #2491 settled for a bridge that was -merely still being prepared. Which side of that line a failure falls on is a declared property of -its code rather than a judgment made at the call site. +merely still being prepared. Which side of that line a failure falls on is declared where the +failure is thrown — `SnapshotSourceFailure.scope` is `capture` or `generation`, and the route reads +the field — rather than judged at the route from the failure's kind or code. What stays unnormalized, and why the consumer rule stays: the runner's query-sweep tier has no window ancestry, so no node in its tree can declare a space, and the provider producers diff --git a/packages/capture-kit/src/snapshot-quality-verdict.test.ts b/packages/capture-kit/src/snapshot-quality-verdict.test.ts index ea8afbc157..a752834264 100644 --- a/packages/capture-kit/src/snapshot-quality-verdict.test.ts +++ b/packages/capture-kit/src/snapshot-quality-verdict.test.ts @@ -30,6 +30,23 @@ test('readSnapshotQualityVerdict accepts a well-formed verdict', () => { }); }); +test('readSnapshotQualityVerdict carries a non-zero unplaced-geometry disclosure and drops a zero', () => { + const disclosed = readSnapshotQualityVerdict({ + state: 'healthy', + backend: 'tree', + unresolvedCoordinateSpaceWindows: 1, + }); + assert.equal(disclosed?.unresolvedCoordinateSpaceWindows, 1); + + const placed = readSnapshotQualityVerdict({ + state: 'healthy', + backend: 'tree', + unresolvedCoordinateSpaceWindows: 0, + }); + assert.equal(placed?.state, 'healthy'); + assert.equal('unresolvedCoordinateSpaceWindows' in placed!, false); +}); + test('readSnapshotQualityVerdict drops incomplete phase timing without dropping the verdict', () => { const verdict = readSnapshotQualityVerdict({ state: 'healthy', diff --git a/packages/capture-kit/src/snapshot-quality-verdict.ts b/packages/capture-kit/src/snapshot-quality-verdict.ts index 2913abe32f..43764ff518 100644 --- a/packages/capture-kit/src/snapshot-quality-verdict.ts +++ b/packages/capture-kit/src/snapshot-quality-verdict.ts @@ -57,6 +57,12 @@ export function readSnapshotQualityVerdict(value: unknown): SnapshotQualityVerdi collapsedLeafIndexes: Array.isArray(raw.collapsedLeafIndexes) ? raw.collapsedLeafIndexes.filter((entry): entry is number => typeof entry === 'number') : undefined, + // A zero is not a disclosure; the runner omits it, and a reader that emitted it would make the + // warning below key off a value instead of a fact. + ...(typeof raw.unresolvedCoordinateSpaceWindows === 'number' && + raw.unresolvedCoordinateSpaceWindows > 0 + ? { unresolvedCoordinateSpaceWindows: raw.unresolvedCoordinateSpaceWindows } + : {}), ...(timing ? { timing } : {}), }; } diff --git a/packages/capture-kit/src/snapshot/snapshot-presentation/quality-warnings.ts b/packages/capture-kit/src/snapshot/snapshot-presentation/quality-warnings.ts index 0f79547323..a5b222a266 100644 --- a/packages/capture-kit/src/snapshot/snapshot-presentation/quality-warnings.ts +++ b/packages/capture-kit/src/snapshot/snapshot-presentation/quality-warnings.ts @@ -7,11 +7,26 @@ export function renderSnapshotQualityWarnings( return [ ...stateWarning(verdict), ...customActionCoverageWarning(verdict), + ...unplacedGeometryWarning(verdict), ...depthWarning(verdict), ...collapsedLeafWarnings(verdict, nodes), ]; } +/** + * The capture publishes one coordinate space, and says so when it could not: a system surface + * hosted in the device's native orientation space that the runner could not turn back, because the + * app's interface orientation was unreadable (#2612). Without this line the tree would hold two + * spaces and look like one. + */ +function unplacedGeometryWarning(verdict: SnapshotQualityVerdict): string[] { + const windows = verdict.unresolvedCoordinateSpaceWindows; + if (windows === undefined) return []; + return [ + `${windows} window(s) on this screen — typically the system keyboard while the app is in landscape — report their geometry in the device's native orientation space, and this capture could not read the app's interface orientation to turn it back. Rects under those windows are not in the same space as the rest of the tree, so distances measured against them and taps aimed at them are unreliable; use screenshot as visual truth for that area and re-run snapshot after the surface is dismissed.`, + ]; +} + function customActionCoverageWarning(verdict: SnapshotQualityVerdict): string[] { const coverage = verdict.customActions; if (!coverage) return []; diff --git a/packages/capture-kit/src/snapshot/snapshot-presentation/warnings.test.ts b/packages/capture-kit/src/snapshot/snapshot-presentation/warnings.test.ts index 978b53969b..0380a90e37 100644 --- a/packages/capture-kit/src/snapshot/snapshot-presentation/warnings.test.ts +++ b/packages/capture-kit/src/snapshot/snapshot-presentation/warnings.test.ts @@ -20,6 +20,23 @@ test("the runner-wire 'deferred' reasonCode survives parsing into warning suppre assert.deepEqual(renderSnapshotQualityWarnings(verdict!, []), []); }); +test('a capture that could not place a turned surface host discloses it through the wire verdict', () => { + // End-to-end through the raw wire parser, like the deferred case above: the count has to be a + // member of the read verdict or the disclosure is silently stripped. + const verdict = readSnapshotQualityVerdict({ + state: 'healthy', + backend: 'tree', + unresolvedCoordinateSpaceWindows: 1, + }); + const warnings = renderSnapshotQualityWarnings(verdict!, []); + assert.equal(warnings.length, 1); + assert.match(warnings[0]!, /^1 window\(s\) on this screen/); + assert.match(warnings[0]!, /could not read the app's interface orientation/); + + const placed = readSnapshotQualityVerdict({ state: 'healthy', backend: 'tree' }); + assert.deepEqual(renderSnapshotQualityWarnings(placed!, []), []); +}); + test('penalty-deferred recovered captures suppress the fallback warning but keep depth copy', () => { // Every capture of a hostile screen re-stamps recovered/deferred; the capture that armed the // penalty already carried the full warning, so repeating it each capture is context noise. diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 30839bd036..ac2b03213a 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -51,6 +51,12 @@ export type SnapshotQualityVerdict = { * pass has to be disclosed rather than left to look complete. */ customActions?: { read: number; candidates: number; truncated: number; blocked: boolean }; + /** + * Surface hosts the published tree still carries in the device's native orientation space: the + * capture could not name the app's interface orientation to turn them back (#2612). Rects under + * those hosts are not in the space the rest of the tree publishes. Present only when non-zero. + */ + unresolvedCoordinateSpaceWindows?: number; /** Response-level phase timing for the backend named by `backend`. */ timing?: SnapshotQualityTiming; }; diff --git a/packages/platform-apple/src/snapshot-observability.test.ts b/packages/platform-apple/src/snapshot-observability.test.ts index 00e4d58c94..881233d69f 100644 --- a/packages/platform-apple/src/snapshot-observability.test.ts +++ b/packages/platform-apple/src/snapshot-observability.test.ts @@ -32,7 +32,7 @@ const failed = ( kind: SnapshotSourceFailure['kind'] = 'unsupported', ): SnapshotSourceOutcome => ({ stage: 'failed', - failure: { kind, code }, + failure: { kind, code, scope: 'generation' }, }); const acquired = (): SnapshotSourceOutcome => ({ stage: 'acquired', diff --git a/packages/platform-apple/src/snapshot-route.test.ts b/packages/platform-apple/src/snapshot-route.test.ts index 4d9c0775b9..9662f4a648 100644 --- a/packages/platform-apple/src/snapshot-route.test.ts +++ b/packages/platform-apple/src/snapshot-route.test.ts @@ -154,7 +154,7 @@ test('a sheet that appears after an absent probe is identified by the surface, n const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { source: sourceReturning({ stage: 'failed', - failure: { kind: 'transport-failure', code: 'bridge-disconnected' }, + failure: { kind: 'transport-failure', code: 'bridge-disconnected', scope: 'generation' }, }), resolveTarget: vi.fn(async () => target), systemSurfacePresent: async () => 'absent', @@ -275,7 +275,7 @@ test('a probe that cannot answer discloses the skipped bridge and stays incompar test('typed bridge failure falls back once and disables retries for that app generation', async () => { const source = sourceReturning({ stage: 'failed', - failure: { kind: 'transport-failure', code: 'bridge-disconnected' }, + failure: { kind: 'transport-failure', code: 'bridge-disconnected', scope: 'generation' }, }); const fallback = vi.fn(async () => runnerResult()); const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { @@ -305,7 +305,7 @@ test('typed bridge failure falls back once and disables retries for that app gen test('a bridge still being prepared sends only that capture to the runner', async () => { const source = sourceReturning({ stage: 'failed', - failure: { kind: 'preparing', code: 'bridge-preparation-pending' }, + failure: { kind: 'preparing', code: 'bridge-preparation-pending', scope: 'capture' }, }); const fallback = vi.fn(async () => runnerResult()); const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { @@ -335,6 +335,7 @@ test('a window in an unresolved coordinate space sends only that capture to the failure: { kind: 'unsupported', code: 'window-coordinate-space-unresolved', + scope: 'capture', details: { windows: 1 }, }, }); @@ -362,7 +363,7 @@ test('a window in an unresolved coordinate space sends only that capture to the test('a new app generation re-enables the bridge', async () => { const source = sourceReturning({ stage: 'failed', - failure: { kind: 'transport-failure', code: 'bridge-disconnected' }, + failure: { kind: 'transport-failure', code: 'bridge-disconnected', scope: 'generation' }, }); const resolveTarget = vi .fn() @@ -386,7 +387,7 @@ test('stale bridge acquisition resolves the current generation before XCTest fal const currentTarget = { ...target, pid: 84, generation: '84:launch-b' }; const source = sourceReturning({ stage: 'failed', - failure: { kind: 'stale-target', code: 'target-generation-changed' }, + failure: { kind: 'stale-target', code: 'target-generation-changed', scope: 'generation' }, }); const resolveTarget = vi.fn().mockResolvedValueOnce(target).mockResolvedValueOnce(currentTarget); const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { @@ -463,11 +464,14 @@ test('cancelled acquisition does not start a fallback after the request aborts', const controller = new AbortController(); const source = sourceReturning({ stage: 'failed', - failure: { kind: 'cancelled', code: 'abort-signal' }, + failure: { kind: 'cancelled', code: 'abort-signal', scope: 'generation' }, }); vi.mocked(source.acquire).mockImplementation(async () => { controller.abort(new DOMException('request ended', 'AbortError')); - return { stage: 'failed', failure: { kind: 'cancelled', code: 'abort-signal' } }; + return { + stage: 'failed', + failure: { kind: 'cancelled', code: 'abort-signal', scope: 'generation' }, + }; }); const fallback = vi.fn(async () => runnerResult()); const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { @@ -556,7 +560,7 @@ test.each([ // gave up on — ~33 acquisitions per `open`, each a fresh connect. const source = sourceReturning({ stage: 'failed', - failure: { kind: 'transport-failure', code }, + failure: { kind: 'transport-failure', code, scope: 'generation' }, }); const route = createAppleSnapshotRoute( { ...platformRuntimeHostFixture(), clock: steppingClock() }, @@ -574,7 +578,10 @@ test.each([ test('a relaunched generation rebaselines the circuit and observes the launch', async () => { const outcomes: SnapshotSourceOutcome[] = [ - { stage: 'failed', failure: { kind: 'transport-failure', code: 'bridge-disconnected' } }, + { + stage: 'failed', + failure: { kind: 'transport-failure', code: 'bridge-disconnected', scope: 'generation' }, + }, bridgeAcquisition(), ]; let acquisitions = 0; diff --git a/packages/platform-apple/src/snapshot-route.ts b/packages/platform-apple/src/snapshot-route.ts index 339f305088..8f7244ef80 100644 --- a/packages/platform-apple/src/snapshot-route.ts +++ b/packages/platform-apple/src/snapshot-route.ts @@ -211,7 +211,7 @@ export function createAppleSnapshotRoute( target, { lineage: target, residue: [] }, request, - { kind: 'malformed-tree', code: 'presentation-invariant' }, + { kind: 'malformed-tree', code: 'presentation-invariant', scope: 'generation' }, disabledGenerations, error, ); @@ -273,13 +273,12 @@ async function fallbackAfterFailure( disabledGenerations: Set, cause?: unknown, ): Promise { - // A failed bridge is evidence about this app generation, so its captures take the runner until the - // generation is rebaselined. Two failures are evidence about something else instead: a bridge that is - // merely still being prepared speaks for the daemon's build queue, and a screen holding a surface in - // another coordinate space speaks for this capture only. Both have to be able to use the bridge on - // the next capture, which is what let one cold host cost every later capture of a stable screen - // (#2491) and would otherwise cost every portrait capture after one landscape keyboard (#2612). - if (opensGenerationCircuit(failure)) disabledGenerations.add(generationKey(failedTarget)); + // A failure scoped to the generation retires it from the bridge until it is rebaselined. One scoped + // to the capture — a bridge still being prepared, a screen holding a surface in another coordinate + // space — leaves the generation on the bridge path, which is what stops one cold host from costing + // every later capture of a stable screen (#2491) and one landscape keyboard from costing every + // portrait capture after it (#2612). The scope is declared where the failure is thrown. + if (failure.scope === 'generation') disabledGenerations.add(generationKey(failedTarget)); emitRouteDiagnostic( failure.code, { id: failedTarget.udid }, @@ -298,19 +297,6 @@ async function fallbackAfterFailure( ); } -/** - * Whether a bridge failure is evidence that this app generation cannot be served by the bridge. - * - * Two failures say something else instead. A bridge that is still being prepared speaks for the - * daemon's build queue, and a screen holding a surface in another coordinate space speaks for this - * capture only: the surface is on screen now and gone after the next keystroke, so retiring the - * generation would move every later capture of a healthy app to the runner to work around one screen - * (#2491 settled the first of these, and #2612 the second). - */ -function opensGenerationCircuit(failure: SnapshotSourceFailure): boolean { - return failure.kind !== 'preparing' && failure.code !== WINDOW_COORDINATE_SPACE_UNRESOLVED; -} - async function runFallback( deviceId: string, input: CaptureSnapshotInput, diff --git a/packages/platform-apple/src/snapshot-source-facade.ts b/packages/platform-apple/src/snapshot-source-facade.ts index dc58e1edcd..3b2b430ee2 100644 --- a/packages/platform-apple/src/snapshot-source-facade.ts +++ b/packages/platform-apple/src/snapshot-source-facade.ts @@ -34,7 +34,7 @@ export function createSimulatorSnapshotSource(): SimulatorSnapshotSource { if (closed) { return { stage: 'failed', - failure: { kind: 'unsupported', code: 'source-closed' }, + failure: { kind: 'unsupported', code: 'source-closed', scope: 'generation' }, }; } return await (await load()).acquire(request); diff --git a/packages/platform-apple/src/snapshot-source/adapter.test.ts b/packages/platform-apple/src/snapshot-source/adapter.test.ts index c6cdab1f88..9004a7d909 100644 --- a/packages/platform-apple/src/snapshot-source/adapter.test.ts +++ b/packages/platform-apple/src/snapshot-source/adapter.test.ts @@ -699,6 +699,7 @@ test('the Simulator AX source refuses a window whose coordinate space it cannot if (outcome.stage === 'failed') { assert.equal(outcome.failure.kind, 'unsupported'); assert.equal(outcome.failure.code, 'window-coordinate-space-unresolved'); + assert.equal(outcome.failure.scope, 'capture'); assert.equal(outcome.failure.details?.windows, 1); } diff --git a/packages/platform-apple/src/snapshot-source/adapter.ts b/packages/platform-apple/src/snapshot-source/adapter.ts index f6dfbebec7..c7c526c8af 100644 --- a/packages/platform-apple/src/snapshot-source/adapter.ts +++ b/packages/platform-apple/src/snapshot-source/adapter.ts @@ -6,7 +6,11 @@ import type { } from '@agent-device/contracts/ios-snapshot'; import { createSnapshotSourceDeadline, remainingSnapshotSourceMs } from './deadline.ts'; import { AcceptedDepthHints, type DepthHintDecision } from './depth-hints.ts'; -import { asSnapshotSourceError, snapshotSourceError } from './errors.ts'; +import { + asSnapshotSourceError, + snapshotSourceCaptureError, + snapshotSourceError, +} from './errors.ts'; import { SnapshotBridgeManager } from './lifecycle.ts'; import { resolveSnapshotSourceLimits } from './limits.ts'; import { readSnapshotBridgeRecovery, type SnapshotBridgeEnvelope } from './protocol.ts'; @@ -99,6 +103,7 @@ export function createSimulatorSnapshotSource( failure: { kind: failure.failureKind, code: failure.failureCode, + scope: failure.failureScope, ...(failure.details ? { details: failure.details } : {}), }, } satisfies SnapshotSourceOutcome; @@ -226,7 +231,7 @@ function createAcquisition( // the source refuses the screen it cannot put in one space and the route serves the runner, which // reads the orientation and rotates what it captured (#2612). if (decoded.unresolvedCoordinateSpaceWindows > 0) { - throw snapshotSourceError('unsupported', 'window-coordinate-space-unresolved', { + throw snapshotSourceCaptureError('unsupported', 'window-coordinate-space-unresolved', { windows: decoded.unresolvedCoordinateSpaceWindows, }); } diff --git a/packages/platform-apple/src/snapshot-source/errors.ts b/packages/platform-apple/src/snapshot-source/errors.ts index bd7671c007..e40fc6df43 100644 --- a/packages/platform-apple/src/snapshot-source/errors.ts +++ b/packages/platform-apple/src/snapshot-source/errors.ts @@ -1,5 +1,5 @@ import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; -import type { SnapshotSourceFailureKind } from './types.ts'; +import type { SnapshotSourceFailureKind, SnapshotSourceFailureScope } from './types.ts'; const APP_ERROR_CODE_BY_KIND: Readonly> = { unsupported: 'UNSUPPORTED_OPERATION', @@ -15,6 +15,7 @@ const APP_ERROR_CODE_BY_KIND: Readonly export class SnapshotSourceError extends AppError { readonly failureKind: SnapshotSourceFailureKind; readonly failureCode: string; + readonly failureScope: SnapshotSourceFailureScope; constructor( kind: SnapshotSourceFailureKind, @@ -22,6 +23,7 @@ export class SnapshotSourceError extends AppError { message = `iOS Simulator snapshot source ${kind}: ${code}`, details: Readonly> = {}, cause?: unknown, + scope: SnapshotSourceFailureScope = 'generation', ) { super( APP_ERROR_CODE_BY_KIND[kind], @@ -37,9 +39,11 @@ export class SnapshotSourceError extends AppError { this.name = 'SnapshotSourceError'; this.failureKind = kind; this.failureCode = code; + this.failureScope = scope; } } +/** A failure that is evidence about the app generation the bridge served. */ export function snapshotSourceError( kind: SnapshotSourceFailureKind, code: string, @@ -49,6 +53,19 @@ export function snapshotSourceError( return new SnapshotSourceError(kind, code, undefined, details, cause); } +/** + * A failure that is evidence about this capture only. The route keeps the generation on the bridge, + * so a throw site reaches for this constructor exactly when the next capture must be able to use the + * bridge again (#2491, #2612). + */ +export function snapshotSourceCaptureError( + kind: SnapshotSourceFailureKind, + code: string, + details: Readonly> = {}, +): SnapshotSourceError { + return new SnapshotSourceError(kind, code, undefined, details, undefined, 'capture'); +} + export function asSnapshotSourceError(error: unknown): SnapshotSourceError { if (error instanceof SnapshotSourceError) return error; if (isRequestCanceledError(error)) { diff --git a/packages/platform-apple/src/snapshot-source/preparation.ts b/packages/platform-apple/src/snapshot-source/preparation.ts index 1126c951f1..c72d55d174 100644 --- a/packages/platform-apple/src/snapshot-source/preparation.ts +++ b/packages/platform-apple/src/snapshot-source/preparation.ts @@ -6,7 +6,7 @@ import { waitForSnapshotSourceDelay, type SnapshotSourceDeadline, } from './deadline.ts'; -import { asSnapshotSourceError, snapshotSourceError } from './errors.ts'; +import { asSnapshotSourceError, snapshotSourceCaptureError } from './errors.ts'; import type { SnapshotSourceBridgeBinary, SnapshotSourceHost, @@ -120,7 +120,7 @@ export function createSnapshotBridgePreparation( // Waiting inside the request's own deadline keeps a genuine client abort typed // `cancelled`: only a preparation that is simply still running is reported as `preparing`. wait: (waitMs, stop) => waitForSnapshotSourceDelay(deadline, waitMs, code, stop), - pending: () => snapshotSourceError('preparing', 'bridge-preparation-pending'), + pending: () => snapshotSourceCaptureError('preparing', 'bridge-preparation-pending'), }); }, close: () => { diff --git a/packages/platform-apple/src/snapshot-source/types.ts b/packages/platform-apple/src/snapshot-source/types.ts index aaae313b98..3662d092b6 100644 --- a/packages/platform-apple/src/snapshot-source/types.ts +++ b/packages/platform-apple/src/snapshot-source/types.ts @@ -47,9 +47,19 @@ export type SnapshotSourceFailureKind = /** The bridge binary is still being prepared by a detached attempt; nothing failed. */ | 'preparing'; +/** + * What a bridge failure is evidence about. `generation`: the app generation the bridge served, so + * the route retires it from the bridge until it relaunches. `capture`: this capture only — a bridge + * still being prepared (#2491), or a screen holding a surface in another coordinate space (#2612) — + * so the next capture asks the bridge again. Declared where the failure is thrown, never judged at + * the route. + */ +export type SnapshotSourceFailureScope = 'capture' | 'generation'; + export type SnapshotSourceFailure = Readonly<{ kind: SnapshotSourceFailureKind; code: string; + scope: SnapshotSourceFailureScope; details?: Readonly>; }>; diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts index d679c9e205..8abe052783 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts @@ -10,9 +10,9 @@ import { // ADR 0011 Layer 2 golden parity table (#2612): the SAME JSON is asserted against the Swift twin // (SnapshotGeometrySpace and CoordinateSpaceRotation in -// apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift, gated XCTest in -// that file), so drift between the host's decoder and the runner's capture turns CI red on whichever -// side changed. +// apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, +// replayed by that package's CoordinateSpaceTests), so drift between the host's decoder and the +// runner's capture turns CI red on whichever side changed. type FixtureFrame = | { diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index 5d16c08995..944b5830af 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -23,7 +23,7 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * The same rule is enforced in Swift for the capture that produces these trees, and one table proves * they are one rule: `contracts/fixtures/window-coordinate-space.json` is replayed here and against * `SnapshotGeometrySpace` in - * `apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift`. The table also + * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`. The table also * pins the way back (`CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`, Swift-only * today); change either rule only through that table. */ From f107d74223e1c16cb687b616233bfc36d4d7623f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 18 Sep 2026 13:24:26 +0200 Subject: [PATCH 2/4] docs(ios): point the bridge refusal decision at the #2659 spike verdict The ADR 0004 refusal decision and the TS detector's header now cite the measurement (issue #2659, diff of #2667): the AX orientation attribute resolves but reads 0 through the guest channel, and the only cheap service read is device orientation, which diverges on a rotation-locked app. --- docs/adr/0004-ios-snapshot-backend-strategy.md | 8 ++++++-- .../src/snapshot-source/window-coordinate-space.ts | 8 ++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index f18fea84bf..9b14b8ccd8 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -416,8 +416,12 @@ the ADR 0011 parity-table shape, which is what stops capture from turning back w than the exact inverse of what dispatch turns forward. Decision. A producer that cannot name the app's interface orientation refuses the screen rather -than publishing two spaces in one tree. The Simulator AX bridge's attribute set carries no -orientation fact, so its decoder counts window roots reporting the app's box quarter-turned and +than publishing two spaces in one tree. The Simulator AX bridge cannot name it: the #2659 spike +(verdict on that issue, write-up in the diff of #2667) measured that the one AX attribute for it, +`XC_kAXXCAttributeApplicationOrientation` (id 1503), resolves but reads 0 through the guest's +snapshot channel and errors `kAXErrorServerNotFound` through XCTest's own reader, and that the only +cheap service read is *device* orientation, which diverges from the app's on a rotation-locked app. +So its decoder counts window roots reporting the app's box quarter-turned and fails that capture (`window-coordinate-space-unresolved`, kind `unsupported`) — the same refusal shape as `remote-content-boundary` above — and the route serves the runner, which reads the orientation. A mixed tree is not half-usable: a published rect is an address, and once some rects diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index 944b5830af..13c58fa59e 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -26,6 +26,14 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`. The table also * pins the way back (`CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`, Swift-only * today); change either rule only through that table. + * + * Why this is a detector and not a repair: the bridge guest cannot read the app's interface + * orientation. Measured in the #2659 spike (verdict on the issue, write-up in the diff of #2667): + * AX attribute `XC_kAXXCAttributeApplicationOrientation` (id 1503) resolves but reads 0 through the + * guest's snapshot channel, and the cheap BackBoard read is device orientation, which turns a + * rotation-locked app's screen that never turned. The bar for retiring this file is a guest read + * that names the foreground app's orientation and survives a rotation-locked app; attribute 1503 is + * the handle to watch. */ /** From 5f93e58abb1640c9db710d04fe009aa956ed2647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 18 Sep 2026 14:27:04 +0200 Subject: [PATCH 3/4] refactor(ios): make failure scope a required constructor arg; count a turned surface once Addresses review on #2664. - SnapshotSourceError takes scope as a required argument (no 'generation' default), positioned after the required code. The two factories name their side: snapshotSourceError -> 'generation', snapshotSourceCaptureError -> 'capture'. New errors.test.ts pins both. - unplacedSurfaceHostCount folds a turned surface's window and the surface directly under it into one host, so the disclosed count is a surface count even when both report the turned box. Package test covers the both-turned pair. --- .../SnapshotCoordinateSpace.swift | 24 ++++++++++++++----- .../CoordinateSpaceTests.swift | 10 ++++++++ .../src/snapshot-source/errors.test.ts | 21 ++++++++++++++++ .../src/snapshot-source/errors.ts | 6 ++--- 4 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 packages/platform-apple/src/snapshot-source/errors.test.ts diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 51b5797920..9f5c5ed491 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -214,17 +214,29 @@ public enum SnapshotGeometrySpace: Equatable { /// two spaces instead of letting rects under those hosts read as addresses (#2612). public static func unplacedSurfaceHostCount(in nodes: [RawAXNode], viewport: CGRect) -> Int { guard isPlottable(viewport) else { return 0 } - var count = 0 - for node in nodes { + // `parentIndex` is a position into this array: every producer appends nodes in traversal order + // with `index == nodes.count` and sets a child's `parentIndex` to the parent's index, so + // `nodes[parentIndex]` is that parent. A turned host is one surface even when both its window + // and the surface directly under it report the turned box, so a turned node whose parent is + // itself turned is folded into that parent rather than counted a second time. + func turnedFrame(_ node: RawAXNode) -> Bool { let parentIsWindow = node.parentIndex.map { index in nodes.indices.contains(index) && isWindowType(nodes[index].type) } ?? false guard isSurfaceHost(isWindow: isWindowType(node.type), parentIsWindow: parentIsWindow) - else { continue } - let frame = CGRect( - x: node.rect.x, y: node.rect.y, width: node.rect.width, height: node.rect.height + else { return false } + return isQuarterTurned( + CGRect(x: node.rect.x, y: node.rect.y, width: node.rect.width, height: node.rect.height), + relativeTo: viewport ) - if isQuarterTurned(frame, relativeTo: viewport) { count += 1 } + } + var count = 0 + for node in nodes { + guard turnedFrame(node) else { continue } + let parentTurned = node.parentIndex.map { index in + nodes.indices.contains(index) && turnedFrame(nodes[index]) + } ?? false + if !parentTurned { count += 1 } } return count } diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index 9664943501..119a7bd20d 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -355,6 +355,16 @@ final class CoordinateSpaceTests: XCTestCase { ] XCTAssertEqual(SnapshotGeometrySpace.unplacedSurfaceHostCount(in: deep, viewport: viewport), 0) XCTAssertEqual(SnapshotGeometrySpace.unplacedSurfaceHostCount(in: unplaced, viewport: .infinite), 0) + + // One turned surface is one host even when the window AND the surface under it both report the + // turned box: the child folds into the turned window rather than counting a second time. + let bothTurned = [ + node(0, type: "Application", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: nil), + node(1, type: "Window", rect: SnapshotRect(x: 0, y: 0, width: 402, height: 874), parent: 0), + node(2, type: "Other", rect: SnapshotRect(x: 0, y: 0, width: 402, height: 874), parent: 1), + node(3, type: "Key", rect: SnapshotRect(x: 154, y: 77, width: 45, height: 72), parent: 2), + ] + XCTAssertEqual(SnapshotGeometrySpace.unplacedSurfaceHostCount(in: bothTurned, viewport: viewport), 1) } /// Golden parity table (#2612): every case in contracts/fixtures/window-coordinate-space.json diff --git a/packages/platform-apple/src/snapshot-source/errors.test.ts b/packages/platform-apple/src/snapshot-source/errors.test.ts new file mode 100644 index 0000000000..4de458b9e2 --- /dev/null +++ b/packages/platform-apple/src/snapshot-source/errors.test.ts @@ -0,0 +1,21 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { snapshotSourceCaptureError, snapshotSourceError } from './errors.ts'; + +// The scope decides whether the route retires the app generation from the bridge, so it is a +// required constructor argument (no default): a failure cannot be built without naming its side of +// the rule. These two factories are the only constructors, and each names its scope. +test('snapshotSourceError is evidence about the app generation', () => { + const error = snapshotSourceError('transport-failure', 'bridge-disconnected'); + assert.equal(error.failureScope, 'generation'); +}); + +test('snapshotSourceCaptureError is evidence about this capture only', () => { + const preparing = snapshotSourceCaptureError('preparing', 'bridge-preparation-pending'); + assert.equal(preparing.failureScope, 'capture'); + const rotated = snapshotSourceCaptureError('unsupported', 'window-coordinate-space-unresolved', { + windows: 1, + }); + assert.equal(rotated.failureScope, 'capture'); + assert.equal(rotated.details?.windows, 1); +}); diff --git a/packages/platform-apple/src/snapshot-source/errors.ts b/packages/platform-apple/src/snapshot-source/errors.ts index e40fc6df43..ec63c2ac34 100644 --- a/packages/platform-apple/src/snapshot-source/errors.ts +++ b/packages/platform-apple/src/snapshot-source/errors.ts @@ -20,10 +20,10 @@ export class SnapshotSourceError extends AppError { constructor( kind: SnapshotSourceFailureKind, code: string, + scope: SnapshotSourceFailureScope, message = `iOS Simulator snapshot source ${kind}: ${code}`, details: Readonly> = {}, cause?: unknown, - scope: SnapshotSourceFailureScope = 'generation', ) { super( APP_ERROR_CODE_BY_KIND[kind], @@ -50,7 +50,7 @@ export function snapshotSourceError( details: Readonly> = {}, cause?: unknown, ): SnapshotSourceError { - return new SnapshotSourceError(kind, code, undefined, details, cause); + return new SnapshotSourceError(kind, code, 'generation', undefined, details, cause); } /** @@ -63,7 +63,7 @@ export function snapshotSourceCaptureError( code: string, details: Readonly> = {}, ): SnapshotSourceError { - return new SnapshotSourceError(kind, code, undefined, details, undefined, 'capture'); + return new SnapshotSourceError(kind, code, 'capture', undefined, details); } export function asSnapshotSourceError(error: unknown): SnapshotSourceError { From 321aef2609c591c67f8c492a3e742816f002fd07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 18 Sep 2026 16:10:41 +0200 Subject: [PATCH 4/4] refactor(ios): keep only the coordinate-space package move; drop scope field and unplaced-geometry disclosure Measured against main with npm pack: the earlier +9.3 kB was 6.8 kB of prose comments in the moved file (the presentation package is written comment-free and copied verbatim), ~1.5 kB of disclosure plumbing, and ~0.2 kB for the failure scope field. The move alone is now +25 bytes. - Moved file rewritten in the package's comment-free style; rationale stays in ADR 0004 and the golden fixture description. - Failure scope field reverted to main's opensGenerationCircuit predicate: a required constructor argument guarded what the route tests already guard, for two fixed codes. - unresolvedCoordinateSpaceWindows disclosure reverted: its only trigger path (private-AX tier with a penalized XCTest channel) anchors on a root frame whose space is unmeasured and is infinite in the common case, so the count is zero by construction there; never observed live. Noted on #2661 where one-pass normalization makes it a measured one-liner if ever wanted. - ADR 0004 keeps the #2659 spike pointer and the package-home sentence only. --- CHANGELOG.md | 12 +- .../RunnerTests+SnapshotCapturePlan.swift | 33 ----- .../SnapshotCoordinateSpace.swift | 117 +----------------- .../CoordinateSpaceTests.swift | 85 ------------- .../adr/0004-ios-snapshot-backend-strategy.md | 20 ++- .../src/snapshot-quality-verdict.test.ts | 17 --- .../src/snapshot-quality-verdict.ts | 6 - .../snapshot-presentation/quality-warnings.ts | 15 --- .../snapshot-presentation/warnings.test.ts | 17 --- packages/kernel/src/snapshot.ts | 6 - .../src/snapshot-observability.test.ts | 2 +- .../platform-apple/src/snapshot-route.test.ts | 25 ++-- packages/platform-apple/src/snapshot-route.ts | 28 +++-- .../src/snapshot-source-facade.ts | 2 +- .../src/snapshot-source/adapter.test.ts | 1 - .../src/snapshot-source/adapter.ts | 9 +- .../src/snapshot-source/errors.test.ts | 21 ---- .../src/snapshot-source/errors.ts | 21 +--- .../src/snapshot-source/preparation.ts | 4 +- .../src/snapshot-source/types.ts | 10 -- 20 files changed, 55 insertions(+), 396 deletions(-) delete mode 100644 packages/platform-apple/src/snapshot-source/errors.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 88e60ccab3..f31b5e1519 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,15 +34,9 @@ in the app's own orientation space.` — and the route sends that one capture to the runner instead of retiring the app generation, so the next capture of a healthy app still uses the bridge. One table proves both languages apply one rule: `contracts/fixtures/window-coordinate-space.json` (#2612). -- Changed (ios): a runner capture that could not turn a rotated system surface back into the app's - orientation space — the interface orientation was unreadable, or the tier had only the bridge's - root box to anchor on — now says so instead of publishing two coordinate spaces silently: the - quality verdict carries `unresolvedCoordinateSpaceWindows` and the snapshot warns that rects under - those windows are not in the same space as the rest of the tree. Whether a Simulator AX bridge - failure retires the app generation from the bridge is now declared where the failure is thrown - (`scope: capture | generation`) rather than matched on its code at the route. The rotation table - and the coordinate-space rule moved from the XCTest bundle into the `AgentDeviceSnapshotPresentation` - package, where `swift test` replays the golden table without a simulator. +- Changed (ios): the rotation table and the coordinate-space rule moved from the XCTest runner bundle + into the `AgentDeviceSnapshotPresentation` package, where `swift test` replays the golden table + without a simulator. No behaviour change. - Added (limrun): `longpress` on Limrun iOS direct sessions. The interactor refused it as unsupported although the SDK exposes the HID primitives; it now holds one touch as a `performActions` batch of `touchDown`, `wait`, `touchUp`, defaulting to the 800 ms the Android diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 5ff3994583..7c3141c7a0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -27,10 +27,6 @@ struct SnapshotQuality: Codable { let collapsedLeafIndexes: [Int]? /// Coverage of the bounded custom-action pass, when the capture asked for one. let customActions: SnapshotCustomActionCoverage? - /// Surface hosts the published tree still carries in the device's native space: the capture could - /// not name the app's interface orientation to turn them back, so rects under them are not in the - /// space the rest of the tree publishes (#2612). Absent when every host was placed. - var unresolvedCoordinateSpaceWindows: Int? = nil /// Response-level timing for the accepted backend attempt, never repeated per node. var timing: SnapshotCaptureTiming? = nil } @@ -65,8 +61,6 @@ struct SnapshotBackendCapture { let effectiveDepth: Int? var customActions: SnapshotCustomActionCoverage? = nil var qualityPayload: DataPayload? = nil - /// Turned surface hosts the acquisition left as reported; nil when the tree holds none. - var unresolvedCoordinateSpaceWindows: Int? = nil var timing: SnapshotCaptureTiming? = nil } @@ -509,13 +503,6 @@ extension RunnerTests { var capture = presented capture.timing = timer.timing - // A host the walk could not place is disclosed, never silently published as an address: the - // orientation read failed, or the tier had only the bridge's root box to anchor on (#2612). - let unplaced = SnapshotGeometrySpace.unplacedSurfaceHostCount( - in: acquisition.nodes, - viewport: acquisition.viewport - ) - capture.unresolvedCoordinateSpaceWindows = unplaced > 0 ? unplaced : nil return SnapshotBackendAttempt( outcome: .captured(capture), timing: timer.timing @@ -638,7 +625,6 @@ extension RunnerTests { effectiveDepth: capture.effectiveDepth, collapsedLeafIndexes: Self.collapsedLeafIndexes(payload.nodes ?? []), customActions: capture.customActions, - unresolvedCoordinateSpaceWindows: capture.unresolvedCoordinateSpaceWindows, timing: capture.timing ) return DataPayload( @@ -890,25 +876,6 @@ extension RunnerTests { XCTAssertEqual(payload.nodes?.count, 1) } - /// A tree that still carries a turned host says so on the verdict, and one that holds none says - /// nothing: the daemon's warning keys off presence, so a zero must not ride the wire as a fact. - func testStampedPayloadDisclosesUnplacedSurfaceHosts() { - let payload = DataPayload( - nodes: [planTestNode(index: 0, type: "Application", label: "App")], - truncated: false - ) - let unplaced = stampedSnapshotPayload( - SnapshotBackendCapture( - payload: payload, effectiveDepth: nil, unresolvedCoordinateSpaceWindows: 1), - backend: .recursiveTree, state: "healthy", reason: nil) - XCTAssertEqual(unplaced.snapshotQuality?.unresolvedCoordinateSpaceWindows, 1) - - let placed = stampedSnapshotPayload( - SnapshotBackendCapture(payload: payload, effectiveDepth: nil), - backend: .recursiveTree, state: "healthy", reason: nil) - XCTAssertNil(placed.snapshotQuality?.unresolvedCoordinateSpaceWindows) - } - func testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance() { let complete = SnapshotBackendCapture( payload: DataPayload( diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 9f5c5ed491..fea57569ec 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -1,28 +1,10 @@ import Foundation import CoreGraphics -// Where a piece of geometry lives, and how it gets where the caller means it. -// -// DISPATCH: synthesized events skip XCTest's orientation handling, so a point planned in an app's -// interface space has to be rotated into the device's native (portrait-up) space before it is -// performed. Without that, a landscape tap lands somewhere else. -// -// CAPTURE: iOS hosts some system surfaces in that native space even while the app is rotated, so -// their whole subtree arrives quarter-turned. Measured on iPhone 17 Pro (iOS 26.2) with the system -// keyboard up in landscape: the app viewport is 874 x 402, `UIRemoteKeyboardWindow` reports its own -// box as 402 x 874, a key arrives 45 x 72 at x 154, and the key plane as 202 x 724 — a strip down -// the left edge — while the screenshot draws a 724 x 204 band docked at y 198, which is exactly -// what the live `app.keyboards` query measures. Rules that read the reported numbers refused app -// content the keyboard was nowhere near and let a tap through into a key (#2612, #2589). -// -// Both directions live in ONE table on purpose: capture rotates back with the exact inverse of what -// dispatch rotates forward, so a performed tap and a reported rect can never disagree about which -// pixel is which. The round-trip test in the package suite is that proof, not a comment asking for -// care. This file is pure geometry, which is why it lives here rather than in the XCTest bundle: -// nothing in it needs a simulator to run. +// Rotation between an app's interface space and the device's native (portrait-up) space, shared by +// synthesized dispatch (forward) and capture (inverse). Rationale and measurements: ADR 0004 +// "the coordinate space of a captured subtree" and contracts/fixtures/window-coordinate-space.json. -/// The UIInterfaceOrientation raw values the rotation table switches on. XCTest exposes the app's -/// interface orientation as an integer, and 0 (unknown) deliberately rotates nothing. public enum RunnerInterfaceOrientation { public static let unknown = 0 public static let portrait = 1 @@ -31,9 +13,7 @@ public enum RunnerInterfaceOrientation { public static let landscapeLeft = 4 } -/// The quarter-turn between an app's interface space and the device's native (portrait-up) space. public enum CoordinateSpaceRotation { - /// An interface-space point, in the native space synthesized events are performed in. public static func native( point: CGPoint, in frame: CGRect, @@ -50,12 +30,11 @@ public enum CoordinateSpaceRotation { return CGPoint(x: localY, y: width - localX) case RunnerInterfaceOrientation.portraitUpsideDown: return CGPoint(x: width - localX, y: height - localY) - default: // portrait, or an orientation the platform did not name + default: return CGPoint(x: localX, y: localY) } } - /// An interface-space translation vector, in the same native space as `native(point:)`. public static func native(vector: CGVector, interfaceOrientation: Int) -> CGVector { switch interfaceOrientation { case RunnerInterfaceOrientation.landscapeRight: @@ -64,12 +43,11 @@ public enum CoordinateSpaceRotation { return CGVector(dx: vector.dy, dy: -vector.dx) case RunnerInterfaceOrientation.portraitUpsideDown: return CGVector(dx: -vector.dx, dy: -vector.dy) - default: // portrait, or an orientation the platform did not name + default: return vector } } - /// The inverse of `native(point:)`: a native-space point, in the app's interface space. public static func oriented( point: CGPoint, in frame: CGRect, @@ -89,16 +67,13 @@ public enum CoordinateSpaceRotation { case RunnerInterfaceOrientation.portraitUpsideDown: localX = width - Double(point.x) localY = height - Double(point.y) - default: // portrait, or an orientation the platform did not name + default: localX = Double(point.x) localY = Double(point.y) } return CGPoint(x: localX + Double(frame.minX), y: localY + Double(frame.minY)) } - /// The inverse of `native(point:)` for a rect. A quarter turn swaps the axes, so opposite corners - /// of the reported rect become opposite corners of the result; ordering them keeps the origin - /// top-left, which is what every consumer of a published rect reads. public static func oriented( rect: CGRect, in frame: CGRect, @@ -123,34 +98,12 @@ public enum CoordinateSpaceRotation { } } -/// The space one captured subtree reports its rects in. -/// -/// RULE: the capture publishes every rect in the app's orientation space. A surface host declares the -/// space of its own subtree — its box is either the app's box, or the app's box turned through a -/// quarter, which is how a surface hosted in the device's native space announces itself — and anything -/// below it inherits that space. Nothing is declared, and every rect stays exactly as the platform -/// reported it, when the app frame is unusable, when the interface orientation names no quarter turn -/// (portrait, or an orientation the platform did not name), or when the app frame is square. Geometry -/// the capture cannot place is not a claim about where it is, and consumers already fail open on it the -/// way they do on any missing platform fact — but the capture says so: `unplacedSurfaceHostCount` -/// counts the hosts a published tree still carries turned, so the daemon can warn instead of -/// publishing two spaces silently. -/// -/// A half turn is a different claim: an upside-down app's own box is already the native box, so no -/// reported window distinguishes the two spaces and nothing is turned back. That is the same limit the -/// box-only rule accepts for a square app frame, and it is why the consumer rules in -/// `tap-keyboard-occlusion.ts` stay rather than becoming an assertion here. public enum SnapshotGeometrySpace: Equatable { - /** The app's own orientation space: the one every consumer of a published tree reads. */ case appOrientation - /** The device's native (portrait-up) space, with the way back to the app's space attached. */ case deviceNative(appFrame: CGRect, interfaceOrientation: Int) - /// How far a window's side lengths may miss the app's swapped side lengths and still be that - /// quarter turn. Measured captures match to the point; this absorbs float representation only. public static let quarterTurnTolerance: Double = 1 - /// The reported rect, in the space the capture publishes. public func orientedFrame(of reportedFrame: CGRect) -> CGRect { switch self { case .appOrientation: @@ -164,14 +117,6 @@ public enum SnapshotGeometrySpace: Equatable { } } - /// The space one node's subtree reports in. - /// - /// Only a surface host may declare a space: the window itself, or the surface the window hands its - /// content to. XCTest reports `UIRemoteKeyboardWindow` with the app's own box and its child with - /// the turned one, so the window alone is not where the turn shows up, while a deep node claiming a - /// turned box is content whose reported bounds happen to be large and is left in the inherited - /// space. A host that reports anything other than a turned box declares the app's space for its - /// subtree, which is how a tree with no rotated surface stays exactly as reported. public static func space( reportedBySurfaceHost isSurfaceHost: Bool, reportedFrame: CGRect, @@ -185,65 +130,15 @@ public enum SnapshotGeometrySpace: Equatable { return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) } - /// Whether this orientation is a turn the reported box can be measured against. Portrait and an - /// unnamed orientation leave the device's native space upright, so nothing under them is turned. public static func namesQuarterTurn(_ interfaceOrientation: Int) -> Bool { interfaceOrientation == RunnerInterfaceOrientation.landscapeLeft || interfaceOrientation == RunnerInterfaceOrientation.landscapeRight } - /// Whether this node, or the window above it, is where a hosted surface's box appears. public static func isSurfaceHost(isWindow: Bool, parentIsWindow: Bool) -> Bool { isWindow || parentIsWindow } - /// The element type names under which a producer publishes the app's windows: the app's own - /// window, `UITextEffectsWindow`, `UIRemoteKeyboardWindow`. XCTest models each as its own element - /// typed `.application` or `.window`, and every producer names them this way in a published tree. - public static func isWindowType(_ type: String) -> Bool { - type == "Application" || type == "Window" - } - - /// Surface hosts a published tree still carries in the device's native space. - /// - /// A capture that could name the app's interface orientation turned every such host back, so its - /// published tree holds none. One that could not — the orientation read failed, or the tier had - /// only the bridge's own root box to anchor on — publishes the host as the platform reported it, - /// and this is where that shows: a host whose box is the viewport quarter-turned. The count is a - /// disclosure, not a repair; it rides the quality verdict so the daemon can say which captures hold - /// two spaces instead of letting rects under those hosts read as addresses (#2612). - public static func unplacedSurfaceHostCount(in nodes: [RawAXNode], viewport: CGRect) -> Int { - guard isPlottable(viewport) else { return 0 } - // `parentIndex` is a position into this array: every producer appends nodes in traversal order - // with `index == nodes.count` and sets a child's `parentIndex` to the parent's index, so - // `nodes[parentIndex]` is that parent. A turned host is one surface even when both its window - // and the surface directly under it report the turned box, so a turned node whose parent is - // itself turned is folded into that parent rather than counted a second time. - func turnedFrame(_ node: RawAXNode) -> Bool { - let parentIsWindow = node.parentIndex.map { index in - nodes.indices.contains(index) && isWindowType(nodes[index].type) - } ?? false - guard isSurfaceHost(isWindow: isWindowType(node.type), parentIsWindow: parentIsWindow) - else { return false } - return isQuarterTurned( - CGRect(x: node.rect.x, y: node.rect.y, width: node.rect.width, height: node.rect.height), - relativeTo: viewport - ) - } - var count = 0 - for node in nodes { - guard turnedFrame(node) else { continue } - let parentTurned = node.parentIndex.map { index in - nodes.indices.contains(index) && turnedFrame(nodes[index]) - } ?? false - if !parentTurned { count += 1 } - } - return count - } - - /// A window that reports the app's two side lengths on the other axes is hosted in the device's - /// native space. A square app frame cannot be told from its own quarter turn, so it is left alone - /// rather than guessed at. private static func isQuarterTurned(_ frame: CGRect, relativeTo appFrame: CGRect) -> Bool { guard isPlottable(frame), isPlottable(appFrame), abs(appFrame.width - appFrame.height) > quarterTurnTolerance diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index 119a7bd20d..c08ff5522b 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -288,9 +288,6 @@ final class CoordinateSpaceTests: XCTestCase { // surface below a window is a host too. XCTAssertTrue(SnapshotGeometrySpace.isSurfaceHost(isWindow: false, parentIsWindow: true)) XCTAssertFalse(SnapshotGeometrySpace.isSurfaceHost(isWindow: false, parentIsWindow: false)) - XCTAssertTrue(SnapshotGeometrySpace.isWindowType("Application")) - XCTAssertTrue(SnapshotGeometrySpace.isWindowType("Window")) - XCTAssertFalse(SnapshotGeometrySpace.isWindowType("Other")) let windowSpace = SnapshotGeometrySpace.space( reportedBySurfaceHost: true, @@ -322,51 +319,6 @@ final class CoordinateSpaceTests: XCTestCase { ) } - /// A published tree that still carries a turned host is one the capture could not place, and the - /// count is what lets the daemon say so. A tree the capture did turn back holds none. - func testUnplacedSurfaceHostCountNamesTheHostsATreeStillCarriesTurned() { - let viewport = CGRect(x: 0, y: 0, width: 874, height: 402) - // Application > [Window(app box) > Button, Window(app box) > Other(turned) > Key] - let unplaced = [ - node(0, type: "Application", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: nil), - node(1, type: "Window", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: 0), - node(2, type: "Button", rect: SnapshotRect(x: 204, y: 323, width: 91, height: 55), parent: 1), - node(3, type: "Window", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: 0), - node(4, type: "Other", rect: SnapshotRect(x: 0, y: 0, width: 402, height: 874), parent: 3), - node(5, type: "Key", rect: SnapshotRect(x: 154, y: 77, width: 45, height: 72), parent: 4), - ] - XCTAssertEqual(SnapshotGeometrySpace.unplacedSurfaceHostCount(in: unplaced, viewport: viewport), 1) - - // The same screen once the capture turned the host back: nothing is left to disclose. - let placed = unplaced.map { entry in - entry.index == 4 - ? node(4, type: "Other", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: 3) - : entry - } - XCTAssertEqual(SnapshotGeometrySpace.unplacedSurfaceHostCount(in: placed, viewport: viewport), 0) - - // A turned box deep in the tree is content, not a host; a viewport the capture could not resolve - // anchors nothing. - let deep = [ - node(0, type: "Application", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: nil), - node(1, type: "Window", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: 0), - node(2, type: "Other", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: 1), - node(3, type: "Other", rect: SnapshotRect(x: 0, y: 0, width: 402, height: 874), parent: 2), - ] - XCTAssertEqual(SnapshotGeometrySpace.unplacedSurfaceHostCount(in: deep, viewport: viewport), 0) - XCTAssertEqual(SnapshotGeometrySpace.unplacedSurfaceHostCount(in: unplaced, viewport: .infinite), 0) - - // One turned surface is one host even when the window AND the surface under it both report the - // turned box: the child folds into the turned window rather than counting a second time. - let bothTurned = [ - node(0, type: "Application", rect: SnapshotRect(x: 0, y: 0, width: 874, height: 402), parent: nil), - node(1, type: "Window", rect: SnapshotRect(x: 0, y: 0, width: 402, height: 874), parent: 0), - node(2, type: "Other", rect: SnapshotRect(x: 0, y: 0, width: 402, height: 874), parent: 1), - node(3, type: "Key", rect: SnapshotRect(x: 154, y: 77, width: 45, height: 72), parent: 2), - ] - XCTAssertEqual(SnapshotGeometrySpace.unplacedSurfaceHostCount(in: bothTurned, viewport: viewport), 1) - } - /// Golden parity table (#2612): every case in contracts/fixtures/window-coordinate-space.json /// must agree with the vitest twin /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). Add @@ -414,14 +366,6 @@ final class CoordinateSpaceTests: XCTestCase { "\(testCase.name) (interfaceOrientation \(interfaceOrientation))" ) } - // The disclosure reads the same rule off a published node: a host the table calls turned is - // the host the count names, whichever way the app is turned. - let host = node(0, type: "Window", rect: snapshotRect(testCase.window.cgRect), parent: nil) - XCTAssertEqual( - SnapshotGeometrySpace.unplacedSurfaceHostCount(in: [host], viewport: appFrame), - testCase.quarterTurned ? 1 : 0, - "\(testCase.name) (unplaced host count)" - ) } for testCase in fixture.rotationCases { XCTAssertEqual( @@ -436,33 +380,4 @@ final class CoordinateSpaceTests: XCTestCase { } } - private func node(_ index: Int, type: String, rect: SnapshotRect, parent: Int?) -> RawAXNode { - RawAXNode( - index: index, - type: type, - label: nil, - identifier: nil, - value: nil, - rect: rect, - enabled: true, - focused: nil, - selected: nil, - hittable: false, - depth: parent == nil ? 0 : 1, - parentIndex: parent, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - } - - private func snapshotRect(_ frame: CGRect) -> SnapshotRect { - guard frame.origin.x.isFinite, frame.origin.y.isFinite, frame.width.isFinite, - frame.height.isFinite - else { - return SnapshotRect(x: 0, y: 0, width: 0, height: 0) - } - return SnapshotRect( - x: Double(frame.minX), y: Double(frame.minY), - width: Double(frame.width), height: Double(frame.height)) - } } diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 9b14b8ccd8..1d9fbe309e 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -405,15 +405,12 @@ oriented frame, so an address and the permission to tap it cannot disagree. Wher named — no usable app frame, an interface orientation the platform did not report, or an app frame square enough to be indistinguishable from its own quarter turn — the capture publishes what the platform reported rather than guessing at a rotation, and consumers fail open on that geometry the -way they do on any missing platform fact. It does not do so silently: a published tree that still -carries a surface host reporting the viewport quarter-turned is counted after every tier's -acquisition and stamped on the quality verdict as `unresolvedCoordinateSpaceWindows`, which the -daemon renders as a warning, so a tree holding two spaces never looks like one. Detection, the way -back and that count share one rotation table with synthesized dispatch — the pure geometry lives in -the `AgentDeviceSnapshotPresentation` package (`SnapshotCoordinateSpace.swift`), not in the XCTest -bundle — and both languages replay it against `contracts/fixtures/window-coordinate-space.json` in -the ADR 0011 parity-table shape, which is what stops capture from turning back with anything other -than the exact inverse of what dispatch turns forward. +way they do on any missing platform fact. Detection and the way back share one rotation table with +synthesized dispatch — pure geometry that lives in the `AgentDeviceSnapshotPresentation` package +(`SnapshotCoordinateSpace.swift`), not in the XCTest bundle — and both languages replay it against +`contracts/fixtures/window-coordinate-space.json` in the ADR 0011 parity-table shape, which is what +stops capture from turning back with anything other than the exact inverse of what dispatch turns +forward. Decision. A producer that cannot name the app's interface orientation refuses the screen rather than publishing two spaces in one tree. The Simulator AX bridge cannot name it: the #2659 spike @@ -435,9 +432,8 @@ hybrid app, so its screens keep the runner until it relaunches. A rotated surfac the screen in front of the reader and about nothing else — it is up now and gone after the next keystroke — so retiring the generation would move every later portrait capture of a healthy app onto the runner to work around one landscape keyboard, the cost #2491 settled for a bridge that was -merely still being prepared. Which side of that line a failure falls on is declared where the -failure is thrown — `SnapshotSourceFailure.scope` is `capture` or `generation`, and the route reads -the field — rather than judged at the route from the failure's kind or code. +merely still being prepared. Which side of that line a failure falls on is a declared property of +its code rather than a judgment made at the call site. What stays unnormalized, and why the consumer rule stays: the runner's query-sweep tier has no window ancestry, so no node in its tree can declare a space, and the provider producers diff --git a/packages/capture-kit/src/snapshot-quality-verdict.test.ts b/packages/capture-kit/src/snapshot-quality-verdict.test.ts index a752834264..ea8afbc157 100644 --- a/packages/capture-kit/src/snapshot-quality-verdict.test.ts +++ b/packages/capture-kit/src/snapshot-quality-verdict.test.ts @@ -30,23 +30,6 @@ test('readSnapshotQualityVerdict accepts a well-formed verdict', () => { }); }); -test('readSnapshotQualityVerdict carries a non-zero unplaced-geometry disclosure and drops a zero', () => { - const disclosed = readSnapshotQualityVerdict({ - state: 'healthy', - backend: 'tree', - unresolvedCoordinateSpaceWindows: 1, - }); - assert.equal(disclosed?.unresolvedCoordinateSpaceWindows, 1); - - const placed = readSnapshotQualityVerdict({ - state: 'healthy', - backend: 'tree', - unresolvedCoordinateSpaceWindows: 0, - }); - assert.equal(placed?.state, 'healthy'); - assert.equal('unresolvedCoordinateSpaceWindows' in placed!, false); -}); - test('readSnapshotQualityVerdict drops incomplete phase timing without dropping the verdict', () => { const verdict = readSnapshotQualityVerdict({ state: 'healthy', diff --git a/packages/capture-kit/src/snapshot-quality-verdict.ts b/packages/capture-kit/src/snapshot-quality-verdict.ts index 43764ff518..2913abe32f 100644 --- a/packages/capture-kit/src/snapshot-quality-verdict.ts +++ b/packages/capture-kit/src/snapshot-quality-verdict.ts @@ -57,12 +57,6 @@ export function readSnapshotQualityVerdict(value: unknown): SnapshotQualityVerdi collapsedLeafIndexes: Array.isArray(raw.collapsedLeafIndexes) ? raw.collapsedLeafIndexes.filter((entry): entry is number => typeof entry === 'number') : undefined, - // A zero is not a disclosure; the runner omits it, and a reader that emitted it would make the - // warning below key off a value instead of a fact. - ...(typeof raw.unresolvedCoordinateSpaceWindows === 'number' && - raw.unresolvedCoordinateSpaceWindows > 0 - ? { unresolvedCoordinateSpaceWindows: raw.unresolvedCoordinateSpaceWindows } - : {}), ...(timing ? { timing } : {}), }; } diff --git a/packages/capture-kit/src/snapshot/snapshot-presentation/quality-warnings.ts b/packages/capture-kit/src/snapshot/snapshot-presentation/quality-warnings.ts index a5b222a266..0f79547323 100644 --- a/packages/capture-kit/src/snapshot/snapshot-presentation/quality-warnings.ts +++ b/packages/capture-kit/src/snapshot/snapshot-presentation/quality-warnings.ts @@ -7,26 +7,11 @@ export function renderSnapshotQualityWarnings( return [ ...stateWarning(verdict), ...customActionCoverageWarning(verdict), - ...unplacedGeometryWarning(verdict), ...depthWarning(verdict), ...collapsedLeafWarnings(verdict, nodes), ]; } -/** - * The capture publishes one coordinate space, and says so when it could not: a system surface - * hosted in the device's native orientation space that the runner could not turn back, because the - * app's interface orientation was unreadable (#2612). Without this line the tree would hold two - * spaces and look like one. - */ -function unplacedGeometryWarning(verdict: SnapshotQualityVerdict): string[] { - const windows = verdict.unresolvedCoordinateSpaceWindows; - if (windows === undefined) return []; - return [ - `${windows} window(s) on this screen — typically the system keyboard while the app is in landscape — report their geometry in the device's native orientation space, and this capture could not read the app's interface orientation to turn it back. Rects under those windows are not in the same space as the rest of the tree, so distances measured against them and taps aimed at them are unreliable; use screenshot as visual truth for that area and re-run snapshot after the surface is dismissed.`, - ]; -} - function customActionCoverageWarning(verdict: SnapshotQualityVerdict): string[] { const coverage = verdict.customActions; if (!coverage) return []; diff --git a/packages/capture-kit/src/snapshot/snapshot-presentation/warnings.test.ts b/packages/capture-kit/src/snapshot/snapshot-presentation/warnings.test.ts index 0380a90e37..978b53969b 100644 --- a/packages/capture-kit/src/snapshot/snapshot-presentation/warnings.test.ts +++ b/packages/capture-kit/src/snapshot/snapshot-presentation/warnings.test.ts @@ -20,23 +20,6 @@ test("the runner-wire 'deferred' reasonCode survives parsing into warning suppre assert.deepEqual(renderSnapshotQualityWarnings(verdict!, []), []); }); -test('a capture that could not place a turned surface host discloses it through the wire verdict', () => { - // End-to-end through the raw wire parser, like the deferred case above: the count has to be a - // member of the read verdict or the disclosure is silently stripped. - const verdict = readSnapshotQualityVerdict({ - state: 'healthy', - backend: 'tree', - unresolvedCoordinateSpaceWindows: 1, - }); - const warnings = renderSnapshotQualityWarnings(verdict!, []); - assert.equal(warnings.length, 1); - assert.match(warnings[0]!, /^1 window\(s\) on this screen/); - assert.match(warnings[0]!, /could not read the app's interface orientation/); - - const placed = readSnapshotQualityVerdict({ state: 'healthy', backend: 'tree' }); - assert.deepEqual(renderSnapshotQualityWarnings(placed!, []), []); -}); - test('penalty-deferred recovered captures suppress the fallback warning but keep depth copy', () => { // Every capture of a hostile screen re-stamps recovered/deferred; the capture that armed the // penalty already carried the full warning, so repeating it each capture is context noise. diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index ac2b03213a..30839bd036 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -51,12 +51,6 @@ export type SnapshotQualityVerdict = { * pass has to be disclosed rather than left to look complete. */ customActions?: { read: number; candidates: number; truncated: number; blocked: boolean }; - /** - * Surface hosts the published tree still carries in the device's native orientation space: the - * capture could not name the app's interface orientation to turn them back (#2612). Rects under - * those hosts are not in the space the rest of the tree publishes. Present only when non-zero. - */ - unresolvedCoordinateSpaceWindows?: number; /** Response-level phase timing for the backend named by `backend`. */ timing?: SnapshotQualityTiming; }; diff --git a/packages/platform-apple/src/snapshot-observability.test.ts b/packages/platform-apple/src/snapshot-observability.test.ts index 881233d69f..00e4d58c94 100644 --- a/packages/platform-apple/src/snapshot-observability.test.ts +++ b/packages/platform-apple/src/snapshot-observability.test.ts @@ -32,7 +32,7 @@ const failed = ( kind: SnapshotSourceFailure['kind'] = 'unsupported', ): SnapshotSourceOutcome => ({ stage: 'failed', - failure: { kind, code, scope: 'generation' }, + failure: { kind, code }, }); const acquired = (): SnapshotSourceOutcome => ({ stage: 'acquired', diff --git a/packages/platform-apple/src/snapshot-route.test.ts b/packages/platform-apple/src/snapshot-route.test.ts index 9662f4a648..4d9c0775b9 100644 --- a/packages/platform-apple/src/snapshot-route.test.ts +++ b/packages/platform-apple/src/snapshot-route.test.ts @@ -154,7 +154,7 @@ test('a sheet that appears after an absent probe is identified by the surface, n const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { source: sourceReturning({ stage: 'failed', - failure: { kind: 'transport-failure', code: 'bridge-disconnected', scope: 'generation' }, + failure: { kind: 'transport-failure', code: 'bridge-disconnected' }, }), resolveTarget: vi.fn(async () => target), systemSurfacePresent: async () => 'absent', @@ -275,7 +275,7 @@ test('a probe that cannot answer discloses the skipped bridge and stays incompar test('typed bridge failure falls back once and disables retries for that app generation', async () => { const source = sourceReturning({ stage: 'failed', - failure: { kind: 'transport-failure', code: 'bridge-disconnected', scope: 'generation' }, + failure: { kind: 'transport-failure', code: 'bridge-disconnected' }, }); const fallback = vi.fn(async () => runnerResult()); const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { @@ -305,7 +305,7 @@ test('typed bridge failure falls back once and disables retries for that app gen test('a bridge still being prepared sends only that capture to the runner', async () => { const source = sourceReturning({ stage: 'failed', - failure: { kind: 'preparing', code: 'bridge-preparation-pending', scope: 'capture' }, + failure: { kind: 'preparing', code: 'bridge-preparation-pending' }, }); const fallback = vi.fn(async () => runnerResult()); const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { @@ -335,7 +335,6 @@ test('a window in an unresolved coordinate space sends only that capture to the failure: { kind: 'unsupported', code: 'window-coordinate-space-unresolved', - scope: 'capture', details: { windows: 1 }, }, }); @@ -363,7 +362,7 @@ test('a window in an unresolved coordinate space sends only that capture to the test('a new app generation re-enables the bridge', async () => { const source = sourceReturning({ stage: 'failed', - failure: { kind: 'transport-failure', code: 'bridge-disconnected', scope: 'generation' }, + failure: { kind: 'transport-failure', code: 'bridge-disconnected' }, }); const resolveTarget = vi .fn() @@ -387,7 +386,7 @@ test('stale bridge acquisition resolves the current generation before XCTest fal const currentTarget = { ...target, pid: 84, generation: '84:launch-b' }; const source = sourceReturning({ stage: 'failed', - failure: { kind: 'stale-target', code: 'target-generation-changed', scope: 'generation' }, + failure: { kind: 'stale-target', code: 'target-generation-changed' }, }); const resolveTarget = vi.fn().mockResolvedValueOnce(target).mockResolvedValueOnce(currentTarget); const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { @@ -464,14 +463,11 @@ test('cancelled acquisition does not start a fallback after the request aborts', const controller = new AbortController(); const source = sourceReturning({ stage: 'failed', - failure: { kind: 'cancelled', code: 'abort-signal', scope: 'generation' }, + failure: { kind: 'cancelled', code: 'abort-signal' }, }); vi.mocked(source.acquire).mockImplementation(async () => { controller.abort(new DOMException('request ended', 'AbortError')); - return { - stage: 'failed', - failure: { kind: 'cancelled', code: 'abort-signal', scope: 'generation' }, - }; + return { stage: 'failed', failure: { kind: 'cancelled', code: 'abort-signal' } }; }); const fallback = vi.fn(async () => runnerResult()); const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { @@ -560,7 +556,7 @@ test.each([ // gave up on — ~33 acquisitions per `open`, each a fresh connect. const source = sourceReturning({ stage: 'failed', - failure: { kind: 'transport-failure', code, scope: 'generation' }, + failure: { kind: 'transport-failure', code }, }); const route = createAppleSnapshotRoute( { ...platformRuntimeHostFixture(), clock: steppingClock() }, @@ -578,10 +574,7 @@ test.each([ test('a relaunched generation rebaselines the circuit and observes the launch', async () => { const outcomes: SnapshotSourceOutcome[] = [ - { - stage: 'failed', - failure: { kind: 'transport-failure', code: 'bridge-disconnected', scope: 'generation' }, - }, + { stage: 'failed', failure: { kind: 'transport-failure', code: 'bridge-disconnected' } }, bridgeAcquisition(), ]; let acquisitions = 0; diff --git a/packages/platform-apple/src/snapshot-route.ts b/packages/platform-apple/src/snapshot-route.ts index 8f7244ef80..339f305088 100644 --- a/packages/platform-apple/src/snapshot-route.ts +++ b/packages/platform-apple/src/snapshot-route.ts @@ -211,7 +211,7 @@ export function createAppleSnapshotRoute( target, { lineage: target, residue: [] }, request, - { kind: 'malformed-tree', code: 'presentation-invariant', scope: 'generation' }, + { kind: 'malformed-tree', code: 'presentation-invariant' }, disabledGenerations, error, ); @@ -273,12 +273,13 @@ async function fallbackAfterFailure( disabledGenerations: Set, cause?: unknown, ): Promise { - // A failure scoped to the generation retires it from the bridge until it is rebaselined. One scoped - // to the capture — a bridge still being prepared, a screen holding a surface in another coordinate - // space — leaves the generation on the bridge path, which is what stops one cold host from costing - // every later capture of a stable screen (#2491) and one landscape keyboard from costing every - // portrait capture after it (#2612). The scope is declared where the failure is thrown. - if (failure.scope === 'generation') disabledGenerations.add(generationKey(failedTarget)); + // A failed bridge is evidence about this app generation, so its captures take the runner until the + // generation is rebaselined. Two failures are evidence about something else instead: a bridge that is + // merely still being prepared speaks for the daemon's build queue, and a screen holding a surface in + // another coordinate space speaks for this capture only. Both have to be able to use the bridge on + // the next capture, which is what let one cold host cost every later capture of a stable screen + // (#2491) and would otherwise cost every portrait capture after one landscape keyboard (#2612). + if (opensGenerationCircuit(failure)) disabledGenerations.add(generationKey(failedTarget)); emitRouteDiagnostic( failure.code, { id: failedTarget.udid }, @@ -297,6 +298,19 @@ async function fallbackAfterFailure( ); } +/** + * Whether a bridge failure is evidence that this app generation cannot be served by the bridge. + * + * Two failures say something else instead. A bridge that is still being prepared speaks for the + * daemon's build queue, and a screen holding a surface in another coordinate space speaks for this + * capture only: the surface is on screen now and gone after the next keystroke, so retiring the + * generation would move every later capture of a healthy app to the runner to work around one screen + * (#2491 settled the first of these, and #2612 the second). + */ +function opensGenerationCircuit(failure: SnapshotSourceFailure): boolean { + return failure.kind !== 'preparing' && failure.code !== WINDOW_COORDINATE_SPACE_UNRESOLVED; +} + async function runFallback( deviceId: string, input: CaptureSnapshotInput, diff --git a/packages/platform-apple/src/snapshot-source-facade.ts b/packages/platform-apple/src/snapshot-source-facade.ts index 3b2b430ee2..dc58e1edcd 100644 --- a/packages/platform-apple/src/snapshot-source-facade.ts +++ b/packages/platform-apple/src/snapshot-source-facade.ts @@ -34,7 +34,7 @@ export function createSimulatorSnapshotSource(): SimulatorSnapshotSource { if (closed) { return { stage: 'failed', - failure: { kind: 'unsupported', code: 'source-closed', scope: 'generation' }, + failure: { kind: 'unsupported', code: 'source-closed' }, }; } return await (await load()).acquire(request); diff --git a/packages/platform-apple/src/snapshot-source/adapter.test.ts b/packages/platform-apple/src/snapshot-source/adapter.test.ts index 9004a7d909..c6cdab1f88 100644 --- a/packages/platform-apple/src/snapshot-source/adapter.test.ts +++ b/packages/platform-apple/src/snapshot-source/adapter.test.ts @@ -699,7 +699,6 @@ test('the Simulator AX source refuses a window whose coordinate space it cannot if (outcome.stage === 'failed') { assert.equal(outcome.failure.kind, 'unsupported'); assert.equal(outcome.failure.code, 'window-coordinate-space-unresolved'); - assert.equal(outcome.failure.scope, 'capture'); assert.equal(outcome.failure.details?.windows, 1); } diff --git a/packages/platform-apple/src/snapshot-source/adapter.ts b/packages/platform-apple/src/snapshot-source/adapter.ts index c7c526c8af..f6dfbebec7 100644 --- a/packages/platform-apple/src/snapshot-source/adapter.ts +++ b/packages/platform-apple/src/snapshot-source/adapter.ts @@ -6,11 +6,7 @@ import type { } from '@agent-device/contracts/ios-snapshot'; import { createSnapshotSourceDeadline, remainingSnapshotSourceMs } from './deadline.ts'; import { AcceptedDepthHints, type DepthHintDecision } from './depth-hints.ts'; -import { - asSnapshotSourceError, - snapshotSourceCaptureError, - snapshotSourceError, -} from './errors.ts'; +import { asSnapshotSourceError, snapshotSourceError } from './errors.ts'; import { SnapshotBridgeManager } from './lifecycle.ts'; import { resolveSnapshotSourceLimits } from './limits.ts'; import { readSnapshotBridgeRecovery, type SnapshotBridgeEnvelope } from './protocol.ts'; @@ -103,7 +99,6 @@ export function createSimulatorSnapshotSource( failure: { kind: failure.failureKind, code: failure.failureCode, - scope: failure.failureScope, ...(failure.details ? { details: failure.details } : {}), }, } satisfies SnapshotSourceOutcome; @@ -231,7 +226,7 @@ function createAcquisition( // the source refuses the screen it cannot put in one space and the route serves the runner, which // reads the orientation and rotates what it captured (#2612). if (decoded.unresolvedCoordinateSpaceWindows > 0) { - throw snapshotSourceCaptureError('unsupported', 'window-coordinate-space-unresolved', { + throw snapshotSourceError('unsupported', 'window-coordinate-space-unresolved', { windows: decoded.unresolvedCoordinateSpaceWindows, }); } diff --git a/packages/platform-apple/src/snapshot-source/errors.test.ts b/packages/platform-apple/src/snapshot-source/errors.test.ts deleted file mode 100644 index 4de458b9e2..0000000000 --- a/packages/platform-apple/src/snapshot-source/errors.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { snapshotSourceCaptureError, snapshotSourceError } from './errors.ts'; - -// The scope decides whether the route retires the app generation from the bridge, so it is a -// required constructor argument (no default): a failure cannot be built without naming its side of -// the rule. These two factories are the only constructors, and each names its scope. -test('snapshotSourceError is evidence about the app generation', () => { - const error = snapshotSourceError('transport-failure', 'bridge-disconnected'); - assert.equal(error.failureScope, 'generation'); -}); - -test('snapshotSourceCaptureError is evidence about this capture only', () => { - const preparing = snapshotSourceCaptureError('preparing', 'bridge-preparation-pending'); - assert.equal(preparing.failureScope, 'capture'); - const rotated = snapshotSourceCaptureError('unsupported', 'window-coordinate-space-unresolved', { - windows: 1, - }); - assert.equal(rotated.failureScope, 'capture'); - assert.equal(rotated.details?.windows, 1); -}); diff --git a/packages/platform-apple/src/snapshot-source/errors.ts b/packages/platform-apple/src/snapshot-source/errors.ts index ec63c2ac34..bd7671c007 100644 --- a/packages/platform-apple/src/snapshot-source/errors.ts +++ b/packages/platform-apple/src/snapshot-source/errors.ts @@ -1,5 +1,5 @@ import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; -import type { SnapshotSourceFailureKind, SnapshotSourceFailureScope } from './types.ts'; +import type { SnapshotSourceFailureKind } from './types.ts'; const APP_ERROR_CODE_BY_KIND: Readonly> = { unsupported: 'UNSUPPORTED_OPERATION', @@ -15,12 +15,10 @@ const APP_ERROR_CODE_BY_KIND: Readonly export class SnapshotSourceError extends AppError { readonly failureKind: SnapshotSourceFailureKind; readonly failureCode: string; - readonly failureScope: SnapshotSourceFailureScope; constructor( kind: SnapshotSourceFailureKind, code: string, - scope: SnapshotSourceFailureScope, message = `iOS Simulator snapshot source ${kind}: ${code}`, details: Readonly> = {}, cause?: unknown, @@ -39,31 +37,16 @@ export class SnapshotSourceError extends AppError { this.name = 'SnapshotSourceError'; this.failureKind = kind; this.failureCode = code; - this.failureScope = scope; } } -/** A failure that is evidence about the app generation the bridge served. */ export function snapshotSourceError( kind: SnapshotSourceFailureKind, code: string, details: Readonly> = {}, cause?: unknown, ): SnapshotSourceError { - return new SnapshotSourceError(kind, code, 'generation', undefined, details, cause); -} - -/** - * A failure that is evidence about this capture only. The route keeps the generation on the bridge, - * so a throw site reaches for this constructor exactly when the next capture must be able to use the - * bridge again (#2491, #2612). - */ -export function snapshotSourceCaptureError( - kind: SnapshotSourceFailureKind, - code: string, - details: Readonly> = {}, -): SnapshotSourceError { - return new SnapshotSourceError(kind, code, 'capture', undefined, details); + return new SnapshotSourceError(kind, code, undefined, details, cause); } export function asSnapshotSourceError(error: unknown): SnapshotSourceError { diff --git a/packages/platform-apple/src/snapshot-source/preparation.ts b/packages/platform-apple/src/snapshot-source/preparation.ts index c72d55d174..1126c951f1 100644 --- a/packages/platform-apple/src/snapshot-source/preparation.ts +++ b/packages/platform-apple/src/snapshot-source/preparation.ts @@ -6,7 +6,7 @@ import { waitForSnapshotSourceDelay, type SnapshotSourceDeadline, } from './deadline.ts'; -import { asSnapshotSourceError, snapshotSourceCaptureError } from './errors.ts'; +import { asSnapshotSourceError, snapshotSourceError } from './errors.ts'; import type { SnapshotSourceBridgeBinary, SnapshotSourceHost, @@ -120,7 +120,7 @@ export function createSnapshotBridgePreparation( // Waiting inside the request's own deadline keeps a genuine client abort typed // `cancelled`: only a preparation that is simply still running is reported as `preparing`. wait: (waitMs, stop) => waitForSnapshotSourceDelay(deadline, waitMs, code, stop), - pending: () => snapshotSourceCaptureError('preparing', 'bridge-preparation-pending'), + pending: () => snapshotSourceError('preparing', 'bridge-preparation-pending'), }); }, close: () => { diff --git a/packages/platform-apple/src/snapshot-source/types.ts b/packages/platform-apple/src/snapshot-source/types.ts index 3662d092b6..aaae313b98 100644 --- a/packages/platform-apple/src/snapshot-source/types.ts +++ b/packages/platform-apple/src/snapshot-source/types.ts @@ -47,19 +47,9 @@ export type SnapshotSourceFailureKind = /** The bridge binary is still being prepared by a detached attempt; nothing failed. */ | 'preparing'; -/** - * What a bridge failure is evidence about. `generation`: the app generation the bridge served, so - * the route retires it from the bridge until it relaunches. `capture`: this capture only — a bridge - * still being prepared (#2491), or a screen holding a surface in another coordinate space (#2612) — - * so the next capture asks the bridge again. Declared where the failure is thrown, never judged at - * the route. - */ -export type SnapshotSourceFailureScope = 'capture' | 'generation'; - export type SnapshotSourceFailure = Readonly<{ kind: SnapshotSourceFailureKind; code: string; - scope: SnapshotSourceFailureScope; details?: Readonly>; }>;