diff --git a/CHANGELOG.md b/CHANGELOG.md index 827ec0abea..f31b5e1519 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ in the app's own orientation space.` — and the route sends that one capture to the runner instead of retiring the app generation, so the next capture of a healthy app still uses the bridge. One table proves both languages apply one rule: `contracts/fixtures/window-coordinate-space.json` (#2612). +- Changed (ios): the rotation table and the coordinate-space rule moved from the XCTest runner bundle + into the `AgentDeviceSnapshotPresentation` package, where `swift test` replays the golden table + without a simulator. No behaviour change. - Added (limrun): `longpress` on Limrun iOS direct sessions. The interactor refused it as unsupported although the SDK exposes the HID primitives; it now holds one touch as a `performActions` batch of `touchDown`, `wait`, `touchUp`, defaulting to the 800 ms the Android diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift index 914cad4cde..18c8638222 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift @@ -1,4 +1,5 @@ import XCTest +import AgentDeviceSnapshotPresentation // The scroll viewport rule the runner shares with the TS runtime (#2500). // diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift index bfdf0db779..b2d8c8ae12 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift @@ -44,7 +44,7 @@ extension RunnerTests { let fields = privateAXFields(raw) let nodeSpace = SnapshotGeometrySpace.space( reportedBySurfaceHost: SnapshotGeometrySpace.isSurfaceHost( - elementType: fields.elementType, + isWindow: isWindowElement(fields.elementType), parentIsWindow: parentIsWindow ), reportedFrame: fields.rect, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 60e2e25e21..ef4eedd522 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -1069,7 +1069,7 @@ extension RunnerTests { ) -> SnapshotGeometrySpace { SnapshotGeometrySpace.space( reportedBySurfaceHost: SnapshotGeometrySpace.isSurfaceHost( - elementType: snapshot.elementType, + isWindow: isWindowElement(snapshot.elementType), parentIsWindow: parentIsWindow ), reportedFrame: snapshot.frame, diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift new file mode 100644 index 0000000000..fea57569ec --- /dev/null +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -0,0 +1,157 @@ +import Foundation +import CoreGraphics + +// Rotation between an app's interface space and the device's native (portrait-up) space, shared by +// synthesized dispatch (forward) and capture (inverse). Rationale and measurements: ADR 0004 +// "the coordinate space of a captured subtree" and contracts/fixtures/window-coordinate-space.json. + +public enum RunnerInterfaceOrientation { + public static let unknown = 0 + public static let portrait = 1 + public static let portraitUpsideDown = 2 + public static let landscapeRight = 3 + public static let landscapeLeft = 4 +} + +public enum CoordinateSpaceRotation { + public static func native( + point: CGPoint, + in frame: CGRect, + interfaceOrientation: Int + ) -> CGPoint { + let localX = Double(point.x) - Double(frame.minX) + let localY = Double(point.y) - Double(frame.minY) + let width = Double(frame.width) + let height = Double(frame.height) + switch interfaceOrientation { + case RunnerInterfaceOrientation.landscapeRight: + return CGPoint(x: height - localY, y: localX) + case RunnerInterfaceOrientation.landscapeLeft: + return CGPoint(x: localY, y: width - localX) + case RunnerInterfaceOrientation.portraitUpsideDown: + return CGPoint(x: width - localX, y: height - localY) + default: + return CGPoint(x: localX, y: localY) + } + } + + public static func native(vector: CGVector, interfaceOrientation: Int) -> CGVector { + switch interfaceOrientation { + case RunnerInterfaceOrientation.landscapeRight: + return CGVector(dx: -vector.dy, dy: vector.dx) + case RunnerInterfaceOrientation.landscapeLeft: + return CGVector(dx: vector.dy, dy: -vector.dx) + case RunnerInterfaceOrientation.portraitUpsideDown: + return CGVector(dx: -vector.dx, dy: -vector.dy) + default: + return vector + } + } + + public static func oriented( + point: CGPoint, + in frame: CGRect, + interfaceOrientation: Int + ) -> CGPoint { + let width = Double(frame.width) + let height = Double(frame.height) + let localX: Double + let localY: Double + switch interfaceOrientation { + case RunnerInterfaceOrientation.landscapeRight: + localX = Double(point.y) + localY = height - Double(point.x) + case RunnerInterfaceOrientation.landscapeLeft: + localX = width - Double(point.y) + localY = Double(point.x) + case RunnerInterfaceOrientation.portraitUpsideDown: + localX = width - Double(point.x) + localY = height - Double(point.y) + default: + localX = Double(point.x) + localY = Double(point.y) + } + return CGPoint(x: localX + Double(frame.minX), y: localY + Double(frame.minY)) + } + + public static func oriented( + rect: CGRect, + in frame: CGRect, + interfaceOrientation: Int + ) -> CGRect { + let leading = oriented( + point: CGPoint(x: rect.minX, y: rect.minY), + in: frame, + interfaceOrientation: interfaceOrientation + ) + let trailing = oriented( + point: CGPoint(x: rect.maxX, y: rect.maxY), + in: frame, + interfaceOrientation: interfaceOrientation + ) + return CGRect( + x: min(leading.x, trailing.x), + y: min(leading.y, trailing.y), + width: abs(trailing.x - leading.x), + height: abs(trailing.y - leading.y) + ) + } +} + +public enum SnapshotGeometrySpace: Equatable { + case appOrientation + case deviceNative(appFrame: CGRect, interfaceOrientation: Int) + + public static let quarterTurnTolerance: Double = 1 + + public func orientedFrame(of reportedFrame: CGRect) -> CGRect { + switch self { + case .appOrientation: + return reportedFrame + case .deviceNative(let appFrame, let interfaceOrientation): + return CoordinateSpaceRotation.oriented( + rect: reportedFrame, + in: appFrame, + interfaceOrientation: interfaceOrientation + ) + } + } + + public static func space( + reportedBySurfaceHost isSurfaceHost: Bool, + reportedFrame: CGRect, + inheritedFrom inherited: SnapshotGeometrySpace, + appFrame: CGRect, + interfaceOrientation: Int + ) -> SnapshotGeometrySpace { + guard isSurfaceHost else { return inherited } + guard namesQuarterTurn(interfaceOrientation) else { return .appOrientation } + guard isQuarterTurned(reportedFrame, relativeTo: appFrame) else { return .appOrientation } + return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) + } + + public static func namesQuarterTurn(_ interfaceOrientation: Int) -> Bool { + interfaceOrientation == RunnerInterfaceOrientation.landscapeLeft + || interfaceOrientation == RunnerInterfaceOrientation.landscapeRight + } + + public static func isSurfaceHost(isWindow: Bool, parentIsWindow: Bool) -> Bool { + isWindow || parentIsWindow + } + + private static func isQuarterTurned(_ frame: CGRect, relativeTo appFrame: CGRect) -> Bool { + guard isPlottable(frame), isPlottable(appFrame), + abs(appFrame.width - appFrame.height) > quarterTurnTolerance + else { + return false + } + return abs(frame.width - appFrame.height) <= quarterTurnTolerance + && abs(frame.height - appFrame.width) <= quarterTurnTolerance + } + + private static func isPlottable(_ frame: CGRect) -> Bool { + frame.origin.x.isFinite && frame.origin.y.isFinite + && frame.width.isFinite && frame.height.isFinite + && frame.width > 0 && frame.height > 0 + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift similarity index 51% rename from apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift rename to apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index f2a06c0415..c08ff5522b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -1,272 +1,61 @@ +import AgentDeviceSnapshotPresentation +import CoreGraphics +import Foundation import XCTest -// Where a piece of geometry lives, and how it gets where the caller means it. -// -// DISPATCH: synthesized events skip XCTest's orientation handling, so a point planned in an app's -// interface space has to be rotated into the device's native (portrait-up) space before it is -// performed. Without that, a landscape tap lands somewhere else. -// -// CAPTURE: iOS hosts some system surfaces in that native space even while the app is rotated, so -// their whole subtree arrives quarter-turned. Measured on iPhone 17 Pro (iOS 26.2) with the system -// keyboard up in landscape: the app viewport is 874 x 402, `UIRemoteKeyboardWindow` reports its own -// box as 402 x 874, a key arrives 45 x 72 at x 154, and the key plane as 202 x 724 — a strip down -// the left edge — while the screenshot draws a 724 x 204 band docked at y 198, which is exactly -// what the live `app.keyboards` query measures. Rules that read the reported numbers refused app -// content the keyboard was nowhere near and let a tap through into a key (#2612, #2589). -// -// Both directions live in ONE table on purpose: capture rotates back with the exact inverse of what -// dispatch rotates forward, so a performed tap and a reported rect can never disagree about which -// pixel is which. The round-trip test below is that proof, not a comment asking for care. - -/// The UIInterfaceOrientation raw values the rotation table switches on. XCTest exposes the app's -/// interface orientation as an integer, and 0 (unknown) deliberately rotates nothing. -enum RunnerInterfaceOrientation { - static let unknown = 0 - static let portrait = 1 - static let portraitUpsideDown = 2 - static let landscapeRight = 3 - static let landscapeLeft = 4 -} - -/// The quarter-turn between an app's interface space and the device's native (portrait-up) space. -enum CoordinateSpaceRotation { - /// An interface-space point, in the native space synthesized events are performed in. - static func native( - point: CGPoint, - in frame: CGRect, - interfaceOrientation: Int - ) -> CGPoint { - let localX = Double(point.x) - Double(frame.minX) - let localY = Double(point.y) - Double(frame.minY) - let width = Double(frame.width) - let height = Double(frame.height) - switch interfaceOrientation { - case RunnerInterfaceOrientation.landscapeRight: - return CGPoint(x: height - localY, y: localX) - case RunnerInterfaceOrientation.landscapeLeft: - return CGPoint(x: localY, y: width - localX) - case RunnerInterfaceOrientation.portraitUpsideDown: - return CGPoint(x: width - localX, y: height - localY) - default: // portrait, or an orientation the platform did not name - return CGPoint(x: localX, y: localY) - } - } - - /// An interface-space translation vector, in the same native space as `native(point:)`. - static func native(vector: CGVector, interfaceOrientation: Int) -> CGVector { - switch interfaceOrientation { - case RunnerInterfaceOrientation.landscapeRight: - return CGVector(dx: -vector.dy, dy: vector.dx) - case RunnerInterfaceOrientation.landscapeLeft: - return CGVector(dx: vector.dy, dy: -vector.dx) - case RunnerInterfaceOrientation.portraitUpsideDown: - return CGVector(dx: -vector.dx, dy: -vector.dy) - default: // portrait, or an orientation the platform did not name - return vector - } - } - - /// The inverse of `native(point:)`: a native-space point, in the app's interface space. - static func oriented( - point: CGPoint, - in frame: CGRect, - interfaceOrientation: Int - ) -> CGPoint { - let width = Double(frame.width) - let height = Double(frame.height) - let localX: Double - let localY: Double - switch interfaceOrientation { - case RunnerInterfaceOrientation.landscapeRight: - localX = Double(point.y) - localY = height - Double(point.x) - case RunnerInterfaceOrientation.landscapeLeft: - localX = width - Double(point.y) - localY = Double(point.x) - case RunnerInterfaceOrientation.portraitUpsideDown: - localX = width - Double(point.x) - localY = height - Double(point.y) - default: // portrait, or an orientation the platform did not name - localX = Double(point.x) - localY = Double(point.y) - } - return CGPoint(x: localX + Double(frame.minX), y: localY + Double(frame.minY)) - } - - /// The inverse of `native(point:)` for a rect. A quarter turn swaps the axes, so opposite corners - /// of the reported rect become opposite corners of the result; ordering them keeps the origin - /// top-left, which is what every consumer of a published rect reads. - static func oriented( - rect: CGRect, - in frame: CGRect, - interfaceOrientation: Int - ) -> CGRect { - let leading = oriented( - point: CGPoint(x: rect.minX, y: rect.minY), - in: frame, - interfaceOrientation: interfaceOrientation - ) - let trailing = oriented( - point: CGPoint(x: rect.maxX, y: rect.maxY), - in: frame, - interfaceOrientation: interfaceOrientation - ) - return CGRect( - x: min(leading.x, trailing.x), - y: min(leading.y, trailing.y), - width: abs(trailing.x - leading.x), - height: abs(trailing.y - leading.y) - ) - } -} - -/// The space one captured subtree reports its rects in. -/// -/// RULE: the capture publishes every rect in the app's orientation space. A surface host declares the -/// space of its own subtree — its box is either the app's box, or the app's box turned through a -/// quarter, which is how a surface hosted in the device's native space announces itself — and anything -/// below it inherits that space. Nothing is declared, and every rect stays exactly as the platform -/// reported it, when the app frame is unusable, when the interface orientation names no quarter turn -/// (portrait, or an orientation the platform did not name), or when the app frame is square. Geometry -/// the capture cannot place is not a claim about where it is, and consumers already fail open on it the -/// way they do on any missing platform fact. -/// -/// A half turn is a different claim: an upside-down app's own box is already the native box, so no -/// reported window distinguishes the two spaces and nothing is turned back. That is the same limit the -/// box-only rule accepts for a square app frame, and it is why the consumer rules in -/// `tap-keyboard-occlusion.ts` stay rather than becoming an assertion here. -enum SnapshotGeometrySpace: Equatable { - /** The app's own orientation space: the one every consumer of a published tree reads. */ - case appOrientation - /** The device's native (portrait-up) space, with the way back to the app's space attached. */ - case deviceNative(appFrame: CGRect, interfaceOrientation: Int) +/// The rotation table and the space a captured window declares (#2612). Pure geometry: it runs here, +/// without a simulator, and the runner's walkers are tested separately for threading it through. +final class CoordinateSpaceTests: XCTestCase { + private struct WindowCoordinateSpaceFixture: Decodable { + /// A frame as JSON can carry one. Infinity has no JSON spelling, so the unusable box a platform + /// hands back is named `{"infinite": true}`: `CGRect.infinite` here, and an infinite rect in the + /// vitest twin. + struct Frame: Decodable { + private enum CodingKeys: String, CodingKey { + case x, y, width, height, infinite + } - /// How far a window's side lengths may miss the app's swapped side lengths and still be that - /// quarter turn. Measured captures match to the point; this absorbs float representation only. - static let quarterTurnTolerance: Double = 1 + let cgRect: CGRect - /// The reported rect, in the space the capture publishes. - func orientedFrame(of reportedFrame: CGRect) -> CGRect { - switch self { - case .appOrientation: - return reportedFrame - case .deviceNative(let appFrame, let interfaceOrientation): - return CoordinateSpaceRotation.oriented( - rect: reportedFrame, - in: appFrame, - interfaceOrientation: interfaceOrientation - ) + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + guard try container.decodeIfPresent(Bool.self, forKey: .infinite) != true else { + self.cgRect = .infinite + return + } + self.cgRect = CGRect( + x: try container.decode(Double.self, forKey: .x), + y: try container.decode(Double.self, forKey: .y), + width: try container.decode(Double.self, forKey: .width), + height: try container.decode(Double.self, forKey: .height) + ) + } } - } - /// The space one node's subtree reports in. - /// - /// Only a surface host may declare a space: the window itself, or the surface the window hands its - /// content to. XCTest reports `UIRemoteKeyboardWindow` with the app's own box and its child with - /// the turned one, so the window alone is not where the turn shows up, while a deep node claiming a - /// turned box is content whose reported bounds happen to be large and is left in the inherited - /// space. A host that reports anything other than a turned box declares the app's space for its - /// subtree, which is how a tree with no rotated surface stays exactly as reported. - static func space( - reportedBySurfaceHost isSurfaceHost: Bool, - reportedFrame: CGRect, - inheritedFrom inherited: SnapshotGeometrySpace, - appFrame: CGRect, - interfaceOrientation: Int - ) -> SnapshotGeometrySpace { - guard isSurfaceHost else { return inherited } - guard namesQuarterTurn(interfaceOrientation) else { return .appOrientation } - guard isQuarterTurned(reportedFrame, relativeTo: appFrame) else { return .appOrientation } - return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) - } - - /// Whether this orientation is a turn the reported box can be measured against. Portrait and an - /// unnamed orientation leave the device's native space upright, so nothing under them is turned. - static func namesQuarterTurn(_ interfaceOrientation: Int) -> Bool { - interfaceOrientation == RunnerInterfaceOrientation.landscapeLeft - || interfaceOrientation == RunnerInterfaceOrientation.landscapeRight - } - - /// Whether this node, or the window above it, is where a hosted surface's box appears. - static func isSurfaceHost( - elementType: XCUIElement.ElementType?, - parentIsWindow: Bool - ) -> Bool { - elementType == .application || elementType == .window || parentIsWindow - } - - /// A window that reports the app's two side lengths on the other axes is hosted in the device's - /// native space. A square app frame cannot be told from its own quarter turn, so it is left alone - /// rather than guessed at. - private static func isQuarterTurned(_ frame: CGRect, relativeTo appFrame: CGRect) -> Bool { - guard isPlottable(frame), isPlottable(appFrame), - abs(appFrame.width - appFrame.height) > quarterTurnTolerance - else { - return false + struct Constants: Decodable { + let quarterTurnTolerance: Double } - return abs(frame.width - appFrame.height) <= quarterTurnTolerance - && abs(frame.height - appFrame.width) <= quarterTurnTolerance - } - private static func isPlottable(_ frame: CGRect) -> Bool { - frame.origin.x.isFinite && frame.origin.y.isFinite - && frame.width.isFinite && frame.height.isFinite - && frame.width > 0 && frame.height > 0 - } -} - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct WindowCoordinateSpaceFixture: Decodable { - /// A frame as JSON can carry one. Infinity has no JSON spelling, so the unusable box a platform - /// hands back is named `{"infinite": true}`: `CGRect.infinite` here, and an infinite rect in the - /// vitest twin. - struct Frame: Decodable { - private enum CodingKeys: String, CodingKey { - case x, y, width, height, infinite + struct QuarterTurnCase: Decodable { + let name: String + let window: Frame + let app: Frame + let quarterTurned: Bool } - let cgRect: CGRect - - init(from decoder: Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - guard try container.decodeIfPresent(Bool.self, forKey: .infinite) != true else { - self.cgRect = .infinite - return - } - self.cgRect = CGRect( - x: try container.decode(Double.self, forKey: .x), - y: try container.decode(Double.self, forKey: .y), - width: try container.decode(Double.self, forKey: .width), - height: try container.decode(Double.self, forKey: .height) - ) + struct RotationCase: Decodable { + let name: String + let interfaceOrientation: Int + let app: Frame + let native: Frame + let oriented: Frame } - } - - struct Constants: Decodable { - let quarterTurnTolerance: Double - } - - struct QuarterTurnCase: Decodable { - let name: String - let window: Frame - let app: Frame - let quarterTurned: Bool - } - struct RotationCase: Decodable { - let name: String - let interfaceOrientation: Int - let app: Frame - let native: Frame - let oriented: Frame + let constants: Constants + let quarterTurnCases: [QuarterTurnCase] + let rotationCases: [RotationCase] } - let constants: Constants - let quarterTurnCases: [QuarterTurnCase] - let rotationCases: [RotationCase] -} - -extension RunnerTests { /// Identity in portrait/unknown, 90° per landscape, 180° upside-down. func testNativeSynthesizedPointRotatesByInterfaceOrientation() { let portrait = CGRect(x: 0, y: 0, width: 834, height: 1210) @@ -429,7 +218,7 @@ extension RunnerTests { XCTAssertEqual( SnapshotGeometrySpace.space( reportedBySurfaceHost: SnapshotGeometrySpace.isSurfaceHost( - elementType: XCUIElement.ElementType(rawValue: 20), + isWindow: false, parentIsWindow: false ), reportedFrame: CGRect(x: 154, y: 77, width: 45, height: 72), @@ -489,6 +278,47 @@ extension RunnerTests { ) } + /// Where a turned box has to appear before it declares a space. + func testOnlyASurfaceHostDeclaresTheCoordinateSpaceOfItsSubtree() { + let appFrame = CGRect(x: 0, y: 0, width: 874, height: 402) + let turned = CGRect(x: 0, y: 0, width: 402, height: 874) + let landscape = RunnerInterfaceOrientation.landscapeRight + + // XCTest gives `UIRemoteKeyboardWindow` the app's own box and its child the turned one, so the + // surface below a window is a host too. + XCTAssertTrue(SnapshotGeometrySpace.isSurfaceHost(isWindow: false, parentIsWindow: true)) + XCTAssertFalse(SnapshotGeometrySpace.isSurfaceHost(isWindow: false, parentIsWindow: false)) + + let windowSpace = SnapshotGeometrySpace.space( + reportedBySurfaceHost: true, + reportedFrame: appFrame, + inheritedFrom: .appOrientation, + appFrame: appFrame, + interfaceOrientation: landscape + ) + XCTAssertEqual(windowSpace, .appOrientation) + let surfaceSpace = SnapshotGeometrySpace.space( + reportedBySurfaceHost: true, + reportedFrame: turned, + inheritedFrom: windowSpace, + appFrame: appFrame, + interfaceOrientation: landscape + ) + XCTAssertEqual(surfaceSpace, .deviceNative(appFrame: appFrame, interfaceOrientation: landscape)) + // Deep in the tree a turned box is content reporting large bounds, not a hosted surface: it keeps + // the space it inherited rather than rewriting the space below it. + XCTAssertEqual( + SnapshotGeometrySpace.space( + reportedBySurfaceHost: false, + reportedFrame: turned, + inheritedFrom: windowSpace, + appFrame: appFrame, + interfaceOrientation: landscape + ), + .appOrientation + ) + } + /// Golden parity table (#2612): every case in contracts/fixtures/window-coordinate-space.json /// must agree with the vitest twin /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). Add @@ -497,9 +327,9 @@ extension RunnerTests { /// rotation rows carry their own orientation. func testWindowCoordinateSpaceMatchesGoldenParityTable() throws { let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner + .deletingLastPathComponent() // AgentDeviceSnapshotPresentationTests + .deletingLastPathComponent() // Tests + .deletingLastPathComponent() // snapshot-presentation .deletingLastPathComponent() // apple .deletingLastPathComponent() // repo root .appendingPathComponent("contracts") @@ -549,60 +379,5 @@ extension RunnerTests { ) } } -} -#endif - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - /// Where a turned box has to appear before it declares a space. - func testOnlyASurfaceHostDeclaresTheCoordinateSpaceOfItsSubtree() { - let appFrame = CGRect(x: 0, y: 0, width: 874, height: 402) - let turned = CGRect(x: 0, y: 0, width: 402, height: 874) - let landscape = RunnerInterfaceOrientation.landscapeRight - - // XCTest gives `UIRemoteKeyboardWindow` the app's own box and its child the turned one, so the - // surface below a window is a host too. - XCTAssertTrue( - SnapshotGeometrySpace.isSurfaceHost( - elementType: XCUIElement.ElementType(rawValue: 21), - parentIsWindow: true - ) - ); - XCTAssertFalse( - SnapshotGeometrySpace.isSurfaceHost( - elementType: XCUIElement.ElementType(rawValue: 21), - parentIsWindow: false - ) - ); - let windowSpace = SnapshotGeometrySpace.space( - reportedBySurfaceHost: true, - reportedFrame: appFrame, - inheritedFrom: .appOrientation, - appFrame: appFrame, - interfaceOrientation: landscape - ) - XCTAssertEqual(windowSpace, .appOrientation) - let surfaceSpace = SnapshotGeometrySpace.space( - reportedBySurfaceHost: true, - reportedFrame: turned, - inheritedFrom: windowSpace, - appFrame: appFrame, - interfaceOrientation: landscape - ) - XCTAssertEqual(surfaceSpace, .deviceNative(appFrame: appFrame, interfaceOrientation: landscape)) - // Deep in the tree a turned box is content reporting large bounds, not a hosted surface: it keeps - // the space it inherited rather than rewriting the space below it. - XCTAssertEqual( - SnapshotGeometrySpace.space( - reportedBySurfaceHost: false, - reportedFrame: turned, - inheritedFrom: windowSpace, - appFrame: appFrame, - interfaceOrientation: landscape - ), - .appOrientation - ) - } } -#endif diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index 1424410655..a8497d9b07 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -1,5 +1,5 @@ { - "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side.", + "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side.", "constants": { "quarterTurnTolerance": 1 }, diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index e566b71489..1d9fbe309e 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -406,14 +406,19 @@ named — no usable app frame, an interface orientation the platform did not rep square enough to be indistinguishable from its own quarter turn — the capture publishes what the platform reported rather than guessing at a rotation, and consumers fail open on that geometry the way they do on any missing platform fact. Detection and the way back share one rotation table with -synthesized dispatch, and both languages replay it against +synthesized dispatch — pure geometry that lives in the `AgentDeviceSnapshotPresentation` package +(`SnapshotCoordinateSpace.swift`), not in the XCTest bundle — and both languages replay it against `contracts/fixtures/window-coordinate-space.json` in the ADR 0011 parity-table shape, which is what stops capture from turning back with anything other than the exact inverse of what dispatch turns forward. Decision. A producer that cannot name the app's interface orientation refuses the screen rather -than publishing two spaces in one tree. The Simulator AX bridge's attribute set carries no -orientation fact, so its decoder counts window roots reporting the app's box quarter-turned and +than publishing two spaces in one tree. The Simulator AX bridge cannot name it: the #2659 spike +(verdict on that issue, write-up in the diff of #2667) measured that the one AX attribute for it, +`XC_kAXXCAttributeApplicationOrientation` (id 1503), resolves but reads 0 through the guest's +snapshot channel and errors `kAXErrorServerNotFound` through XCTest's own reader, and that the only +cheap service read is *device* orientation, which diverges from the app's on a rotation-locked app. +So its decoder counts window roots reporting the app's box quarter-turned and fails that capture (`window-coordinate-space-unresolved`, kind `unsupported`) — the same refusal shape as `remote-content-boundary` above — and the route serves the runner, which reads the orientation. A mixed tree is not half-usable: a published rect is an address, and once some rects diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts index d679c9e205..8abe052783 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts @@ -10,9 +10,9 @@ import { // ADR 0011 Layer 2 golden parity table (#2612): the SAME JSON is asserted against the Swift twin // (SnapshotGeometrySpace and CoordinateSpaceRotation in -// apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift, gated XCTest in -// that file), so drift between the host's decoder and the runner's capture turns CI red on whichever -// side changed. +// apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, +// replayed by that package's CoordinateSpaceTests), so drift between the host's decoder and the +// runner's capture turns CI red on whichever side changed. type FixtureFrame = | { diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index 5d16c08995..13c58fa59e 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -23,9 +23,17 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * The same rule is enforced in Swift for the capture that produces these trees, and one table proves * they are one rule: `contracts/fixtures/window-coordinate-space.json` is replayed here and against * `SnapshotGeometrySpace` in - * `apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift`. The table also + * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`. The table also * pins the way back (`CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`, Swift-only * today); change either rule only through that table. + * + * Why this is a detector and not a repair: the bridge guest cannot read the app's interface + * orientation. Measured in the #2659 spike (verdict on the issue, write-up in the diff of #2667): + * AX attribute `XC_kAXXCAttributeApplicationOrientation` (id 1503) resolves but reads 0 through the + * guest's snapshot channel, and the cheap BackBoard read is device orientation, which turns a + * rotation-locked app's screen that never turned. The bar for retiring this file is a guest read + * that names the foreground app's orientation and survives a rotation-locked app; attribute 1503 is + * the handle to watch. */ /**