diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index 13244aeaf2..df933a24ab 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -217,12 +217,11 @@ extension RunnerTests { } let rootFrame = privateAXRect(root["frame"]) - let geometry = privateAXSnapshotGeometry( + let viewport = privateAXSnapshotViewport( app: app, bundleId: target.bundleId, rootFrame: rootFrame ) - let viewport = geometry.viewport let nodes = privateAXAcquisition( rawRoot: root, hint: hint @@ -259,8 +258,7 @@ extension RunnerTests { customActions: Self.privateAXCustomActionCoverage( response[RunnerAXSnapshotCustomActionsKey] ), - viewport: viewport, - interfaceOrientation: geometry.interfaceOrientation + viewport: viewport ) #else return nil @@ -275,37 +273,32 @@ extension RunnerTests { !hasAbandonedMainThreadWork() && !isSnapshotXCTestChannelPenalized(bundleId: bundleId) } - /// The geometry this tier may anchor a rotation on. The bridge's own root frame is one more - /// reported box rather than the app's frame, so a capture anchored on it reports no interface - /// orientation and normalizes nothing: rotated system surfaces then stay as reported, which the - /// consumers already treat as geometry they cannot measure (#2612). - private func privateAXSnapshotGeometry( + /// The app's reported viewport when XCTest can read it, else the bridge's own root frame declared + /// `.derived`, which cannot anchor a rotation: rotated system surfaces then stay as reported (#2612). + private func privateAXSnapshotViewport( app: XCUIApplication, bundleId: String?, rootFrame: CGRect - ) -> (viewport: CGRect, interfaceOrientation: Int) { - let fallback = rootFrame.isEmpty ? CGRect.infinite : rootFrame + ) -> SnapshotViewport { + let fallback = SnapshotViewport.derived(box: rootFrame) guard shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId) else { - return (fallback, RunnerInterfaceOrientation.unknown) + return fallback } do { - let anchor = try runMainThreadWork( + let reported = try runMainThreadWork( "private_ax_viewport", timeout: 1, timeoutError: snapshotMainThreadTimeoutError("reading private AX viewport") ) { - ( - viewport: self.safeSnapshotViewport(app: app), - interfaceOrientation: self.capturedInterfaceOrientation(app: app) - ) + self.safeSnapshotViewport(app: app, readingOrientation: true) } - if anchor.viewport.isInfinite || anchor.viewport.isNull || anchor.viewport.isEmpty { - return (fallback, RunnerInterfaceOrientation.unknown) + if case .missing = reported { + return fallback } - return (anchor.viewport, anchor.interfaceOrientation) + return reported } catch { NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_VIEWPORT_FALLBACK=%@", String(describing: error)) - return (fallback, RunnerInterfaceOrientation.unknown) + return fallback } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index da185b037e..3ab477ff74 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -1,3 +1,4 @@ +import AgentDeviceSnapshotPresentation import XCTest extension RunnerTests { @@ -120,16 +121,9 @@ extension RunnerTests { return navigationBackKeywords.firstIndex { text.contains($0) } } - // isFinite/>0 alone don't reject CGRect.infinite — its origin (~-9e307) is finite. - static func isUsableNavigationFrame(_ frame: CGRect) -> Bool { - guard frame.width.isFinite, frame.height.isFinite, frame.width > 0, frame.height > 0 else { - return false - } - return !frame.isInfinite - } - static func isTopNavigationControlFrame(_ candidate: CGRect, in window: CGRect) -> Bool { - guard isUsableNavigationFrame(candidate), isUsableNavigationFrame(window) else { + guard SnapshotGeometry.isPositiveFinite(candidate), SnapshotGeometry.isPositiveFinite(window) + else { return false } // Accept the compact navigation/search header band without matching deep content controls. @@ -138,7 +132,7 @@ extension RunnerTests { } static func topLeadingNavigationFallbackPoint(in frame: CGRect) -> CGPoint? { - guard isUsableNavigationFrame(frame) else { + guard SnapshotGeometry.isPositiveFinite(frame) else { return nil } // Aim at the standard leading navigation slot, bounded for compact and tablet widths. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 3d469bf868..f87b5b0648 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -21,9 +21,8 @@ extension RunnerTests { struct SnapshotTraversalContext { let queryRoot: XCUIElement let rootSnapshot: XCUIElementSnapshot - let viewport: CGRect - /** Which way the app's interface is turned from the device's native space (#2612). */ - let interfaceOrientation: Int + /** Carries which way the app's interface is turned from the device's native space (#2612). */ + let viewport: SnapshotViewport /** * The keyboard band this capture measured, published beside the tree so the daemon's tap guard * measures against the producer's own reading rather than a band it derives from these rects @@ -278,8 +277,7 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: context.viewport, - interfaceOrientation: context.interfaceOrientation + viewport: context.viewport ) } @@ -435,8 +433,7 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: context.viewport, - interfaceOrientation: context.interfaceOrientation + viewport: context.viewport ) } @@ -455,14 +452,13 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: .infinite, - interfaceOrientation: RunnerInterfaceOrientation.unknown + viewport: .missing(reason: .notProvided) ), .completed ) } - let viewport = safeSnapshotViewport(app: app) + let viewport = safeSnapshotViewport(app: app, readingOrientation: false) var seen = Set() var candidates: [RawAXNode] = [] let flatElements = flatInteractiveElements(app: app, deadline: deadline) @@ -492,11 +488,9 @@ extension RunnerTests { } // The synthetic root doubles as the daemon's viewport (find.ts prefers on-screen matches - // inside nodes[0].rect): use the real screen viewport when capture produced a finite one, - // so off-screen candidates can never inflate the root and masquerade as on-screen. - let rootRect = viewport.isInfinite || viewport.isNull || viewport.isEmpty - ? interactiveRootFrame(for: candidates) - : viewport + // inside nodes[0].rect): use the real screen viewport when the capture resolved one, so + // off-screen candidates can never inflate the root and masquerade as on-screen. + let rootRect = viewport.rect ?? interactiveRootFrame(for: candidates) nodes[0] = interactiveRootNode(rect: rootRect) for candidate in candidates { nodes.append( @@ -524,8 +518,7 @@ extension RunnerTests { nodes: nodes, truncated: outcome == .deadlineExhausted, effectiveDepth: nil, - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.unknown + viewport: viewport ), outcome ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 26572f56fa..7ebc82596f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -41,18 +41,13 @@ extension RunnerTests { // The viewport and the interface orientation are one hop: geometry that arrives in the device's // native space can only be placed relative to the app's own frame and rotation, and asking for // the pair twice would read them at two different moments of a rotation. - let geometry = try runMainThreadWork( + let viewport = try runMainThreadWork( "snapshot_viewport", timeout: min(1.0, max(0.1, captureDeadline.timeIntervalSinceNow)), timeoutError: snapshotMainThreadTimeoutError("preparing tree snapshot") ) { - ( - viewport: self.safeSnapshotViewport(app: app), - interfaceOrientation: self.capturedInterfaceOrientation(app: app) - ) + self.safeSnapshotViewport(app: app, readingOrientation: true) } - let viewport = geometry.viewport - let interfaceOrientation = geometry.interfaceOrientation let treeSliceBudget = treeCaptureSliceBudgetOverride ?? treeCaptureSliceBudget let slice = min(treeSliceBudget, max(0.5, captureDeadline.timeIntervalSinceNow)) guard let rootSnapshot = try captureSnapshotRootBounded(app, sliceSeconds: slice) else { @@ -70,7 +65,6 @@ extension RunnerTests { queryRoot: app, rootSnapshot: rootSnapshot, viewport: viewport, - interfaceOrientation: interfaceOrientation, keyboardBand: keyboardBand ) } @@ -140,8 +134,18 @@ extension RunnerTests { return nil } - func safeSnapshotViewport(app: XCUIApplication) -> CGRect { - safely("SNAPSHOT_VIEWPORT", CGRect.infinite) { snapshotViewport(app: app) } + /// The viewport as a declared fact; a read that raises is `.missing(reason: .notProvided)` (#2891). + /// `readingOrientation` reads the interface orientation in the same hop, for tiers whose frames can + /// arrive in the device's native space; without it the viewport cannot anchor a rotation. + func safeSnapshotViewport(app: XCUIApplication, readingOrientation: Bool) -> SnapshotViewport { + safely("SNAPSHOT_VIEWPORT", .missing(reason: .notProvided)) { + .reported( + box: snapshotAppFrame(app: app), + interfaceOrientation: readingOrientation + ? capturedInterfaceOrientation(app: app) + : RunnerInterfaceOrientation.unknown + ) + } } private func describeSnapshotError(_ error: Error) -> String { @@ -231,16 +235,12 @@ extension RunnerTests { return text.isEmpty ? nil : text } - private func snapshotViewport(app: XCUIApplication) -> CGRect { + private func snapshotAppFrame(app: XCUIApplication) -> CGRect { #if os(iOS) - let appFrame = onScreenWindowFrame(app: app) + return onScreenWindowFrame(app: app) #else - let appFrame = app.frame + return app.frame #endif - if !appFrame.isNull && !appFrame.isEmpty { - return appFrame - } - return .infinite } static func snapshotTraversalIdentity( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index e7f3898652..f1547821ba 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -513,8 +513,7 @@ extension RunnerTests { let normalizedAcquisition = acquisition.replacingNodes( SnapshotGeometrySpace.normalized( nodes: acquisition.nodes, - viewport: acquisition.viewport, - interfaceOrientation: acquisition.interfaceOrientation + viewport: acquisition.viewport ) ) @@ -641,7 +640,7 @@ extension RunnerTests { guard Self.structuralOnlyNodeTypes.contains(node.type) else { return false } guard !isRootContainer else { return true } - let isFullScreenContainer = !node.hittable && rootRects.contains { rootRect in + let isFullScreenContainer = node.hittable != true && rootRects.contains { rootRect in rootRect.x == node.rect.x && rootRect.y == node.rect.y && rootRect.width == node.rect.width && rootRect.height == node.rect.height } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift index 82f6bec10b..4e4fd9ff44 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift @@ -464,15 +464,14 @@ extension RunnerTests { interactiveOnly: true, customActions: false) let acquired = SnapshotGeometrySpace.normalized( nodes: privateAXAcquisition(rawRoot: tree, hint: hint), - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait + viewport: .reported(box: viewport, interfaceOrientation: RunnerInterfaceOrientation.portrait) ) // Acquisition serializes the drawer too; the shared fold is what hides it (#1797). XCTAssertTrue(acquired.compactMap(\.label).contains("Admin settings")) let capture = try SnapshotPresentation.presentRegular( SnapshotAcquisition( - hint: hint, nodes: acquired, truncated: false, effectiveDepth: nil, viewport: viewport), + hint: hint, nodes: acquired, truncated: false, effectiveDepth: nil, viewport: .reported(box: viewport)), options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false), policy: .cursorProjected ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift index 23ee5db993..3dcd2908dd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift @@ -42,8 +42,11 @@ extension RunnerTests { interfaceOrientation: RunnerInterfaceOrientation.portrait) return try SnapshotPresentation.presentRegular( SnapshotAcquisition( - hint: hint, nodes: nodes, truncated: false, effectiveDepth: nil, viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait), + hint: hint, nodes: nodes, truncated: false, effectiveDepth: nil, + viewport: .reported( + box: viewport, + interfaceOrientation: RunnerInterfaceOrientation.portrait + )), options: PresentationOptions( interactiveOnly: interactiveOnly, depth: nil, scope: nil, raw: false), policy: .cursorProjected @@ -59,8 +62,7 @@ extension RunnerTests { ) -> [RawAXNode] { SnapshotGeometrySpace.normalized( nodes: privateAXAcquisition(rawRoot: rawRoot, hint: hint), - viewport: viewport, - interfaceOrientation: interfaceOrientation + viewport: .reported(box: viewport, interfaceOrientation: interfaceOrientation) ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift index edec6e8333..7f12cb87b6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift @@ -94,8 +94,7 @@ extension RunnerTests { XCTAssertFalse(app.frame.isEmpty) // The traversal context reads the viewport under a 1 s cap; a cold first read can overrun it // and abandon the wrong block, so pay it here, uncapped. - _ = safeSnapshotViewport(app: app) - _ = capturedInterfaceOrientation(app: app) + _ = safeSnapshotViewport(app: app, readingOrientation: true) currentApp = app currentBundleId = "com.callstack.agentdevice.runner.tree-capture-test" snapshotXCTestPenaltyWarmupExemption.isPending = true diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift index a2c709eeb9..87d3ad061e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift @@ -282,7 +282,7 @@ extension RunnerTests { nodes: [], truncated: false, effectiveDepth: nil, - viewport: .infinite + viewport: .reported(box: CGRect(x: 0, y: 0, width: 402, height: 874)) ), options: options ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotHittabilityTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotHittabilityTests.swift index 61e0024121..89828f62f7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotHittabilityTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotHittabilityTests.swift @@ -97,7 +97,7 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 100, height: 100) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: options ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationConformanceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationConformanceTests.swift index 718eb5651f..2c16b49851 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationConformanceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationConformanceTests.swift @@ -97,7 +97,7 @@ extension RunnerTests { }, truncated: false, effectiveDepth: nil, - viewport: fixture.viewport.cgRect + viewport: .reported(box: fixture.viewport.cgRect) ) let options = PresentationOptions( interactiveOnly: false, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationGeometryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationGeometryTests.swift index 087f00e510..db52ad5d2f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationGeometryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationGeometryTests.swift @@ -6,7 +6,7 @@ extension RunnerTests { func testEffectiveGeometryIntersectsViewportAndAncestorClip() { let effective = SnapshotGeometry.effectiveFrame( reportedFrame: CGRect(x: 350, y: 80, width: 100, height: 100), - viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + viewport: .reported(box: CGRect(x: 0, y: 0, width: 402, height: 874)), ancestorClip: CGRect(x: 300, y: 100, width: 80, height: 80) ) @@ -17,7 +17,7 @@ extension RunnerTests { let reported = CGRect(x: 500, y: 120, width: 100, height: 44) let effective = SnapshotGeometry.effectiveFrame( reportedFrame: reported, - viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + viewport: .reported(box: CGRect(x: 0, y: 0, width: 402, height: 874)), ancestorClip: nil ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift index c0c739b86d..9aeea4f490 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift @@ -97,7 +97,7 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: viewport + viewport: .reported(box: viewport) ) } @@ -147,7 +147,7 @@ extension RunnerTests { ], truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 320, height: 240) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 320, height: 240)) ) let options = PresentationOptions( @@ -203,7 +203,7 @@ extension RunnerTests { ], truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 320, height: 240) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 320, height: 240)) ) let options = PresentationOptions( @@ -254,7 +254,7 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: .infinite + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: options ).nodes) @@ -309,7 +309,7 @@ extension RunnerTests { XCTAssertThrowsError( try SnapshotPresentationInvariant.validateRegular( folded, - viewport: viewport, + viewport: .reported(box: viewport), policy: .cursorProjected ) ) { error in diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift index c103678186..855abbd25b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift @@ -40,7 +40,7 @@ extension RunnerTests { truncated: 0, blocked: false ), - viewport: .infinite + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: PresentationOptions( interactiveOnly: true, @@ -113,7 +113,7 @@ extension RunnerTests { projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, customActions: false), nodes: acquired, truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 1_000, height: 1_000)), + viewport: .reported(box: CGRect(x: 0, y: 0, width: 1_000, height: 1_000))), options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) ).nodes ) @@ -124,7 +124,7 @@ extension RunnerTests { projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: true, customActions: false), nodes: acquired, truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 1_000, height: 1_000)), + viewport: .reported(box: CGRect(x: 0, y: 0, width: 1_000, height: 1_000))), options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false) ).nodes ) @@ -147,7 +147,8 @@ extension RunnerTests { hint: CaptureHint( projection: .raw, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, customActions: false), - nodes: acquired, truncated: false, effectiveDepth: nil, viewport: .infinite), + nodes: acquired, truncated: false, effectiveDepth: nil, + viewport: .reported(box: CGRect(x: 0, y: 0, width: 1_000, height: 1_000)) ), options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: true) ).nodes ) @@ -176,7 +177,7 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 100, height: 100) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ) let presented = try SnapshotPresentation.presentRegular( @@ -211,7 +212,7 @@ extension RunnerTests { nodes: acquired, truncated: false, effectiveDepth: nil, - viewport: viewport + viewport: .reported(box: viewport) ), options: regularOptions ).nodes @@ -231,7 +232,7 @@ extension RunnerTests { nodes: acquired, truncated: false, effectiveDepth: nil, - viewport: .infinite + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: rawOptions ).nodes @@ -286,7 +287,7 @@ extension RunnerTests { ], truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 1_000, height: 1_000) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 1_000, height: 1_000)) ) let options = PresentationOptions( interactiveOnly: true, @@ -313,7 +314,7 @@ extension RunnerTests { nodes: acquisition.nodes, truncated: false, effectiveDepth: nil, - viewport: .infinite + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: PresentationOptions( interactiveOnly: true, @@ -374,10 +375,11 @@ extension RunnerTests { let regularAcquisition = SnapshotAcquisition( hint: SnapshotPresentation.captureHint(for: regularRequest), nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 100, height: 100)) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100))) let rawAcquisition = SnapshotAcquisition( hint: SnapshotPresentation.captureHint(for: rawRequest), - nodes: nodes, truncated: false, effectiveDepth: nil, viewport: .infinite) + nodes: nodes, truncated: false, effectiveDepth: nil, + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100))) let regularCaptureForRawRequest = try SnapshotPresentation.present( regularAcquisition, options: rawRequest) @@ -470,7 +472,7 @@ extension RunnerTests { ], truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 100, height: 100) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: options ) @@ -552,7 +554,7 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: viewport + viewport: .reported(box: viewport) ) let presented = try XCTUnwrap( SnapshotPresentation.present(acquisition, options: options)?.nodes) @@ -596,8 +598,10 @@ extension RunnerTests { ] let normalized = SnapshotGeometrySpace.normalized( nodes: acquired, - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + viewport: .reported( + box: viewport, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) ) let options = PresentationOptions(interactiveOnly: false, depth: 3, scope: nil, raw: false) let hint = SnapshotPresentation.captureHint(for: options) @@ -608,7 +612,7 @@ extension RunnerTests { nodes: normalized, truncated: false, effectiveDepth: nil, - viewport: viewport + viewport: .reported(box: viewport) ), options: options )?.nodes diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotVisibilityFoldTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotVisibilityFoldTests.swift index 2063c56f14..ee6c4400dd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotVisibilityFoldTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotVisibilityFoldTests.swift @@ -25,7 +25,11 @@ extension RunnerTests { policy: SnapshotVisibilityFold.Policy = .cursorProjected ) -> [SnapshotPresentationNode] { SnapshotVisibilityFold.fold( - nodes, viewport: viewport, interactiveOnly: interactiveOnly, policy: policy) + nodes, + viewport: .reported(box: viewport), + interactiveOnly: interactiveOnly, + policy: policy + ) } func testRegularFoldClipsScrollOverflowReparentsAndBooksHints() { diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 561861bc29..300b8c910e 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -105,7 +105,7 @@ public enum CoordinateSpaceRotation { public enum SnapshotGeometrySpace: Equatable { case appOrientation - case deviceNative(appFrame: CGRect, interfaceOrientation: Int) + case deviceNative(appFrame: SnapshotViewport.Box, interfaceOrientation: Int) public static let quarterTurnTolerance: Double = 1 @@ -116,22 +116,26 @@ public enum SnapshotGeometrySpace: Equatable { case .deviceNative(let appFrame, let interfaceOrientation): return CoordinateSpaceRotation.oriented( rect: reportedFrame, - in: appFrame, + in: appFrame.rect, interfaceOrientation: interfaceOrientation ) } } + /// Only a `.reported` viewport carries an orientation, so only it can anchor a rotation. public static func space( reportedBySurfaceHost isSurfaceHost: Bool, reportedFrame: CGRect, inheritedFrom inherited: SnapshotGeometrySpace, - appFrame: CGRect, - interfaceOrientation: Int + viewport: SnapshotViewport ) -> SnapshotGeometrySpace { guard isSurfaceHost else { return inherited } - guard namesQuarterTurn(interfaceOrientation) else { return .appOrientation } - guard isQuarterTurned(reportedFrame, relativeTo: appFrame) else { return .appOrientation } + guard case .reported(let appFrame, let interfaceOrientation) = viewport, + namesQuarterTurn(interfaceOrientation), + isQuarterTurned(reportedFrame, relativeTo: appFrame.rect) + else { + return .appOrientation + } return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) } @@ -145,7 +149,7 @@ public enum SnapshotGeometrySpace: Equatable { } private static func isQuarterTurned(_ frame: CGRect, relativeTo appFrame: CGRect) -> Bool { - guard isPlottable(frame), isPlottable(appFrame), + guard SnapshotGeometry.isPositiveFinite(frame), abs(appFrame.width - appFrame.height) > quarterTurnTolerance else { return false @@ -153,19 +157,12 @@ public enum SnapshotGeometrySpace: Equatable { return abs(frame.width - appFrame.height) <= quarterTurnTolerance && abs(frame.height - appFrame.width) <= quarterTurnTolerance } - - private static func isPlottable(_ frame: CGRect) -> Bool { - frame.origin.x.isFinite && frame.origin.y.isFinite - && frame.width.isFinite && frame.height.isFinite - && frame.width > 0 && frame.height > 0 - } } extension SnapshotGeometrySpace { public static func normalized( nodes: [RawAXNode], - viewport: CGRect, - interfaceOrientation: Int + viewport: SnapshotViewport ) -> [RawAXNode] { let carriers = SnapshotVisibilityFold.visibilityExemptCarrierTypes var spaces = [SnapshotGeometrySpace](repeating: .appOrientation, count: nodes.count) @@ -180,16 +177,16 @@ extension SnapshotGeometrySpace { ), reportedFrame: node.rect.cgRect, inheritedFrom: parentIndex.map { spaces[$0] } ?? .appOrientation, - appFrame: viewport, - interfaceOrientation: interfaceOrientation + viewport: viewport ) spaces[position] = nodeSpace let frame = nodeSpace.orientedFrame(of: node.rect.cgRect) result.append( node.replacing( rect: SnapshotRect(frame), - hittable: node.parentIndex != nil - && SnapshotGeometry.isGeometricallyActionable( + hittable: node.parentIndex == nil + ? false + : SnapshotGeometry.isGeometricallyActionable( enabled: node.enabled, frame: frame, viewport: viewport diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotGeometry.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotGeometry.swift index ec8bc50863..d0db9a8e24 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotGeometry.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotGeometry.swift @@ -2,14 +2,23 @@ import Foundation import CoreGraphics public enum SnapshotGeometry { + /// Twin of `isPositiveFiniteRect` in `packages/kernel/src/rect.ts`. `CGRect.infinite` is built + /// from finite components, so it is refused by identity. + public static func isPositiveFinite(_ rect: CGRect) -> Bool { + !rect.isInfinite + && rect.origin.x.isFinite && rect.origin.y.isFinite + && rect.size.width.isFinite && rect.size.height.isFinite + && rect.size.width > 0 && rect.size.height > 0 + } + public static func effectiveFrame( reportedFrame: CGRect, - viewport: CGRect, + viewport: SnapshotViewport, ancestorClip: CGRect? ) -> CGRect { var frame = reportedFrame - if !viewport.isInfinite { - frame = clipped(frame, to: viewport) + if let box = viewport.rect { + frame = clipped(frame, to: box) } if let ancestorClip { frame = clipped(frame, to: ancestorClip) @@ -34,15 +43,18 @@ public enum SnapshotGeometry { ) } + /// The shared `hittable` predicate (#1933), twin of `isGeometricallyActionable` in + /// `packages/kernel/src/rect.ts`, with `CGRect.contains`'s half-open right and bottom edges. + /// `nil` when only containment is left to decide and the capture has no viewport box: the node's + /// `hittable` is then absent on the wire, as it is on the host bridge (#2891). public static func isGeometricallyActionable( enabled: Bool, frame: CGRect, - viewport: CGRect - ) -> Bool { - guard enabled, !frame.isNull, !frame.isEmpty else { return false } - if viewport.isInfinite { return true } - let center = CGPoint(x: frame.midX, y: frame.midY) - return viewport.contains(center) + viewport: SnapshotViewport + ) -> Bool? { + guard enabled, isPositiveFinite(frame) else { return false } + guard let box = viewport.rect else { return nil } + return box.contains(CGPoint(x: frame.midX, y: frame.midY)) } private static func clipped(_ frame: CGRect, to clip: CGRect) -> CGRect { diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift index 95a3cf39d7..d0421aab6d 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift @@ -38,7 +38,8 @@ public struct RawAXNode: Equatable { public let enabled: Bool public let focused: Bool? public let selected: Bool? - public var hittable: Bool + /// Geometric actionability; `nil` when the capture has no viewport box to decide it against. + public var hittable: Bool? public let depth: Int public let parentIndex: Int? public let hiddenContentAbove: Bool? @@ -55,7 +56,7 @@ public struct RawAXNode: Equatable { enabled: Bool, focused: Bool?, selected: Bool?, - hittable: Bool, + hittable: Bool?, depth: Int, parentIndex: Int?, hiddenContentAbove: Bool?, @@ -79,7 +80,7 @@ public struct RawAXNode: Equatable { self.actions = actions } - func replacing(rect: SnapshotRect, hittable: Bool) -> RawAXNode { + func replacing(rect: SnapshotRect, hittable: Bool?) -> RawAXNode { var updated = self updated.rect = rect updated.hittable = hittable @@ -153,15 +154,65 @@ public struct PresentationOptions: Equatable { } } +/// What a capture knows about the viewport hosting its tree: the three cases of the host's +/// `IosViewportEvidence` (#2891). No rectangle stands for "unknown". +public enum SnapshotViewport: Equatable { + /// A box `SnapshotGeometry.isPositiveFinite` accepted. Only the factories below construct one. + public struct Box: Equatable { + public let rect: CGRect + + init(positiveFinite rect: CGRect) { + self.rect = rect + } + } + + /// The platform's box for the app's surface, with the interface orientation read in the same hop. + /// Only this case can anchor a rotation (#2612). + case reported(Box, interfaceOrientation: Int) + /// A box the capture inferred from its own root element. It clips and contains, and carries no + /// orientation, so it cannot anchor a rotation. + case derived(Box) + case missing(reason: MissingReason) + + public enum MissingReason: Equatable { + /// The read was skipped or raised. + case notProvided + /// The box read is one `SnapshotGeometry.isPositiveFinite` refuses. + case invalid + } + + public var rect: CGRect? { + switch self { + case .reported(let box, _), .derived(let box): + return box.rect + case .missing: + return nil + } + } + + public static func reported( + box: CGRect, + interfaceOrientation: Int = RunnerInterfaceOrientation.unknown + ) -> SnapshotViewport { + SnapshotGeometry.isPositiveFinite(box) + ? .reported(Box(positiveFinite: box), interfaceOrientation: interfaceOrientation) + : .missing(reason: .invalid) + } + + public static func derived(box: CGRect) -> SnapshotViewport { + SnapshotGeometry.isPositiveFinite(box) + ? .derived(Box(positiveFinite: box)) + : .missing(reason: .invalid) + } +} + public struct SnapshotAcquisition { public let hint: CaptureHint public var nodes: [RawAXNode] public let truncated: Bool public let effectiveDepth: Int? public var customActions: SnapshotCustomActionCoverage? - public let viewport: CGRect - /// The app's interface orientation, consumed by the one `normalized` pass; `unknown` turns nothing. - public let interfaceOrientation: Int + public let viewport: SnapshotViewport public init( hint: CaptureHint, @@ -169,8 +220,7 @@ public struct SnapshotAcquisition { truncated: Bool, effectiveDepth: Int?, customActions: SnapshotCustomActionCoverage? = nil, - viewport: CGRect, - interfaceOrientation: Int = 0 + viewport: SnapshotViewport ) { self.hint = hint self.nodes = nodes @@ -178,7 +228,6 @@ public struct SnapshotAcquisition { self.effectiveDepth = effectiveDepth self.customActions = customActions self.viewport = viewport - self.interfaceOrientation = interfaceOrientation } public func replacingNodes(_ nodes: [RawAXNode]) -> SnapshotAcquisition { @@ -226,7 +275,7 @@ public struct PresentedNode: Codable, Equatable { public let enabled: Bool public let focused: Bool? public let selected: Bool? - public let hittable: Bool + public let hittable: Bool? public let depth: Int public let parentIndex: Int? public let hiddenContentAbove: Bool? diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationInvariant.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationInvariant.swift index 7ce2f25589..a28ba33b2c 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationInvariant.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationInvariant.swift @@ -1,6 +1,7 @@ import Foundation import CoreGraphics +/// With no viewport box the cumulative clip has no root, so containment has nothing to violate. public enum SnapshotPresentationInvariant { struct ValidationStats: Equatable { let parentClipLookups: Int @@ -12,7 +13,7 @@ public enum SnapshotPresentationInvariant { public static func validateRegular( _ nodes: [SnapshotPresentationNode], - viewport: CGRect, + viewport: SnapshotViewport, policy: SnapshotVisibilityFold.Policy ) throws { _ = try validateRegularWithStats(nodes, viewport: viewport, policy: policy) @@ -20,24 +21,25 @@ public enum SnapshotPresentationInvariant { static func validateRegularWithStats( _ nodes: [SnapshotPresentationNode], - viewport: CGRect, + viewport: SnapshotViewport, policy: SnapshotVisibilityFold.Policy ) throws -> ValidationStats { var parentClipLookups = 0 - var clipIncludingNodeByIndex: [Int: CGRect] = [:] + var clipIncludingNodeByIndex: [Int: CGRect?] = [:] clipIncludingNodeByIndex.reserveCapacity(nodes.count) + let rootClip = viewport.rect for node in nodes { - let ancestorClip: CGRect + let ancestorClip: CGRect? if let parentIndex = node.raw.parentIndex { parentClipLookups += 1 - ancestorClip = clipIncludingNodeByIndex[parentIndex] ?? viewport + ancestorClip = clipIncludingNodeByIndex[parentIndex] ?? rootClip } else { - ancestorClip = viewport + ancestorClip = rootClip } let frame = node.effectiveRect.cgRect - let clipIncludingNode: CGRect + let clipIncludingNode: CGRect? if policy == .cursorProjected, SnapshotVisibilityFold.scrollContainerTypeNames.contains(node.raw.type), !frame.isNull, @@ -50,7 +52,7 @@ public enum SnapshotPresentationInvariant { clipIncludingNodeByIndex[node.raw.index] = clipIncludingNode guard !frame.isNull, !frame.isEmpty else { - if node.raw.hittable { + if node.raw.hittable == true { throw SnapshotPresentationFailure.regularDegenerateNodeIsActionable( index: node.raw.index, frame: node.effectiveRect @@ -59,11 +61,11 @@ public enum SnapshotPresentationInvariant { continue } - guard contains(frame, in: ancestorClip) else { + if let clip = ancestorClip, !contains(frame, in: clip) { throw SnapshotPresentationFailure.regularNodeOutsideCumulativeClip( index: node.raw.index, frame: node.effectiveRect, - clip: SnapshotRect(ancestorClip) + clip: SnapshotRect(clip) ) } } diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift index afd0b479c4..9f9a4d3ea3 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift @@ -100,7 +100,7 @@ public enum SnapshotVisibilityFold { public static func traversalDecision( for node: RawAXNode, parent: TraversalState, - viewport: CGRect, + viewport: SnapshotViewport, interactiveOnly: Bool, hasChildren: Bool, policy: Policy @@ -153,7 +153,7 @@ public enum SnapshotVisibilityFold { public static func fold( _ nodes: [RawAXNode], - viewport: CGRect, + viewport: SnapshotViewport, interactiveOnly: Bool, policy: Policy ) -> [SnapshotPresentationNode] { @@ -203,12 +203,17 @@ public enum SnapshotVisibilityFold { enabled: node.enabled, focused: node.focused, selected: node.selected, - hittable: node.parentIndex != nil && node.hittable - && SnapshotGeometry.isGeometricallyActionable( - enabled: node.enabled, - frame: decision.effectiveFrame, - viewport: viewport - ), + hittable: node.parentIndex == nil + ? false + : node.hittable.flatMap { sourceHittable in + sourceHittable + ? SnapshotGeometry.isGeometricallyActionable( + enabled: node.enabled, + frame: decision.effectiveFrame, + viewport: viewport + ) + : false + }, depth: outDepth, parentIndex: keptIndex, hiddenContentAbove: node.hiddenContentAbove, diff --git a/apple/snapshot-presentation/Sources/SnapshotPresentationConformance/main.swift b/apple/snapshot-presentation/Sources/SnapshotPresentationConformance/main.swift index 89bd98c4a1..005386ba10 100644 --- a/apple/snapshot-presentation/Sources/SnapshotPresentationConformance/main.swift +++ b/apple/snapshot-presentation/Sources/SnapshotPresentationConformance/main.swift @@ -75,7 +75,8 @@ private func acquisition(for input: ConformanceInput) -> SnapshotAcquisition { }, truncated: false, effectiveDepth: nil, - viewport: input.viewport.cgRect + // The host engine refuses to fold without a viewport, so the differential always reports one. + viewport: .reported(box: input.viewport.cgRect) ) } diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/ActionabilityPolicyTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/ActionabilityPolicyTests.swift new file mode 100644 index 0000000000..27882e1442 --- /dev/null +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/ActionabilityPolicyTests.swift @@ -0,0 +1,118 @@ +import AgentDeviceSnapshotPresentation +import CoreGraphics +import Foundation +import XCTest + +/// Replays `contracts/fixtures/snapshot-actionability-policy.json` against the Swift predicate. The +/// TypeScript twin replays the same rows in `scripts/ios-snapshot-differential.test.ts`. +final class ActionabilityPolicyTests: XCTestCase { + private struct Table: Decodable { + let cases: [PolicyCase] + } + + private struct ViewportFact: Decodable { + private enum CodingKeys: String, CodingKey { + case kind, rect, reason + } + + let declaredKind: String + let viewport: SnapshotViewport + + /// A row whose box the factory refuses would silently test `missing` under another label. + var matchesDeclaredKind: Bool { + switch (declaredKind, viewport) { + case ("reported", .reported), ("derived", .derived), ("missing", .missing): + return true + default: + return false + } + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let kind = try container.decode(String.self, forKey: .kind) + self.declaredKind = kind + switch kind { + case "reported": + self.viewport = .reported(box: try container.decode(FixtureRect.self, forKey: .rect).cgRect) + case "derived": + self.viewport = .derived(box: try container.decode(FixtureRect.self, forKey: .rect).cgRect) + case "missing": + switch try container.decode(String.self, forKey: .reason) { + case "not-provided": + self.viewport = .missing(reason: .notProvided) + case "invalid": + self.viewport = .missing(reason: .invalid) + default: + throw DecodingError.dataCorruptedError( + forKey: .reason, + in: container, + debugDescription: "unknown missing-viewport reason" + ) + } + default: + throw DecodingError.dataCorruptedError( + forKey: .kind, + in: container, + debugDescription: "unknown viewport kind" + ) + } + } + } + + private struct PolicyCase: Decodable { + let name: String + let swift: Bool + let typescript: Bool + let asymmetry: String? + let enabled: Bool + let node: FixtureRect + let viewport: ViewportFact + /// `nil` is the absent bit. + let hittable: Bool? + let nodeRectGuardPasses: Bool + + var declaresItsAsymmetry: Bool { + (swift && typescript) != (asymmetry?.isEmpty == false) + } + } + + func testActionabilityPolicyAgreesWithEveryGoldenVector() throws { + let table = try JSONDecoder().decode( + Table.self, + from: Data(contentsOf: contractsFixtureURL("snapshot-actionability-policy.json")) + ) + XCTAssertEqual(Set(table.cases.map(\.name)).count, table.cases.count, "names must be unique") + let swiftCases = table.cases.filter(\.swift) + XCTAssertEqual( + Set(swiftCases.map(\.viewport.declaredKind)), + ["reported", "derived", "missing"] + ) + for testCase in table.cases { + XCTAssertTrue( + testCase.declaresItsAsymmetry, + "\(testCase.name): a row both languages do not share must name the asymmetry" + ) + } + for testCase in swiftCases { + XCTAssertTrue( + testCase.viewport.matchesDeclaredKind, + "\(testCase.name): declared \(testCase.viewport.declaredKind) must survive declaration" + ) + XCTAssertEqual( + SnapshotGeometry.isPositiveFinite(testCase.node.cgRect), + testCase.nodeRectGuardPasses, + "\(testCase.name): node-rect guard" + ) + XCTAssertEqual( + SnapshotGeometry.isGeometricallyActionable( + enabled: testCase.enabled, + frame: testCase.node.cgRect, + viewport: testCase.viewport.viewport + ), + testCase.hittable, + testCase.name + ) + } + } +} diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/ConformanceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/ConformanceTests.swift index 2e526095bf..d99fdda405 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/ConformanceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/ConformanceTests.swift @@ -49,7 +49,7 @@ final class ConformanceTests: XCTestCase { ], truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 100, height: 100) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: options, policy: .cursorProjected diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index 90e67e938d..c5df2bc22c 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -7,30 +7,7 @@ import XCTest /// without a simulator, and the runner's walkers are tested separately for threading it through. final class CoordinateSpaceTests: XCTestCase { private struct WindowCoordinateSpaceFixture: Decodable { - /// A frame as JSON can carry one. Infinity has no JSON spelling, so the unusable box a platform - /// hands back is named `{"infinite": true}`: `CGRect.infinite` here, and an infinite rect in the - /// vitest twin. - struct Frame: Decodable { - private enum CodingKeys: String, CodingKey { - case x, y, width, height, infinite - } - - let cgRect: CGRect - - init(from decoder: Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - guard try container.decodeIfPresent(Bool.self, forKey: .infinite) != true else { - self.cgRect = .infinite - return - } - self.cgRect = CGRect( - x: try container.decode(Double.self, forKey: .x), - y: try container.decode(Double.self, forKey: .y), - width: try container.decode(Double.self, forKey: .width), - height: try container.decode(Double.self, forKey: .height) - ) - } - } + typealias Frame = FixtureRect struct Constants: Decodable { let quarterTurnTolerance: Double @@ -161,10 +138,9 @@ final class CoordinateSpaceTests: XCTestCase { reportedBySurfaceHost: true, reportedFrame: CGRect(x: 0, y: 0, width: 402, height: 874), inheritedFrom: .appOrientation, - appFrame: appFrame, - interfaceOrientation: landscape + viewport: .reported(box: appFrame, interfaceOrientation: landscape) ) - XCTAssertEqual(space, .deviceNative(appFrame: appFrame, interfaceOrientation: landscape)) + XCTAssertEqual(space, deviceNative(appFrame, landscape)) // Reported on iPhone 17 Pro (iOS 26.2), landscape, system keyboard over the fixture's form. XCTAssertEqual( @@ -198,8 +174,10 @@ final class CoordinateSpaceTests: XCTestCase { reportedBySurfaceHost: true, reportedFrame: rotated, inheritedFrom: .appOrientation, - appFrame: appFrame, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + viewport: .reported( + box: appFrame, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) ) // The app's own window reports the app's box, and so does a hosted surface that already @@ -209,8 +187,10 @@ final class CoordinateSpaceTests: XCTestCase { reportedBySurfaceHost: true, reportedFrame: appFrame, inheritedFrom: nativeSpace, - appFrame: appFrame, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + viewport: .reported( + box: appFrame, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) ), .appOrientation ) @@ -223,8 +203,10 @@ final class CoordinateSpaceTests: XCTestCase { ), reportedFrame: CGRect(x: 154, y: 77, width: 45, height: 72), inheritedFrom: nativeSpace, - appFrame: appFrame, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + viewport: .reported( + box: appFrame, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) ), nativeSpace ) @@ -234,8 +216,7 @@ final class CoordinateSpaceTests: XCTestCase { reportedBySurfaceHost: true, reportedFrame: rotated, inheritedFrom: .appOrientation, - appFrame: appFrame, - interfaceOrientation: RunnerInterfaceOrientation.unknown + viewport: .reported(box: appFrame, interfaceOrientation: RunnerInterfaceOrientation.unknown) ) XCTAssertEqual(unnamed, .appOrientation) XCTAssertEqual( @@ -248,22 +229,26 @@ final class CoordinateSpaceTests: XCTestCase { reportedBySurfaceHost: true, reportedFrame: rotated, inheritedFrom: .appOrientation, - appFrame: CGRect(x: 16, y: 24, width: 874, height: 402), - interfaceOrientation: RunnerInterfaceOrientation.portrait - ), - .appOrientation - ) - // An app frame the capture could not resolve cannot anchor a rotation. - XCTAssertEqual( - SnapshotGeometrySpace.space( - reportedBySurfaceHost: true, - reportedFrame: rotated, - inheritedFrom: .appOrientation, - appFrame: .infinite, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + viewport: .reported( + box: CGRect(x: 16, y: 24, width: 874, height: 402), + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) ), .appOrientation ) + // Only a reported viewport carries an orientation: no box, or a box the capture derived from + // its own root, cannot anchor a rotation (#2891). + for viewport in [SnapshotViewport.missing(reason: .notProvided), .derived(box: appFrame)] { + XCTAssertEqual( + SnapshotGeometrySpace.space( + reportedBySurfaceHost: true, + reportedFrame: rotated, + inheritedFrom: .appOrientation, + viewport: viewport + ), + .appOrientation + ) + } // A square app cannot be told from its own quarter turn, so its geometry is left alone. let square = CGRect(x: 0, y: 0, width: 800, height: 800) XCTAssertEqual( @@ -271,8 +256,10 @@ final class CoordinateSpaceTests: XCTestCase { reportedBySurfaceHost: true, reportedFrame: square, inheritedFrom: .appOrientation, - appFrame: square, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + viewport: .reported( + box: square, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) ), .appOrientation ) @@ -293,18 +280,16 @@ final class CoordinateSpaceTests: XCTestCase { reportedBySurfaceHost: true, reportedFrame: appFrame, inheritedFrom: .appOrientation, - appFrame: appFrame, - interfaceOrientation: landscape + viewport: .reported(box: appFrame, interfaceOrientation: landscape) ) XCTAssertEqual(windowSpace, .appOrientation) let surfaceSpace = SnapshotGeometrySpace.space( reportedBySurfaceHost: true, reportedFrame: turned, inheritedFrom: windowSpace, - appFrame: appFrame, - interfaceOrientation: landscape + viewport: .reported(box: appFrame, interfaceOrientation: landscape) ) - XCTAssertEqual(surfaceSpace, .deviceNative(appFrame: appFrame, interfaceOrientation: landscape)) + XCTAssertEqual(surfaceSpace, deviceNative(appFrame, landscape)) // Deep in the tree a turned box is content reporting large bounds, not a hosted surface: it keeps // the space it inherited rather than rewriting the space below it. XCTAssertEqual( @@ -312,8 +297,7 @@ final class CoordinateSpaceTests: XCTestCase { reportedBySurfaceHost: false, reportedFrame: turned, inheritedFrom: windowSpace, - appFrame: appFrame, - interfaceOrientation: landscape + viewport: .reported(box: appFrame, interfaceOrientation: landscape) ), .appOrientation ) @@ -340,15 +324,14 @@ final class CoordinateSpaceTests: XCTestCase { RunnerInterfaceOrientation.landscapeRight, RunnerInterfaceOrientation.landscapeLeft ] { let expected: SnapshotGeometrySpace = testCase.quarterTurned - ? .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) + ? deviceNative(appFrame, interfaceOrientation) : .appOrientation XCTAssertEqual( SnapshotGeometrySpace.space( reportedBySurfaceHost: true, reportedFrame: testCase.window.cgRect, inheritedFrom: .appOrientation, - appFrame: appFrame, - interfaceOrientation: interfaceOrientation + viewport: .reported(box: appFrame, interfaceOrientation: interfaceOrientation) ), expected, "\(testCase.name) (interfaceOrientation \(interfaceOrientation))" @@ -368,16 +351,15 @@ final class CoordinateSpaceTests: XCTestCase { } } + private func deviceNative(_ appFrame: CGRect, _ interfaceOrientation: Int) -> SnapshotGeometrySpace { + guard case .reported(let box, _) = SnapshotViewport.reported(box: appFrame) else { + preconditionFailure("\(appFrame) is not a viewport box") + } + return .deviceNative(appFrame: box, interfaceOrientation: interfaceOrientation) + } + private func loadWindowCoordinateSpaceFixture() throws -> WindowCoordinateSpaceFixture { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceSnapshotPresentationTests - .deletingLastPathComponent() // Tests - .deletingLastPathComponent() // snapshot-presentation - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("window-coordinate-space.json") + let fixtureURL = contractsFixtureURL("window-coordinate-space.json") return try JSONDecoder().decode( WindowCoordinateSpaceFixture.self, from: Data(contentsOf: fixtureURL) @@ -421,8 +403,7 @@ final class CoordinateSpaceTests: XCTestCase { let expected = (namesQuarterTurn && !squareApp) ? testCase.oriented : testCase.native let normalized = SnapshotGeometrySpace.normalized( nodes: turnedSubtree(app: app, reportedLeaf: testCase.native.cgRect), - viewport: app, - interfaceOrientation: testCase.interfaceOrientation + viewport: .reported(box: app, interfaceOrientation: testCase.interfaceOrientation) ) XCTAssertEqual(normalized.count, 4) XCTAssertEqual(normalized[3].rect.cgRect, expected.cgRect, testCase.name) @@ -443,8 +424,7 @@ final class CoordinateSpaceTests: XCTestCase { ] let normalized = SnapshotGeometrySpace.normalized( nodes: acquired, - viewport: app, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + viewport: .reported(box: app, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight) ) XCTAssertEqual( normalized.first { $0.label == "planeBand" }?.rect, @@ -470,8 +450,7 @@ final class CoordinateSpaceTests: XCTestCase { ] let normalized = SnapshotGeometrySpace.normalized( nodes: acquired, - viewport: app, - interfaceOrientation: RunnerInterfaceOrientation.unknown + viewport: .reported(box: app, interfaceOrientation: RunnerInterfaceOrientation.unknown) ) XCTAssertEqual(normalized.map(\.rect), acquired.map(\.rect)) XCTAssertEqual(normalized[1].hittable, true) @@ -479,12 +458,42 @@ final class CoordinateSpaceTests: XCTestCase { XCTAssertEqual(normalized[0].hittable, false) } + /// Without a viewport box the pass turns nothing and cannot decide containment, so a child's + /// `hittable` is absent rather than declared; a disabled child is still declared `false` (#2891). + func testNormalizedWithoutAViewportLeavesContainmentUndecided() { + let app = CGRect(x: 0, y: 0, width: 874, height: 402) + let acquired = turnedSubtree(app: app, reportedLeaf: CGRect(x: 100, y: 100, width: 40, height: 20)) + let normalized = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .missing(reason: .notProvided) + ) + XCTAssertEqual(normalized.map(\.rect), acquired.map(\.rect)) + XCTAssertEqual(normalized[0].hittable, false) + XCTAssertNil(normalized[3].hittable) + + let disabled = RawAXNode( + index: 1, type: "Button", label: nil, identifier: nil, value: nil, + rect: SnapshotRect(x: 100, y: 100, width: 40, height: 20), + enabled: false, focused: nil, selected: nil, hittable: true, + depth: 1, parentIndex: 0, hiddenContentAbove: nil, hiddenContentBelow: nil + ) + XCTAssertEqual( + SnapshotGeometrySpace.normalized( + nodes: [acquired[0], disabled], + viewport: .missing(reason: .notProvided) + )[1].hittable, + false + ) + } + func testNormalizedOfAnEmptyArrayIsEmpty() { XCTAssertEqual( SnapshotGeometrySpace.normalized( nodes: [], - viewport: CGRect(x: 0, y: 0, width: 874, height: 402), - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + viewport: .reported( + box: CGRect(x: 0, y: 0, width: 874, height: 402), + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) ), [] ) diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/FixtureRect.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/FixtureRect.swift new file mode 100644 index 0000000000..f3152d43b4 --- /dev/null +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/FixtureRect.swift @@ -0,0 +1,41 @@ +import CoreGraphics +import Foundation + +/// A rect as a `contracts/fixtures/` table spells it. JSON has no infinity, so the two unusable +/// boxes are named: `{"infinite": true}` is `CGRect.infinite`, and `{"nonFinite": true}` is a +/// box whose components are infinite. +struct FixtureRect: Decodable { + private enum CodingKeys: String, CodingKey { + case x, y, width, height, infinite, nonFinite + } + + let cgRect: CGRect + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + if try container.decodeIfPresent(Bool.self, forKey: .infinite) == true { + self.cgRect = .infinite + } else if try container.decodeIfPresent(Bool.self, forKey: .nonFinite) == true { + self.cgRect = CGRect(x: -.infinity, y: -.infinity, width: .infinity, height: .infinity) + } else { + self.cgRect = CGRect( + x: try container.decode(Double.self, forKey: .x), + y: try container.decode(Double.self, forKey: .y), + width: try container.decode(Double.self, forKey: .width), + height: try container.decode(Double.self, forKey: .height) + ) + } + } +} + +func contractsFixtureURL(_ name: String, from filePath: String = #filePath) -> URL { + URL(fileURLWithPath: filePath) + .deletingLastPathComponent() // AgentDeviceSnapshotPresentationTests + .deletingLastPathComponent() // Tests + .deletingLastPathComponent() // snapshot-presentation + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent(name) +} diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/InvariantTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/InvariantTests.swift index 6b06f8c5a4..2a2eee8f48 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/InvariantTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/InvariantTests.swift @@ -28,7 +28,7 @@ final class InvariantTests: XCTestCase { let stats = try SnapshotPresentationInvariant.validateRegularWithStats( nodes, - viewport: viewport, + viewport: .reported(box: viewport), policy: .cursorProjected ) diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/RegularDepthTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/RegularDepthTests.swift index 6d7bc3f529..5589a9c304 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/RegularDepthTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/RegularDepthTests.swift @@ -21,7 +21,7 @@ final class RegularDepthTests: XCTestCase { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: viewport + viewport: .reported(box: viewport) ) let result = try XCTUnwrap(SnapshotPresentation.present(acquisition, options: options)) diff --git a/contracts/fixtures/snapshot-actionability-policy.json b/contracts/fixtures/snapshot-actionability-policy.json new file mode 100644 index 0000000000..5a567f6495 --- /dev/null +++ b/contracts/fixtures/snapshot-actionability-policy.json @@ -0,0 +1,220 @@ +{ + "description": "Golden vector table for the shared `hittable` predicate (#1933, #2891), for input shapes the fixed 320x240 fold fixture in ios-snapshot-engine-conformance.json cannot reach. RULE: a node is actionable when it is enabled, its own rect is positive and finite, and its center falls inside the viewport box, half-open on the right and bottom edges. Implementations: SnapshotGeometry.isGeometricallyActionable (Swift, replayed by ActionabilityPolicyTests) and isGeometricallyActionable in packages/kernel/src/rect.ts (replayed by scripts/ios-snapshot-differential.test.ts). The viewport is the declared fact `reported`, `derived`, or `missing` (IosViewportEvidence). `hittable: null` is the absent bit: with no viewport box, containment is undecided. Every row declares `swift` and `typescript`; a row one side skips carries `asymmetry`.", + "cases": [ + { + "name": "a center strictly inside the reported box is actionable", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": 100, "y": 100, "width": 40, "height": 20 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": true, + "nodeRectGuardPasses": true + }, + { + "name": "a center one point short of the right edge is actionable", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": 239, "y": 110, "width": 160, "height": 20 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": true, + "nodeRectGuardPasses": true + }, + { + "name": "a center landing exactly on the right edge is not actionable: the edge is half-open", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": 240, "y": 110, "width": 160, "height": 20 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": false, + "nodeRectGuardPasses": true + }, + { + "name": "a center one point short of the bottom edge is actionable", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": 100, "y": 159, "width": 40, "height": 160 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": true, + "nodeRectGuardPasses": true + }, + { + "name": "a center landing exactly on the bottom edge is not actionable: the edge is half-open", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": 100, "y": 160, "width": 40, "height": 160 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": false, + "nodeRectGuardPasses": true + }, + { + "name": "a center landing exactly on the left edge is actionable: that edge is inclusive", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": -20, "y": 100, "width": 40, "height": 20 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": true, + "nodeRectGuardPasses": true + }, + { + "name": "a center landing exactly on the top edge is actionable: that edge is inclusive", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": 100, "y": -10, "width": 40, "height": 20 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": true, + "nodeRectGuardPasses": true + }, + { + "name": "a disabled node is not actionable even with its center inside the box", + "swift": true, + "typescript": true, + "enabled": false, + "node": { "x": 100, "y": 100, "width": 40, "height": 20 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": false, + "nodeRectGuardPasses": true + }, + { + "name": "a center inside the box reached through a canonical box is actionable", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": 80, "y": 100, "width": 20, "height": 20 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": true, + "nodeRectGuardPasses": true + }, + { + "name": "the same center reached through a negative width is not actionable: the node guard decides, and Swift used to answer actionable", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": 100, "y": 100, "width": -20, "height": 20 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": false, + "nodeRectGuardPasses": false + }, + { + "name": "a center inside the box reached through a zero width is not actionable", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": 100, "y": 110, "width": 0, "height": 20 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": false, + "nodeRectGuardPasses": false + }, + { + "name": "that zero-area box's center reached through a canonical box is actionable", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": 90, "y": 110, "width": 20, "height": 20 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": true, + "nodeRectGuardPasses": true + }, + { + "name": "Apple's no-box sentinel is not actionable: CGRect.infinite is made of finite Doubles whose center is (0,0), so only the platform that owns that value can refuse it by identity", + "swift": true, + "typescript": false, + "asymmetry": "Swift decodes {\"infinite\": true} as CGRect.infinite; TypeScript has no such value, and frameFromGuest in packages/platform-apple/src/snapshot-source/tree.ts refuses a frame whose components are not Number.isFinite before the predicate ever sees it, which is the row below. The sentinel's components are finite, so TypeScript would call them actionable: the refusal belongs to the producer that parses an Apple frame, not to a magic value inside the shared rule.", + "enabled": true, + "node": { "infinite": true }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": false, + "nodeRectGuardPasses": false + }, + { + "name": "a node rect whose components are not finite is not actionable on either producer, and the component checks refuse it before any center is computed", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "nonFinite": true }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": false, + "nodeRectGuardPasses": false + }, + { + "name": "the center a box spelled infinite lands on is actionable when a real box reaches it", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": -10, "y": -10, "width": 20, "height": 20 }, + "viewport": { "kind": "reported", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": true, + "nodeRectGuardPasses": true + }, + { + "name": "the same node whose center sits inside a reported box has no hittable bit once the viewport goes missing", + "swift": true, + "typescript": false, + "asymmetry": "The TypeScript predicate takes a box. Without one, each host path withholds the bit before asking it: the bridge reader publishes derived hittability only for a reported viewport, and resolveViewportEvidence refuses to fold a regular presentation.", + "enabled": true, + "node": { "x": 100, "y": 100, "width": 40, "height": 20 }, + "viewport": { "kind": "missing", "reason": "not-provided" }, + "hittable": null, + "nodeRectGuardPasses": true + }, + { + "name": "a viewport declared invalid leaves the bit absent the same way", + "swift": true, + "typescript": false, + "asymmetry": "The TypeScript predicate takes a box. Without one, each host path withholds the bit before asking it: the bridge reader publishes derived hittability only for a reported viewport, and resolveViewportEvidence refuses to fold a regular presentation.", + "enabled": true, + "node": { "x": 100, "y": 100, "width": 40, "height": 20 }, + "viewport": { "kind": "missing", "reason": "invalid" }, + "hittable": null, + "nodeRectGuardPasses": true + }, + { + "name": "a disabled node is not actionable without a viewport: that half of the rule needs no box", + "swift": true, + "typescript": false, + "asymmetry": "The TypeScript predicate takes a box. Without one, each host path withholds the bit before asking it: the bridge reader publishes derived hittability only for a reported viewport, and resolveViewportEvidence refuses to fold a regular presentation.", + "enabled": false, + "node": { "x": 100, "y": 100, "width": 40, "height": 20 }, + "viewport": { "kind": "missing", "reason": "not-provided" }, + "hittable": false, + "nodeRectGuardPasses": true + }, + { + "name": "a zero-area node is not actionable without a viewport: the node guard needs no box", + "swift": true, + "typescript": false, + "asymmetry": "The TypeScript predicate takes a box. Without one, each host path withholds the bit before asking it: the bridge reader publishes derived hittability only for a reported viewport, and resolveViewportEvidence refuses to fold a regular presentation.", + "enabled": true, + "node": { "x": 100, "y": 100, "width": 0, "height": 20 }, + "viewport": { "kind": "missing", "reason": "not-provided" }, + "hittable": false, + "nodeRectGuardPasses": false + }, + { + "name": "a derived box the capture read from its own root still answers containment", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": 100, "y": 100, "width": 40, "height": 20 }, + "viewport": { "kind": "derived", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": true, + "nodeRectGuardPasses": true + }, + { + "name": "a derived box still refuses a node whose center is off that box", + "swift": true, + "typescript": true, + "enabled": true, + "node": { "x": 900, "y": 10, "width": 40, "height": 20 }, + "viewport": { "kind": "derived", "rect": { "x": 0, "y": 0, "width": 320, "height": 240 } }, + "hittable": false, + "nodeRectGuardPasses": true + } + ] +} diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index dc46f6db6c..9610cc8bb9 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -304,6 +304,16 @@ a typed `IOS_SNAPSHOT_PRESENTATION_FAILED` capture failure with the named `prese snapshot-quality reason, preserved through recovery and the existing TypeScript verdict/warning contract. +Inside the runner the viewport is a declared fact, not a rectangle: `SnapshotViewport` is +`reported(box, interfaceOrientation)`, `derived(box)`, or `missing(reason)`, the cases of the host's +`IosViewportEvidence` (#2891). Only `reported` carries an orientation, so only it can anchor a +rotation in `SnapshotGeometrySpace`. With no box the clip skips, the cumulative-clip invariant has no +root clip to violate, and a node whose actionability depends on containment has no `hittable` on the +wire, as on the host bridge; disabled or degenerate nodes stay declared `false`. The runner route's +host evidence comes from the payload's root nodes (`resolveIosViewportEvidenceFromRoots` in +`packages/capture-kit/src/ios-snapshot-acquisition.ts`). `contracts/fixtures/snapshot-actionability-policy.json` +pins the predicate for shapes the 320x240 fold fixture cannot reach. + A regular `--depth` request is a presentation cut, not an acquisition bound. `CaptureHint` keeps raw traversal depth (`--raw --depth`) separate from regular presented depth, but the recursive tree walk no longer reads presented depth (or any geometry) while descending: for a regular capture it diff --git a/packages/kernel/src/rect.ts b/packages/kernel/src/rect.ts index fe25acfe7d..a6ebb19bc5 100644 --- a/packages/kernel/src/rect.ts +++ b/packages/kernel/src/rect.ts @@ -1,5 +1,6 @@ import type { Rect } from './snapshot.ts'; +/** Twin of `SnapshotGeometry.isPositiveFinite` on the runner (#2891). */ export function isPositiveFiniteRect(rect: Rect | undefined): rect is Rect { return Boolean( rect && @@ -29,13 +30,12 @@ export function containsPoint(rect: Rect, x: number, y: number): boolean { /** * The shared `hittable` predicate every iOS snapshot producer publishes (#1933): an enabled node - * with a positive frame whose center falls inside the viewport. It is the TypeScript twin of the - * runner's Swift `SnapshotGeometry.isGeometricallyActionable`, including `CGRect.contains`'s - * half-open right/bottom edges — a center landing exactly on the viewport's right or bottom edge is - * not hittable on either producer. The host AX bridge derives the source bit from the node's own - * frame and the fold intersects it with the clipped frame, so a `hittable:` selector cannot tell the - * two producers apart. Kept here so both packages read one definition rather than each re-encoding - * the rule. + * with a positive finite frame whose center falls inside the viewport. It is the TypeScript twin of + * the runner's Swift `SnapshotGeometry.isGeometricallyActionable`, including `CGRect.contains`'s + * half-open right/bottom edges; `contracts/fixtures/snapshot-actionability-policy.json` pins both. + * Callers without a viewport box withhold the bit instead of asking. The host AX bridge derives the + * source bit from the node's own frame and the fold intersects it with the clipped frame, so a + * `hittable:` selector cannot tell the two producers apart. */ export function isGeometricallyActionable( enabled: boolean, diff --git a/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts b/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts index a21f130552..917efddbd9 100644 --- a/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts +++ b/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts @@ -207,6 +207,33 @@ test('a healthy payload with valid viewport roots still presents', () => { ); }); +// A runner capture with no viewport box omits `hittable` on the nodes it could not decide (#2891); +// the host presents them undecided rather than minting a value. +test('a runner payload with the hittable bit absent presents without declaring it', () => { + const nodes: RawSnapshotNode[] = [ + { + index: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: false, + }, + { index: 1, parentIndex: 0, type: 'Other', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 2, + parentIndex: 1, + type: 'Button', + label: 'Not Now', + rect: { x: 16, y: 400, width: 80, height: 32 }, + }, + ]; + for (const interactiveOnly of [false, true]) { + const presented = presentAppleRunnerSnapshot('device-1', { interactiveOnly }, { nodes }); + const button = presented.find((node) => node.label === 'Not Now'); + assert.ok(button, `interactiveOnly=${interactiveOnly}: the undecided button is presented`); + assert.equal('hittable' in button, false); + } +}); + // The keyboard band the runner measured for a capture (#2660). The reader is the only place a wire // fact becomes a daemon fact, so it owns the whole strictness budget: what cannot be placed is // restated as `unmeasurable` with a reason, never as a band and never as silence. diff --git a/scripts/check-affected/model.ts b/scripts/check-affected/model.ts index d21e10de21..259e53fb68 100644 --- a/scripts/check-affected/model.ts +++ b/scripts/check-affected/model.ts @@ -453,7 +453,9 @@ const BUILD_OWNERSHIP: ReadonlyArray<{ owns: (file) => file.startsWith('packages/capture-kit/src/ios-snapshot-engine/') || file.startsWith('apple/snapshot-presentation/') || - file === 'contracts/fixtures/ios-snapshot-engine-conformance.json', + file === 'packages/kernel/src/rect.ts' || + file === 'contracts/fixtures/ios-snapshot-engine-conformance.json' || + file === 'contracts/fixtures/snapshot-actionability-policy.json', }, // Both platform builds compile the same runner sources, and each is a separate // gate in a separate lane, so a Swift change owns both. diff --git a/scripts/ios-snapshot-differential.test.ts b/scripts/ios-snapshot-differential.test.ts index a8c8d30414..60f3fdf8bc 100644 --- a/scripts/ios-snapshot-differential.test.ts +++ b/scripts/ios-snapshot-differential.test.ts @@ -1,6 +1,10 @@ import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; import { test } from 'node:test'; import fc from 'fast-check'; +import { isGeometricallyActionable, isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import type { Rect } from '@agent-device/kernel/snapshot'; import { compareDifferentialCases, swiftToolchainAvailable, @@ -124,3 +128,107 @@ function assertWithinKillCriterion(startedAt: number, seed: number): void { String(seed), ); } + +type ActionabilityRect = Readonly<{ x: number; y: number; width: number; height: number }>; +type ActionabilityUnusableRect = Readonly<{ infinite: true }> | Readonly<{ nonFinite: true }>; +type ActionabilityViewport = + | Readonly<{ kind: 'reported' | 'derived'; rect: ActionabilityRect }> + | Readonly<{ kind: 'missing'; reason: 'not-provided' | 'invalid' }>; +type ActionabilityVector = Readonly<{ + name: string; + swift: boolean; + typescript: boolean; + asymmetry?: string; + enabled: boolean; + node: ActionabilityRect | ActionabilityUnusableRect; + viewport: ActionabilityViewport; + /** `null` is the absent bit. */ + hittable: boolean | null; + nodeRectGuardPasses: boolean; +}>; + +const ACTIONABILITY_POLICY_PATH = path.resolve( + import.meta.dirname, + '..', + 'contracts', + 'fixtures', + 'snapshot-actionability-policy.json', +); + +/** A box whose components are not numbers anything may plot. JSON has no literal for infinity. */ +const NON_FINITE_RECT: Rect = { + x: Number.NEGATIVE_INFINITY, + y: Number.NEGATIVE_INFINITY, + width: Number.POSITIVE_INFINITY, + height: Number.POSITIVE_INFINITY, +}; + +function declaresItsAsymmetry(vector: ActionabilityVector): boolean { + const hasReason = typeof vector.asymmetry === 'string' && vector.asymmetry.length > 0; + return (vector.swift && vector.typescript) !== hasReason; +} + +function readActionabilityVectors(): readonly ActionabilityVector[] { + const table = JSON.parse(fs.readFileSync(ACTIONABILITY_POLICY_PATH, 'utf8')) as { + cases: readonly ActionabilityVector[]; + }; + assert.ok(table.cases.length > 0, 'actionability vector table must not be empty'); + assert.equal( + new Set(table.cases.map((vector) => vector.name)).size, + table.cases.length, + 'actionability vector names must be unique', + ); + for (const vector of table.cases) { + assert.equal(typeof vector.swift, 'boolean', `${vector.name}: row must declare the Swift side`); + assert.equal( + typeof vector.typescript, + 'boolean', + `${vector.name}: row must declare the TypeScript side`, + ); + assert.ok( + declaresItsAsymmetry(vector), + `${vector.name}: a row both languages do not share must name the asymmetry`, + ); + } + return table.cases; +} + +function toRect(node: ActionabilityVector['node']): Rect { + if ('nonFinite' in node) return NON_FINITE_RECT; + if ('infinite' in node) { + throw new Error( + "CGRect.infinite is Apple's value and no row reaching TypeScript may stand for it: " + + 'that row belongs to the Swift side alone', + ); + } + return node; +} + +// The Swift twin of these rows is ActionabilityPolicyTests, run by `swift test --package-path +// apple/snapshot-presentation` in this same command. +test('the shared hittable predicate agrees with every golden actionability vector', () => { + for (const vector of readActionabilityVectors().filter((row) => row.typescript)) { + const node = toRect(vector.node); + assert.equal( + isPositiveFiniteRect(node), + vector.nodeRectGuardPasses, + `${vector.name}: node-rect guard`, + ); + if (vector.viewport.kind === 'missing') { + throw new Error(`${vector.name}: the TypeScript predicate takes a box`); + } + assert.equal( + isGeometricallyActionable(vector.enabled, node, vector.viewport.rect), + vector.hittable, + vector.name, + ); + } +}); + +test('the TypeScript rows cover every viewport kind that carries a box', () => { + const vectors = readActionabilityVectors().filter((row) => row.typescript); + assert.deepEqual([...new Set(vectors.map((vector) => vector.viewport.kind))].sort(), [ + 'derived', + 'reported', + ]); +});