From fcc735008ffbd8a2e895081654fe1591f4b68bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 19:08:45 +0200 Subject: [PATCH 1/2] fix(ios): cut modal-contained presentations from bridge snapshots The host AX bridge reports every sibling container in a window, so each UIKit modal presentation left the screens it presents over published as live content. A sheet covers only part of the screen, so the geometric occlusion pass left the covered controls actionable and selectors resolved them: a three-screen stack published 567 nodes across three screens where the same app under the XCTest runner published 76 across the top one. The presentation cut now applies modal containment: when the last transition view under a container carries UIKit's dimming view as its direct child, the earlier transition views that dimmed area spans, and their subtrees, are cut from the regular and interactive snapshots. Raw snapshots still report them, and producers that report no UIKit class names never trigger the rule, so it stays inert on the runner. Every removed source is counted in the fold's presentation stats (ADR 0026). The rule joins `geometry-policy.ts`, the module the fold already evaluates for its per-node decisions, rather than arriving as a new eager module: the eager-closure budget holds this entry surface at its merge-base size. --- CHANGELOG.md | 19 ++ .../adr/0004-ios-snapshot-backend-strategy.md | 39 +++ .../src/ios-snapshot-engine/engine.ts | 2 + .../geometry-policy.test.ts | 223 ++++++++++++++++++ .../ios-snapshot-engine/geometry-policy.ts | 77 +++++- .../src/ios-snapshot-engine/geometry.ts | 8 +- .../runner-presentation.ts | 1 + .../src/ios-snapshot-engine/tree.ts | 21 +- .../src/ios-snapshot-engine/types.ts | 2 + 9 files changed, 388 insertions(+), 4 deletions(-) create mode 100644 packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index f4ee6cdf56..e4f5d370f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,25 @@ through the refusal; the runner transport never resends it. Interactions that mutate without a leading read (`press`, a coordinate `fill`, `swipe`, `scroll`, hardware keys) keep the foreground repair and still bring a stopped app up. +- Fixed (ios): a local Simulator snapshot taken through the host AX bridge no longer publishes the + screens a modal presentation covers, so a selector resolves the control the user can reach instead + of the stale copy underneath it. UIKit appends each presentation as a later sibling of what it + presents over, and the bridge reports every sibling, so a React Navigation stack that presents its + pushed routes as sheets kept all three screens live: one capture of a three-screen stack published + 567 nodes across three screens where the same app under the XCTest runner published 76 across the + top one. Maestro's visibility predicate is geometric, so `- tapOn: "Push article"` matched the + covered screen's button first and the flow failed on an assertion about the screen that never + arrived, and `presentation: "formSheet"` fields matched intermittently (#2638). Geometry cannot + close this: a sheet leaves most of the covered screen uncovered, and the occlusion pass only + treats floating chrome as a cover. The presentation cut now applies modal containment — when the + last transition view under a container carries UIKit's dimming view as its direct child, the + earlier transition views that dimmed area spans, and everything under them, are cut from the + regular and interactive snapshots. A presentation UIKit does not dim is left as it was, so the cut + asserts containment only where the producer states it. `snapshot --raw` still reports the covered + screens, and producers that report no UIKit class names — the XCTest runner, whose own queries + answered 76 nodes for that same state, plus `appium-source` and `limrun-ios-tree` — never trigger + the cut. All 39 flows of React Navigation's Maestro suite pass on an iPhone 17 Simulator running + iOS 26.2 with this change, including two that never passed on the bridge. - Fixed (ios): a local Simulator snapshot taken through the host AX bridge once again publishes the geometric `hittable` fact, so `is hittable` and a `hittable:` selector resolve the same controls on the bridge and the XCTest runner. The snapshot capability table has declared `hittable = diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 46042f3e3b..093a36932a 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -475,3 +475,42 @@ last reader that can still refuse geometry it cannot place is the tap-path keybo width rule therefore remains. The rule detects un-normalized arrival, not a standing fact about iOS: the producers above do normalize, and a band taller than it is wide is what one that did not looks like. + +## Amendment: modal containment in the presentation cut + +The bridge reads one window and reports every container hanging off it, while XCTest's own queries +answer only the presentation the user can reach. React Navigation's card-plus-modal example is the +measured case: three routes presented as sheets left three transition-view containers in one window +and one capture published 567 nodes across three screens, where the same app state under the runner +published 76 across the top screen alone. Maestro's visibility test is geometric, so a `tapOn` +resolved the covered screen's button and the flow failed on an assertion about the screen that never +arrived, and a `presentation: "formSheet"` route's fields matched intermittently (#2638). + +Decision. The fold applies modal containment: when the last transition view under a container carries +UIKit's dimming view as its direct child, the earlier transition views whose frame that dimmed area +spans are cut, with their subtrees, from the regular and interactive projections. Both ends of the +claim are producer facts rather than an ordering guess — the dimming view is UIKit's own declaration +that it dims what sits behind, and its frame is the producer's rectangle, which is what a covered +container must be inside. A presentation with no dimming view, and any container whose earlier +sibling is not a transition view, keep today's behavior: containment is asserted only where the +producer states it, so the rule fails closed rather than guessing which siblings are shadows. +Every source the cut removes is counted in the presentation's +`stats.modalContainedNodeCount` — internal evidence, per ADR 0026, never wire vocabulary — because +the same screen either side of an animation otherwise moves hundreds of comparable lines with nothing +to attribute them to. + +Why this seam and not the neighbouring ones. The semantic presentation rules +(`IOS_PRESENTATION_RULES`) run only for the interactive projection, so a suppression rule there could +not deliver the same claim to a regular `snapshot`; the regular eligibility table is pinned against +its Swift twin, so widening it would move a cross-language contract that this rule does not touch; the +occlusion pass *marks* a node covered and keeps it published, and a published-but-marked node is +exactly the 567-versus-76 divergence being fixed, since visibility filtering reads geometry and not +the mark; and cutting at acquisition would leak the decision into `--raw`, which owes the reader the +tree the platform reported. Raw therefore still carries the covered screens. + +Producers that report no UIKit class names never trigger the cut, and that is a fact about their +output rather than a backend exception in the fold: the runner already omits modal-contained content +from its own queries, `appium-source` and `limrun-ios-tree` report element types and no classes, and +the macOS desktop surface arrives already presented. A scope naming a modal-contained screen now +publishes an empty projection, which is the same healthy empty answer any other unmatched scope gives +— the screen is not what is presented. diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.ts index a8df22b5d0..5a22ae4a7b 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.ts @@ -101,6 +101,7 @@ function presentAcquiredSnapshot( presentedNodeCount: projected.nodes.length, sourceNodeCount: acquisition.nodes.length, parentClipLookups: 0, + modalContainedNodeCount: 0, }, }; } @@ -153,6 +154,7 @@ function presentAcquiredSnapshot( presentedNodeCount: compacted.nodes.length, sourceNodeCount: acquisition.nodes.length, parentClipLookups: folded.stats.parentClipLookups + validation.parentClipLookups, + modalContainedNodeCount: folded.stats.modalContainedNodeCount, }, }; } diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts new file mode 100644 index 0000000000..2daf508e40 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts @@ -0,0 +1,223 @@ +import { expect, test } from 'vitest'; +import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; +import { foldIosSnapshot } from './geometry.ts'; +import { collectModalContainedIndexes } from './geometry-policy.ts'; +import { projectIosSnapshot } from './projection.ts'; + +const VIEWPORT = { x: 0, y: 0, width: 402, height: 874 }; +const SHEET: Rect = { x: 16, y: 62, width: 370, height: 747 }; + +type PresentationShape = Readonly<{ + title: string; + containerRole?: string; + rect?: Rect; + dimming?: 'direct' | 'nested'; +}>; + +/** + * Mirrors a react-navigation capture: the presenting container carries only a drop shadow with + * UIKit's dimming view below it, while each modal presentation carries a dimming view as a + * direct child of its transition view. + */ +function presentationSubtree( + start: number, + depth: number, + parentIndex: number, + shape: PresentationShape, +): RawSnapshotNode[] { + const rect = shape.rect ?? VIEWPORT; + const dimmingRect: Rect = { + x: -rect.width, + y: -rect.height, + width: rect.width * 3, + height: rect.height * 3, + }; + const nodes: RawSnapshotNode[] = [ + snapshotNode(start, depth, parentIndex, 'UITransitionView', 'Other'), + snapshotNode(start + 1, depth + 1, start, 'UIDropShadowView', 'Other', undefined, rect), + snapshotNode( + start + 2, + depth + 2, + start + 1, + shape.containerRole ?? 'RNSModalScreen', + 'Other', + shape.title, + rect, + ), + snapshotNode(start + 3, depth + 3, start + 2, undefined, 'Button', `Push from ${shape.title}`, { + x: rect.x + 12, + y: rect.y + 92, + width: 120, + height: 40, + }), + ]; + if ((shape.dimming ?? 'direct') === 'direct') { + nodes.splice( + 1, + 0, + snapshotNode(start + 4, depth + 1, start, 'UIDimmingView', 'Other', undefined, dimmingRect), + ); + } else { + nodes.splice( + 2, + 0, + snapshotNode( + start + 4, + depth + 2, + start + 1, + 'UIDimmingView', + 'Other', + undefined, + dimmingRect, + ), + ); + } + return nodes; +} + +function snapshotNode( + index: number, + depth: number, + parentIndex: number | undefined, + role: string | undefined, + type: string, + label?: string, + rect: Rect = VIEWPORT, +): RawSnapshotNode { + return { + index, + depth, + ...(parentIndex === undefined ? {} : { parentIndex }), + type, + ...(role ? { role } : {}), + ...(label ? { label } : {}), + rect, + hittable: true, + }; +} + +function windowWith(children: RawSnapshotNode[]): RawSnapshotNode[] { + return [snapshotNode(0, 0, undefined, 'UIWindow', 'Window'), ...children]; +} + +function stackedPresentations(): RawSnapshotNode[] { + return windowWith([ + ...presentationSubtree(1, 1, 0, { + title: 'Article by Dalek', + containerRole: 'RCTSurfaceHostingProxyRootView', + dimming: 'nested', + }), + ...presentationSubtree(6, 1, 0, { title: 'Albums', rect: SHEET }), + ...presentationSubtree(11, 1, 0, { title: 'Article by The Doctor' }), + ]); +} + +test('dims away every earlier presentation the covering dimming view spans', () => { + const contained = collectModalContainedIndexes(stackedPresentations()); + expect([...contained].sort((left, right) => left - right)).toEqual([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + ]); +}); + +test('keeps the covering presentation and everything it contains', () => { + const contained = collectModalContainedIndexes(stackedPresentations()); + expect([...contained].filter((index) => index >= 11)).toEqual([]); +}); + +test('presents only the covering subtree through the fold', () => { + const folded = foldIosSnapshot(stackedPresentations(), VIEWPORT, false, 'cursor-projected'); + expect(folded.nodes.flatMap((node) => (node.raw.label ? [node.raw.label] : []))).toEqual([ + 'Article by The Doctor', + 'Push from Article by The Doctor', + ]); +}); + +test('records how many sources the cut removed', () => { + const folded = foldIosSnapshot(stackedPresentations(), VIEWPORT, false, 'cursor-projected'); + expect(folded.stats.modalContainedNodeCount).toBe(10); + expect(folded.stats.sourceNodeCount).toBe(16); +}); + +test('publishes an empty projection when a scope names a modal-contained screen', () => { + const folded = foldIosSnapshot(stackedPresentations(), VIEWPORT, false, 'cursor-projected'); + const covered = projectIosSnapshot({ + nodes: folded.nodes, + projection: 'regular', + scope: 'Albums', + depth: null, + foldPolicy: 'cursor-projected', + }); + const covering = projectIosSnapshot({ + nodes: folded.nodes, + projection: 'regular', + scope: 'Article by The Doctor', + depth: null, + foldPolicy: 'cursor-projected', + }); + expect(covered.nodes).toEqual([]); + expect(covering.nodes.length).toBeGreaterThan(0); +}); + +test('spares an earlier sibling that is not a presentation container', () => { + const nodes = windowWith([ + snapshotNode(1, 1, 0, 'RCTSurfaceHostingProxyRootView', 'Other'), + snapshotNode(2, 2, 1, undefined, 'Button', 'Live control', { + x: 12, + y: 154, + width: 120, + height: 40, + }), + ...presentationSubtree(3, 1, 0, { title: 'Albums' }), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); +}); + +test('fails closed when the last sibling is not a presentation container', () => { + const nodes = windowWith([ + ...presentationSubtree(1, 1, 0, { title: 'Article by Dalek' }), + snapshotNode(6, 1, 0, '_UIAlertControllerView', 'Other', 'Don’t leave'), + snapshotNode(7, 2, 6, 'UIDimmingView', 'Other'), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); +}); + +test('fails closed when the dimming view sits below the presentation container', () => { + const nodes = windowWith([ + ...presentationSubtree(1, 1, 0, { title: 'Article by Dalek' }), + ...presentationSubtree(6, 1, 0, { title: 'Albums', dimming: 'nested' }), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); +}); + +test('fails closed when the dimming view does not span the earlier container', () => { + const nodes = windowWith([ + ...presentationSubtree(1, 1, 0, { title: 'Article by Dalek' }), + snapshotNode(6, 1, 0, 'UITransitionView', 'Other'), + snapshotNode(7, 2, 6, 'UIDimmingView', 'Other', undefined, { + x: 0, + y: 0, + width: 10, + height: 10, + }), + snapshotNode(8, 2, 6, 'UIDropShadowView', 'Other', undefined, SHEET), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); +}); + +test('stays inert on a tree that reports no UIKit class names', () => { + const nodes = windowWith([ + snapshotNode(1, 1, 0, undefined, 'Other', 'Article by Dalek'), + snapshotNode(2, 1, 0, undefined, 'Other', 'Article by The Doctor'), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); +}); + +test('contains a full-screen presentation that a sheet does not geometrically cover', () => { + const nodes = windowWith([ + ...presentationSubtree(1, 1, 0, { title: 'Article by Dalek' }), + ...presentationSubtree(6, 1, 0, { title: 'Albums', rect: SHEET }), + ]); + expect([...collectModalContainedIndexes(nodes)].sort((left, right) => left - right)).toEqual([ + 1, 2, 3, 4, 5, + ]); +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts index 7f0108a68e..6e85222605 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts @@ -1,12 +1,21 @@ import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import { isPositiveFiniteRect, rectContains } from '@agent-device/kernel/rect'; import { normalizeType } from '@agent-device/contracts/snapshot'; +import { collectChildrenByParent, collectSubtreeIndexes } from './tree.ts'; import type { IosSnapshotFoldPolicy } from './types.ts'; const SCROLL_CONTAINER_TYPES = new Set(['collectionview', 'scrollview', 'table']); const VISIBILITY_CARRIER_TYPES = new Set(['application', 'window']); const NEGLIGIBLE_DECORATION_TOLERANCE = 1; +/** + * Private UIKit classes are matched by exact name, unlike the substring role tests the occlusion + * pass uses: this rule decides node membership, so a near miss on a class name has to fail closed + * rather than widen the cut. + */ +const PRESENTATION_CONTAINER_ROLE = 'uitransitionview'; +const PRESENTATION_DIMMING_ROLE = 'uidimmingview'; + export type TraversalState = Readonly<{ projectedOut: boolean; ancestorClip?: Rect; @@ -223,3 +232,69 @@ function intersectRect(left: Rect, right: Rect): Rect { height: bottomEdge - y, }; } + +type CoveringPresentation = Readonly<{ + container: RawSnapshotNode; + dimmingRect: Rect; +}>; + +/** + * UIKit appends each modal presentation as a later sibling of the container it presents over, + * dims the content behind it with a dimming view that is a direct child of the presentation + * container, and the host accessibility snapshot reports siblings in that subview order. A + * presenting container carries only a drop shadow, so a direct-child dimming view separates a + * modal presentation from an ordinary container, and its dimmed area says which earlier + * presentation the user can no longer reach. + * + * Producers that report no UIKit class names — the XCTest runner, whose own queries already + * omit modal-contained content — never trigger this rule. + */ +export function collectModalContainedIndexes( + nodes: readonly RawSnapshotNode[], +): ReadonlySet { + const childrenByParent = collectChildrenByParent(nodes); + const contained = new Set(); + for (const siblings of childrenByParent.values()) { + const covering = findCoveringPresentation(siblings, childrenByParent); + if (!covering) continue; + for (const sibling of siblings.slice(0, -1)) { + if (isDimmedByPresentation(covering, sibling)) { + for (const index of collectSubtreeIndexes(sibling.index, childrenByParent)) { + contained.add(index); + } + } + } + } + return contained; +} + +function findCoveringPresentation( + siblings: readonly RawSnapshotNode[], + childrenByParent: ReadonlyMap, +): CoveringPresentation | undefined { + const last = siblings.at(-1); + if (!last || !isPresentationContainer(last)) return undefined; + const dimming = (childrenByParent.get(last.index) ?? []).find(isPresentationDimmingView); + const dimmingRect = dimming && isPositiveFiniteRect(dimming.rect) ? dimming.rect : undefined; + return dimmingRect ? { container: last, dimmingRect } : undefined; +} + +function isDimmedByPresentation(covering: CoveringPresentation, sibling: RawSnapshotNode): boolean { + return ( + isPresentationContainer(sibling) && + isPositiveFiniteRect(sibling.rect) && + rectContains(covering.dimmingRect, sibling.rect) + ); +} + +function isPresentationContainer(node: RawSnapshotNode): boolean { + return hasRole(node, PRESENTATION_CONTAINER_ROLE); +} + +function isPresentationDimmingView(node: RawSnapshotNode): boolean { + return hasRole(node, PRESENTATION_DIMMING_ROLE); +} + +function hasRole(node: RawSnapshotNode, role: string): boolean { + return normalizeType(node.role ?? '') === role; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts index 61b6345cea..f69739b62c 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts @@ -2,6 +2,7 @@ import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { isGeometricallyActionable } from '@agent-device/kernel/rect'; import { validateIosSnapshotGraph } from './graph.ts'; import { + collectModalContainedIndexes, rootTraversal, traversalDecision, type BranchState, @@ -22,13 +23,15 @@ export function foldIosSnapshot( options: IosSnapshotFoldOptions = {}, ): { nodes: IosSnapshotPresentationNode[]; stats: IosSnapshotPresentationStats } { validateIosSnapshotGraph(nodes); - const hasChildren = buildChildPresence(nodes); + const modalContained = collectModalContainedIndexes(nodes); + const presentable = nodes.filter((node) => !modalContained.has(node.index)); + const hasChildren = buildChildPresence(presentable); const states = new Map(); const kept: IosSnapshotPresentationNode[] = []; const hints = new Map(); let parentClipLookups = 0; - for (const node of nodes) { + for (const node of presentable) { const parentState = readParentState(node, states, () => { parentClipLookups += 1; }); @@ -61,6 +64,7 @@ export function foldIosSnapshot( presentedNodeCount: presented.length, sourceNodeCount: nodes.length, parentClipLookups, + modalContainedNodeCount: modalContained.size, }, }; } diff --git a/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts index 4a48b11284..cbae8a042d 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts @@ -42,6 +42,7 @@ export function presentIosRunnerSnapshot( presentedNodeCount: compacted.nodes.length, sourceNodeCount: input.presentation.payload.nodes.length, parentClipLookups: validationStats.parentClipLookups, + modalContainedNodeCount: 0, }, }; } diff --git a/packages/capture-kit/src/ios-snapshot-engine/tree.ts b/packages/capture-kit/src/ios-snapshot-engine/tree.ts index 8cd1d5c77e..d728a59803 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/tree.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/tree.ts @@ -12,7 +12,9 @@ export type SnapshotTreeRuleContext = { suppressNode: (source: RawSnapshotNode, representatives: readonly RawSnapshotNode[]) => void; }; -export function collectChildrenByParent(nodes: RawSnapshotNode[]): Map { +export function collectChildrenByParent( + nodes: readonly RawSnapshotNode[], +): Map { const childrenByParent = new Map(); for (const node of nodes) { if (typeof node.parentIndex !== 'number') continue; @@ -23,6 +25,23 @@ export function collectChildrenByParent(nodes: RawSnapshotNode[]): Map, +): number[] { + const indexes: number[] = []; + const pending = [startIndex]; + while (pending.length > 0) { + const index = pending.pop()!; + indexes.push(index); + for (const child of childrenByParent.get(index) ?? []) { + pending.push(child.index); + } + } + return indexes; +} + const descendantEndPositionCache = new WeakMap(); export function collectDescendants( diff --git a/packages/capture-kit/src/ios-snapshot-engine/types.ts b/packages/capture-kit/src/ios-snapshot-engine/types.ts index 29e4fb5bdc..91fd4392bc 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/types.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/types.ts @@ -20,6 +20,8 @@ export type IosSnapshotPresentationStats = Readonly<{ presentedNodeCount: number; sourceNodeCount: number; parentClipLookups: number; + /** Sources the presentation cut removed because a later modal presentation dims them. */ + modalContainedNodeCount: number; }>; export type IosSnapshotEnginePresentation = Readonly<{ From 31defc4c14f982b0dfd71b6efe1f886434984fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 09:35:38 +0200 Subject: [PATCH 2/2] fix(ios): cut a presentation only when its dimming view takes touches A sheet resting at an undimmed detent keeps UIKit's dimming view as a direct child of its transition view, at the same window-sized frame, while the presenting screen stays reachable. The modal-containment rule treated any such dimming view as proof of modality, so the bridge snapshot cut the presenting screen's controls: on react-navigation's form-sheet example, `snapshot -i` over the `Custom Dimming` sheet published only the sheet, although a press on the presenting `Height Steps` button navigates. The Simulator AX bridge now reads `XC_kAXXCAttributeIsUserInteractionEnabled` for dimming views (source version v1.7.0) and the decoder publishes it as `userInteractionEnabled`. It reads false for the undimmed detent and true for a dimmed form sheet and a card modal; no attribute the bridge already read tells them apart, and `IsVisible` reads false for both the undimmed sheet and the card modal. The read targets dimming views alone: requesting the attribute for every node cost about half again the capture time on a 492-node tree. The rule now asserts containment only when the dimming view takes touches, and fails closed when the producer did not read it. The modal rule and the structural identifier rule now share one parent-link subtree walker in tree.ts. --- CHANGELOG.md | 16 +-- apple/snapshot-bridge/SnapshotBridgeRuntime.m | 48 ++++++++- .../adr/0004-ios-snapshot-backend-strategy.md | 32 ++++-- .../geometry-policy.test.ts | 102 ++++++++++++------ .../ios-snapshot-engine/geometry-policy.ts | 24 +++-- .../ios-snapshot-engine/noise-structural.ts | 24 +---- .../src/ios-snapshot-engine/tree.ts | 26 ++--- packages/kernel/src/snapshot.ts | 2 + .../fixtures/wire-vocabulary.json | 3 +- .../src/snapshot-source/protocol.test.ts | 2 +- .../src/snapshot-source/protocol.ts | 3 +- .../src/snapshot-source/tree.test.ts | 24 +++++ .../src/snapshot-source/tree.ts | 11 ++ .../capture/runtime/snapshot-unchanged.ts | 1 + 14 files changed, 224 insertions(+), 94 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4f5d370f0..d63c20cf2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,13 +55,15 @@ arrived, and `presentation: "formSheet"` fields matched intermittently (#2638). Geometry cannot close this: a sheet leaves most of the covered screen uncovered, and the occlusion pass only treats floating chrome as a cover. The presentation cut now applies modal containment — when the - last transition view under a container carries UIKit's dimming view as its direct child, the - earlier transition views that dimmed area spans, and everything under them, are cut from the - regular and interactive snapshots. A presentation UIKit does not dim is left as it was, so the cut - asserts containment only where the producer states it. `snapshot --raw` still reports the covered - screens, and producers that report no UIKit class names — the XCTest runner, whose own queries - answered 76 nodes for that same state, plus `appium-source` and `limrun-ios-tree` — never trigger - the cut. All 39 flows of React Navigation's Maestro suite pass on an iPhone 17 Simulator running + last transition view under a container carries UIKit's dimming view as its direct child and that + dimming view takes touches, the earlier transition views that dimmed area spans, and everything + under them, are cut from the regular and interactive snapshots. The bridge now reads + `userInteractionEnabled` on dimming views (source version v1.7.0) because a sheet resting at an + undimmed detent keeps its dimming view while the screen under it stays reachable; that sheet, and + any presentation UIKit does not dim, is left as it was, so the cut asserts containment only where + the producer states it. `snapshot --raw` still reports the covered screens, and producers that + report no UIKit class names — the XCTest runner, whose own queries answered 76 nodes for that same + state, plus `appium-source` and `limrun-ios-tree` — never trigger the cut. All 39 flows of React Navigation's Maestro suite pass on an iPhone 17 Simulator running iOS 26.2 with this change, including two that never passed on the bridge. - Fixed (ios): a local Simulator snapshot taken through the host AX bridge once again publishes the geometric `hittable` fact, so `is hittable` and a `hittable:` selector resolve the same controls on diff --git a/apple/snapshot-bridge/SnapshotBridgeRuntime.m b/apple/snapshot-bridge/SnapshotBridgeRuntime.m index 6f79767b9a..a4e8da8eb3 100644 --- a/apple/snapshot-bridge/SnapshotBridgeRuntime.m +++ b/apple/snapshot-bridge/SnapshotBridgeRuntime.m @@ -18,7 +18,7 @@ NSString *const kProtocolVersionKey = @"protocolVersion"; NSString *const kSourceVersionKey = @"sourceVersion"; NSString *const kRequestIdKey = @"requestId"; -NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.6.0"; +NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.7.0"; const NSUInteger kProtocolVersion = 1; const uint32_t kMaximumFrameBytes = 16 * 1024 * 1024; const NSUInteger kMaximumDepth = 128; @@ -34,8 +34,11 @@ static NSString *const kAttributeAutomationType = @"XC_kAXXCAttributeAutomationType"; static NSString *const kAttributeTraits = @"XC_kAXXCAttributeTraits"; static NSString *const kAttributeChildren = @"XC_kAXXCAttributeChildren"; +static NSString *const kAttributeIsUserInteractionEnabled = @"XC_kAXXCAttributeIsUserInteractionEnabled"; +static NSString *const kPresentationDimmingViewClass = @"UIDimmingView"; static NSString *const kSnapshotAttributes = @"UIAccessibilitySnapshotKeyAttributes"; static NSString *const kSnapshotChildren = @"UIAccessibilitySnapshotKeyChildren"; +static NSString *const kSnapshotElement = @"UIAccessibilitySnapshotKeyElement"; static NSString *const kXctAutomationSupportPath = @"/Developer/Library/PrivateFrameworks/XCTAutomationSupport.framework/XCTAutomationSupport"; static NSString *const kAxRuntimePath = @@ -202,8 +205,35 @@ - (nullable id)jsonValue:(id)value name:(NSString *)name return nil; } +/* + * Whether a presentation's dimming view takes touches is what separates a sheet that blocks the + * content under it from one resting at an undimmed detent, and nothing else in the tree says so. + * It is read for dimming views alone: requesting it for every node costs about half again the + * capture time. + */ +- (nullable NSNumber *)userInteractionEnabledForSnapshot:(NSDictionary *)snapshot + options:(nullable NSDictionary *)options +{ + id element = snapshot[kSnapshotElement]; + NSNumber *attribute = [options[@"attributes"] firstObject]; + if (!element || ![attribute isKindOfClass:NSNumber.class]) return nil; + @try { + NSError *failure = nil; + NSDictionary *read = [_framework userTestingSnapshotForElement:element options:options error:&failure]; + if (![read isKindOfClass:NSDictionary.class]) return nil; + NSDictionary *attributes = read[kSnapshotAttributes]; + if (![attributes isKindOfClass:NSDictionary.class]) return nil; + id value = attributes[attribute]; + return [value isKindOfClass:NSNumber.class] ? @([(NSNumber *)value boolValue]) : nil; + } @catch (NSException *exception) { + (void)exception; + return nil; + } +} + - (nullable NSDictionary *)nodeFromSnapshot:(id)snapshot namesByNumber:(NSDictionary *)namesByNumber + interactionOptions:(nullable NSDictionary *)interactionOptions depth:(NSUInteger)depth maxDepth:(NSUInteger)maxDepth maxNodes:(NSUInteger)maxNodes @@ -233,6 +263,11 @@ - (nullable NSDictionary *)nodeFromSnapshot:(id)snapshot id safe = [self jsonValue:attributes[number] name:name]; if (safe) node[name] = safe; } + if ([node[kAttributeElementType] isEqual:kPresentationDimmingViewClass]) { + NSNumber *interaction = [self userInteractionEnabledForSnapshot:(NSDictionary *)snapshot + options:interactionOptions]; + if (interaction) node[kAttributeIsUserInteractionEnabled] = interaction; + } NSArray *children = ((NSDictionary *)snapshot)[kSnapshotChildren]; if (![children isKindOfClass:NSArray.class]) { @@ -246,6 +281,7 @@ - (nullable NSDictionary *)nodeFromSnapshot:(id)snapshot for (id child in children) { NSDictionary *built = [self nodeFromSnapshot:child namesByNumber:namesByNumber + interactionOptions:interactionOptions depth:depth + 1 maxDepth:maxDepth maxNodes:maxNodes @@ -317,6 +353,15 @@ - (nullable NSDictionary *)snapshotForProcess:(pid_t)pid options[@"maxDepth"] = @(maxDepth); options[@"maxChildren"] = @(maxNodes); options[@"maxArrayCount"] = @(maxNodes); + NSArray *interactionAttribute = _attributeNumbersForNames(@[ kAttributeIsUserInteractionEnabled ]); + NSMutableDictionary *interactionOptions = nil; + if ([interactionAttribute isKindOfClass:NSArray.class] && interactionAttribute.count == 1) { + interactionOptions = [options mutableCopy]; + interactionOptions[@"attributes"] = interactionAttribute; + interactionOptions[@"maxDepth"] = @0; + interactionOptions[@"maxChildren"] = @0; + interactionOptions[@"maxArrayCount"] = @0; + } BOOL automationEnabled = [self assertAutomationMode:YES]; NSError *runtimeError = nil; id snapshot = nil; @@ -372,6 +417,7 @@ - (nullable NSDictionary *)snapshotForProcess:(pid_t)pid NSUInteger count = 0; NSDictionary *tree = [self nodeFromSnapshot:snapshot namesByNumber:namesByNumber + interactionOptions:interactionOptions depth:0 maxDepth:maxDepth maxNodes:maxNodes diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 093a36932a..5e8fe6e66a 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -487,13 +487,27 @@ resolved the covered screen's button and the flow failed on an assertion about t arrived, and a `presentation: "formSheet"` route's fields matched intermittently (#2638). Decision. The fold applies modal containment: when the last transition view under a container carries -UIKit's dimming view as its direct child, the earlier transition views whose frame that dimmed area -spans are cut, with their subtrees, from the regular and interactive projections. Both ends of the -claim are producer facts rather than an ordering guess — the dimming view is UIKit's own declaration -that it dims what sits behind, and its frame is the producer's rectangle, which is what a covered -container must be inside. A presentation with no dimming view, and any container whose earlier -sibling is not a transition view, keep today's behavior: containment is asserted only where the -producer states it, so the rule fails closed rather than guessing which siblings are shadows. +UIKit's dimming view as its direct child and the producer reports that dimming view takes touches +(`userInteractionEnabled: true`), the earlier transition views whose frame that dimmed area spans are +cut, with their subtrees, from the regular and interactive projections. Every part of the claim is a +producer fact rather than an ordering guess — the dimming view is UIKit's own declaration that it dims +what sits behind, its interaction state says whether a touch there reaches what sits behind, and its +frame is the producer's rectangle, which is what a covered container must be inside. A presentation +with no dimming view, a dimming view the producer did not read or reports passing touches through, +and any container whose earlier sibling is not a transition view keep today's behavior: containment +is asserted only where the producer states it, so the rule fails closed rather than guessing which +siblings are shadows. + +A sheet resting at an undimmed detent (`largestUndimmedDetentIdentifier`, react-native-screens +`sheetLargestUndimmedDetentIndex`) is why the interaction state is part of the claim. UIKit keeps the +dimming view in the tree at the same window-sized frame, and the presenting screen stays reachable: on +react-navigation's form-sheet example a press on the presenting screen's `Height Steps` button +navigates while the `Custom Dimming` sheet rests at its smallest detent. The bridge reports that +dimming view `userInteractionEnabled: false`, and `true` for the dimmed form sheet and the card modal. +No attribute the bridge already read told them apart, and `XC_kAXXCAttributeIsVisible` cannot either: +it reads false for the undimmed sheet's dimming view and for the card modal's. The bridge reads the +attribute for dimming views alone, in one follow-up read per view: requesting it on every node cost +about half again the capture time on a 492-node tree, while the targeted read did not move it. Every source the cut removes is counted in the presentation's `stats.modalContainedNodeCount` — internal evidence, per ADR 0026, never wire vocabulary — because the same screen either side of an animation otherwise moves hundreds of comparable lines with nothing @@ -508,8 +522,8 @@ exactly the 567-versus-76 divergence being fixed, since visibility filtering rea the mark; and cutting at acquisition would leak the decision into `--raw`, which owes the reader the tree the platform reported. Raw therefore still carries the covered screens. -Producers that report no UIKit class names never trigger the cut, and that is a fact about their -output rather than a backend exception in the fold: the runner already omits modal-contained content +Producers that report no UIKit class names or no dimming-view interaction state never trigger the +cut, and that is a fact about their output rather than a backend exception in the fold: the runner already omits modal-contained content from its own queries, `appium-source` and `limrun-ios-tree` report element types and no classes, and the macOS desktop surface arrives already presented. A scope naming a modal-contained screen now publishes an empty projection, which is the same healthy empty answer any other unmatched scope gives diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts index 2daf508e40..9afe2542b1 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts @@ -12,12 +12,15 @@ type PresentationShape = Readonly<{ containerRole?: string; rect?: Rect; dimming?: 'direct' | 'nested'; + /** What the producer read from the dimming view; `undefined` means it did not read it. */ + dimmingTakesTouches?: boolean; }>; /** * Mirrors a react-navigation capture: the presenting container carries only a drop shadow with * UIKit's dimming view below it, while each modal presentation carries a dimming view as a - * direct child of its transition view. + * direct child of its transition view. The Simulator AX bridge reads `userInteractionEnabled` + * on every dimming view, so the fixture defaults to the dimmed state it reports. */ function presentationSubtree( start: number, @@ -26,11 +29,12 @@ function presentationSubtree( shape: PresentationShape, ): RawSnapshotNode[] { const rect = shape.rect ?? VIEWPORT; + const takesTouches = 'dimmingTakesTouches' in shape ? shape.dimmingTakesTouches : true; const dimmingRect: Rect = { - x: -rect.width, - y: -rect.height, - width: rect.width * 3, - height: rect.height * 3, + x: -VIEWPORT.width, + y: -VIEWPORT.height, + width: VIEWPORT.width * 3, + height: VIEWPORT.height * 3, }; const nodes: RawSnapshotNode[] = [ snapshotNode(start, depth, parentIndex, 'UITransitionView', 'Other'), @@ -51,27 +55,20 @@ function presentationSubtree( height: 40, }), ]; - if ((shape.dimming ?? 'direct') === 'direct') { - nodes.splice( - 1, - 0, - snapshotNode(start + 4, depth + 1, start, 'UIDimmingView', 'Other', undefined, dimmingRect), - ); - } else { - nodes.splice( - 2, - 0, - snapshotNode( - start + 4, - depth + 2, - start + 1, - 'UIDimmingView', - 'Other', - undefined, - dimmingRect, - ), - ); - } + const direct = (shape.dimming ?? 'direct') === 'direct'; + const dimming = snapshotNode( + start + 4, + depth + (direct ? 1 : 2), + direct ? start : start + 1, + 'UIDimmingView', + 'Other', + undefined, + dimmingRect, + ); + nodes.splice(direct ? 1 : 2, 0, { + ...dimming, + ...(takesTouches === undefined ? {} : { userInteractionEnabled: takesTouches }), + }); return nodes; } @@ -176,7 +173,7 @@ test('fails closed when the last sibling is not a presentation container', () => const nodes = windowWith([ ...presentationSubtree(1, 1, 0, { title: 'Article by Dalek' }), snapshotNode(6, 1, 0, '_UIAlertControllerView', 'Other', 'Don’t leave'), - snapshotNode(7, 2, 6, 'UIDimmingView', 'Other'), + { ...snapshotNode(7, 2, 6, 'UIDimmingView', 'Other'), userInteractionEnabled: true }, ]); expect(collectModalContainedIndexes(nodes).size).toBe(0); }); @@ -193,12 +190,15 @@ test('fails closed when the dimming view does not span the earlier container', ( const nodes = windowWith([ ...presentationSubtree(1, 1, 0, { title: 'Article by Dalek' }), snapshotNode(6, 1, 0, 'UITransitionView', 'Other'), - snapshotNode(7, 2, 6, 'UIDimmingView', 'Other', undefined, { - x: 0, - y: 0, - width: 10, - height: 10, - }), + { + ...snapshotNode(7, 2, 6, 'UIDimmingView', 'Other', undefined, { + x: 0, + y: 0, + width: 10, + height: 10, + }), + userInteractionEnabled: true, + }, snapshotNode(8, 2, 6, 'UIDropShadowView', 'Other', undefined, SHEET), ]); expect(collectModalContainedIndexes(nodes).size).toBe(0); @@ -221,3 +221,39 @@ test('contains a full-screen presentation that a sheet does not geometrically co 1, 2, 3, 4, 5, ]); }); + +test('leaves the presenting screen reachable under a sheet at an undimmed detent', () => { + const nodes = windowWith([ + ...presentationSubtree(1, 1, 0, { + title: 'Form Sheet', + containerRole: 'RCTSurfaceHostingProxyRootView', + dimming: 'nested', + dimmingTakesTouches: false, + }), + ...presentationSubtree(6, 1, 0, { + title: 'Custom Dimming', + rect: { x: 8, y: 654, width: 386, height: 212 }, + dimmingTakesTouches: false, + }), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); + const folded = foldIosSnapshot(nodes, VIEWPORT, false, 'cursor-projected'); + expect(folded.nodes.flatMap((node) => (node.raw.label ? [node.raw.label] : []))).toEqual([ + 'Form Sheet', + 'Push from Form Sheet', + 'Custom Dimming', + 'Push from Custom Dimming', + ]); +}); + +test('fails closed when the producer did not read whether the dimming view takes touches', () => { + const nodes = windowWith([ + ...presentationSubtree(1, 1, 0, { title: 'Article by Dalek' }), + ...presentationSubtree(6, 1, 0, { + title: 'Albums', + rect: SHEET, + dimmingTakesTouches: undefined, + }), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts index 6e85222605..8d1ffd155f 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts @@ -1,7 +1,7 @@ import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { isPositiveFiniteRect, rectContains } from '@agent-device/kernel/rect'; import { normalizeType } from '@agent-device/contracts/snapshot'; -import { collectChildrenByParent, collectSubtreeIndexes } from './tree.ts'; +import { collectChildrenByParent, collectSubtreeByParentLinks } from './tree.ts'; import type { IosSnapshotFoldPolicy } from './types.ts'; const SCROLL_CONTAINER_TYPES = new Set(['collectionview', 'scrollview', 'table']); @@ -244,10 +244,12 @@ type CoveringPresentation = Readonly<{ * container, and the host accessibility snapshot reports siblings in that subview order. A * presenting container carries only a drop shadow, so a direct-child dimming view separates a * modal presentation from an ordinary container, and its dimmed area says which earlier - * presentation the user can no longer reach. + * presentation the user can no longer reach. A sheet resting at an undimmed detent keeps that + * dimming view with user interaction disabled and touches reach the content under it, so the + * rule asserts containment only when the producer states the dimming view takes touches. * - * Producers that report no UIKit class names — the XCTest runner, whose own queries already - * omit modal-contained content — never trigger this rule. + * Producers that report no UIKit class names or no `userInteractionEnabled` — the XCTest runner, + * whose own queries already omit modal-contained content — never trigger this rule. */ export function collectModalContainedIndexes( nodes: readonly RawSnapshotNode[], @@ -258,10 +260,10 @@ export function collectModalContainedIndexes( const covering = findCoveringPresentation(siblings, childrenByParent); if (!covering) continue; for (const sibling of siblings.slice(0, -1)) { - if (isDimmedByPresentation(covering, sibling)) { - for (const index of collectSubtreeIndexes(sibling.index, childrenByParent)) { - contained.add(index); - } + if (!isDimmedByPresentation(covering, sibling)) continue; + contained.add(sibling.index); + for (const descendant of collectSubtreeByParentLinks(sibling, childrenByParent)) { + contained.add(descendant.index); } } } @@ -275,8 +277,10 @@ function findCoveringPresentation( const last = siblings.at(-1); if (!last || !isPresentationContainer(last)) return undefined; const dimming = (childrenByParent.get(last.index) ?? []).find(isPresentationDimmingView); - const dimmingRect = dimming && isPositiveFiniteRect(dimming.rect) ? dimming.rect : undefined; - return dimmingRect ? { container: last, dimmingRect } : undefined; + if (dimming?.userInteractionEnabled !== true || !isPositiveFiniteRect(dimming.rect)) { + return undefined; + } + return { container: last, dimmingRect: dimming.rect }; } function isDimmedByPresentation(covering: CoveringPresentation, sibling: RawSnapshotNode): boolean { diff --git a/packages/capture-kit/src/ios-snapshot-engine/noise-structural.ts b/packages/capture-kit/src/ios-snapshot-engine/noise-structural.ts index cafa3847d4..12a55ef743 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/noise-structural.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/noise-structural.ts @@ -1,6 +1,10 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { normalizeType } from '@agent-device/contracts/snapshot'; -import { collectChildrenByParent, type SnapshotTreeRuleContext } from './tree.ts'; +import { + collectChildrenByParent, + collectSubtreeByParentLinks, + type SnapshotTreeRuleContext, +} from './tree.ts'; export function collectIosStructuralIdentifierSuppression( nodes: RawSnapshotNode[], @@ -27,21 +31,3 @@ export function collectIosStructuralIdentifierSuppression( context.suppressNode(node, content); } } - -function collectSubtreeByParentLinks( - root: RawSnapshotNode, - childrenByParent: ReadonlyMap, -): RawSnapshotNode[] { - const descendants: RawSnapshotNode[] = []; - const visited = new Set([root.index]); - const pending = [...(childrenByParent.get(root.index) ?? [])]; - while (pending.length > 0) { - const current = pending.pop(); - if (!current || visited.has(current.index)) continue; - visited.add(current.index); - descendants.push(current); - const children = childrenByParent.get(current.index); - if (children) pending.push(...children); - } - return descendants; -} diff --git a/packages/capture-kit/src/ios-snapshot-engine/tree.ts b/packages/capture-kit/src/ios-snapshot-engine/tree.ts index d728a59803..9b4621c63f 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/tree.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/tree.ts @@ -25,21 +25,23 @@ export function collectChildrenByParent( return childrenByParent; } -/** Walks a subtree through parent links only, so it does not require reported depths. */ -export function collectSubtreeIndexes( - startIndex: number, +/** A node's descendants, walked through parent links only so reported depths are not needed. */ +export function collectSubtreeByParentLinks( + root: RawSnapshotNode, childrenByParent: ReadonlyMap, -): number[] { - const indexes: number[] = []; - const pending = [startIndex]; +): RawSnapshotNode[] { + const descendants: RawSnapshotNode[] = []; + const visited = new Set([root.index]); + const pending = [...(childrenByParent.get(root.index) ?? [])]; while (pending.length > 0) { - const index = pending.pop()!; - indexes.push(index); - for (const child of childrenByParent.get(index) ?? []) { - pending.push(child.index); - } + const current = pending.pop(); + if (!current || visited.has(current.index)) continue; + visited.add(current.index); + descendants.push(current); + const children = childrenByParent.get(current.index); + if (children) pending.push(...children); } - return indexes; + return descendants; } const descendantEndPositionCache = new WeakMap(); diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 33eeafed20..ff57222c86 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -239,6 +239,8 @@ export type RawSnapshotNode = { selectionStart?: number; selectionEnd?: number; visibleToUser?: boolean; + /** UIKit `isUserInteractionEnabled`; absent means the producer did not read it, not false. */ + userInteractionEnabled?: boolean; hittable?: boolean; depth?: number; parentIndex?: number; diff --git a/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json b/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json index 3b78906a73..6eacf669d6 100644 --- a/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json +++ b/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json @@ -1,6 +1,6 @@ { "protocolVersion": 1, - "sourceVersion": "agent-device-simulator-ax-v1.6.0", + "sourceVersion": "agent-device-simulator-ax-v1.7.0", "requestKeys": [ "verb", "requestId", @@ -38,6 +38,7 @@ "XC_kAXXCAttributeFrame", "XC_kAXXCAttributeAutomationType", "XC_kAXXCAttributeTraits", + "XC_kAXXCAttributeIsUserInteractionEnabled", "XC_kAXXCAttributeChildren" ] } diff --git a/packages/platform-apple/src/snapshot-source/protocol.test.ts b/packages/platform-apple/src/snapshot-source/protocol.test.ts index a2839aed01..4fda71db5f 100644 --- a/packages/platform-apple/src/snapshot-source/protocol.test.ts +++ b/packages/platform-apple/src/snapshot-source/protocol.test.ts @@ -142,7 +142,7 @@ test('wire vocabulary guard keeps TS and Objective-C literals aligned', async () assert.deepEqual(wireVocabulary.responseKeys, SNAPSHOT_SOURCE_RESPONSE_KEYS); assert.deepEqual(wireVocabulary.attributeKeys, SNAPSHOT_SOURCE_ATTRIBUTE_KEYS); assert.match(nativeSource, /kProtocolVersion = 1/); - assert.match(nativeSource, /kSourceVersion = @"agent-device-simulator-ax-v1\.6\.0"/); + assert.match(nativeSource, /kSourceVersion = @"agent-device-simulator-ax-v1\.7\.0"/); for (const key of [ ...wireVocabulary.requestKeys, ...wireVocabulary.responseKeys, diff --git a/packages/platform-apple/src/snapshot-source/protocol.ts b/packages/platform-apple/src/snapshot-source/protocol.ts index cb2f887f30..ce663f86ec 100644 --- a/packages/platform-apple/src/snapshot-source/protocol.ts +++ b/packages/platform-apple/src/snapshot-source/protocol.ts @@ -3,7 +3,7 @@ import { snapshotSourceError } from './errors.ts'; import type { SnapshotSourceLimits } from './types.ts'; export const SNAPSHOT_SOURCE_PROTOCOL_VERSION = 1; -export const SNAPSHOT_SOURCE_VERSION = 'agent-device-simulator-ax-v1.6.0'; +export const SNAPSHOT_SOURCE_VERSION = 'agent-device-simulator-ax-v1.7.0'; const FRAME_HEADER_BYTES = 4; export const SNAPSHOT_SOURCE_WIRE_KEYS = Object.freeze([ @@ -45,6 +45,7 @@ export const SNAPSHOT_SOURCE_ATTRIBUTE_KEYS = Object.freeze([ 'XC_kAXXCAttributeFrame', 'XC_kAXXCAttributeAutomationType', 'XC_kAXXCAttributeTraits', + 'XC_kAXXCAttributeIsUserInteractionEnabled', 'XC_kAXXCAttributeChildren', ] as const); diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index 23853eddc0..553c3a4e98 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -429,3 +429,27 @@ test('a window reporting the app box quarter-turned is counted as an unresolved 0, ); }); + +test('the bridge tree publishes whether a dimming view takes touches', () => { + const dimming = (enabled?: unknown) => ({ + [application]: 'UIDimmingView', + [frame]: { X: -390, Y: -844, Width: 1170, Height: 2532 }, + ...(enabled === undefined ? {} : { XC_kAXXCAttributeIsUserInteractionEnabled: enabled }), + [children]: [], + }); + const decode = (enabled?: unknown) => + decodeSnapshotBridgeTree( + { [application]: 'Application', [children]: [dimming(enabled)] }, + { truncated: false }, + limits, + ).nodes[1]; + + assert.equal(decode(true)?.userInteractionEnabled, true); + assert.equal(decode(false)?.userInteractionEnabled, false, 'a sheet at an undimmed detent'); + assert.equal(decode()?.userInteractionEnabled, undefined, 'an unread fact stays unknown'); + assert.throws( + () => decode(1), + (error: unknown) => + error instanceof SnapshotSourceError && error.failureCode === 'user-interaction-invalid', + ); +}); diff --git a/packages/platform-apple/src/snapshot-source/tree.ts b/packages/platform-apple/src/snapshot-source/tree.ts index dd8322f78f..8fe1f6518f 100644 --- a/packages/platform-apple/src/snapshot-source/tree.ts +++ b/packages/platform-apple/src/snapshot-source/tree.ts @@ -20,6 +20,7 @@ const ATTRIBUTE = Object.freeze({ frame: 'XC_kAXXCAttributeFrame', automationType: 'XC_kAXXCAttributeAutomationType', traits: 'XC_kAXXCAttributeTraits', + userInteractionEnabled: 'XC_kAXXCAttributeIsUserInteractionEnabled', children: 'XC_kAXXCAttributeChildren', }); @@ -245,6 +246,7 @@ function nodeFacts( // Publishes `selected: true` only when the selected bit is set and omits it otherwise — the same // shape the XCTest tree produces, so a `selected:` selector cannot tell the producers apart. const selected = traits === undefined || (traits & SELECTED_TRAIT) === 0n ? undefined : true; + const userInteractionEnabled = optionalBoolean(value[ATTRIBUTE.userInteractionEnabled]); return { index, ...(parentIndex === undefined ? {} : { parentIndex }), @@ -265,6 +267,7 @@ function nodeFacts( ...(frame ? { rect: frame } : {}), ...(enabled === undefined ? {} : { enabled }), ...(selected === undefined ? {} : { selected }), + ...(userInteractionEnabled === undefined ? {} : { userInteractionEnabled }), depth, }; } @@ -395,6 +398,14 @@ function traitsFromGuest(value: unknown): bigint | undefined { return BigInt(value); } +function optionalBoolean(value: unknown): boolean | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value !== 'boolean') { + throw snapshotSourceError('malformed-tree', 'user-interaction-invalid'); + } + return value; +} + function optionalInteger(value: unknown): number | undefined { if (value === undefined || value === null) return undefined; if (!Number.isSafeInteger(value)) diff --git a/src/commands/capture/runtime/snapshot-unchanged.ts b/src/commands/capture/runtime/snapshot-unchanged.ts index dab0fe96a8..aa249709ce 100644 --- a/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/src/commands/capture/runtime/snapshot-unchanged.ts @@ -88,6 +88,7 @@ type ComparableSnapshotNode = Omit< | 'selectionStart' | 'selectionEnd' | 'visibleToUser' + | 'userInteractionEnabled' | 'inheritsLabel' | 'inheritsIdentifier' >;