diff --git a/CHANGELOG.md b/CHANGELOG.md index f4ee6cdf56..d63c20cf2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,27 @@ through the refusal; the runner transport never resends it. Interactions that mutate without a leading read (`press`, a coordinate `fill`, `swipe`, `scroll`, hardware keys) keep the foreground repair and still bring a stopped app up. +- Fixed (ios): a local Simulator snapshot taken through the host AX bridge no longer publishes the + screens a modal presentation covers, so a selector resolves the control the user can reach instead + of the stale copy underneath it. UIKit appends each presentation as a later sibling of what it + presents over, and the bridge reports every sibling, so a React Navigation stack that presents its + pushed routes as sheets kept all three screens live: one capture of a three-screen stack published + 567 nodes across three screens where the same app under the XCTest runner published 76 across the + top one. Maestro's visibility predicate is geometric, so `- tapOn: "Push article"` matched the + covered screen's button first and the flow failed on an assertion about the screen that never + arrived, and `presentation: "formSheet"` fields matched intermittently (#2638). Geometry cannot + close this: a sheet leaves most of the covered screen uncovered, and the occlusion pass only + treats floating chrome as a cover. The presentation cut now applies modal containment — when the + last transition view under a container carries UIKit's dimming view as its direct child and that + dimming view takes touches, the earlier transition views that dimmed area spans, and everything + under them, are cut from the regular and interactive snapshots. The bridge now reads + `userInteractionEnabled` on dimming views (source version v1.7.0) because a sheet resting at an + undimmed detent keeps its dimming view while the screen under it stays reachable; that sheet, and + any presentation UIKit does not dim, is left as it was, so the cut asserts containment only where + the producer states it. `snapshot --raw` still reports the covered screens, and producers that + report no UIKit class names — the XCTest runner, whose own queries answered 76 nodes for that same + state, plus `appium-source` and `limrun-ios-tree` — never trigger the cut. All 39 flows of React Navigation's Maestro suite pass on an iPhone 17 Simulator running + iOS 26.2 with this change, including two that never passed on the bridge. - Fixed (ios): a local Simulator snapshot taken through the host AX bridge once again publishes the geometric `hittable` fact, so `is hittable` and a `hittable:` selector resolve the same controls on the bridge and the XCTest runner. The snapshot capability table has declared `hittable = diff --git a/apple/snapshot-bridge/SnapshotBridgeRuntime.m b/apple/snapshot-bridge/SnapshotBridgeRuntime.m index 6f79767b9a..a4e8da8eb3 100644 --- a/apple/snapshot-bridge/SnapshotBridgeRuntime.m +++ b/apple/snapshot-bridge/SnapshotBridgeRuntime.m @@ -18,7 +18,7 @@ NSString *const kProtocolVersionKey = @"protocolVersion"; NSString *const kSourceVersionKey = @"sourceVersion"; NSString *const kRequestIdKey = @"requestId"; -NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.6.0"; +NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.7.0"; const NSUInteger kProtocolVersion = 1; const uint32_t kMaximumFrameBytes = 16 * 1024 * 1024; const NSUInteger kMaximumDepth = 128; @@ -34,8 +34,11 @@ static NSString *const kAttributeAutomationType = @"XC_kAXXCAttributeAutomationType"; static NSString *const kAttributeTraits = @"XC_kAXXCAttributeTraits"; static NSString *const kAttributeChildren = @"XC_kAXXCAttributeChildren"; +static NSString *const kAttributeIsUserInteractionEnabled = @"XC_kAXXCAttributeIsUserInteractionEnabled"; +static NSString *const kPresentationDimmingViewClass = @"UIDimmingView"; static NSString *const kSnapshotAttributes = @"UIAccessibilitySnapshotKeyAttributes"; static NSString *const kSnapshotChildren = @"UIAccessibilitySnapshotKeyChildren"; +static NSString *const kSnapshotElement = @"UIAccessibilitySnapshotKeyElement"; static NSString *const kXctAutomationSupportPath = @"/Developer/Library/PrivateFrameworks/XCTAutomationSupport.framework/XCTAutomationSupport"; static NSString *const kAxRuntimePath = @@ -202,8 +205,35 @@ - (nullable id)jsonValue:(id)value name:(NSString *)name return nil; } +/* + * Whether a presentation's dimming view takes touches is what separates a sheet that blocks the + * content under it from one resting at an undimmed detent, and nothing else in the tree says so. + * It is read for dimming views alone: requesting it for every node costs about half again the + * capture time. + */ +- (nullable NSNumber *)userInteractionEnabledForSnapshot:(NSDictionary *)snapshot + options:(nullable NSDictionary *)options +{ + id element = snapshot[kSnapshotElement]; + NSNumber *attribute = [options[@"attributes"] firstObject]; + if (!element || ![attribute isKindOfClass:NSNumber.class]) return nil; + @try { + NSError *failure = nil; + NSDictionary *read = [_framework userTestingSnapshotForElement:element options:options error:&failure]; + if (![read isKindOfClass:NSDictionary.class]) return nil; + NSDictionary *attributes = read[kSnapshotAttributes]; + if (![attributes isKindOfClass:NSDictionary.class]) return nil; + id value = attributes[attribute]; + return [value isKindOfClass:NSNumber.class] ? @([(NSNumber *)value boolValue]) : nil; + } @catch (NSException *exception) { + (void)exception; + return nil; + } +} + - (nullable NSDictionary *)nodeFromSnapshot:(id)snapshot namesByNumber:(NSDictionary *)namesByNumber + interactionOptions:(nullable NSDictionary *)interactionOptions depth:(NSUInteger)depth maxDepth:(NSUInteger)maxDepth maxNodes:(NSUInteger)maxNodes @@ -233,6 +263,11 @@ - (nullable NSDictionary *)nodeFromSnapshot:(id)snapshot id safe = [self jsonValue:attributes[number] name:name]; if (safe) node[name] = safe; } + if ([node[kAttributeElementType] isEqual:kPresentationDimmingViewClass]) { + NSNumber *interaction = [self userInteractionEnabledForSnapshot:(NSDictionary *)snapshot + options:interactionOptions]; + if (interaction) node[kAttributeIsUserInteractionEnabled] = interaction; + } NSArray *children = ((NSDictionary *)snapshot)[kSnapshotChildren]; if (![children isKindOfClass:NSArray.class]) { @@ -246,6 +281,7 @@ - (nullable NSDictionary *)nodeFromSnapshot:(id)snapshot for (id child in children) { NSDictionary *built = [self nodeFromSnapshot:child namesByNumber:namesByNumber + interactionOptions:interactionOptions depth:depth + 1 maxDepth:maxDepth maxNodes:maxNodes @@ -317,6 +353,15 @@ - (nullable NSDictionary *)snapshotForProcess:(pid_t)pid options[@"maxDepth"] = @(maxDepth); options[@"maxChildren"] = @(maxNodes); options[@"maxArrayCount"] = @(maxNodes); + NSArray *interactionAttribute = _attributeNumbersForNames(@[ kAttributeIsUserInteractionEnabled ]); + NSMutableDictionary *interactionOptions = nil; + if ([interactionAttribute isKindOfClass:NSArray.class] && interactionAttribute.count == 1) { + interactionOptions = [options mutableCopy]; + interactionOptions[@"attributes"] = interactionAttribute; + interactionOptions[@"maxDepth"] = @0; + interactionOptions[@"maxChildren"] = @0; + interactionOptions[@"maxArrayCount"] = @0; + } BOOL automationEnabled = [self assertAutomationMode:YES]; NSError *runtimeError = nil; id snapshot = nil; @@ -372,6 +417,7 @@ - (nullable NSDictionary *)snapshotForProcess:(pid_t)pid NSUInteger count = 0; NSDictionary *tree = [self nodeFromSnapshot:snapshot namesByNumber:namesByNumber + interactionOptions:interactionOptions depth:0 maxDepth:maxDepth maxNodes:maxNodes diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 46042f3e3b..5e8fe6e66a 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -475,3 +475,56 @@ last reader that can still refuse geometry it cannot place is the tap-path keybo width rule therefore remains. The rule detects un-normalized arrival, not a standing fact about iOS: the producers above do normalize, and a band taller than it is wide is what one that did not looks like. + +## Amendment: modal containment in the presentation cut + +The bridge reads one window and reports every container hanging off it, while XCTest's own queries +answer only the presentation the user can reach. React Navigation's card-plus-modal example is the +measured case: three routes presented as sheets left three transition-view containers in one window +and one capture published 567 nodes across three screens, where the same app state under the runner +published 76 across the top screen alone. Maestro's visibility test is geometric, so a `tapOn` +resolved the covered screen's button and the flow failed on an assertion about the screen that never +arrived, and a `presentation: "formSheet"` route's fields matched intermittently (#2638). + +Decision. The fold applies modal containment: when the last transition view under a container carries +UIKit's dimming view as its direct child and the producer reports that dimming view takes touches +(`userInteractionEnabled: true`), the earlier transition views whose frame that dimmed area spans are +cut, with their subtrees, from the regular and interactive projections. Every part of the claim is a +producer fact rather than an ordering guess — the dimming view is UIKit's own declaration that it dims +what sits behind, its interaction state says whether a touch there reaches what sits behind, and its +frame is the producer's rectangle, which is what a covered container must be inside. A presentation +with no dimming view, a dimming view the producer did not read or reports passing touches through, +and any container whose earlier sibling is not a transition view keep today's behavior: containment +is asserted only where the producer states it, so the rule fails closed rather than guessing which +siblings are shadows. + +A sheet resting at an undimmed detent (`largestUndimmedDetentIdentifier`, react-native-screens +`sheetLargestUndimmedDetentIndex`) is why the interaction state is part of the claim. UIKit keeps the +dimming view in the tree at the same window-sized frame, and the presenting screen stays reachable: on +react-navigation's form-sheet example a press on the presenting screen's `Height Steps` button +navigates while the `Custom Dimming` sheet rests at its smallest detent. The bridge reports that +dimming view `userInteractionEnabled: false`, and `true` for the dimmed form sheet and the card modal. +No attribute the bridge already read told them apart, and `XC_kAXXCAttributeIsVisible` cannot either: +it reads false for the undimmed sheet's dimming view and for the card modal's. The bridge reads the +attribute for dimming views alone, in one follow-up read per view: requesting it on every node cost +about half again the capture time on a 492-node tree, while the targeted read did not move it. +Every source the cut removes is counted in the presentation's +`stats.modalContainedNodeCount` — internal evidence, per ADR 0026, never wire vocabulary — because +the same screen either side of an animation otherwise moves hundreds of comparable lines with nothing +to attribute them to. + +Why this seam and not the neighbouring ones. The semantic presentation rules +(`IOS_PRESENTATION_RULES`) run only for the interactive projection, so a suppression rule there could +not deliver the same claim to a regular `snapshot`; the regular eligibility table is pinned against +its Swift twin, so widening it would move a cross-language contract that this rule does not touch; the +occlusion pass *marks* a node covered and keeps it published, and a published-but-marked node is +exactly the 567-versus-76 divergence being fixed, since visibility filtering reads geometry and not +the mark; and cutting at acquisition would leak the decision into `--raw`, which owes the reader the +tree the platform reported. Raw therefore still carries the covered screens. + +Producers that report no UIKit class names or no dimming-view interaction state never trigger the +cut, and that is a fact about their output rather than a backend exception in the fold: the runner already omits modal-contained content +from its own queries, `appium-source` and `limrun-ios-tree` report element types and no classes, and +the macOS desktop surface arrives already presented. A scope naming a modal-contained screen now +publishes an empty projection, which is the same healthy empty answer any other unmatched scope gives +— the screen is not what is presented. diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.ts index a8df22b5d0..5a22ae4a7b 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.ts @@ -101,6 +101,7 @@ function presentAcquiredSnapshot( presentedNodeCount: projected.nodes.length, sourceNodeCount: acquisition.nodes.length, parentClipLookups: 0, + modalContainedNodeCount: 0, }, }; } @@ -153,6 +154,7 @@ function presentAcquiredSnapshot( presentedNodeCount: compacted.nodes.length, sourceNodeCount: acquisition.nodes.length, parentClipLookups: folded.stats.parentClipLookups + validation.parentClipLookups, + modalContainedNodeCount: folded.stats.modalContainedNodeCount, }, }; } diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts new file mode 100644 index 0000000000..9afe2542b1 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts @@ -0,0 +1,259 @@ +import { expect, test } from 'vitest'; +import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; +import { foldIosSnapshot } from './geometry.ts'; +import { collectModalContainedIndexes } from './geometry-policy.ts'; +import { projectIosSnapshot } from './projection.ts'; + +const VIEWPORT = { x: 0, y: 0, width: 402, height: 874 }; +const SHEET: Rect = { x: 16, y: 62, width: 370, height: 747 }; + +type PresentationShape = Readonly<{ + title: string; + containerRole?: string; + rect?: Rect; + dimming?: 'direct' | 'nested'; + /** What the producer read from the dimming view; `undefined` means it did not read it. */ + dimmingTakesTouches?: boolean; +}>; + +/** + * Mirrors a react-navigation capture: the presenting container carries only a drop shadow with + * UIKit's dimming view below it, while each modal presentation carries a dimming view as a + * direct child of its transition view. The Simulator AX bridge reads `userInteractionEnabled` + * on every dimming view, so the fixture defaults to the dimmed state it reports. + */ +function presentationSubtree( + start: number, + depth: number, + parentIndex: number, + shape: PresentationShape, +): RawSnapshotNode[] { + const rect = shape.rect ?? VIEWPORT; + const takesTouches = 'dimmingTakesTouches' in shape ? shape.dimmingTakesTouches : true; + const dimmingRect: Rect = { + x: -VIEWPORT.width, + y: -VIEWPORT.height, + width: VIEWPORT.width * 3, + height: VIEWPORT.height * 3, + }; + const nodes: RawSnapshotNode[] = [ + snapshotNode(start, depth, parentIndex, 'UITransitionView', 'Other'), + snapshotNode(start + 1, depth + 1, start, 'UIDropShadowView', 'Other', undefined, rect), + snapshotNode( + start + 2, + depth + 2, + start + 1, + shape.containerRole ?? 'RNSModalScreen', + 'Other', + shape.title, + rect, + ), + snapshotNode(start + 3, depth + 3, start + 2, undefined, 'Button', `Push from ${shape.title}`, { + x: rect.x + 12, + y: rect.y + 92, + width: 120, + height: 40, + }), + ]; + const direct = (shape.dimming ?? 'direct') === 'direct'; + const dimming = snapshotNode( + start + 4, + depth + (direct ? 1 : 2), + direct ? start : start + 1, + 'UIDimmingView', + 'Other', + undefined, + dimmingRect, + ); + nodes.splice(direct ? 1 : 2, 0, { + ...dimming, + ...(takesTouches === undefined ? {} : { userInteractionEnabled: takesTouches }), + }); + return nodes; +} + +function snapshotNode( + index: number, + depth: number, + parentIndex: number | undefined, + role: string | undefined, + type: string, + label?: string, + rect: Rect = VIEWPORT, +): RawSnapshotNode { + return { + index, + depth, + ...(parentIndex === undefined ? {} : { parentIndex }), + type, + ...(role ? { role } : {}), + ...(label ? { label } : {}), + rect, + hittable: true, + }; +} + +function windowWith(children: RawSnapshotNode[]): RawSnapshotNode[] { + return [snapshotNode(0, 0, undefined, 'UIWindow', 'Window'), ...children]; +} + +function stackedPresentations(): RawSnapshotNode[] { + return windowWith([ + ...presentationSubtree(1, 1, 0, { + title: 'Article by Dalek', + containerRole: 'RCTSurfaceHostingProxyRootView', + dimming: 'nested', + }), + ...presentationSubtree(6, 1, 0, { title: 'Albums', rect: SHEET }), + ...presentationSubtree(11, 1, 0, { title: 'Article by The Doctor' }), + ]); +} + +test('dims away every earlier presentation the covering dimming view spans', () => { + const contained = collectModalContainedIndexes(stackedPresentations()); + expect([...contained].sort((left, right) => left - right)).toEqual([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + ]); +}); + +test('keeps the covering presentation and everything it contains', () => { + const contained = collectModalContainedIndexes(stackedPresentations()); + expect([...contained].filter((index) => index >= 11)).toEqual([]); +}); + +test('presents only the covering subtree through the fold', () => { + const folded = foldIosSnapshot(stackedPresentations(), VIEWPORT, false, 'cursor-projected'); + expect(folded.nodes.flatMap((node) => (node.raw.label ? [node.raw.label] : []))).toEqual([ + 'Article by The Doctor', + 'Push from Article by The Doctor', + ]); +}); + +test('records how many sources the cut removed', () => { + const folded = foldIosSnapshot(stackedPresentations(), VIEWPORT, false, 'cursor-projected'); + expect(folded.stats.modalContainedNodeCount).toBe(10); + expect(folded.stats.sourceNodeCount).toBe(16); +}); + +test('publishes an empty projection when a scope names a modal-contained screen', () => { + const folded = foldIosSnapshot(stackedPresentations(), VIEWPORT, false, 'cursor-projected'); + const covered = projectIosSnapshot({ + nodes: folded.nodes, + projection: 'regular', + scope: 'Albums', + depth: null, + foldPolicy: 'cursor-projected', + }); + const covering = projectIosSnapshot({ + nodes: folded.nodes, + projection: 'regular', + scope: 'Article by The Doctor', + depth: null, + foldPolicy: 'cursor-projected', + }); + expect(covered.nodes).toEqual([]); + expect(covering.nodes.length).toBeGreaterThan(0); +}); + +test('spares an earlier sibling that is not a presentation container', () => { + const nodes = windowWith([ + snapshotNode(1, 1, 0, 'RCTSurfaceHostingProxyRootView', 'Other'), + snapshotNode(2, 2, 1, undefined, 'Button', 'Live control', { + x: 12, + y: 154, + width: 120, + height: 40, + }), + ...presentationSubtree(3, 1, 0, { title: 'Albums' }), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); +}); + +test('fails closed when the last sibling is not a presentation container', () => { + const nodes = windowWith([ + ...presentationSubtree(1, 1, 0, { title: 'Article by Dalek' }), + snapshotNode(6, 1, 0, '_UIAlertControllerView', 'Other', 'Don’t leave'), + { ...snapshotNode(7, 2, 6, 'UIDimmingView', 'Other'), userInteractionEnabled: true }, + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); +}); + +test('fails closed when the dimming view sits below the presentation container', () => { + const nodes = windowWith([ + ...presentationSubtree(1, 1, 0, { title: 'Article by Dalek' }), + ...presentationSubtree(6, 1, 0, { title: 'Albums', dimming: 'nested' }), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); +}); + +test('fails closed when the dimming view does not span the earlier container', () => { + const nodes = windowWith([ + ...presentationSubtree(1, 1, 0, { title: 'Article by Dalek' }), + snapshotNode(6, 1, 0, 'UITransitionView', 'Other'), + { + ...snapshotNode(7, 2, 6, 'UIDimmingView', 'Other', undefined, { + x: 0, + y: 0, + width: 10, + height: 10, + }), + userInteractionEnabled: true, + }, + snapshotNode(8, 2, 6, 'UIDropShadowView', 'Other', undefined, SHEET), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); +}); + +test('stays inert on a tree that reports no UIKit class names', () => { + const nodes = windowWith([ + snapshotNode(1, 1, 0, undefined, 'Other', 'Article by Dalek'), + snapshotNode(2, 1, 0, undefined, 'Other', 'Article by The Doctor'), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); +}); + +test('contains a full-screen presentation that a sheet does not geometrically cover', () => { + const nodes = windowWith([ + ...presentationSubtree(1, 1, 0, { title: 'Article by Dalek' }), + ...presentationSubtree(6, 1, 0, { title: 'Albums', rect: SHEET }), + ]); + expect([...collectModalContainedIndexes(nodes)].sort((left, right) => left - right)).toEqual([ + 1, 2, 3, 4, 5, + ]); +}); + +test('leaves the presenting screen reachable under a sheet at an undimmed detent', () => { + const nodes = windowWith([ + ...presentationSubtree(1, 1, 0, { + title: 'Form Sheet', + containerRole: 'RCTSurfaceHostingProxyRootView', + dimming: 'nested', + dimmingTakesTouches: false, + }), + ...presentationSubtree(6, 1, 0, { + title: 'Custom Dimming', + rect: { x: 8, y: 654, width: 386, height: 212 }, + dimmingTakesTouches: false, + }), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); + const folded = foldIosSnapshot(nodes, VIEWPORT, false, 'cursor-projected'); + expect(folded.nodes.flatMap((node) => (node.raw.label ? [node.raw.label] : []))).toEqual([ + 'Form Sheet', + 'Push from Form Sheet', + 'Custom Dimming', + 'Push from Custom Dimming', + ]); +}); + +test('fails closed when the producer did not read whether the dimming view takes touches', () => { + const nodes = windowWith([ + ...presentationSubtree(1, 1, 0, { title: 'Article by Dalek' }), + ...presentationSubtree(6, 1, 0, { + title: 'Albums', + rect: SHEET, + dimmingTakesTouches: undefined, + }), + ]); + expect(collectModalContainedIndexes(nodes).size).toBe(0); +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts index 7f0108a68e..8d1ffd155f 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts @@ -1,12 +1,21 @@ import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import { isPositiveFiniteRect, rectContains } from '@agent-device/kernel/rect'; import { normalizeType } from '@agent-device/contracts/snapshot'; +import { collectChildrenByParent, collectSubtreeByParentLinks } from './tree.ts'; import type { IosSnapshotFoldPolicy } from './types.ts'; const SCROLL_CONTAINER_TYPES = new Set(['collectionview', 'scrollview', 'table']); const VISIBILITY_CARRIER_TYPES = new Set(['application', 'window']); const NEGLIGIBLE_DECORATION_TOLERANCE = 1; +/** + * Private UIKit classes are matched by exact name, unlike the substring role tests the occlusion + * pass uses: this rule decides node membership, so a near miss on a class name has to fail closed + * rather than widen the cut. + */ +const PRESENTATION_CONTAINER_ROLE = 'uitransitionview'; +const PRESENTATION_DIMMING_ROLE = 'uidimmingview'; + export type TraversalState = Readonly<{ projectedOut: boolean; ancestorClip?: Rect; @@ -223,3 +232,73 @@ function intersectRect(left: Rect, right: Rect): Rect { height: bottomEdge - y, }; } + +type CoveringPresentation = Readonly<{ + container: RawSnapshotNode; + dimmingRect: Rect; +}>; + +/** + * UIKit appends each modal presentation as a later sibling of the container it presents over, + * dims the content behind it with a dimming view that is a direct child of the presentation + * container, and the host accessibility snapshot reports siblings in that subview order. A + * presenting container carries only a drop shadow, so a direct-child dimming view separates a + * modal presentation from an ordinary container, and its dimmed area says which earlier + * presentation the user can no longer reach. A sheet resting at an undimmed detent keeps that + * dimming view with user interaction disabled and touches reach the content under it, so the + * rule asserts containment only when the producer states the dimming view takes touches. + * + * Producers that report no UIKit class names or no `userInteractionEnabled` — the XCTest runner, + * whose own queries already omit modal-contained content — never trigger this rule. + */ +export function collectModalContainedIndexes( + nodes: readonly RawSnapshotNode[], +): ReadonlySet { + const childrenByParent = collectChildrenByParent(nodes); + const contained = new Set(); + for (const siblings of childrenByParent.values()) { + const covering = findCoveringPresentation(siblings, childrenByParent); + if (!covering) continue; + for (const sibling of siblings.slice(0, -1)) { + if (!isDimmedByPresentation(covering, sibling)) continue; + contained.add(sibling.index); + for (const descendant of collectSubtreeByParentLinks(sibling, childrenByParent)) { + contained.add(descendant.index); + } + } + } + return contained; +} + +function findCoveringPresentation( + siblings: readonly RawSnapshotNode[], + childrenByParent: ReadonlyMap, +): CoveringPresentation | undefined { + const last = siblings.at(-1); + if (!last || !isPresentationContainer(last)) return undefined; + const dimming = (childrenByParent.get(last.index) ?? []).find(isPresentationDimmingView); + if (dimming?.userInteractionEnabled !== true || !isPositiveFiniteRect(dimming.rect)) { + return undefined; + } + return { container: last, dimmingRect: dimming.rect }; +} + +function isDimmedByPresentation(covering: CoveringPresentation, sibling: RawSnapshotNode): boolean { + return ( + isPresentationContainer(sibling) && + isPositiveFiniteRect(sibling.rect) && + rectContains(covering.dimmingRect, sibling.rect) + ); +} + +function isPresentationContainer(node: RawSnapshotNode): boolean { + return hasRole(node, PRESENTATION_CONTAINER_ROLE); +} + +function isPresentationDimmingView(node: RawSnapshotNode): boolean { + return hasRole(node, PRESENTATION_DIMMING_ROLE); +} + +function hasRole(node: RawSnapshotNode, role: string): boolean { + return normalizeType(node.role ?? '') === role; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts index 61b6345cea..f69739b62c 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts @@ -2,6 +2,7 @@ import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { isGeometricallyActionable } from '@agent-device/kernel/rect'; import { validateIosSnapshotGraph } from './graph.ts'; import { + collectModalContainedIndexes, rootTraversal, traversalDecision, type BranchState, @@ -22,13 +23,15 @@ export function foldIosSnapshot( options: IosSnapshotFoldOptions = {}, ): { nodes: IosSnapshotPresentationNode[]; stats: IosSnapshotPresentationStats } { validateIosSnapshotGraph(nodes); - const hasChildren = buildChildPresence(nodes); + const modalContained = collectModalContainedIndexes(nodes); + const presentable = nodes.filter((node) => !modalContained.has(node.index)); + const hasChildren = buildChildPresence(presentable); const states = new Map(); const kept: IosSnapshotPresentationNode[] = []; const hints = new Map(); let parentClipLookups = 0; - for (const node of nodes) { + for (const node of presentable) { const parentState = readParentState(node, states, () => { parentClipLookups += 1; }); @@ -61,6 +64,7 @@ export function foldIosSnapshot( presentedNodeCount: presented.length, sourceNodeCount: nodes.length, parentClipLookups, + modalContainedNodeCount: modalContained.size, }, }; } diff --git a/packages/capture-kit/src/ios-snapshot-engine/noise-structural.ts b/packages/capture-kit/src/ios-snapshot-engine/noise-structural.ts index cafa3847d4..12a55ef743 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/noise-structural.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/noise-structural.ts @@ -1,6 +1,10 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { normalizeType } from '@agent-device/contracts/snapshot'; -import { collectChildrenByParent, type SnapshotTreeRuleContext } from './tree.ts'; +import { + collectChildrenByParent, + collectSubtreeByParentLinks, + type SnapshotTreeRuleContext, +} from './tree.ts'; export function collectIosStructuralIdentifierSuppression( nodes: RawSnapshotNode[], @@ -27,21 +31,3 @@ export function collectIosStructuralIdentifierSuppression( context.suppressNode(node, content); } } - -function collectSubtreeByParentLinks( - root: RawSnapshotNode, - childrenByParent: ReadonlyMap, -): RawSnapshotNode[] { - const descendants: RawSnapshotNode[] = []; - const visited = new Set([root.index]); - const pending = [...(childrenByParent.get(root.index) ?? [])]; - while (pending.length > 0) { - const current = pending.pop(); - if (!current || visited.has(current.index)) continue; - visited.add(current.index); - descendants.push(current); - const children = childrenByParent.get(current.index); - if (children) pending.push(...children); - } - return descendants; -} diff --git a/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts index 4a48b11284..cbae8a042d 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts @@ -42,6 +42,7 @@ export function presentIosRunnerSnapshot( presentedNodeCount: compacted.nodes.length, sourceNodeCount: input.presentation.payload.nodes.length, parentClipLookups: validationStats.parentClipLookups, + modalContainedNodeCount: 0, }, }; } diff --git a/packages/capture-kit/src/ios-snapshot-engine/tree.ts b/packages/capture-kit/src/ios-snapshot-engine/tree.ts index 8cd1d5c77e..9b4621c63f 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/tree.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/tree.ts @@ -12,7 +12,9 @@ export type SnapshotTreeRuleContext = { suppressNode: (source: RawSnapshotNode, representatives: readonly RawSnapshotNode[]) => void; }; -export function collectChildrenByParent(nodes: RawSnapshotNode[]): Map { +export function collectChildrenByParent( + nodes: readonly RawSnapshotNode[], +): Map { const childrenByParent = new Map(); for (const node of nodes) { if (typeof node.parentIndex !== 'number') continue; @@ -23,6 +25,25 @@ export function collectChildrenByParent(nodes: RawSnapshotNode[]): Map, +): RawSnapshotNode[] { + const descendants: RawSnapshotNode[] = []; + const visited = new Set([root.index]); + const pending = [...(childrenByParent.get(root.index) ?? [])]; + while (pending.length > 0) { + const current = pending.pop(); + if (!current || visited.has(current.index)) continue; + visited.add(current.index); + descendants.push(current); + const children = childrenByParent.get(current.index); + if (children) pending.push(...children); + } + return descendants; +} + const descendantEndPositionCache = new WeakMap(); export function collectDescendants( diff --git a/packages/capture-kit/src/ios-snapshot-engine/types.ts b/packages/capture-kit/src/ios-snapshot-engine/types.ts index 29e4fb5bdc..91fd4392bc 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/types.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/types.ts @@ -20,6 +20,8 @@ export type IosSnapshotPresentationStats = Readonly<{ presentedNodeCount: number; sourceNodeCount: number; parentClipLookups: number; + /** Sources the presentation cut removed because a later modal presentation dims them. */ + modalContainedNodeCount: number; }>; export type IosSnapshotEnginePresentation = Readonly<{ diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 33eeafed20..ff57222c86 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -239,6 +239,8 @@ export type RawSnapshotNode = { selectionStart?: number; selectionEnd?: number; visibleToUser?: boolean; + /** UIKit `isUserInteractionEnabled`; absent means the producer did not read it, not false. */ + userInteractionEnabled?: boolean; hittable?: boolean; depth?: number; parentIndex?: number; diff --git a/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json b/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json index 3b78906a73..6eacf669d6 100644 --- a/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json +++ b/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json @@ -1,6 +1,6 @@ { "protocolVersion": 1, - "sourceVersion": "agent-device-simulator-ax-v1.6.0", + "sourceVersion": "agent-device-simulator-ax-v1.7.0", "requestKeys": [ "verb", "requestId", @@ -38,6 +38,7 @@ "XC_kAXXCAttributeFrame", "XC_kAXXCAttributeAutomationType", "XC_kAXXCAttributeTraits", + "XC_kAXXCAttributeIsUserInteractionEnabled", "XC_kAXXCAttributeChildren" ] } diff --git a/packages/platform-apple/src/snapshot-source/protocol.test.ts b/packages/platform-apple/src/snapshot-source/protocol.test.ts index a2839aed01..4fda71db5f 100644 --- a/packages/platform-apple/src/snapshot-source/protocol.test.ts +++ b/packages/platform-apple/src/snapshot-source/protocol.test.ts @@ -142,7 +142,7 @@ test('wire vocabulary guard keeps TS and Objective-C literals aligned', async () assert.deepEqual(wireVocabulary.responseKeys, SNAPSHOT_SOURCE_RESPONSE_KEYS); assert.deepEqual(wireVocabulary.attributeKeys, SNAPSHOT_SOURCE_ATTRIBUTE_KEYS); assert.match(nativeSource, /kProtocolVersion = 1/); - assert.match(nativeSource, /kSourceVersion = @"agent-device-simulator-ax-v1\.6\.0"/); + assert.match(nativeSource, /kSourceVersion = @"agent-device-simulator-ax-v1\.7\.0"/); for (const key of [ ...wireVocabulary.requestKeys, ...wireVocabulary.responseKeys, diff --git a/packages/platform-apple/src/snapshot-source/protocol.ts b/packages/platform-apple/src/snapshot-source/protocol.ts index cb2f887f30..ce663f86ec 100644 --- a/packages/platform-apple/src/snapshot-source/protocol.ts +++ b/packages/platform-apple/src/snapshot-source/protocol.ts @@ -3,7 +3,7 @@ import { snapshotSourceError } from './errors.ts'; import type { SnapshotSourceLimits } from './types.ts'; export const SNAPSHOT_SOURCE_PROTOCOL_VERSION = 1; -export const SNAPSHOT_SOURCE_VERSION = 'agent-device-simulator-ax-v1.6.0'; +export const SNAPSHOT_SOURCE_VERSION = 'agent-device-simulator-ax-v1.7.0'; const FRAME_HEADER_BYTES = 4; export const SNAPSHOT_SOURCE_WIRE_KEYS = Object.freeze([ @@ -45,6 +45,7 @@ export const SNAPSHOT_SOURCE_ATTRIBUTE_KEYS = Object.freeze([ 'XC_kAXXCAttributeFrame', 'XC_kAXXCAttributeAutomationType', 'XC_kAXXCAttributeTraits', + 'XC_kAXXCAttributeIsUserInteractionEnabled', 'XC_kAXXCAttributeChildren', ] as const); diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index 23853eddc0..553c3a4e98 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -429,3 +429,27 @@ test('a window reporting the app box quarter-turned is counted as an unresolved 0, ); }); + +test('the bridge tree publishes whether a dimming view takes touches', () => { + const dimming = (enabled?: unknown) => ({ + [application]: 'UIDimmingView', + [frame]: { X: -390, Y: -844, Width: 1170, Height: 2532 }, + ...(enabled === undefined ? {} : { XC_kAXXCAttributeIsUserInteractionEnabled: enabled }), + [children]: [], + }); + const decode = (enabled?: unknown) => + decodeSnapshotBridgeTree( + { [application]: 'Application', [children]: [dimming(enabled)] }, + { truncated: false }, + limits, + ).nodes[1]; + + assert.equal(decode(true)?.userInteractionEnabled, true); + assert.equal(decode(false)?.userInteractionEnabled, false, 'a sheet at an undimmed detent'); + assert.equal(decode()?.userInteractionEnabled, undefined, 'an unread fact stays unknown'); + assert.throws( + () => decode(1), + (error: unknown) => + error instanceof SnapshotSourceError && error.failureCode === 'user-interaction-invalid', + ); +}); diff --git a/packages/platform-apple/src/snapshot-source/tree.ts b/packages/platform-apple/src/snapshot-source/tree.ts index dd8322f78f..8fe1f6518f 100644 --- a/packages/platform-apple/src/snapshot-source/tree.ts +++ b/packages/platform-apple/src/snapshot-source/tree.ts @@ -20,6 +20,7 @@ const ATTRIBUTE = Object.freeze({ frame: 'XC_kAXXCAttributeFrame', automationType: 'XC_kAXXCAttributeAutomationType', traits: 'XC_kAXXCAttributeTraits', + userInteractionEnabled: 'XC_kAXXCAttributeIsUserInteractionEnabled', children: 'XC_kAXXCAttributeChildren', }); @@ -245,6 +246,7 @@ function nodeFacts( // Publishes `selected: true` only when the selected bit is set and omits it otherwise — the same // shape the XCTest tree produces, so a `selected:` selector cannot tell the producers apart. const selected = traits === undefined || (traits & SELECTED_TRAIT) === 0n ? undefined : true; + const userInteractionEnabled = optionalBoolean(value[ATTRIBUTE.userInteractionEnabled]); return { index, ...(parentIndex === undefined ? {} : { parentIndex }), @@ -265,6 +267,7 @@ function nodeFacts( ...(frame ? { rect: frame } : {}), ...(enabled === undefined ? {} : { enabled }), ...(selected === undefined ? {} : { selected }), + ...(userInteractionEnabled === undefined ? {} : { userInteractionEnabled }), depth, }; } @@ -395,6 +398,14 @@ function traitsFromGuest(value: unknown): bigint | undefined { return BigInt(value); } +function optionalBoolean(value: unknown): boolean | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value !== 'boolean') { + throw snapshotSourceError('malformed-tree', 'user-interaction-invalid'); + } + return value; +} + function optionalInteger(value: unknown): number | undefined { if (value === undefined || value === null) return undefined; if (!Number.isSafeInteger(value)) diff --git a/src/commands/capture/runtime/snapshot-unchanged.ts b/src/commands/capture/runtime/snapshot-unchanged.ts index dab0fe96a8..aa249709ce 100644 --- a/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/src/commands/capture/runtime/snapshot-unchanged.ts @@ -88,6 +88,7 @@ type ComparableSnapshotNode = Omit< | 'selectionStart' | 'selectionEnd' | 'visibleToUser' + | 'userInteractionEnabled' | 'inheritsLabel' | 'inheritsIdentifier' >;