Skip to content

Commit e2a72ae

Browse files
committed
feat(ios): decide the presented surface from a captured tree (#3565)
Add the rule that answers whether an iOS capture read a modal presentation over the app's own content, the golden table that pins it, and the publication that carries its answer, so the rule has a production consumer the moment it lands. The rule reads a dimming view without its enablement, unlike the occlusion cut that shares the same container class: a sheet resting at an undimmed detent keeps a dimming view that takes no touches and is still presented. `absent` is the strong answer and never the default, because a consumer arms a dismissal confirmation on it. It requires that the producer stated the capture was not capped, that every node carries the UIKit class name the rule reads, that each node is numbered once so the parent links the walk follows name the node they mean, and that the one content window under the app root holds exactly one presentation container. Positive evidence — the dimming view a presentation owns, an alert's own root class, or an element type that names a presentation — answers `presented` from a single node, so a capped capture still reports what it read. Producers that publish no class names, which is every XCTest runner tier, answer `presented` or `unknown` and never `absent`. The rule lives in `geometry-policy.ts` beside the occlusion cut rather than in `engine.ts`, because that module already owns the UIKit presentation vocabulary both readers consume and `geometry.ts` already imports it on the engine's eager path, so the placement costs nothing and leaves one module to change when UIKit does. Both readers now match class names through the same normalization, so a producer that spells a class with a module prefix cannot reach one answer in the occlusion cut and another here. Evidence is matched on the node occurrence a walk stood on, not on the number it carries, so a keyboard host's subtree cannot swallow an app window's own dimming view; only `absent`, a claim about every window the producer read, requires numbering that identifies each node once. The publication is in this layer rather than one up because `check:production-exports` refuses an exported rule with no production consumer, and a layer that ships a decision nobody can read is not a layer that passes. The Apple runner's Swift twin and its own consumer land in the next layer, where the runner acquires trees; both languages read the one fixture published here.
1 parent 518f568 commit e2a72ae

9 files changed

Lines changed: 960 additions & 9 deletions

File tree

‎contracts/fixtures/ios-presented-surface.json‎

Lines changed: 343 additions & 0 deletions
Large diffs are not rendered by default.

‎packages/capture-kit/src/ios-snapshot-engine/engine.test.ts‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,76 @@ function acquisition(
397397
};
398398
}
399399

400+
test("the published presented-surface fact is the acquisition's, for every projection", () => {
401+
const requests = [
402+
createIosSnapshotRequest(),
403+
createIosSnapshotRequest({ interactiveOnly: true }),
404+
createIosSnapshotRequest({ raw: true, depth: 1 }),
405+
createIosSnapshotRequest({ scope: 'Automation lab' }),
406+
];
407+
const published = requests.map((request) =>
408+
publishIosSnapshot(acquiredInput(request, presentedTreeNodes()), request),
409+
);
410+
411+
// One screen answers one value: the fact is read on the acquired tree, so a projection that dropped
412+
// the presentation from what it shows still reports the presentation the producer saw.
413+
assert.deepEqual(
414+
published.map((result) => result.presentedSurface),
415+
['presented', 'presented', 'presented', 'presented'],
416+
);
417+
418+
// The depth-1 raw cut is a real cut: the presentation sits at depth 2 and 3, so what that capture
419+
// publishes holds no trace of it. Were the fact read off the published nodes, this projection would
420+
// have to answer `absent` — which is the failure mode the fact exists to close.
421+
const depthCut = published[2]!;
422+
const publishedRoles = (depthCut.payload.nodes ?? []).map((node) => node.role);
423+
assert.ok(
424+
!publishedRoles.includes('UIDimmingView') && !publishedRoles.includes('UITransitionView'),
425+
`expected the depth-1 cut to drop the presentation, saw ${JSON.stringify(publishedRoles)}`,
426+
);
427+
assert.equal(depthCut.presentedSurface, 'presented');
428+
});
429+
430+
test('an acquisition that states no completeness cannot be read as un-presented', () => {
431+
const request = createIosSnapshotRequest();
432+
const unstated: IosSnapshotAcquisition = {
433+
...acquisition(request, ordinaryTreeNodes()),
434+
truncated: undefined,
435+
};
436+
437+
assert.equal(
438+
publishIosSnapshot({ stage: 'acquired', acquisition: unstated }, request).presentedSurface,
439+
'unknown',
440+
);
441+
assert.equal(
442+
publishIosSnapshot(acquiredInput(request, ordinaryTreeNodes()), request).presentedSurface,
443+
'absent',
444+
);
445+
});
446+
447+
test('a runner-declared publication states no presented-surface fact of its own', () => {
448+
const request = createIosSnapshotRequest();
449+
const input: IosSnapshotInput = {
450+
stage: 'presented',
451+
presentation: {
452+
producer: 'apple-runner',
453+
intent: 'full',
454+
payload: { nodes: presentedTreeNodes(), truncated: false },
455+
},
456+
validation: validationFacts(request),
457+
};
458+
459+
// The runner classifies its own pre-presentation acquisition and publishes the reading beside the
460+
// payload; re-deriving one from the presented tree here would be a second, weaker answer. The same tree
461+
// answers at the acquired stage, so this pins the stage gate and not a classifier that reads neither:
462+
// a publication path that stopped classifying at all would fail the second assertion, not this one.
463+
assert.equal(publishIosSnapshot(input, request).presentedSurface, undefined);
464+
assert.equal(
465+
publishIosSnapshot(acquiredInput(request, presentedTreeNodes()), request).presentedSurface,
466+
'presented',
467+
);
468+
});
469+
400470
function acquiredInput(request: IosSnapshotRequest, nodes: RawSnapshotNode[]): IosSnapshotInput {
401471
return { stage: 'acquired', acquisition: acquisition(request, nodes) };
402472
}
@@ -421,6 +491,53 @@ function nestedNodes(): RawSnapshotNode[] {
421491
];
422492
}
423493

494+
/** A UIKit-shaped tree over which a presentation is up: a second container carrying a dimming view. */
495+
function presentedTreeNodes(): RawSnapshotNode[] {
496+
return [
497+
appNode(0, 'UIApplication', 'Application', viewport, undefined),
498+
appNode(1, 'UIWindow', 'Window', viewport, 0),
499+
appNode(2, 'UITransitionView', 'Other', viewport, 1, 2),
500+
node(3, 'StaticText', 'Automation lab', { x: 10, y: 20, width: 200, height: 20 }, 2, 3),
501+
appNode(4, 'UITransitionView', 'Other', viewport, 1, 2),
502+
{
503+
...appNode(5, 'UIDimmingView', 'Other', viewport, 4, 3),
504+
userInteractionEnabled: true,
505+
},
506+
node(6, 'StaticText', 'Automation sheet', { x: 10, y: 120, width: 200, height: 20 }, 4, 3),
507+
];
508+
}
509+
510+
/** A UIKit-shaped tree with nothing presenting: one window, its own single container. */
511+
function ordinaryTreeNodes(): RawSnapshotNode[] {
512+
return [
513+
appNode(0, 'UIApplication', 'Application', viewport, undefined),
514+
appNode(1, 'UIWindow', 'Window', viewport, 0),
515+
appNode(2, 'UITransitionView', 'Other', viewport, 1),
516+
appNode(
517+
3,
518+
'RCTParagraphComponentView',
519+
'StaticText',
520+
{ x: 10, y: 20, width: 200, height: 20 },
521+
2,
522+
),
523+
];
524+
}
525+
526+
/** A node shaped like the simulator AX bridge reports one: UIKit class name in `role`. */
527+
function appNode(
528+
index: number,
529+
role: string,
530+
type: string,
531+
rect: Rect,
532+
parentIndex: number | undefined,
533+
depth?: number,
534+
): RawSnapshotNode {
535+
return {
536+
...node(index, type, undefined, rect, parentIndex, depth),
537+
role,
538+
};
539+
}
540+
424541
function scopedNodes(): RawSnapshotNode[] {
425542
return [
426543
node(0, 'Application', 'App', viewport),

‎packages/capture-kit/src/ios-snapshot-engine/engine.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'
1313
import { buildIosInteractiveSnapshotPresentation } from './semantic-index.ts';
1414
import { validateIosSnapshotGraph } from './graph.ts';
1515
import { foldIosSnapshot } from './geometry.ts';
16+
import { collectPresentedSurface, type PresentedSurfaceCompleteness } from './geometry-policy.ts';
1617
import { resolveIosViewport, validateIosPayload } from './invariants.ts';
1718
import { projectIosQualitySnapshot, projectIosSnapshot } from './projection.ts';
1819
import { presentIosRunnerSnapshot } from './runner-presentation.ts';
@@ -49,9 +50,29 @@ export function publishIosSnapshot(
4950
...(presentation.validatedViewport
5051
? { validatedViewport: presentation.validatedViewport }
5152
: {}),
53+
...(input.stage === 'acquired'
54+
? {
55+
presentedSurface: collectPresentedSurface(
56+
input.acquisition.nodes,
57+
acquisitionCompleteness(input.acquisition),
58+
),
59+
}
60+
: {}),
5261
};
5362
}
5463

64+
/**
65+
* What an acquisition claims about its own completeness, from the producer's truncation statement and
66+
* never from the tree: a producer that reports nothing about truncation has stated nothing, and a
67+
* presented-surface classification may not read a silent field as "complete" (#2188 invariant 5).
68+
*/
69+
function acquisitionCompleteness(
70+
acquisition: IosSnapshotAcquisition,
71+
): PresentedSurfaceCompleteness {
72+
if (acquisition.truncated === undefined) return 'unstated';
73+
return acquisition.truncated ? 'capped' : 'complete';
74+
}
75+
5576
export function presentIosSnapshot(
5677
input: IosSnapshotInput,
5778
request: IosSnapshotRequest,
Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
import fs from 'node:fs';
2+
import path from 'node:path';
3+
import { expect, test } from 'vitest';
4+
import type { PresentedSurface, RawSnapshotNode } from '@agent-device/kernel/snapshot';
5+
import {
6+
ALERT_ROOT_CLASS_PREFIX,
7+
PRESENTATION_CONTAINER_CLASS,
8+
PRESENTATION_DIMMING_VIEW_CLASS,
9+
PRESENTING_ELEMENT_TYPES,
10+
SYSTEM_WINDOW_CLASSES,
11+
collectPresentedSurface,
12+
type PresentedSurfaceCompleteness,
13+
} from './geometry-policy.ts';
14+
15+
const FIXTURE_PATH = path.resolve(
16+
import.meta.dirname,
17+
'..',
18+
'..',
19+
'..',
20+
'..',
21+
'contracts',
22+
'fixtures',
23+
'ios-presented-surface.json',
24+
);
25+
26+
type FixtureNode = {
27+
index: number;
28+
parentIndex?: number;
29+
type: string;
30+
role?: string;
31+
label?: string;
32+
userInteractionEnabled?: boolean;
33+
};
34+
35+
type FixtureCase = {
36+
name: string;
37+
truncated: boolean | null;
38+
expected: PresentedSurface;
39+
classBlindExpected: PresentedSurface;
40+
nodes: FixtureNode[];
41+
};
42+
43+
type Fixture = {
44+
vocabulary: {
45+
presentationContainerClass: string;
46+
dimmingViewClass: string;
47+
alertRootClassPrefix: string;
48+
systemWindowClasses: string[];
49+
presentingElementTypes: Array<{ name: string; elementType: number }>;
50+
};
51+
cases: FixtureCase[];
52+
};
53+
54+
function readFixture(): Fixture {
55+
return JSON.parse(fs.readFileSync(FIXTURE_PATH, 'utf8')) as Fixture;
56+
}
57+
58+
// Cross-language parity: the classifier must mirror the canonical table the Swift
59+
// SnapshotPresentedSurface also reads, and the vocabulary it mirrors by hand. A drift on either side
60+
// fails here or in the Swift test.
61+
test('the classifier mirrors the canonical vocabulary exactly', () => {
62+
const vocabulary = readFixture().vocabulary;
63+
expect(vocabulary.presentationContainerClass).toBe(PRESENTATION_CONTAINER_CLASS);
64+
expect(vocabulary.dimmingViewClass).toBe(PRESENTATION_DIMMING_VIEW_CLASS);
65+
expect(vocabulary.alertRootClassPrefix).toBe(ALERT_ROOT_CLASS_PREFIX);
66+
expect(new Set(vocabulary.systemWindowClasses)).toEqual(SYSTEM_WINDOW_CLASSES);
67+
expect(Object.fromEntries(PRESENTING_ELEMENT_TYPES)).toEqual(
68+
Object.fromEntries(
69+
vocabulary.presentingElementTypes.map((entry) => [entry.name, entry.elementType]),
70+
),
71+
);
72+
});
73+
74+
// Every case runs twice: once with the UIKit class names the table spells, once with every class name
75+
// stripped, which is what a capture from a producer that reports none — every XCTest runner tier —
76+
// actually looks like. That reading is what `classBlindExpected` states, and it is why the table can
77+
// hold a case a bridge capture answers `presented` and a runner capture answers `unknown`.
78+
test('every golden case classifies as the table states', () => {
79+
for (const testCase of readFixture().cases) {
80+
const completeness = completenessOf(testCase);
81+
expect(collectPresentedSurface(nodesOf(testCase), completeness)).toBe(testCase.expected);
82+
expect(collectPresentedSurface(classBlindNodesOf(testCase), completeness)).toBe(
83+
testCase.classBlindExpected,
84+
);
85+
}
86+
});
87+
88+
/**
89+
* The producer's own completeness claim, which the table states as `truncated`: a null is a producer
90+
* that observes nothing about truncation, and cannot be read as complete.
91+
*/
92+
function completenessOf(testCase: FixtureCase): PresentedSurfaceCompleteness {
93+
if (testCase.truncated === null) return 'unstated';
94+
return testCase.truncated ? 'capped' : 'complete';
95+
}
96+
97+
function nodesOf(testCase: FixtureCase): RawSnapshotNode[] {
98+
return testCase.nodes.map((node) => ({ ...node }));
99+
}
100+
101+
function classBlindNodesOf(testCase: FixtureCase): RawSnapshotNode[] {
102+
return testCase.nodes.map(({ role: _role, ...node }) => ({ ...node }));
103+
}

0 commit comments

Comments
 (0)