Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions packages/daemon/src/execution/interaction/internal/find.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pip
import { coveredInteractionDetails } from '@agent-device/selectors/interaction-error';
import {
centerOfRect,
type SnapshotAddressableNode,
type SnapshotKeyboardBandFact,
type SnapshotState,
} from '@agent-device/kernel/snapshot';
Expand Down Expand Up @@ -62,7 +63,7 @@ type FindContext = {

type ResolvedMatch = {
node: SnapshotState['nodes'][number];
resolvedNode: SnapshotState['nodes'][number];
resolvedNode: SnapshotAddressableNode;
ref: string;
nodes: SnapshotState['nodes'];
/** The in-place iOS system surface the target capture described (#2438), if any. */
Expand All @@ -75,7 +76,7 @@ type ResolvedMatch = {
* seam surfaces it: click/fill re-enter the interaction leaf, which owns
* that refusal's shape and raises it against the same node.
*/
occludedNode?: SnapshotState['nodes'][number];
occludedNode?: SnapshotAddressableNode;
};

export async function handleFindCommands(params: FindRouteInput): Promise<DaemonResponse | null> {
Expand Down
4 changes: 3 additions & 1 deletion packages/kernel/src/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,8 @@ export type SnapshotNode = RawSnapshotNode & {
inheritsIdentifier?: true;
};

export type SnapshotAddressableNode = SnapshotNode & { ref: string };

/**
* Platform role vocabulary → the text presenter's normalized label. Owned here (rather than
* capture-kit) so `attachRefs` can publish the same `kind` on every node without capture-kit
Expand Down Expand Up @@ -837,7 +839,7 @@ export function attachRefs(
* address (an overlay label, a settle tail entry, a ref pin). A ref-less node is readable and
* searchable, never addressable.
*/
export function hasSnapshotRef(node: SnapshotNode): node is SnapshotNode & { ref: string } {
export function hasSnapshotRef(node: SnapshotNode): node is SnapshotAddressableNode {
return node.ref !== undefined;
}

Expand Down
8 changes: 6 additions & 2 deletions packages/selectors/src/interaction-error.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
import type { Rect, SnapshotCoveringNode, SnapshotNode } from '@agent-device/kernel/snapshot';
import type {
Rect,
SnapshotAddressableNode,
SnapshotCoveringNode,
} from '@agent-device/kernel/snapshot';

/**
* Machine-readable `error.details.reason` values shared by interaction producers and adapters.
Expand Down Expand Up @@ -44,7 +48,7 @@ export const INTERACTION_ERROR_REASONS = {
} as const;

/** The part of a covered node a refusal reports. */
type CoveredNodeFacts = Pick<SnapshotNode, 'ref' | 'interactionBlocked' | 'coveredBy'>;
type CoveredNodeFacts = Pick<SnapshotAddressableNode, 'ref' | 'interactionBlocked' | 'coveredBy'>;

/**
* The whole `error.details` shape of a covered refusal, shared by every producer (the interaction
Expand Down
10 changes: 5 additions & 5 deletions packages/selectors/src/internal/public-resolution-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,21 @@ export type SelectorDiagnostics = {
* The disclosure for an ambiguous selector that was resolved by the heuristic;
* present only when the heuristic picked among N>1 matches (ADR 0012).
*/
export type SelectorDisambiguationDisclosure = {
export type SelectorDisambiguationDisclosure<TNode extends SnapshotNode = SnapshotNode> = {
matchCount: number;
tiebreak: DisambiguationTiebreak;
/** Every losing matched node, document order, uncapped (response layer caps). */
alternatives: SnapshotNode[];
alternatives: TNode[];
};

/** String-only resolution result; the parser representation remains package-private. */
export type SelectorResolution = {
node: SnapshotNode;
export type SelectorResolution<TNode extends SnapshotNode = SnapshotNode> = {
node: TNode;
selector: string;
selectorIndex: number;
matches: number;
diagnostics: SelectorDiagnostics[];
disambiguation?: SelectorDisambiguationDisclosure;
disambiguation?: SelectorDisambiguationDisclosure<TNode>;
};

/**
Expand Down
223 changes: 222 additions & 1 deletion packages/selectors/src/selector-pipeline.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
import assert from 'node:assert/strict';
import { test } from 'vitest';
import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot';
import type {
RawSnapshotNode,
SnapshotAddressableNode,
SnapshotNode,
} from '@agent-device/kernel/snapshot';
import { SELECTOR_RESOLUTION_POLICIES } from '@agent-device/selectors';
import { makeSnapshotState } from './snapshot-geometry.fixtures.ts';
import {
Expand All @@ -14,6 +18,7 @@ import {
} from './interaction-targeting.fixtures.ts';
import {
listSelectorPipelineMatches,
resolveActionSelectorPipeline,
resolveSelectorPipeline,
runNodePipelineStages,
selectorPollBudget,
Expand Down Expand Up @@ -121,6 +126,90 @@ const ROOT_ANCESTOR_TREE: RawSnapshotNode[] = [
},
];

const REFLESS_RECT_TREE: SnapshotNode[] = [
{
index: 0,
depth: 0,
type: 'XCUIElementTypeButton',
label: 'Addressless',
rect: { x: 20, y: 30, width: 100, height: 40 },
hittable: true,
kind: 'button',
},
];

const REFLESS_PROMOTION_TREE: SnapshotNode[] = [
{
index: 0,
depth: 0,
type: 'XCUIElementTypeApplication',
rect: { x: 0, y: 0, width: 390, height: 844 },
hittable: true,
kind: 'application',
ref: 'e1',
},
{
index: 1,
depth: 1,
parentIndex: 0,
type: 'XCUIElementTypeCell',
label: 'Addressless Row',
rect: { x: 20, y: 30, width: 180, height: 50 },
hittable: true,
kind: 'cell',
},
{
index: 2,
depth: 2,
parentIndex: 1,
type: 'XCUIElementTypeStaticText',
label: 'Addressless Row',
rect: { x: 30, y: 40, width: 120, height: 20 },
hittable: false,
kind: 'text',
ref: 'e3',
},
];

const REFLESS_STRUCTURAL_ANCESTOR_TREE: SnapshotNode[] = [
{
index: 0,
depth: 0,
type: 'XCUIElementTypeApplication',
rect: { x: 0, y: 0, width: 390, height: 844 },
kind: 'application',
ref: 'e1',
},
{
index: 1,
depth: 1,
parentIndex: 0,
type: 'XCUIElementTypeOther',
identifier: 'bridged-control',
rect: { x: 20, y: 30, width: 36, height: 36 },
kind: 'other',
ref: 'e2',
},
{
index: 2,
depth: 2,
parentIndex: 1,
type: 'XCUIElementTypeOther',
rect: { x: 20, y: 30, width: 36, height: 36 },
kind: 'other',
},
{
index: 3,
depth: 3,
parentIndex: 2,
type: 'XCUIElementTypeButton',
identifier: 'bridged-control',
rect: { x: 20, y: 30, width: 36, height: 36 },
kind: 'button',
ref: 'e4',
},
];

function nodesOf(raw: RawSnapshotNode[]): SnapshotNode[] {
return makeSnapshotState(raw).nodes;
}
Expand Down Expand Up @@ -455,6 +544,138 @@ test('a listing row cannot be handed the node stages it does not declare', () =>
void [nodeStages, pollBudget];
});

test('rect-requiring pipeline entries narrow outcome nodes to addressable nodes', async () => {
const nodes = nodesOf(PROMOTABLE_TREE);
const staged = await runNodePipelineStages(
SELECTOR_PIPELINE_POLICIES.promotedTarget,
nodes,
nodes[0]!,
{ offscreen: async (node) => node },
);
const stagedAddressable: SnapshotAddressableNode = staged.node;
const stagedRef: string = staged.node.ref;

const resolved = await resolveSelectorPipeline(
SELECTOR_PIPELINE_POLICIES.coveredDiagnosis,
nodes,
'label="Account"',
MATCH,
);
if (resolved.kind === 'target') {
const resolvedAddressable: SnapshotAddressableNode = resolved.node;
const resolvedRef: string = resolved.node.ref;
void [resolvedAddressable, resolvedRef];
}

const listed = listSelectorPipelineMatches(
SELECTOR_PIPELINE_POLICIES.findAct,
nodes,
'label="Account"',
MATCH,
);
const listedAddressable: SnapshotAddressableNode = listed.list!.matchedNodes[0]!;
const listedRef: string = listed.list!.matchedNodes[0]!.ref;

const action = resolveActionSelectorPipeline(
SELECTOR_PIPELINE_POLICIES.promotedTarget,
nodes,
'label="Account"',
MATCH,
);
if (action.kind === 'resolved') {
const actionAddressable: SnapshotAddressableNode = action.resolution.node;
const actionRef: string = action.resolution.node.ref;
void [actionAddressable, actionRef];
}

const read = await runNodePipelineStages(SELECTOR_PIPELINE_POLICIES.readAny, nodes, nodes[0]!);
// @ts-expect-error read rows do not prove that a selectable node is addressable.
const readRef: string = read.node.ref;

const readList = listSelectorPipelineMatches(
SELECTOR_PIPELINE_POLICIES.readList,
nodes,
'label="Account"',
MATCH,
);
// @ts-expect-error listing rows without rect requirements do not prove addressability.
const readListRef: string = readList.list!.matchedNodes[0]!.ref;
void [stagedAddressable, stagedRef, listedAddressable, listedRef, readRef, readListRef];
});

test('rect-requiring pipeline entries do not expose ref-less rect nodes as addressable', async () => {
const listed = listSelectorPipelineMatches(
SELECTOR_PIPELINE_POLICIES.findAct,
REFLESS_RECT_TREE,
'label="Addressless"',
MATCH,
);
assert.equal(listed.list, null);

const resolved = await resolveSelectorPipeline(
SELECTOR_PIPELINE_POLICIES.coveredDiagnosis,
REFLESS_RECT_TREE,
'label="Addressless"',
MATCH,
);
assert.equal(resolved.kind, 'none');

const read = await resolveSelectorPipeline(
SELECTOR_PIPELINE_POLICIES.readAny,
REFLESS_RECT_TREE,
'label="Addressless"',
MATCH,
);
assert.equal(read.kind, 'target');
if (read.kind === 'target') assert.equal(read.node.ref, undefined);

const action = resolveActionSelectorPipeline(
SELECTOR_PIPELINE_POLICIES.promotedTarget,
REFLESS_RECT_TREE,
'label="Addressless"',
MATCH,
);
assert.equal(action.kind, 'none');
});

test('acting selector classification keeps full ancestry after candidate addressability filtering', () => {
const action = resolveActionSelectorPipeline(
SELECTOR_PIPELINE_POLICIES.promotedTarget,
REFLESS_STRUCTURAL_ANCESTOR_TREE,
'id="bridged-control"',
MATCH,
);
assert.equal(action.kind, 'resolved');
if (action.kind !== 'resolved') return;
assert.equal(action.resolution.node.index, 3);
assert.equal(action.resolution.matches, 2);
assert.equal(action.resolution.disambiguation?.tiebreak, 'structural-equivalence');
});

test('rect-requiring node stages fail closed if promotion lands on a ref-less node', async () => {
await assert.rejects(
() =>
runNodePipelineStages(
SELECTOR_PIPELINE_POLICIES.resolvedTarget,
REFLESS_RECT_TREE,
REFLESS_RECT_TREE[0]!,
{ offscreen: async (node) => node },
),
/requires an addressable target/,
);

await assert.rejects(
() =>
runNodePipelineStages(
SELECTOR_PIPELINE_POLICIES.promotedTarget,
REFLESS_PROMOTION_TREE,
REFLESS_PROMOTION_TREE[2]!,
{ offscreen: async (node) => node },
),
/requires an addressable target/,
);
});

test('pipeline rows declare only the stages their kind enforces', () => {
// The #1649 rule, carried to this table: a column nothing consumes is an
// unverifiable claim that reads as truth. A row declares the stages ITS KIND
Expand Down
Loading
Loading