From f7a36b4413d52fdea2a57051b8a508fdb1e4ff42 Mon Sep 17 00:00:00 2001 From: Huangshuo Kuang <141250392+kkkhs@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:58:41 +0000 Subject: [PATCH] fix(selectors): type rect-requiring targets as addressable --- .../execution/interaction/internal/find.ts | 5 +- packages/kernel/src/snapshot.ts | 4 +- packages/selectors/src/interaction-error.ts | 8 +- .../src/internal/public-resolution-types.ts | 10 +- .../selectors/src/selector-pipeline.test.ts | 223 +++++++++++++++++- packages/selectors/src/selector-pipeline.ts | 167 ++++++++++--- .../interaction/runtime/keyboard-occlusion.ts | 8 +- .../runtime/ref-target-resolution.ts | 17 +- .../runtime/resolution-disclosure.test.ts | 2 + .../runtime/resolution-disclosure.ts | 21 +- .../interaction/runtime/resolution.ts | 4 +- .../runtime/selector-action-resolution.ts | 51 +--- .../interaction/runtime/selector-readiness.ts | 12 +- .../runtime/target-visibility-stages.ts | 13 +- 14 files changed, 431 insertions(+), 114 deletions(-) diff --git a/packages/daemon/src/execution/interaction/internal/find.ts b/packages/daemon/src/execution/interaction/internal/find.ts index 62be98f335..3c9546e043 100644 --- a/packages/daemon/src/execution/interaction/internal/find.ts +++ b/packages/daemon/src/execution/interaction/internal/find.ts @@ -10,6 +10,7 @@ import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pip import { coveredInteractionDetails } from '@agent-device/selectors/interaction-error'; import { centerOfRect, + type SnapshotAddressableNode, type SnapshotKeyboardBandFact, type SnapshotState, } from '@agent-device/kernel/snapshot'; @@ -62,7 +63,7 @@ type FindContext = { type ResolvedMatch = { node: SnapshotState['nodes'][number]; - resolvedNode: SnapshotState['nodes'][number]; + resolvedNode: SnapshotAddressableNode; ref: string; nodes: SnapshotState['nodes']; /** The in-place iOS system surface the target capture described (#2438), if any. */ @@ -75,7 +76,7 @@ type ResolvedMatch = { * seam surfaces it: click/fill re-enter the interaction leaf, which owns * that refusal's shape and raises it against the same node. */ - occludedNode?: SnapshotState['nodes'][number]; + occludedNode?: SnapshotAddressableNode; }; export async function handleFindCommands(params: FindRouteInput): Promise { diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index ce00c6c59f..1d35a7403e 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -447,6 +447,8 @@ export type SnapshotNode = RawSnapshotNode & { inheritsIdentifier?: true; }; +export type SnapshotAddressableNode = SnapshotNode & { ref: string }; + /** * Platform role vocabulary → the text presenter's normalized label. Owned here (rather than * capture-kit) so `attachRefs` can publish the same `kind` on every node without capture-kit @@ -837,7 +839,7 @@ export function attachRefs( * address (an overlay label, a settle tail entry, a ref pin). A ref-less node is readable and * searchable, never addressable. */ -export function hasSnapshotRef(node: SnapshotNode): node is SnapshotNode & { ref: string } { +export function hasSnapshotRef(node: SnapshotNode): node is SnapshotAddressableNode { return node.ref !== undefined; } diff --git a/packages/selectors/src/interaction-error.ts b/packages/selectors/src/interaction-error.ts index d02455a144..06fa0ab672 100644 --- a/packages/selectors/src/interaction-error.ts +++ b/packages/selectors/src/interaction-error.ts @@ -1,4 +1,8 @@ -import type { Rect, SnapshotCoveringNode, SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { + Rect, + SnapshotAddressableNode, + SnapshotCoveringNode, +} from '@agent-device/kernel/snapshot'; /** * Machine-readable `error.details.reason` values shared by interaction producers and adapters. @@ -44,7 +48,7 @@ export const INTERACTION_ERROR_REASONS = { } as const; /** The part of a covered node a refusal reports. */ -type CoveredNodeFacts = Pick; +type CoveredNodeFacts = Pick; /** * The whole `error.details` shape of a covered refusal, shared by every producer (the interaction diff --git a/packages/selectors/src/internal/public-resolution-types.ts b/packages/selectors/src/internal/public-resolution-types.ts index c7a365ca04..aed590406b 100644 --- a/packages/selectors/src/internal/public-resolution-types.ts +++ b/packages/selectors/src/internal/public-resolution-types.ts @@ -12,21 +12,21 @@ export type SelectorDiagnostics = { * The disclosure for an ambiguous selector that was resolved by the heuristic; * present only when the heuristic picked among N>1 matches (ADR 0012). */ -export type SelectorDisambiguationDisclosure = { +export type SelectorDisambiguationDisclosure = { matchCount: number; tiebreak: DisambiguationTiebreak; /** Every losing matched node, document order, uncapped (response layer caps). */ - alternatives: SnapshotNode[]; + alternatives: TNode[]; }; /** String-only resolution result; the parser representation remains package-private. */ -export type SelectorResolution = { - node: SnapshotNode; +export type SelectorResolution = { + node: TNode; selector: string; selectorIndex: number; matches: number; diagnostics: SelectorDiagnostics[]; - disambiguation?: SelectorDisambiguationDisclosure; + disambiguation?: SelectorDisambiguationDisclosure; }; /** diff --git a/packages/selectors/src/selector-pipeline.test.ts b/packages/selectors/src/selector-pipeline.test.ts index cd7ac88f5b..12de4bdf63 100644 --- a/packages/selectors/src/selector-pipeline.test.ts +++ b/packages/selectors/src/selector-pipeline.test.ts @@ -1,6 +1,10 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { + RawSnapshotNode, + SnapshotAddressableNode, + SnapshotNode, +} from '@agent-device/kernel/snapshot'; import { SELECTOR_RESOLUTION_POLICIES } from '@agent-device/selectors'; import { makeSnapshotState } from './snapshot-geometry.fixtures.ts'; import { @@ -14,6 +18,7 @@ import { } from './interaction-targeting.fixtures.ts'; import { listSelectorPipelineMatches, + resolveActionSelectorPipeline, resolveSelectorPipeline, runNodePipelineStages, selectorPollBudget, @@ -121,6 +126,90 @@ const ROOT_ANCESTOR_TREE: RawSnapshotNode[] = [ }, ]; +const REFLESS_RECT_TREE: SnapshotNode[] = [ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeButton', + label: 'Addressless', + rect: { x: 20, y: 30, width: 100, height: 40 }, + hittable: true, + kind: 'button', + }, +]; + +const REFLESS_PROMOTION_TREE: SnapshotNode[] = [ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: true, + kind: 'application', + ref: 'e1', + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeCell', + label: 'Addressless Row', + rect: { x: 20, y: 30, width: 180, height: 50 }, + hittable: true, + kind: 'cell', + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'XCUIElementTypeStaticText', + label: 'Addressless Row', + rect: { x: 30, y: 40, width: 120, height: 20 }, + hittable: false, + kind: 'text', + ref: 'e3', + }, +]; + +const REFLESS_STRUCTURAL_ANCESTOR_TREE: SnapshotNode[] = [ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + kind: 'application', + ref: 'e1', + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeOther', + identifier: 'bridged-control', + rect: { x: 20, y: 30, width: 36, height: 36 }, + kind: 'other', + ref: 'e2', + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'XCUIElementTypeOther', + rect: { x: 20, y: 30, width: 36, height: 36 }, + kind: 'other', + }, + { + index: 3, + depth: 3, + parentIndex: 2, + type: 'XCUIElementTypeButton', + identifier: 'bridged-control', + rect: { x: 20, y: 30, width: 36, height: 36 }, + kind: 'button', + ref: 'e4', + }, +]; + function nodesOf(raw: RawSnapshotNode[]): SnapshotNode[] { return makeSnapshotState(raw).nodes; } @@ -455,6 +544,138 @@ test('a listing row cannot be handed the node stages it does not declare', () => void [nodeStages, pollBudget]; }); +test('rect-requiring pipeline entries narrow outcome nodes to addressable nodes', async () => { + const nodes = nodesOf(PROMOTABLE_TREE); + const staged = await runNodePipelineStages( + SELECTOR_PIPELINE_POLICIES.promotedTarget, + nodes, + nodes[0]!, + { offscreen: async (node) => node }, + ); + const stagedAddressable: SnapshotAddressableNode = staged.node; + const stagedRef: string = staged.node.ref; + + const resolved = await resolveSelectorPipeline( + SELECTOR_PIPELINE_POLICIES.coveredDiagnosis, + nodes, + 'label="Account"', + MATCH, + ); + if (resolved.kind === 'target') { + const resolvedAddressable: SnapshotAddressableNode = resolved.node; + const resolvedRef: string = resolved.node.ref; + void [resolvedAddressable, resolvedRef]; + } + + const listed = listSelectorPipelineMatches( + SELECTOR_PIPELINE_POLICIES.findAct, + nodes, + 'label="Account"', + MATCH, + ); + const listedAddressable: SnapshotAddressableNode = listed.list!.matchedNodes[0]!; + const listedRef: string = listed.list!.matchedNodes[0]!.ref; + + const action = resolveActionSelectorPipeline( + SELECTOR_PIPELINE_POLICIES.promotedTarget, + nodes, + 'label="Account"', + MATCH, + ); + if (action.kind === 'resolved') { + const actionAddressable: SnapshotAddressableNode = action.resolution.node; + const actionRef: string = action.resolution.node.ref; + void [actionAddressable, actionRef]; + } + + const read = await runNodePipelineStages(SELECTOR_PIPELINE_POLICIES.readAny, nodes, nodes[0]!); + // @ts-expect-error read rows do not prove that a selectable node is addressable. + const readRef: string = read.node.ref; + + const readList = listSelectorPipelineMatches( + SELECTOR_PIPELINE_POLICIES.readList, + nodes, + 'label="Account"', + MATCH, + ); + // @ts-expect-error listing rows without rect requirements do not prove addressability. + const readListRef: string = readList.list!.matchedNodes[0]!.ref; + void [stagedAddressable, stagedRef, listedAddressable, listedRef, readRef, readListRef]; +}); + +test('rect-requiring pipeline entries do not expose ref-less rect nodes as addressable', async () => { + const listed = listSelectorPipelineMatches( + SELECTOR_PIPELINE_POLICIES.findAct, + REFLESS_RECT_TREE, + 'label="Addressless"', + MATCH, + ); + assert.equal(listed.list, null); + + const resolved = await resolveSelectorPipeline( + SELECTOR_PIPELINE_POLICIES.coveredDiagnosis, + REFLESS_RECT_TREE, + 'label="Addressless"', + MATCH, + ); + assert.equal(resolved.kind, 'none'); + + const read = await resolveSelectorPipeline( + SELECTOR_PIPELINE_POLICIES.readAny, + REFLESS_RECT_TREE, + 'label="Addressless"', + MATCH, + ); + assert.equal(read.kind, 'target'); + if (read.kind === 'target') assert.equal(read.node.ref, undefined); + + const action = resolveActionSelectorPipeline( + SELECTOR_PIPELINE_POLICIES.promotedTarget, + REFLESS_RECT_TREE, + 'label="Addressless"', + MATCH, + ); + assert.equal(action.kind, 'none'); +}); + +test('acting selector classification keeps full ancestry after candidate addressability filtering', () => { + const action = resolveActionSelectorPipeline( + SELECTOR_PIPELINE_POLICIES.promotedTarget, + REFLESS_STRUCTURAL_ANCESTOR_TREE, + 'id="bridged-control"', + MATCH, + ); + assert.equal(action.kind, 'resolved'); + if (action.kind !== 'resolved') return; + assert.equal(action.resolution.node.index, 3); + assert.equal(action.resolution.matches, 2); + assert.equal(action.resolution.disambiguation?.tiebreak, 'structural-equivalence'); +}); + +test('rect-requiring node stages fail closed if promotion lands on a ref-less node', async () => { + await assert.rejects( + () => + runNodePipelineStages( + SELECTOR_PIPELINE_POLICIES.resolvedTarget, + REFLESS_RECT_TREE, + REFLESS_RECT_TREE[0]!, + { offscreen: async (node) => node }, + ), + /requires an addressable target/, + ); + + await assert.rejects( + () => + runNodePipelineStages( + SELECTOR_PIPELINE_POLICIES.promotedTarget, + REFLESS_PROMOTION_TREE, + REFLESS_PROMOTION_TREE[2]!, + { offscreen: async (node) => node }, + ), + /requires an addressable target/, + ); +}); + test('pipeline rows declare only the stages their kind enforces', () => { // The #1649 rule, carried to this table: a column nothing consumes is an // unverifiable claim that reads as truth. A row declares the stages ITS KIND diff --git a/packages/selectors/src/selector-pipeline.ts b/packages/selectors/src/selector-pipeline.ts index 1309af9e28..98fd653697 100644 --- a/packages/selectors/src/selector-pipeline.ts +++ b/packages/selectors/src/selector-pipeline.ts @@ -1,16 +1,26 @@ -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; -import type { SelectorChainMatchList, SelectorMatchOptions } from '@agent-device/selectors'; +import { + hasSnapshotRef, + type SnapshotAddressableNode, + type SnapshotNode, +} from '@agent-device/kernel/snapshot'; +import type { + SelectorChainMatchList, + SelectorMatchOptions, + SelectorResolution, +} from '@agent-device/selectors'; import { listSelectorChainMatches, resolveSelectorChainWithPolicy, } from '@agent-device/selectors/engine'; import { isSnapshotNodeInteractionBlocked } from '@agent-device/capture-kit/snapshot-occlusion'; import { + classifyActionableTouchCandidates, isRootInteractionContainer, resolveActionableTouchResolution, resolveElementReportedTwice, } from './interaction-targeting.ts'; import type { + ActingPipelinePolicy, CandidateSetPipelinePolicy, SelectorListPolicy, SelectorPipelinePolicy, @@ -45,6 +55,9 @@ import type { * A row whose RESULT is the candidate set (`find list`) declares only the * two stages a listing can run, so the node-stage entries below reject it at * the type level rather than skipping stages it claimed. + * - `resolveActionSelectorPipeline` — acting rows whose candidate set may + * collapse through actionable equivalence; the collapse stays inside this + * owner so addressability is checked at the same exit as every other row. * - `runNodePipelineStages` — the node stages for a target that came from a * non-chain matcher (`@ref` lookup, find's fuzzy locator) or from a narrowed * candidate set. Not a bypass: it runs every stage the row declares. @@ -66,18 +79,24 @@ export type SelectorPipelineHooks = { offscreen?: (node: SnapshotNode, nodes: SnapshotNode[]) => Promise; }; -export type SelectorPipelineTarget = - | { kind: 'target'; node: SnapshotNode } - | { kind: 'occluded'; node: SnapshotNode }; +export type SelectorPipelineNodeForPolicy = TPolicy extends { + resolution: { requireRect: true }; +} + ? SnapshotAddressableNode + : SnapshotNode; -export type SelectorPipelineOutcome = +export type SelectorPipelineTarget = + | { kind: 'target'; node: TNode } + | { kind: 'occluded'; node: TNode }; + +export type SelectorPipelineOutcome = | { kind: 'none' } | { kind: 'ambiguous'; selector: string; selectorIndex: number; matchedNodes: SnapshotNode[] } /** `selector` is the ALTERNATIVE that matched, which is what a refusal names. */ - | { kind: 'occluded'; node: SnapshotNode; selector: string } + | { kind: 'occluded'; node: TNode; selector: string } | { kind: 'target'; - node: SnapshotNode; + node: TNode; selector: string; selectorIndex: number; matches: number; @@ -85,13 +104,13 @@ export type SelectorPipelineOutcome = }; /** Rows that resolve one target: candidacy, ambiguity, then every node stage. */ -export async function resolveSelectorPipeline( - policy: SingleTargetPipelinePolicy, +export async function resolveSelectorPipeline( + policy: TPolicy, nodes: SnapshotNode[], selector: string, match: SelectorMatchOptions, hooks: SelectorPipelineHooks = {}, -): Promise { +): Promise>> { const outcome = resolveSelectorChainWithPolicy( pipelineCandidates(policy, nodes), selector, @@ -164,12 +183,12 @@ function resolveEquivalentControlTarget( } /** Every node stage this row declares, applied to a resolved target. */ -async function stageResolvedRow( - policy: SingleTargetPipelinePolicy, +async function stageResolvedRow( + policy: TPolicy, nodes: SnapshotNode[], resolved: ResolvedRowTarget, hooks: SelectorPipelineHooks, -): Promise { +): Promise>> { const staged = await runNodePipelineStages(policy, nodes, resolved.node, hooks); if (staged.kind === 'occluded') { return { kind: 'occluded', node: staged.node, selector: resolved.selector }; @@ -182,53 +201,133 @@ async function stageResolvedRow( * first matching alternative and must narrow or refuse it, then bring the node * it chose back through `runNodePipelineStages`. * - * `candidates` is the tree as this row sees it — the candidacy stage's output. - * A caller that reasons about the surroundings of a match (find's ranking, the - * acting equivalence classification) must reason about THAT tree, or an - * overlay-covered node would come back as a competitor the row already excluded. + * `candidates` is the node set as this row lets selectors see it. Acting + * equivalence still receives the full tree for structural ancestry, then + * brings the winning node back through the checked pipeline exit. */ -export type SelectorPipelineMatches = { +export type SelectorPipelineMatches = { candidates: SnapshotNode[]; - list: SelectorChainMatchList | null; + list: (Omit & { matchedNodes: TNode[] }) | null; }; -export function listSelectorPipelineMatches( - policy: CandidateSetPipelinePolicy, +export type SelectorActionPipelineOutcome = + | { kind: 'none' } + | { kind: 'ambiguous'; selector: string; candidates: SnapshotNode[] } + | { kind: 'resolved'; resolution: SelectorResolution }; + +export function listSelectorPipelineMatches( + policy: TPolicy, nodes: SnapshotNode[], selector: string, match: SelectorMatchOptions, -): SelectorPipelineMatches { +): SelectorPipelineMatches> { const candidates = pipelineCandidates(policy, nodes); + const list = listSelectorChainMatches(candidates, selector, { + ...match, + requireRect: policy.resolution.requireRect, + }); return { candidates, - list: listSelectorChainMatches(candidates, selector, { - ...match, - requireRect: policy.resolution.requireRect, - }), + list: pipelineMatchListForPolicy(policy, list), + }; +} + +export function resolveActionSelectorPipeline( + policy: TPolicy, + nodes: SnapshotNode[], + selectorExpression: string, + match: SelectorMatchOptions, +): SelectorActionPipelineOutcome> { + const { list } = listSelectorPipelineMatches(policy, nodes, selectorExpression, match); + const matched = list?.matchedNodes ?? []; + if (!list || matched.length === 0) return { kind: 'none' }; + + const diagnostics = [{ selector: list.selector, matches: matched.length }]; + if (matched.length === 1) { + return { + kind: 'resolved', + resolution: { + node: matched[0]!, + selector: list.selector, + selectorIndex: list.selectorIndex, + matches: 1, + diagnostics, + }, + }; + } + + const classification = classifyActionableTouchCandidates(nodes, matched); + if (classification.kind === 'ambiguous') { + return { kind: 'ambiguous', selector: list.selector, candidates: classification.candidates }; + } + + return { + kind: 'resolved', + resolution: { + node: nodeForPolicy(policy, classification.node), + selector: list.selector, + selectorIndex: list.selectorIndex, + matches: matched.length, + diagnostics, + disambiguation: { + matchCount: matched.length, + tiebreak: 'structural-equivalence', + alternatives: matched.filter((candidate) => candidate.index !== classification.node.index), + }, + }, }; } /** Every node stage this row declares, in the order acting routes need them. */ -export async function runNodePipelineStages( - policy: SelectorPipelinePolicy, +export async function runNodePipelineStages( + policy: TPolicy, nodes: SnapshotNode[], node: SnapshotNode, hooks: SelectorPipelineHooks = {}, -): Promise { +): Promise>> { hooks.onResolved?.(node, nodes); const promoted = promoteTarget(policy.promotion, nodes, node); if (policy.occlusion !== 'ignore') { if (promoted.covered || isSnapshotNodeInteractionBlocked(promoted.node)) { - return { kind: 'occluded', node: promoted.node }; + return { kind: 'occluded', node: nodeForPolicy(policy, promoted.node) }; } } - return { kind: 'target', node: await guardOffscreen(policy, nodes, promoted.node, hooks) }; + return { + kind: 'target', + node: nodeForPolicy(policy, await guardOffscreen(policy, nodes, promoted.node, hooks)), + }; +} + +function nodeForPolicy( + policy: TPolicy, + node: SnapshotNode, +): SelectorPipelineNodeForPolicy { + if (policy.resolution.requireRect && !hasSnapshotRef(node)) { + throw new Error( + 'selector pipeline row requires an addressable target, but resolved a ref-less node', + ); + } + return node as SelectorPipelineNodeForPolicy; +} + +function pipelineMatchListForPolicy( + policy: TPolicy, + list: SelectorChainMatchList | null, +): SelectorPipelineMatches>['list'] { + if (!list) return null; + return { + ...list, + matchedNodes: list.matchedNodes.map((node) => nodeForPolicy(policy, node)), + }; } /** The nodes this row lets a selector match against. */ function pipelineCandidates(policy: SelectorListPolicy, nodes: SnapshotNode[]): SnapshotNode[] { - if (policy.occlusion !== 'exclude-and-refuse') return nodes; - return nodes.filter((candidate) => !isSnapshotNodeInteractionBlocked(candidate)); + const addressableCandidates = policy.resolution.requireRect + ? nodes.filter(hasSnapshotRef) + : nodes; + if (policy.occlusion !== 'exclude-and-refuse') return addressableCandidates; + return addressableCandidates.filter((candidate) => !isSnapshotNodeInteractionBlocked(candidate)); } /** diff --git a/src/commands/interaction/runtime/keyboard-occlusion.ts b/src/commands/interaction/runtime/keyboard-occlusion.ts index d9d1599297..bd45fa0bda 100644 --- a/src/commands/interaction/runtime/keyboard-occlusion.ts +++ b/src/commands/interaction/runtime/keyboard-occlusion.ts @@ -1,6 +1,7 @@ import type { Point, Rect, + SnapshotAddressableNode, SnapshotKeyboardBandFact, SnapshotNode, SnapshotState, @@ -33,7 +34,7 @@ import type { InteractionAction } from './interaction-resolution-request.ts'; */ export function assertTapTargetClearOfVisibleKeyboard(params: { nodes: SnapshotState['nodes']; - node: SnapshotNode; + node: SnapshotAddressableNode; action: InteractionAction; /** How the caller named the target, e.g. `Ref @e40` or `Selector text=Form`. */ label: string; @@ -106,8 +107,7 @@ function keyboardOcclusionWarning(point: Point, surface: KeyboardSurface): strin function buildKeyboardOcclusionError(params: { label: string; action: InteractionAction; - /** The ref the target was named by; omitted with the detail when nothing addresses the node. */ - ref: string | undefined; + ref: string; targetRect: NonNullable; surface: KeyboardSurface; }): AppError { @@ -117,7 +117,7 @@ function buildKeyboardOcclusionError(params: { `${params.label} is behind the visible keyboard and cannot ${interactionVerb(params.action)} safely`, { ...TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS, - ...(params.ref === undefined ? {} : { ref: `@${params.ref}` }), + ref: `@${params.ref}`, rect: params.targetRect, keyboardFrame: params.surface.frame, }, diff --git a/src/commands/interaction/runtime/ref-target-resolution.ts b/src/commands/interaction/runtime/ref-target-resolution.ts index 5c08d51696..a44c56362c 100644 --- a/src/commands/interaction/runtime/ref-target-resolution.ts +++ b/src/commands/interaction/runtime/ref-target-resolution.ts @@ -1,10 +1,11 @@ import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; import type { + SnapshotAddressableNode, SnapshotKeyboardBandFact, SnapshotNode, SnapshotState, } from '@agent-device/kernel/snapshot'; -import { findNodeByRef, normalizeRef } from '@agent-device/kernel/snapshot'; +import { findNodeByRef, hasSnapshotRef, normalizeRef } from '@agent-device/kernel/snapshot'; import { resolveRectCenter } from '@agent-device/kernel/rect-center'; import type { AgentDeviceRuntime, @@ -68,8 +69,12 @@ function adoptPreresolvedRefTarget( const ref = normalizeRef(target.ref); if (!ref) throw new AppError('INVALID_ARGS', `Invalid ref: ${target.ref}`); const carriedRef = normalizeRef(preresolved.ref); - const nodeRef = preresolved.node.ref ? normalizeRef(preresolved.node.ref) : null; - if (carriedRef !== ref || nodeRef !== ref || !preresolved.nodes.includes(preresolved.node)) { + if ( + carriedRef !== ref || + !hasSnapshotRef(preresolved.node) || + normalizeRef(preresolved.node.ref) !== ref || + !preresolved.nodes.includes(preresolved.node) + ) { throw new AppError( 'COMMAND_FAILED', 'Internal find target provenance does not match the interaction ref', @@ -276,7 +281,9 @@ function refMissRefusal( return discloseDispatch(refusal, 'no'); } -function isUsableResolvedNode(node: SnapshotNode | null | undefined): node is SnapshotNode { +function isUsableResolvedNode( + node: SnapshotNode | null | undefined, +): node is SnapshotAddressableNode { if (!node) return false; - return resolveRectCenter(node.rect) !== null; + return hasSnapshotRef(node) && resolveRectCenter(node.rect) !== null; } diff --git a/src/commands/interaction/runtime/resolution-disclosure.test.ts b/src/commands/interaction/runtime/resolution-disclosure.test.ts index bb3daff2d7..8658afc27a 100644 --- a/src/commands/interaction/runtime/resolution-disclosure.test.ts +++ b/src/commands/interaction/runtime/resolution-disclosure.test.ts @@ -1,10 +1,12 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import { hasSnapshotRef } from '@agent-device/kernel/snapshot'; import { buildRefResolution } from './resolution-disclosure.ts'; import { selectorSnapshot } from './__tests__/test-utils/index.ts'; test('buildRefResolution is the shared exact and label-fallback disclosure constructor', () => { const node = selectorSnapshot().nodes[0]!; + assert.ok(hasSnapshotRef(node)); assert.deepEqual(buildRefResolution('e1', node, 'exact').resolution, { source: 'ref', diff --git a/src/commands/interaction/runtime/resolution-disclosure.ts b/src/commands/interaction/runtime/resolution-disclosure.ts index bcebe03b0b..6789851e5c 100644 --- a/src/commands/interaction/runtime/resolution-disclosure.ts +++ b/src/commands/interaction/runtime/resolution-disclosure.ts @@ -1,10 +1,15 @@ -import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; +import type { + SnapshotAddressableNode, + SnapshotNode, + SnapshotState, +} from '@agent-device/kernel/snapshot'; import type { AgentDeviceRuntime } from '@agent-device/contracts/runtime-contract'; -import { type SelectorResolution, buildSelectorChainForNode } from '@agent-device/selectors'; +import { buildSelectorChainForNode } from '@agent-device/selectors'; import { resolvePressRecordingTarget } from '@agent-device/selectors/press-retarget'; import { resolveRefLabel } from '@agent-device/capture-kit/snapshot-node-lookup'; import { normalizeType } from '@agent-device/contracts/snapshot'; import { truncateUtf8 } from './truncate-utf8.ts'; +import type { ActionSelectorResolution } from './selector-action-resolution.ts'; import type { RecordingTargetOverride, ResolutionDiagnosticEntry, @@ -15,7 +20,7 @@ import type { InteractionAction } from './interaction-resolution-request.ts'; export type ResolvedRefNode = { ref: string; - node: SnapshotNode; + node: SnapshotAddressableNode; resolution: ResolutionDisclosure; }; @@ -42,7 +47,7 @@ const LABEL_FALLBACK_REF_RESOLUTION: ResolutionDisclosure = { /** Shared construction site for every runtime-ref resolution disclosure. */ export function buildRefResolution( ref: string, - node: SnapshotNode, + node: SnapshotAddressableNode, kind: 'exact' | 'label-fallback', ): ResolvedRefNode { return { @@ -60,7 +65,7 @@ const UNIQUE_RUNTIME_RESOLUTION: ResolutionDisclosure = { // Disclosure only: the winner stays resolveSelectorChain's pick (ADR 0012). export function buildSelectorResolutionDisclosure( - resolved: SelectorResolution, + resolved: ActionSelectorResolution, nodes: SnapshotState['nodes'], ): ResolutionDisclosure { if (!resolved.disambiguation) return UNIQUE_RUNTIME_RESOLUTION; @@ -78,7 +83,7 @@ export function buildSelectorResolutionDisclosure( } function buildResolutionDiagnosticEntry( - node: SnapshotNode, + node: SnapshotAddressableNode, nodes: SnapshotState['nodes'], ): ResolutionDiagnosticEntry { const role = normalizeType(node.type ?? ''); @@ -99,12 +104,12 @@ function buildResolutionDiagnosticEntry( // resolved from, and becomes the pre-action baseline this publishes. export function describeResolvedInteractionNode( runtime: AgentDeviceRuntime, - node: SnapshotNode, + node: SnapshotAddressableNode, tree: SurfaceScopedNodes, action: InteractionAction, resolution: ResolutionDisclosure, ): { - node: SnapshotNode; + node: SnapshotAddressableNode; selectorChain: string[]; refLabel: string | undefined; targetHittable?: boolean; diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index a2255c6569..5497782369 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -1,6 +1,5 @@ import { AppError } from '@agent-device/kernel/errors'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; -import type { SelectorResolution } from '@agent-device/selectors'; import { readinessScheduleFor } from '@agent-device/selectors/selector-pipeline-policy'; import { attemptSelectorResolution, @@ -32,6 +31,7 @@ import { buildSelectorResolutionDisclosure, describeResolvedInteractionNode, } from './resolution-disclosure.ts'; +import type { ActionSelectorResolution } from './selector-action-resolution.ts'; import { assertVisibleSelectorTarget, runInteractionPipelineStages, @@ -138,7 +138,7 @@ async function resolveSelectorInteractionTarget( ): Promise { const selectorExpression = target.selector; let capture: InteractionSnapshot; - let resolved: SelectorResolution | null; + let resolved: ActionSelectorResolution | null; let readiness: SelectorReadinessWait | undefined; const readinessSchedule = readinessScheduleFor(params.pipeline.poll, params.readinessTimeoutMs); if (!readinessSchedule) { diff --git a/src/commands/interaction/runtime/selector-action-resolution.ts b/src/commands/interaction/runtime/selector-action-resolution.ts index fd6837efd9..4288c8e6f4 100644 --- a/src/commands/interaction/runtime/selector-action-resolution.ts +++ b/src/commands/interaction/runtime/selector-action-resolution.ts @@ -1,12 +1,13 @@ import { AppError } from '@agent-device/kernel/errors'; import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { SnapshotAddressableNode, SnapshotNode } from '@agent-device/kernel/snapshot'; import type { SelectorResolution } from '@agent-device/selectors'; -import { classifyActionableTouchCandidates } from '@agent-device/selectors/interaction-targeting'; -import { listSelectorPipelineMatches } from '@agent-device/selectors/selector-pipeline'; +import { resolveActionSelectorPipeline } from '@agent-device/selectors/selector-pipeline'; import type { ActingPipelinePolicy } from '@agent-device/selectors/selector-pipeline-policy'; import { elementMatchCandidateDetails } from '@agent-device/capture-kit/snapshot-lines'; +export type ActionSelectorResolution = SelectorResolution; + /** * How an acting row narrows its candidate set: wrapper duplicates may collapse * through structural/actionable equivalence; matches in distinct branches are @@ -22,46 +23,18 @@ export function resolveActionSelector( selectorExpression: string, platform: Platform | PublicPlatform, policy: ActingPipelinePolicy, -): SelectorResolution | null { - const { candidates, list } = listSelectorPipelineMatches(policy, nodes, selectorExpression, { - platform, - }); - const matched = list?.matchedNodes ?? []; - if (!list || matched.length === 0) return null; - - const diagnostics = [{ selector: list.selector, matches: matched.length }]; - if (matched.length === 1) { - return { - node: matched[0]!, - selector: list.selector, - selectorIndex: list.selectorIndex, - matches: 1, - diagnostics, - }; - } - - const classification = classifyActionableTouchCandidates(candidates, matched); - if (classification.kind === 'ambiguous') { +): ActionSelectorResolution | null { + const outcome = resolveActionSelectorPipeline(policy, nodes, selectorExpression, { platform }); + if (outcome.kind === 'none') return null; + if (outcome.kind === 'ambiguous') { throw new AppError( 'AMBIGUOUS_MATCH', - `Selector matched ${classification.candidates.length} distinct actionable elements: ${list.selector}`, + `Selector matched ${outcome.candidates.length} distinct actionable elements: ${outcome.selector}`, { - selector: list.selector, - ...elementMatchCandidateDetails(classification.candidates), + selector: outcome.selector, + ...elementMatchCandidateDetails(outcome.candidates), }, ); } - - return { - node: classification.node, - selector: list.selector, - selectorIndex: list.selectorIndex, - matches: matched.length, - diagnostics, - disambiguation: { - matchCount: matched.length, - tiebreak: 'structural-equivalence', - alternatives: matched.filter((candidate) => candidate.index !== classification.node.index), - }, - }; + return outcome.resolution; } diff --git a/src/commands/interaction/runtime/selector-readiness.ts b/src/commands/interaction/runtime/selector-readiness.ts index 1380c41bff..4e0b1ade11 100644 --- a/src/commands/interaction/runtime/selector-readiness.ts +++ b/src/commands/interaction/runtime/selector-readiness.ts @@ -1,7 +1,6 @@ import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; import type { SnapshotState } from '@agent-device/kernel/snapshot'; import { inheritPostGestureOutcome } from '@agent-device/kernel/snapshot'; -import type { SelectorResolution } from '@agent-device/selectors'; import { resolveSelectorPipeline } from '@agent-device/selectors/selector-pipeline'; import { SELECTOR_PIPELINE_POLICIES, @@ -17,7 +16,10 @@ import { type InteractionSnapshot, } from './interaction-snapshot-capture.ts'; import { buildCoveredInteractionError } from './target-visibility-stages.ts'; -import { resolveActionSelector } from './selector-action-resolution.ts'; +import { + resolveActionSelector, + type ActionSelectorResolution, +} from './selector-action-resolution.ts'; import { selectorNotFoundFailure } from './selector-read-shared.ts'; import type { InteractionAction, @@ -34,13 +36,13 @@ import type { /** One poll's outcome: the capture it read the tree from, and what it resolved (if anything). */ export type SelectorResolutionAttempt = { capture: InteractionSnapshot; - resolved: SelectorResolution | null; + resolved: ActionSelectorResolution | null; }; /** The narrowed attempt a readiness poll accepts: a resolution with a usable rect. */ export type ResolvedSelectorAttempt = { capture: InteractionSnapshot; - resolved: SelectorResolution; + resolved: ActionSelectorResolution; /** The wait that preceded the hit; absent when the first capture resolved. */ readiness?: SelectorReadinessWait; }; @@ -137,7 +139,7 @@ export async function selectorInteractionFailure(params: { nodes: SnapshotState['nodes']; selectorExpression: string; action: InteractionAction; - resolved: SelectorResolution | null; + resolved: ActionSelectorResolution | null; }): Promise { const { runtime, nodes, selectorExpression, action, resolved } = params; const covered = await detectCoveredSelectorTarget({ runtime, nodes, selectorExpression, action }); diff --git a/src/commands/interaction/runtime/target-visibility-stages.ts b/src/commands/interaction/runtime/target-visibility-stages.ts index b107fc2c65..25d5ad6fcd 100644 --- a/src/commands/interaction/runtime/target-visibility-stages.ts +++ b/src/commands/interaction/runtime/target-visibility-stages.ts @@ -1,6 +1,7 @@ import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; import type { Point, + SnapshotAddressableNode, SnapshotKeyboardBandFact, SnapshotNode, SnapshotState, @@ -11,7 +12,7 @@ import { runNodePipelineStages, type SelectorPipelineHooks, } from '@agent-device/selectors/selector-pipeline'; -import type { SelectorPipelinePolicy } from '@agent-device/selectors/selector-pipeline-policy'; +import type { ActingPipelinePolicy } from '@agent-device/selectors/selector-pipeline-policy'; import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; import { coveredInteractionDetails } from '@agent-device/selectors/interaction-error'; import { @@ -37,7 +38,7 @@ import type { InteractionAction } from './interaction-resolution-request.ts'; */ export function buildCoveredInteractionError(params: { label: string; - node: SnapshotNode; + node: SnapshotAddressableNode; action: InteractionAction; selector?: string; }): AppError { @@ -63,7 +64,7 @@ export function buildCoveredInteractionError(params: { * ref, reading the rect center the platform aims at — says so in its resolver's return type. */ export async function runInteractionPipelineStages(params: { - policy: SelectorPipelinePolicy; + policy: ActingPipelinePolicy; nodes: SnapshotState['nodes']; /** The keyboard band `nodes`' capture measured, when it measured one (#2660). */ keyboard?: SnapshotKeyboardBandFact; @@ -71,8 +72,8 @@ export async function runInteractionPipelineStages( action: InteractionAction; label: string; hooks: SelectorPipelineHooks; - resolveTapPoint: (node: SnapshotNode) => TPoint; -}): Promise<{ node: SnapshotNode; tapPoint: TPoint }> { + resolveTapPoint: (node: SnapshotAddressableNode) => TPoint; +}): Promise<{ node: SnapshotAddressableNode; tapPoint: TPoint }> { const target = await runNodePipelineStages( params.policy, params.nodes, @@ -103,7 +104,7 @@ export async function runInteractionPipelineStages( * owner, and only for rows whose off-screen stage refuses — the row's decision * is made there, so this builds the message and never re-decides. */ -type OffscreenStageParams = { action: InteractionAction; pipeline: SelectorPipelinePolicy }; +type OffscreenStageParams = { action: InteractionAction; pipeline: ActingPipelinePolicy }; // Selector parity for the @ref off-screen guard: without it, a selector // resolving to a closed drawer/carousel item "succeeds" by tapping coordinates