From 6539b3ece957996bb389e6dadda60b0943af2e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 5 Oct 2026 16:30:42 +0200 Subject: [PATCH 1/7] feat(selectors): match role= against the snapshot kind vocabulary with a node-scoped alias window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #3021: the role= selector term and the find role= locator each normalized node.type into a raw leaf class name (statictext, edittext), diverging from the coarse kind vocabulary formatRole publishes on every snapshot node. Both surfaces now read one kernel-owned reader, roleSpellingsOfNode: the canonical kind (the published kind, formatRole-derived only for nodes that bypass attachRefs) plus a deprecation window carrying exactly the node's retired spelling, so role=text and role=text-field start matching while released role=statictext/role=edittext/role=textview scripts keep resolving — on the nodes that actually carried the leaf, never a kind sibling's (role=linearlayout still refuses a FrameLayout, pinned by the #1280 daemon fixtures). No new runtime warning or response shape: the window is disclosed in the versioned find help, user docs, and pinned by the alias tests. contracts normalizeType (fillable/occlusion) and provider-webdriver roleFromWebDriverType (the iOS role field) serve different fields and stay untouched. --- .../kernel/src/snapshot-role-kind.test.ts | 108 ++++++++++++++++ packages/kernel/src/snapshot.ts | 121 +++++++++++++++--- packages/selectors/src/internal/find.test.ts | 104 +++++++++++++++ packages/selectors/src/internal/find.ts | 25 ++-- packages/selectors/src/internal/match.test.ts | 115 +++++++++++++++++ packages/selectors/src/internal/match.ts | 19 ++- .../internal/replay-role-vocabulary.test.ts | 57 +++++++++ src/commands/interaction/metadata.ts | 2 +- website/docs/docs/client-api.md | 2 +- website/docs/docs/selectors.md | 1 + website/docs/docs/snapshots.md | 13 +- 11 files changed, 525 insertions(+), 42 deletions(-) create mode 100644 packages/kernel/src/snapshot-role-kind.test.ts create mode 100644 packages/selectors/src/internal/replay-role-vocabulary.test.ts diff --git a/packages/kernel/src/snapshot-role-kind.test.ts b/packages/kernel/src/snapshot-role-kind.test.ts new file mode 100644 index 0000000000..8b48407a02 --- /dev/null +++ b/packages/kernel/src/snapshot-role-kind.test.ts @@ -0,0 +1,108 @@ +// #3021: the kernel-owned `role=` vocabulary shared by `kind`, the `role=` +// selector term, and the `find role=` locator. Everything is pinned through +// `roleSpellingsOfNode` — the ONE public reader both selector surfaces consume. +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { attachRefs, formatRole, roleSpellingsOfNode } from './snapshot.ts'; + +test('roleSpellingsOfNode leads with the kind the capture published', () => { + const [node] = attachRefs([{ index: 0, type: 'XCUIElementTypeStaticText' }]); + assert.ok(node); + assert.equal(node.kind, 'text'); + // A production node built by `attachRefs` matches on its published `kind` + // first; the documented meaning of `role=` is that vocabulary. + assert.deepEqual(roleSpellingsOfNode(node), ['text', 'statictext']); +}); + +test('roleSpellingsOfNode reuses formatRole for a node that predates kind publication', () => { + // Fixtures that never route through `attachRefs` carry no `kind`; the reader + // computes it with the SAME function that publishes it, not a second + // normalization. + assert.equal(roleSpellingsOfNode({ type: 'android.widget.EditText' })[0], 'text-field'); + assert.equal(roleSpellingsOfNode({})[0], formatRole('Element')); +}); + +test('a renamed leaf stays windowed on the node that carries it', () => { + assert.deepEqual(roleSpellingsOfNode({ type: 'XCUIElementTypeStaticText' }), [ + 'text', + 'statictext', + ]); + assert.deepEqual(roleSpellingsOfNode({ type: 'android.widget.EditText' }), [ + 'text-field', + 'edittext', + ]); + assert.deepEqual(roleSpellingsOfNode({ type: 'android.widget.FrameLayout' }), [ + 'group', + 'framelayout', + ]); + assert.deepEqual(roleSpellingsOfNode({ type: 'XCUIElementTypeSearchField' }), [ + 'search', + 'searchfield', + ]); +}); + +test('the window is node-scoped: sibling leaves of one coarse kind never alias each other', () => { + // The nearest-negative that makes the alias window a window and not a + // widening: `group` renames many leaves (`framelayout`, `linearlayout`, + // `viewgroup`, …), and #3021 must not make `role=linearlayout` match a + // FrameLayout node. Each node carries only ITS OWN old spelling. + assert.deepEqual(roleSpellingsOfNode({ type: 'android.widget.LinearLayout' }), [ + 'group', + 'linearlayout', + ]); + assert.ok(!roleSpellingsOfNode({ type: 'android.widget.FrameLayout' }).includes('linearlayout')); +}); + +test('the context-coarse `textview` leaf stays windowed on both spellings that emitted it', () => { + // `formatRole` answers `text-view` for the desktop/iOS spelling and `text` + // for the Android class; both nodes carried the matchable leaf `textview`. + assert.deepEqual(roleSpellingsOfNode({ type: 'XCUIElementTypeTextView' }), [ + 'text-view', + 'textview', + ]); + assert.deepEqual(roleSpellingsOfNode({ type: 'android.widget.TextView' }), ['text', 'textview']); +}); + +test('AX-prefixed class types carry the AX-stripped spelling the old normalizer matched', () => { + // The macOS helper's `subrole ?? role` fallback emits raw `AX…` classes whose + // canonical kind keeps the prefix while the retired matcher accepted the + // AX-stripped spelling; renumbered leaves like `AXStaticText` window to the + // leaf instead. + assert.deepEqual(roleSpellingsOfNode({ type: 'AXFloatingWindow' }), [ + 'axfloatingwindow', + 'floatingwindow', + ]); + assert.deepEqual(roleSpellingsOfNode({ type: 'AXUnknown' }), ['axunknown', 'unknown']); + assert.deepEqual(roleSpellingsOfNode({ type: 'AXStaticText' }), ['axstatictext', 'statictext']); +}); + +test('the window replays the retired normalizer for arbitrary custom class names', () => { + // The retired spelling took the last `.`-separated segment of any class, so + // a released script could name a custom view class; the window must not + // silently drop spellings formatRole's own leaf step would not produce. + // (formatRole keeps the full dotted spelling as the kind for a non-Android + // class it does not recognize; the old last-segment leaf windows beside it.) + assert.deepEqual(roleSpellingsOfNode({ type: 'io.foo.app.MyButton' }), [ + 'io.foo.app.mybutton', + 'mybutton', + ]); +}); + +test('a kind that equals its leaf carries no alias of its own', () => { + // The nearest-negative for the window itself: `button` and unmapped leaves + // like `gridview` are both the leaf and the kind, so the window adds nothing + // and no vocabulary entry leaks in. + assert.deepEqual(roleSpellingsOfNode({ type: 'XCUIElementTypeButton' }), ['button']); + assert.deepEqual(roleSpellingsOfNode({ type: 'android.widget.GridView' }), ['gridview']); +}); + +test('formatRole keeps its published vocabulary across the leaf-factorization', () => { + // The #3021 refactor moved formatRole's leaf step into a shared helper; the + // coarse answers every backend depends on must not move. + assert.equal(formatRole('XCUIElementTypeStaticText'), 'text'); + assert.equal(formatRole('android.widget.TextView'), 'text'); + assert.equal(formatRole('XCUIElementTypeTextView'), 'text-view'); + assert.equal(formatRole('androidx.recyclerview.widget.RecyclerView'), 'list'); + assert.equal(formatRole('Button'), 'button'); + assert.equal(formatRole(''), 'element'); +}); diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 6691445f25..1ec3e2a66c 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -450,26 +450,113 @@ function lookupRoleLabel(normalized: string): string | undefined { : undefined; } -export function formatRole(type: string): string { - const raw = type; +function isAndroidRoleClass(type: string): boolean { + return ( + type.includes('.') && + (type.startsWith('android.') || type.startsWith('androidx.') || type.startsWith('com.')) + ); +} + +/** + * The leaf normalization {@link formatRole} runs before it consults + * {@link ROLE_LABELS}: strip `XCUIElementType`, lowercase, and reduce Android + * class packages to their leaf. Factored out for readability; the `role=` + * deprecation window below deliberately does NOT share this step — it replays + * the retired normalizer's spelling, not formatRole's leaf. + */ +function normalizeRoleLeaf(type: string): string { let normalized = type.replaceAll(/XCUIElementType/gi, '').toLowerCase(); - const isAndroidClass = - raw.includes('.') && - (raw.startsWith('android.') || raw.startsWith('androidx.') || raw.startsWith('com.')); - if (normalized.includes('.')) { - normalized = normalized - .replace(/^android\.widget\./, '') - .replace(/^android\.view\./, '') - .replace(/^android\.webkit\./, '') - .replace(/^androidx\./, '') - .replace(/^com\.google\.android\./, '') - .replace(/^com\.android\./, ''); - if (isAndroidClass && normalized.includes('.')) { - normalized = normalized.slice(normalized.lastIndexOf('.') + 1); - } + if (!normalized.includes('.')) { + return normalized; + } + const isAndroidClass = isAndroidRoleClass(type); + normalized = normalized + .replace(/^android\.widget\./, '') + .replace(/^android\.view\./, '') + .replace(/^android\.webkit\./, '') + .replace(/^androidx\./, '') + .replace(/^com\.google\.android\./, '') + .replace(/^com\.android\./, ''); + if (isAndroidClass && normalized.includes('.')) { + normalized = normalized.slice(normalized.lastIndexOf('.') + 1); + } + return normalized; +} + +/** + * The `role=` deprecation window (#3021): the legacy spellings this node still + * accepts beside its canonical {@link roleKindOfNode} kind. One entry: the + * node's retired role spelling — what `role=` compared before #3021 — computed + * by {@link legacyRoleSpellingOfType} below and dropped when it already equals + * the canonical kind (the vocabulary-identity case, which needs no window). + * + * The window is deliberately NODE-scoped, not a kind→leaves alias table. One + * coarse kind renames many distinct leaves (`group` covers `framelayout`, + * `linearlayout`, `viewgroup`, …); a kind-keyed table would make + * `role=linearlayout` match a `FrameLayout` — silently WIDENING every released + * script that names a leaf, which is a new breaking change measured against + * the old matcher rather than a compatibility window. So the leaf stays + * matchable only on the nodes that actually carried it: the window is exactly + * the old spelling of THIS node, never a sibling's. + */ +function legacyRoleAliasesForNode( + node: Pick & { kind?: string }, +): readonly string[] { + const alias = legacyRoleSpellingOfType(node.type ?? ''); + return alias && alias !== roleKindOfNode(node) ? [alias] : []; +} + +/** + * The ONE owner of the pre-#3021 `role=` spelling: the retired normalizer's + * exact step — strip `XCUIElementType` and a leading `AX`, lowercase, take the + * last `.`/`/`-separated segment — kept here so the deprecation window can + * replay the old meaning of `role=` for the nodes that carry it while the + * canonical vocabulary stays `formatRole` alone. The retired matcher consumers + * (the selector term and `find`'s deleted `normalizeRole`) are gone; this + * window is their spelling's only remaining reader, and when the window closes + * this function leaves with it. + */ +function legacyRoleSpellingOfType(type: string): string { + let normalized = type + .trim() + .replaceAll(/XCUIElementType/gi, '') + .replace(/^AX/, '') + .toLowerCase(); + const lastSeparator = Math.max(normalized.lastIndexOf('.'), normalized.lastIndexOf('/')); + if (lastSeparator !== -1) { + normalized = normalized.slice(lastSeparator + 1); } + return normalized; +} + +/** + * Every `role=` spelling this node accepts, in ONE place (#3021): the + * canonical kind first, then its windowed legacy aliases. The `role=` selector + * term and the `find role=` locator both read this and nothing else, which is + * what keeps the two — and `kind` itself — unable to disagree. + */ +export function roleSpellingsOfNode( + node: Pick & { kind?: string }, +): readonly string[] { + return [roleKindOfNode(node), ...legacyRoleAliasesForNode(node)]; +} + +/** + * The canonical role kind of a snapshot node — the ONE node-role reader for + * `role=` selector matching (#3021). It prefers the `kind` the capture + * published through {@link attachRefs} and reuses this module's own + * {@link formatRole} when a node predates or bypasses that construction path; + * `kind` is by definition `formatRole(type)`, so there is no second + * normalization to drift from. + */ +function roleKindOfNode(node: Pick & { kind?: string }): string { + return node.kind ?? formatRole(node.type ?? 'Element'); +} + +export function formatRole(type: string): string { + const normalized = normalizeRoleLeaf(type); if (normalized === 'textview') { - return isAndroidClass ? 'text' : 'text-view'; + return isAndroidRoleClass(type) ? 'text' : 'text-view'; } return lookupRoleLabel(normalized) || normalized || 'element'; } diff --git a/packages/selectors/src/internal/find.test.ts b/packages/selectors/src/internal/find.test.ts index 3e172126c8..1bd376cb9f 100644 --- a/packages/selectors/src/internal/find.test.ts +++ b/packages/selectors/src/internal/find.test.ts @@ -2,6 +2,8 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import { findBestMatchesByLocator } from './find.ts'; +import { parseSelectorChain } from './parse.ts'; +import { resolveSelectorChain } from './resolve.ts'; import { formatRole } from '@agent-device/kernel/snapshot'; function makeNode(ref: string, label?: string, identifier?: string): SnapshotNode { @@ -28,3 +30,105 @@ test('findBestMatchesByLocator returns all best-scored matches', () => { assert.equal(result.matches[0]?.ref, 'e1'); assert.equal(result.matches[1]?.ref, 'e2'); }); + +// ── #3021: the `find role=` locator shares the matcher's kind vocabulary ── + +function makeTypedNode(ref: string, type: string): SnapshotNode { + return { + index: Number(ref.replace('e', '')) || 0, + ref, + type, + kind: formatRole(type), + rect: { x: 0, y: 0, width: 100, height: 20 }, + }; +} + +function roleMatches(query: string, types: string[]) { + const nodes = types.map((type, idx) => makeTypedNode(`e${idx + 1}`, type)); + return findBestMatchesByLocator(nodes, 'role', query); +} + +test('find role= scores the coarse kind vocabulary exactly', () => { + // Forward direction: the kind a snapshot publishes for the node. + const text = roleMatches('text', ['android.widget.TextView', 'XCUIElementTypeStaticText']); + assert.equal(text.score, 2); + assert.deepEqual( + text.matches.map((node) => node.ref), + ['e1', 'e2'], + ); +}); + +test('find role= still scores legacy leaf spellings exactly during the window', () => { + // Compatibility direction: `textview` was the old exact leaf for both class + // spellings; the kind is context-coarse (`text` vs `text-view`), and the + // window covers both. + const textView = roleMatches('textview', ['android.widget.TextView', 'XCUIElementTypeTextView']); + assert.equal(textView.score, 2); + assert.deepEqual( + textView.matches.map((node) => node.ref), + ['e1', 'e2'], + ); + const editText = roleMatches('edittext', ['android.widget.EditText']); + assert.equal(editText.score, 2); +}); + +test('find role= keeps the historical substring score for legacy prefixes', () => { + // Before #3021 the locator substring-matched the leaf (`statictext`.includes + // ('static')); the alias window must keep that ranking, at score 1, not + // promote it to an exact hit. + const partial = roleMatches('static', ['XCUIElementTypeStaticText']); + assert.equal(partial.score, 1); + assert.equal(partial.matches[0]?.ref, 'e1'); +}); + +test('find role= refuses sibling vocabulary words with no kind or alias match', () => { + // Nearest-negative: a kind one dash away and an unrelated kind score 0. + assert.equal(roleMatches('text-field', ['XCUIElementTypeStaticText']).score, 0); + assert.equal(roleMatches('switch', ['XCUIElementTypeStaticText']).score, 0); +}); + +test('find role= does not widen a legacy leaf onto kind siblings that never carried it', () => { + // The node-scoped window at the locator: `linearlayout` and `framelayout` + // share the coarse kind `group`, but the legacy leaf was the EXACT class + // leaf, and widening it onto the kind's other members would silently change + // what a released `find role linearlayout` click targets. + const both = roleMatches('linearlayout', [ + 'android.widget.LinearLayout', + 'android.widget.FrameLayout', + ]); + assert.equal(both.score, 2); + assert.deepEqual( + both.matches.map((node) => node.ref), + ['e1'], + ); +}); + +test('find role= and the role= selector term agree on every kind, alias, and negative', () => { + // Locator/matcher parity is the issue's both-paths-reachable proof: the same + // (query, type) pairs resolve through `resolveSelectorChain` exactly where + // the locator scored them exactly 2. + const cases: readonly [string, string][] = [ + ['text', 'XCUIElementTypeStaticText'], + ['statictext', 'XCUIElementTypeStaticText'], + ['textview', 'android.widget.TextView'], + ['text-field', 'android.widget.EditText'], + ['edittext', 'android.widget.EditText'], + ['button', 'XCUIElementTypeButton'], + ['imagebutton', 'XCUIElementTypeOther'], // kind of Other is `other`; must NOT match + ['text-field', 'XCUIElementTypeStaticText'], // sibling kind: must NOT match + ]; + for (const [query, type] of cases) { + const nodes = [makeTypedNode('e1', type)]; + const locatorScore = findBestMatchesByLocator(nodes, 'role', query).score; + const chain = parseSelectorChain(`role=${query}`); + const resolved = resolveSelectorChain(nodes, chain, { + platform: 'android', + requireUnique: true, + }); + assert.equal( + Boolean(resolved), + locatorScore === 2, + `query "${query}" against type "${type}" diverged: locator score ${locatorScore}, selector ${resolved ? 'matched' : 'missed'}`, + ); + } +}); diff --git a/packages/selectors/src/internal/find.ts b/packages/selectors/src/internal/find.ts index 31352ffa76..4b96c2fd37 100644 --- a/packages/selectors/src/internal/find.ts +++ b/packages/selectors/src/internal/find.ts @@ -1,4 +1,4 @@ -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { roleSpellingsOfNode, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { AppError } from '@agent-device/kernel/errors'; import { tryParseSelectorChain } from './parse.ts'; @@ -95,7 +95,7 @@ export function findBestMatchesByLocator( function matchNode(node: SnapshotNode, locator: FindLocator, query: string): number { switch (locator) { case 'role': - return matchRole(node.type, query); + return matchRole(node, query); case 'label': return matchText(node.label, query); case 'value': @@ -121,11 +121,14 @@ function matchText(value: string | undefined, query: string): number { return 0; } -function matchRole(value: string | undefined, query: string): number { - const normalized = normalizeRole(value ?? ''); - if (!normalized) return 0; - if (normalized === query) return 2; - if (normalized.includes(query)) return 1; +function matchRole(node: SnapshotNode, query: string): number { + // The kernel's one role-spelling reader (#3021): the node's canonical `kind` + // first, then its windowed legacy leaf. Exact-vs-substring scoring is the + // historical one, so `find role=static` keeps ranking exactly as it did + // before the kind vocabulary arrived. + const spellings = roleSpellingsOfNode(node); + if (spellings.includes(query)) return 2; + if (spellings.some((spelling) => spelling.includes(query))) return 1; return 0; } @@ -133,14 +136,6 @@ export function normalizeText(value: string): string { return value.trim().toLowerCase().replaceAll(/\s+/g, ' '); } -function normalizeRole(value: string): string { - let normalized = value.trim(); - if (!normalized) return ''; - const lastSegment = normalized.split('.').pop() ?? normalized; - normalized = lastSegment.replaceAll(/XCUIElementType/gi, '').toLowerCase(); - return normalized; -} - export type ParsedFindArgs = { locator: FindLocator; query: string; diff --git a/packages/selectors/src/internal/match.test.ts b/packages/selectors/src/internal/match.test.ts index 2eb1537a9d..c9cf5ef8c1 100644 --- a/packages/selectors/src/internal/match.test.ts +++ b/packages/selectors/src/internal/match.test.ts @@ -71,6 +71,121 @@ test('role selector normalization matches Android class names by leaf type', () assert.equal(resolved.node.ref, 'a1'); }); +// ── #3021: `role=` shares `kind`'s vocabulary, with a legacy-alias window ── + +// One tree covering the three vocabulary decisions at once: the coarse kind +// `kind` publishes, the legacy leaf still windowed for it, and a sibling +// vocabulary word that must match NOTHING here. +const ROLE_VOCABULARY_NODES: SnapshotState['nodes'] = [ + { + ref: 'text1', + index: 0, + type: 'XCUIElementTypeStaticText', + kind: formatRole('XCUIElementTypeStaticText'), + label: 'Total', + rect: { x: 0, y: 0, width: 120, height: 20 }, + hittable: true, + }, + { + ref: 'field1', + index: 1, + type: 'android.widget.EditText', + kind: formatRole('android.widget.EditText'), + label: 'Email', + rect: { x: 0, y: 30, width: 120, height: 44 }, + hittable: true, + }, + { + ref: 'button1', + index: 2, + type: 'XCUIElementTypeButton', + kind: formatRole('XCUIElementTypeButton'), + label: 'Continue', + rect: { x: 0, y: 90, width: 120, height: 44 }, + hittable: true, + }, +]; + +function resolveRole(role: string, platform: 'ios' | 'android' = 'ios') { + const chain = parseSelectorChain(`role=${role}`); + return resolveSelectorChain(ROLE_VOCABULARY_NODES, chain, { + platform, + requireRect: true, + requireUnique: true, + }); +} + +test('role= matches the coarse kind vocabulary: kind=text and kind=text-field resolve', () => { + // The migration's forward direction: these DID NOT match before #3021, + // because the matcher saw the raw leaf (`statictext`, `edittext`). + assert.equal(resolveRole('text')?.node.ref, 'text1'); + assert.equal(resolveRole('text-field', 'android')?.node.ref, 'field1'); +}); + +test('role= keeps matching legacy leaf spellings during the alias window', () => { + // The compatibility direction: released scripts write these; they must keep + // resolving to the same nodes they resolved to before #3021. + assert.equal(resolveRole('statictext')?.node.ref, 'text1'); + assert.equal(resolveRole('edittext', 'android')?.node.ref, 'field1'); + assert.equal(resolveRole('button')?.node.ref, 'button1'); + // The window is the node's OWN old spelling: `textfield` was the old leaf of + // the iOS `TextField` class, never of `EditText`, and must not widen onto it. + assert.equal(resolveRole('textfield', 'android'), null); +}); + +test('role= refuses a sibling vocabulary word that matches no kind or windowed alias', () => { + // Nearest-negatives inside the same vocabulary: `text-view` is one dash from + // `text`, and `switch`/`searchfield` are vocabulary entries (a kind and a + // leaf windowed under kind `search`) that no node here carries. + assert.equal(resolveRole('text-view'), null); + assert.equal(resolveRole('switch'), null); + assert.equal(resolveRole('searchfield'), null); +}); + +test('role= does not widen a legacy leaf onto kind siblings that never carried it', () => { + // The window is the node's OWN old spelling. `linearlayout` and + // `framelayout` share the coarse kind `group`; matching the leaf may not + // pull in the kind's other members, or every released `role=linearlayout` + // script would silently start hitting FrameLayout rows. + const layoutNodes: SnapshotState['nodes'] = [ + 'android.widget.LinearLayout', + 'android.widget.FrameLayout', + ].map((type, idx) => ({ + ref: `row${idx + 1}`, + index: idx, + type, + kind: formatRole(type), + rect: { x: 0, y: idx * 50, width: 300, height: 48 }, + hittable: true, + })); + const chain = parseSelectorChain('role=linearlayout'); + const resolved = resolveSelectorChain(layoutNodes, chain, { + platform: 'android', + requireRect: true, + requireUnique: true, + }); + assert.ok(resolved); + assert.equal(resolved.node.ref, 'row1'); +}); + +test('role= agrees with the published kind for every node it matches (#3021 parity)', () => { + // The architecture pin: for every node in the tree, matching by that node's + // own published `kind` must resolve to exactly that node. This is what makes + // the selector/kind divergence that #3021 closes impossible to reintroduce + // silently for any vocabulary entry. + for (const node of ROLE_VOCABULARY_NODES) { + assert.ok(node.kind, 'fixture nodes carry a published kind'); + const chain = parseSelectorChain(`role=${JSON.stringify(node.kind)}`); + const resolved = resolveSelectorChain(ROLE_VOCABULARY_NODES, chain, { + platform: 'ios', + requireRect: true, + requireUnique: false, + }); + assert.ok(resolved, `role=${node.kind} resolved nothing`); + assert.equal(resolved.node.ref, node.ref, `role=${node.kind} resolved the wrong node`); + } +}); + test('focused selector matches snapshot focus state', () => { const tvNodes: SnapshotState['nodes'] = [ { diff --git a/packages/selectors/src/internal/match.ts b/packages/selectors/src/internal/match.ts index e3d8e1cc21..1b0fcc41d8 100644 --- a/packages/selectors/src/internal/match.ts +++ b/packages/selectors/src/internal/match.ts @@ -1,7 +1,7 @@ import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { roleSpellingsOfNode, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { isNodeEditable, isNodeVisible } from './node.ts'; -import { extractNodeText, normalizeType } from '@agent-device/contracts/snapshot'; +import { extractNodeText } from '@agent-device/contracts/snapshot'; import { normalizeText } from './find.ts'; import type { Selector, SelectorTerm } from './parse.ts'; @@ -22,7 +22,7 @@ function matchesTerm( case 'id': return textEquals(node.identifier, String(term.value)); case 'role': - return textEquals(normalizeType(node.type ?? ''), String(term.value)); + return matchesRoleKind(node, String(term.value)); case 'label': return textEquals(node.label, String(term.value)); case 'value': @@ -55,3 +55,16 @@ function matchesTerm( function textEquals(value: string | undefined, query: string): boolean { return normalizeText(value ?? '') === normalizeText(query); } + +/** + * `role=` matches the canonical `kind` vocabulary (#3021): a term value equals + * the node's role kind, or one of the legacy leaf spellings still windowed for + * that node. The kind and the window both come from the kernel's role + * vocabulary — the same authority `attachRefs` uses to publish `kind` — so the + * selector, the `find role=` locator, and snapshot output cannot disagree + * about a node's role again. + */ +function matchesRoleKind(node: SnapshotNode, query: string): boolean { + const normalizedQuery = normalizeText(query); + return roleSpellingsOfNode(node).some((spelling) => normalizeText(spelling) === normalizedQuery); +} diff --git a/packages/selectors/src/internal/replay-role-vocabulary.test.ts b/packages/selectors/src/internal/replay-role-vocabulary.test.ts new file mode 100644 index 0000000000..69b2194693 --- /dev/null +++ b/packages/selectors/src/internal/replay-role-vocabulary.test.ts @@ -0,0 +1,57 @@ +// #3021 at the replay seam: `role=` vocabulary is resolved by the same +// `resolveRecordedTarget` path `replay` dispatches recorded targets through, so +// the alias window and the kind vocabulary are proven where released scripts +// actually run — not only at the matcher unit. ADR 0012 is untouched: nothing +// here rewrites a recorded expression; a miss still reports `no-match`. +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { attachRefs, type RawSnapshotNode, type SnapshotNode } from '@agent-device/kernel/snapshot'; +import { resolveRecordedTarget, type ReplayRecordedTargetPolicy } from './replay.ts'; + +const POLICY: ReplayRecordedTargetPolicy = { + platform: 'ios', + requireRect: true, + allowDisambiguation: false, +}; + +// The production construction path, so the tree carries published `kind` the +// way every captured session tree does. +const rawNodes: RawSnapshotNode[] = [ + { + index: 0, + type: 'XCUIElementTypeStaticText', + label: 'Total', + rect: { x: 0, y: 0, width: 120, height: 20 }, + }, + { + index: 1, + type: 'XCUIElementTypeButton', + label: 'Continue', + rect: { x: 0, y: 30, width: 120, height: 44 }, + }, +]; +const nodes: SnapshotNode[] = attachRefs(rawNodes); + +test('recorded legacy role= scripts keep resolving through the replay seam', () => { + // `role=statictext` is the spelling released .ad scripts carry; the window + // keeps them replaying against the same node they recorded against. + const legacy = resolveRecordedTarget('role="statictext" label="Total"', nodes, POLICY); + assert.equal(legacy.kind, 'resolved'); + if (legacy.kind === 'resolved') assert.equal(legacy.winner.ref, 'e1'); +}); + +test('coarse kind selectors resolve through the replay seam too', () => { + // The forward direction on the SAME seam a repaired/healed script would use. + const coarse = resolveRecordedTarget('role="text" label="Total"', nodes, POLICY); + assert.equal(coarse.kind, 'resolved'); + if (coarse.kind === 'resolved') assert.equal(coarse.winner.ref, 'e1'); +}); + +test('a sibling kind stays a real miss at the replay seam — no silent widening', () => { + // Nearest-negative: the divergence machinery downstream keys on + // `reason: 'no-match'`; widening the vocabulary to a dash-sibling kind would + // resolve this to the wrong node instead of diverging honestly. + const miss = resolveRecordedTarget('role="text-field"', nodes, POLICY); + assert.equal(miss.kind, 'unresolved'); + if (miss.kind === 'unresolved') assert.equal(miss.reason, 'no-match'); +}); diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 62a5b4ff26..54c607af99 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -73,7 +73,7 @@ const interactionCommandDescriptions = { 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to reach an off-screen target in one command rather than a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset. A visible keyboard shortens the swiped band instead of being dismissed; when too little is left, the command refuses with scroll_keyboard_occludes_surface. A directional scroll also reports the movement it observed as movement: moved, at-edge, unchanged, or unobserved when the two reads could not back a claim either way; an unchanged surface inside a container that still hides content in that direction refuses with scroll_no_progress rather than repeating the requested distance. The movement field is absent where a tier verifies per pass (top/bottom, until), where the runtime cannot read a screen, or where a settle observation or a replay already owns that observation.', get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.', is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, settled, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.', - find: 'Find by text/label/value/role/id and run action', + find: 'Find by text/label/value/role/id and run action. The role locator matches snapshot kind vocabulary (button, text, text-field, switch, ...); pre-reconciliation leaf spellings (statictext, edittext, textview) stay accepted on the nodes that carried them during a deprecation window.', gesture: 'Perform a structured pan, fling, swipe, pinch, rotate, transform, or drag gesture. Select the gesture kind, then provide only the inputs that apply to that kind.', } as const; diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 076f57e8a4..0d068b15d4 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -510,7 +510,7 @@ Use `agent-device/remote-config` for profile loading and path resolution, `agent ## Selector helpers -Use `agent-device/selectors` when a remote daemon or bridge needs to parse and match selector expressions without deep-importing daemon internals. Matching is platform-aware because role normalization and editability checks differ by backend. +Use `agent-device/selectors` when a remote daemon or bridge needs to parse and match selector expressions without deep-importing daemon internals. The `role=` term matches the platform-neutral `kind` vocabulary that `snapshot --json` publishes (see [Snapshots](./snapshots.md#structured-node-fields--json)): pass the nodes as captured and a node matches `role=text` exactly when its `kind` is `text`. Pre-reconciliation leaf spellings (`statictext`, `edittext`, `textview`, …) still match during a deprecation window, each on the nodes that actually carried that class. Matching stays platform-aware because editability checks differ by backend. `listSelectorChainMatches(nodes, chain, options)` returns every node the winning selector alternative matches, in snapshot order, plus that alternative and its index — the same first-match domain `findSelectorChainMatch` uses, without uniqueness refusal, so a runner applies its own strictness to the same nodes the CLI matched. `resolveSelectorChain` can name a LATER alternative: by default it refuses an ambiguous one and keeps walking, so the indices agree when it passes `requireUnique: false`, when the first matching alternative is unique, or when `disambiguateAmbiguous: true` resolves that alternative in place. It returns `null` when no alternative matches. `options` is `{ platform, requireRect? }`; the matched `SnapshotNode` objects are the ones passed in. diff --git a/website/docs/docs/selectors.md b/website/docs/docs/selectors.md index 1c148d66c6..4d0574b139 100644 --- a/website/docs/docs/selectors.md +++ b/website/docs/docs/selectors.md @@ -18,6 +18,7 @@ agent-device find id "com.example:id/login" click Tips: +- `find role ` and the `role=` selector term match the platform-neutral `kind` vocabulary that `snapshot --json` publishes (`button`, `text`, `text-field`, `switch`, …). Older leaf spellings (`statictext`, `edittext`, `textview`) still match during a deprecation window, each on the nodes that actually carried that class. - Use `find ... wait ` to wait for UI to appear. - Combine with scoped snapshots using `snapshot -s "