diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java
index c5c1da5249..775d622f7b 100644
--- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java
+++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java
@@ -69,6 +69,12 @@ static void appendNode(
// unselected control reports selected=false and a helper older than this attribute reports
// nothing at all.
appendAttribute(xml, "selected", Boolean.toString(node.isSelected()));
+ // Present only on a checkable control, with both answers: an unchecked switch reports
+ // checked=false, while a node that cannot be checked reports nothing, like a helper older than
+ // this attribute.
+ if (node.isCheckable()) {
+ appendAttribute(xml, "checked", Boolean.toString(node.isChecked()));
+ }
boolean scrollable = node.isScrollable();
if (scrollable) {
appendAttribute(xml, "scrollable", "true");
@@ -164,9 +170,8 @@ private static boolean isHeading(AccessibilityNodeInfo node) {
return Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && node.isHeading();
}
- // Declared residue (agent-device #1832): checked / checkable / long-clickable are not serialized,
- // so toggle state is invisible to agents. Adding them is a helper protocol change (new attributes
- // + host parser + fields on the wire node), tracked there.
+ // Declared residue (agent-device #1832): long-clickable is not serialized. Adding it is a helper
+ // protocol change (new attribute + host parser + field on the wire node).
private static void appendDrawingOrderAttribute(StringBuilder xml, AccessibilityNodeInfo node) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder()));
diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-diff-checked.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-diff-checked.test.ts
new file mode 100644
index 0000000000..bf95506373
--- /dev/null
+++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-diff-checked.test.ts
@@ -0,0 +1,49 @@
+import assert from 'node:assert/strict';
+import { test } from 'vitest';
+import type { SnapshotNode } from '@agent-device/kernel/snapshot';
+import { buildSnapshotDiff } from '../snapshot-diff.ts';
+
+function toggle(checked?: boolean): SnapshotNode {
+ return {
+ ref: 'e12',
+ index: 0,
+ depth: 0,
+ type: 'android.widget.Switch',
+ label: 'Wi-Fi switch',
+ enabled: true,
+ hittable: true,
+ ...(checked === undefined ? {} : { checked }),
+ };
+}
+
+test('a checked-only flip diffs as changed lines that read differently', () => {
+ const diff = buildSnapshotDiff([toggle(false)], [toggle(true)]);
+ const changed = diff.lines.filter((line) => line.kind !== 'unchanged');
+
+ assert.equal(diff.summary.unchanged, 0);
+ assert.equal(changed.length > 0, true);
+ // The comparable key carries the checked state, so the rendered line has to as well, and both
+ // answers render: the pair reads `[unchecked]` -> `[checked]`, not two identical lines.
+ assert.match(changed[0]!.text, /\[unchecked\]/);
+ assert.match(changed.at(-1)!.text, /\[checked\]/);
+});
+
+test.each([
+ ['both unchecked', [toggle(false)], [toggle(false)]],
+ ['both unreported', [toggle()], [toggle()]],
+])('a still toggle with %s diffs as unchanged', (_label, previous, current) => {
+ const diff = buildSnapshotDiff(previous, current);
+
+ assert.equal(
+ diff.lines.every((line) => line.kind === 'unchanged'),
+ true,
+ );
+});
+
+test('a node that cannot be checked never reads as unchecked', () => {
+ const diff = buildSnapshotDiff([toggle()], [toggle()]);
+ assert.equal(
+ diff.lines.every((line) => !line.text.includes('checked')),
+ true,
+ );
+});
diff --git a/packages/capture-kit/src/snapshot/snapshot-diff.ts b/packages/capture-kit/src/snapshot/snapshot-diff.ts
index 3c7bf020b8..2b28a9198b 100644
--- a/packages/capture-kit/src/snapshot/snapshot-diff.ts
+++ b/packages/capture-kit/src/snapshot/snapshot-diff.ts
@@ -5,6 +5,7 @@ import {
displayLabel,
formatRole,
formatSnapshotLine,
+ stateMarkers,
} from './snapshot-lines.ts';
export type SnapshotDiffResult = {
@@ -27,8 +28,9 @@ type SnapshotComparableLine = {
function snapshotNodeToComparableLine(node: SnapshotNode, depthOverride?: number): string {
const role = formatRole(node.type ?? 'Element');
const textPart = displayLabel(node, role);
- const enabledPart = node.enabled === false ? 'disabled' : 'enabled';
- const selectedPart = node.selected === true ? 'selected' : 'unselected';
+ // The states the rendered line prints are the states the key compares, from one function, so a
+ // fact the diff weighs is always visible in the line it prints.
+ const statePart = stateMarkers(node).join(',');
const hittablePart = node.hittable === true ? 'hittable' : 'not-hittable';
const depthPart = String(depthOverride ?? node.depth ?? 0);
// The rendered line carries the actions list, so the comparable key has to as
@@ -36,9 +38,7 @@ function snapshotNodeToComparableLine(node: SnapshotNode, depthOverride?: number
// silently differs from the baseline's. JSON-encoded because the names are
// app-authored and may contain the field separator.
const actionsPart = node.actions ? JSON.stringify(node.actions) : '';
- return [depthPart, role, textPart, enabledPart, selectedPart, hittablePart, actionsPart].join(
- '|',
- );
+ return [depthPart, role, textPart, statePart, hittablePart, actionsPart].join('|');
}
export function buildSnapshotDiff(
diff --git a/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts b/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts
index 8421f6236c..26955e8ab5 100644
--- a/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts
+++ b/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts
@@ -28,11 +28,12 @@ export function isNavigationSensitiveAction(command: string): boolean {
/**
* Route signature of an Android snapshot, from the fields the Android backend actually carries.
- * The helper serializes no `role`, `checked` or `long-clickable` (declared residue,
- * #1832), so a signature keying on them would compare constants and claim discrimination it does
- * not have. `selected` is left out as a judgement call rather than an inability: a tab-bar tap
- * flips it on two nodes, which the 90%-identical threshold below absorbs at every tree size this
- * check runs on, so keying on it would only add capture retries.
+ * The helper serializes no `role` or `long-clickable` (declared residue, #1832), so a signature
+ * keying on them would compare constants and claim discrimination it does not have. `selected`
+ * and `checked` are left out as a judgement call rather than an inability: a tab-bar tap flips
+ * selection on two nodes and a toggle tap flips one checked state, which the 90%-identical
+ * threshold below absorbs at every tree size this check runs on, so keying on them would only add
+ * capture retries.
*/
export function buildSnapshotSignatures(nodes: SnapshotState['nodes']): string[] {
return nodes.map((node) =>
diff --git a/packages/capture-kit/src/snapshot/snapshot-lines.ts b/packages/capture-kit/src/snapshot/snapshot-lines.ts
index f6004dc995..0a2c598579 100644
--- a/packages/capture-kit/src/snapshot/snapshot-lines.ts
+++ b/packages/capture-kit/src/snapshot/snapshot-lines.ts
@@ -195,6 +195,21 @@ export function formatRole(type: string): string {
return lookupRoleLabel(normalized) || normalized || 'element';
}
+/**
+ * The state markers every rendering path prints, and the states a snapshot diff compares: the diff
+ * renders its lines without text-surface summarizing, and a fact it weighs has to be visible in the
+ * line it prints, or a flip reads as a changed pair whose two lines look identical. Both checked
+ * answers render, since a checkable control shown plain would hide that it toggles; a node that
+ * cannot be checked carries neither.
+ */
+export function stateMarkers(node: SnapshotNode): string[] {
+ const markers: string[] = [];
+ if (node.enabled === false) markers.push('disabled');
+ if (node.selected === true) markers.push('selected');
+ if (node.checked !== undefined) markers.push(node.checked ? 'checked' : 'unchecked');
+ return markers;
+}
+
function lookupRoleLabel(normalized: string): string | undefined {
return Object.prototype.hasOwnProperty.call(ROLE_LABELS, normalized)
? ROLE_LABELS[normalized]
@@ -238,13 +253,7 @@ function buildLineMetadata(
options: SnapshotLineFormatOptions,
textSurface: { text: string; isLargeSurface: boolean; shouldSummarize: boolean },
): string[] {
- const metadata: string[] = [];
- if (node.enabled === false) metadata.push('disabled');
- // Selection is a state a snapshot diff can report as changed, and the diff renders its lines
- // without text-surface summarizing. A fact the diff compares has to be visible in the line it
- // prints, or a selection flip reads as a changed pair whose two lines look identical.
- if (node.selected === true) metadata.push('selected');
- metadata.push(...(node.presentationHints ?? []));
+ const metadata = [...stateMarkers(node), ...(node.presentationHints ?? [])];
if (!options.summarizeTextSurfaces) {
return uniqueMetadata(metadata);
}
diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts
index a6a108d9c9..e1367ff19b 100644
--- a/packages/kernel/src/snapshot.ts
+++ b/packages/kernel/src/snapshot.ts
@@ -251,6 +251,8 @@ export type RawSnapshotNode = {
rect?: Rect;
enabled?: boolean;
selected?: boolean;
+ /** Checked state of a checkable control (switch, checkbox, radio); absent means not checkable or unavailable. */
+ checked?: boolean;
focused?: boolean;
/** Accessibility heading flag an app set on the node; absent means not a heading or unavailable. */
heading?: boolean;
diff --git a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-observation.test.ts b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-observation.test.ts
index 8d58d117d0..22e6b64b19 100644
--- a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-observation.test.ts
+++ b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-observation.test.ts
@@ -359,12 +359,26 @@ test('normalizes absent attributes like Maestro iOS hierarchy mapping', () => {
enabled: false,
selected: false,
focused: false,
+ checked: false,
},
]);
expect(maestroSnapshotSignature(first)).toBe(maestroSnapshotSignature(second));
});
+test('a checked-only flip changes the snapshot signature', () => {
+ // An Android switch tapped on: nothing about it moves or renames, only `checked` flips. The
+ // same signature before and after would read the tap as a no-op and retap it off again.
+ const off = makeSnapshot([
+ { index: 0, type: 'android.widget.Switch', label: 'Wi-Fi', checked: false },
+ ]);
+ const on = makeSnapshot([
+ { index: 0, type: 'android.widget.Switch', label: 'Wi-Fi', checked: true },
+ ]);
+
+ expect(maestroSnapshotSignature(off)).not.toBe(maestroSnapshotSignature(on));
+});
+
test('excludes agent-device presentation metadata from Maestro hierarchy signatures', () => {
const first = makeSnapshot([
{
diff --git a/packages/maestro/src/daemon-port/daemon-runtime-port-observation.ts b/packages/maestro/src/daemon-port/daemon-runtime-port-observation.ts
index c90f173917..f6ce5ffd6d 100644
--- a/packages/maestro/src/daemon-port/daemon-runtime-port-observation.ts
+++ b/packages/maestro/src/daemon-port/daemon-runtime-port-observation.ts
@@ -408,6 +408,7 @@ export function maestroSnapshotSignature(snapshot: SnapshotState): string {
enabled: node.enabled ?? false,
selected: node.selected ?? false,
focused: node.focused ?? false,
+ checked: node.checked ?? false,
bounds: maestroSnapshotBounds(node.rect),
})),
),
diff --git a/packages/platform-android/src/__tests__/ui-hierarchy-checked.test.ts b/packages/platform-android/src/__tests__/ui-hierarchy-checked.test.ts
new file mode 100644
index 0000000000..fe0d5b29d9
--- /dev/null
+++ b/packages/platform-android/src/__tests__/ui-hierarchy-checked.test.ts
@@ -0,0 +1,70 @@
+import { expect, test } from 'vitest';
+import { buildUiHierarchySnapshot, parseUiHierarchyTree } from '../ui-hierarchy.ts';
+
+// A settings row: a switch the helper reports as checkable beside a label that is not. The helper
+// writes `checked` only on the checkable node, with both answers.
+function togglesXml(wifiChecked: boolean): string {
+ return `
+
+
+
+
+
+ `;
+}
+
+// A helper older than the `checked` attribute, or a switch it did not report as checkable.
+const UNREPORTED_CHECKED_XML =
+ '';
+
+function toggleNodes(raw: boolean, interactiveOnly = false, wifiChecked = true) {
+ const { nodes } = buildUiHierarchySnapshot(
+ parseUiHierarchyTree(togglesXml(wifiChecked)),
+ undefined,
+ { raw, interactiveOnly },
+ );
+ const byId = (identifier: string) => nodes.find((node) => node.identifier === identifier);
+ return { label: byId('wifi-label'), wifi: byId('wifi-switch'), small: byId('size-small') };
+}
+
+test.each([
+ { raw: false, interactiveOnly: false },
+ { raw: false, interactiveOnly: true },
+ { raw: true, interactiveOnly: false },
+ { raw: true, interactiveOnly: true },
+])(
+ 'checked state reaches snapshot nodes in every projection (raw=$raw, -i=$interactiveOnly)',
+ ({ raw, interactiveOnly }) => {
+ const { wifi, small } = toggleNodes(raw, interactiveOnly);
+ expect(wifi?.checked).toBe(true);
+ expect(small?.checked).toBe(false);
+ },
+);
+
+test.each([true, false])('the switch answers the state the helper observed (%s)', (wifiChecked) => {
+ expect(toggleNodes(false, false, wifiChecked).wifi?.checked).toBe(wifiChecked);
+});
+
+test('attrs answer explicit false where a node that cannot be checked answers nothing', () => {
+ // Serialized, because that is the answer an agent reads: an unavailable fact drops the key
+ // while JSON encodes an observed `false`.
+ const { label, small } = toggleNodes(false);
+ expect(JSON.parse(JSON.stringify(small)).checked).toBe(false);
+ expect(JSON.parse(JSON.stringify(label))).not.toHaveProperty('checked');
+});
+
+test('an unreported checked state stays unknown instead of becoming false', () => {
+ const node = buildUiHierarchySnapshot(parseUiHierarchyTree(UNREPORTED_CHECKED_XML), undefined, {
+ raw: false,
+ }).nodes.find((node) => node.identifier === 'legacy-switch');
+ expect(node).toBeDefined();
+ expect(node?.checked).toBeUndefined();
+});
diff --git a/packages/platform-android/src/ui-hierarchy-builder.ts b/packages/platform-android/src/ui-hierarchy-builder.ts
index e4f0ffaffb..087fb44bf2 100644
--- a/packages/platform-android/src/ui-hierarchy-builder.ts
+++ b/packages/platform-android/src/ui-hierarchy-builder.ts
@@ -356,6 +356,7 @@ function createAndroidRawSnapshotNode(
selected: node.selected,
heading: node.heading,
roleDescription: node.roleDescription,
+ checked: node.checked,
editable: node.editable,
password: node.password,
hintShowing: node.hintShowing,
diff --git a/packages/platform-android/src/ui-hierarchy-node.ts b/packages/platform-android/src/ui-hierarchy-node.ts
index 205c56ef4a..9dfb81cbc5 100644
--- a/packages/platform-android/src/ui-hierarchy-node.ts
+++ b/packages/platform-android/src/ui-hierarchy-node.ts
@@ -19,6 +19,7 @@ export type AndroidUiHierarchy = {
selected?: boolean;
heading?: boolean;
roleDescription?: string;
+ checked?: boolean;
editable?: boolean;
password?: boolean;
hintShowing?: boolean;
diff --git a/packages/platform-android/src/ui-hierarchy.ts b/packages/platform-android/src/ui-hierarchy.ts
index 7317d057ba..508939aa37 100644
--- a/packages/platform-android/src/ui-hierarchy.ts
+++ b/packages/platform-android/src/ui-hierarchy.ts
@@ -39,6 +39,8 @@ export type AndroidUiNodeMetadata = {
heading?: boolean;
/** Helper-only: the localized role description an app set beside the class, verbatim. */
roleDescription?: string;
+ /** Helper-only, present on a checkable control: the checked state of a switch, checkbox, or radio. */
+ checked?: boolean;
password?: boolean;
editable?: boolean;
selectionStart?: number;
@@ -175,6 +177,7 @@ function readNodeAttributes(node: string): Omit {
...optionalBoolAttr('selected', 'selected'),
...optionalBoolAttr('heading', 'heading'),
...optionalStringAttr('roleDescription', 'role-description'),
+ ...optionalBoolAttr('checked', 'checked'),
...optionalNumberAttr('drawingOrder', 'drawing-order'),
...optionalBoolAttr('scrollable', 'scrollable'),
...optionalBoolAttr('canScrollForward', 'can-scroll-forward'),
@@ -331,6 +334,7 @@ function normalizeAndroidUiHierarchyNode(
selected: attrs.selected,
heading: attrs.heading,
roleDescription: attrs.roleDescription,
+ checked: attrs.checked,
editable: attrs.editable,
password: attrs.password,
hintShowing: attrs.hintShowing,
diff --git a/src/__tests__/android-ui-hierarchy.test.ts b/src/__tests__/android-ui-hierarchy.test.ts
index d9edc4296d..e9149fd58c 100644
--- a/src/__tests__/android-ui-hierarchy.test.ts
+++ b/src/__tests__/android-ui-hierarchy.test.ts
@@ -272,6 +272,29 @@ test('a published Android snapshot carries the heading flag and the role descrip
);
});
+// A settings screen: the helper writes `checked` on the checkable switch and radio, with both
+// answers, and nothing on the label beside them.
+const ANDROID_TOGGLES_XML = `
+
+
+
+
+
+`;
+
+test('a published Android snapshot carries the checked state of checkable controls only', () => {
+ const nodes = publishUiHierarchy(ANDROID_TOGGLES_XML).nodes;
+ const byId = (identifier: string) => nodes.find((node) => node.identifier === identifier)!;
+
+ assert.equal(byId('com.example.app:id/wifi-switch').checked, true);
+ assert.equal(byId('com.example.app:id/size-small').checked, false);
+ assert.equal(byId('com.example.app:id/wifi-label').checked, undefined);
+ assert.deepEqual(
+ Array.from(androidUiNodes(ANDROID_TOGGLES_XML)).map((node) => node.checked),
+ [undefined, undefined, true, false],
+ );
+});
+
test('parseUiHierarchy discards stale inactive Android application windows', () => {
const xml = `
diff --git a/src/commands/capture/runtime/snapshot-unchanged.test.ts b/src/commands/capture/runtime/snapshot-unchanged.test.ts
index 1b3cf7ce6f..32f5adfd61 100644
--- a/src/commands/capture/runtime/snapshot-unchanged.test.ts
+++ b/src/commands/capture/runtime/snapshot-unchanged.test.ts
@@ -73,6 +73,8 @@ test.each>([
{ contentDescription: 'Create a draft' },
{ enabled: false },
{ selected: true },
+ { checked: true },
+ { checked: false },
{ focused: true },
{ placeholder: 'Key echo' },
{ hittable: false },
diff --git a/src/commands/capture/runtime/snapshot-unchanged.ts b/src/commands/capture/runtime/snapshot-unchanged.ts
index da581313bc..c03287df28 100644
--- a/src/commands/capture/runtime/snapshot-unchanged.ts
+++ b/src/commands/capture/runtime/snapshot-unchanged.ts
@@ -111,6 +111,7 @@ const PRESENTATION_SCALAR_FIELDS = {
identifier: true,
enabled: true,
selected: true,
+ checked: true,
focused: true,
heading: true,
roleDescription: true,
diff --git a/src/commands/output/snapshot.test.ts b/src/commands/output/snapshot.test.ts
index 8440986243..ccaff38177 100644
--- a/src/commands/output/snapshot.test.ts
+++ b/src/commands/output/snapshot.test.ts
@@ -820,3 +820,28 @@ test('formatSnapshotLine marks selection anywhere, and keeps text-surface metada
assert.doesNotMatch(line, /\[editable\]/);
assert.doesNotMatch(line, /\[scrollable\]/);
});
+
+test.each([
+ [true, /\[checked\]/],
+ [false, /\[unchecked\]/],
+])(
+ 'formatSnapshotLine renders both checked answers on the default path (%s)',
+ (checked, marker) => {
+ const line = formatSnapshotLine(
+ {
+ ref: 'e2',
+ index: 0,
+ depth: 0,
+ type: 'Switch',
+ label: 'Wi-Fi switch',
+ enabled: true,
+ checked,
+ },
+ 0,
+ false,
+ );
+ // A checkable control rendered as plain would hide that it toggles, and the diff compares the
+ // state, so the line prints whichever answer the helper observed.
+ assert.match(line, marker);
+ },
+);
diff --git a/src/daemon/__tests__/interaction-outcome-policy.test.ts b/src/daemon/__tests__/interaction-outcome-policy.test.ts
index 9cd0d34c5a..20ef6154b2 100644
--- a/src/daemon/__tests__/interaction-outcome-policy.test.ts
+++ b/src/daemon/__tests__/interaction-outcome-policy.test.ts
@@ -467,3 +467,55 @@ test('discriminatingSurfaceChangedWithinRect counts content appearing inside the
assert.equal(discriminatingSurfaceChangedWithinRect(before, after, LIST_RECT), true);
});
+
+// Android is the only producer of `checked`, and a tap whose only effect is a toggle changes nothing
+// else on a screen without a mirrored label. The outcome lane has to read the flip as a change, or a
+// no-change retry taps the switch straight back.
+test('classifyInteractionSurfaceChange reads a checked-only flip as a change', () => {
+ const before = buildInteractionSurfaceSignature(makeToggleSnapshot(false).nodes);
+ const after = buildInteractionSurfaceSignature(makeToggleSnapshot(true).nodes);
+
+ assert.equal(classifyInteractionSurfaceChange(before, after), 'changed');
+});
+
+test.each([
+ { anonymous: false, label: 'a labelled switch' },
+ { anonymous: true, label: 'an anonymous switch' },
+])(
+ 'discriminatingSurfaceChangedWithinRect reads a flip of $label at the same rect as no movement, and a moved one as movement',
+ ({ anonymous }) => {
+ const rect = { x: 0, y: 0, width: 390, height: 844 };
+ const signature = (checked: boolean, y?: number) =>
+ buildInteractionSurfaceSignature(makeToggleSnapshot(checked, y, anonymous).nodes);
+
+ assert.equal(
+ discriminatingSurfaceChangedWithinRect(signature(false), signature(true), rect),
+ false,
+ );
+ assert.equal(
+ discriminatingSurfaceChangedWithinRect(signature(false, 300), signature(false, 200), rect),
+ true,
+ );
+ },
+);
+
+// An anonymous switch has no identity, so its content is its type: the flip still changes only the
+// key, and a swipe that brushed it must not read as the list moving.
+function makeToggleSnapshot(checked: boolean, y = 300, anonymous = false): SnapshotState {
+ const base = makeSnapshot('Inbox');
+ return {
+ ...base,
+ nodes: [
+ ...base.nodes,
+ {
+ ref: 'e3',
+ index: 2,
+ parentIndex: 0,
+ type: 'android.widget.Switch',
+ ...(anonymous ? {} : { identifier: 'wifi-switch', label: 'Wi-Fi switch' }),
+ checked,
+ rect: { x: 300, y, width: 60, height: 40 },
+ },
+ ],
+ };
+}
diff --git a/src/daemon/__tests__/response-views.test.ts b/src/daemon/__tests__/response-views.test.ts
index 4d7cea294d..4b3a103026 100644
--- a/src/daemon/__tests__/response-views.test.ts
+++ b/src/daemon/__tests__/response-views.test.ts
@@ -338,6 +338,18 @@ test('attrs digest keeps explicit false/zero/empty field facts; unavailable ones
}
});
+test('attrs digest carries both checked answers, and none for a node that cannot be checked', () => {
+ for (const checked of [true, false]) {
+ const digest = getView!({ ref: 'e7', node: { ...MATCHED_NODE, checked } }, 'digest');
+ expect(digest.node).toEqual({ ...COMPACT_NODE, checked });
+ }
+ const plain = getView!({ ref: 'e7', node: MATCHED_NODE }, 'digest').node as Record<
+ string,
+ unknown
+ >;
+ expect('checked' in plain).toBe(false);
+});
+
test('find/get default and full return today’s shape unchanged (same reference)', () => {
const data: DaemonResponseData = { ref: '@e7', text: 'Sign in', node: MATCHED_NODE };
expect(findView!(data, 'default')).toBe(data);
diff --git a/src/daemon/__tests__/scroll-movement.test.ts b/src/daemon/__tests__/scroll-movement.test.ts
index bf46f3cbdc..1097fa2ad3 100644
--- a/src/daemon/__tests__/scroll-movement.test.ts
+++ b/src/daemon/__tests__/scroll-movement.test.ts
@@ -388,6 +388,34 @@ test('a difference outside the scrolled container does not buy the movement clai
assert.equal(spy.calls(), 1);
});
+// Android reports `checked`, and a swipe that starts on a switch can flip it. On its own the flip is
+// identity-invariant and reads as an unchanged surface. Beside a change elsewhere, such as the status
+// clock ticking, the pair reads as changed, and the within-container check then sees a switch whose
+// key differs at the same rect: a state flip, not the list moving, so it buys no movement claim.
+test('a toggle the swipe flipped inside the container does not buy the movement claim', async () => {
+ const chrome = {
+ type: 'Image',
+ identifier: 'status-clock',
+ label: '2:40',
+ rect: { x: 20, y: 20, width: 60, height: 20 },
+ } as SnapshotNode;
+ const toggle = (checked: boolean) =>
+ ({
+ type: 'android.widget.Switch',
+ identifier: 'wifi-switch',
+ label: 'Wi-Fi switch',
+ checked,
+ rect: { x: 300, y: 320, width: 60, height: 40 },
+ }) as SnapshotNode;
+ const { observation } = observe({
+ baseline: baselineOf([...screen(0), chrome, toggle(false)]),
+ screens: [[...screen(0), { ...chrome, label: '2:41' } as SnapshotNode, toggle(true)]],
+ });
+
+ assert.equal(await observation, 'unobserved');
+ assertWithheld('change-outside-container');
+});
+
test('a surface with no container to confine the claim to keeps the whole-surface answer', async () => {
const rowsOnly = (offset: number) => screen(offset).filter((node) => node.type !== 'ScrollView');
const { observation } = observe({
diff --git a/src/daemon/handlers/__tests__/snapshot-handler-capture-retry.test.ts b/src/daemon/handlers/__tests__/snapshot-handler-capture-retry.test.ts
index 59350cb063..0d00967083 100644
--- a/src/daemon/handlers/__tests__/snapshot-handler-capture-retry.test.ts
+++ b/src/daemon/handlers/__tests__/snapshot-handler-capture-retry.test.ts
@@ -96,7 +96,9 @@ test('captureSnapshot lazily retries pending no-change touch before returning fr
attemptsRemaining: 2,
preSignature: [
{
- key: 'open-feed|Open feed||Button||enabled|unselected|hittable|#0',
+ key: 'open-feed|Open feed||Button||hittable|#0',
+ identity: 'open-feed|Open feed||Button',
+ content: 'open-feed|Open feed||Button',
x: 20,
y: 120,
width: 160,
@@ -239,7 +241,9 @@ test('captureSnapshot retries pending tap outcome before post-gesture stabilizat
attemptsRemaining: 2,
preSignature: [
{
- key: '|Navigate to Third||android.widget.Button||enabled|unselected|hittable|#0',
+ key: '|Navigate to Third||android.widget.Button||hittable|#0',
+ identity: '|Navigate to Third||android.widget.Button',
+ content: '|Navigate to Third||android.widget.Button',
x: 302,
y: 1301,
width: 476,
diff --git a/src/daemon/interaction-outcome-policy.ts b/src/daemon/interaction-outcome-policy.ts
index 3519a01e83..40c78cea55 100644
--- a/src/daemon/interaction-outcome-policy.ts
+++ b/src/daemon/interaction-outcome-policy.ts
@@ -2,6 +2,7 @@ import type { CommandFlags } from '@agent-device/contracts/command';
import { isMobilePlatform } from '@agent-device/kernel/device';
import type { Rect, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot';
import { collectKeyboardChromeRefs } from '@agent-device/capture-kit/snapshot-chrome';
+import { stateMarkers } from '@agent-device/capture-kit/snapshot-lines';
import { emitDiagnostic } from '@agent-device/host-kit/diagnostics';
import { isViewportRootNode } from '@agent-device/contracts/snapshot';
import { contextFromFlags, type DaemonCommandContext } from './context.ts';
@@ -465,8 +466,11 @@ export function summarizeDiscriminatingSurfaceDivergence(
}
/**
- * Whether the DISCRIMINATING entries inside `rect` differ across a gesture, on the same key-matched,
- * rect-tolerant view `haveIdenticalDiscriminatingSurfaces` uses — restricted to one region.
+ * Whether the DISCRIMINATING entries inside `rect` moved across a gesture: one left or entered the
+ * region, or its rect moved beyond tolerance. Entries match on `content`: the flip-tolerant
+ * `identity` where they have one, the type and role of an anonymous node otherwise, told apart by
+ * document order when repeated. A scroll moves content, while a state flip inside the container (a
+ * switch the swipe brushed, a row it selected) changes the key at the same rect and is not movement.
*
* A whole-surface difference is not automatically the gesture's doing. A captured tree carries system
* chrome with it, and on Android the status bar clocks and icons change on their own while the app's
@@ -478,17 +482,29 @@ export function discriminatingSurfaceChangedWithinRect(
after: InteractionSurfaceSignature,
rect: Rect,
): boolean {
- const beforeInRect = discriminatingEntriesWithinRect(before, rect);
- const afterByKey = new Map(
- discriminatingEntriesWithinRect(after, rect).map((entry) => [entry.key, entry]),
- );
- for (const entry of beforeInRect) {
- const other = afterByKey.get(entry.key);
+ const beforeInRect = contentKeyed(discriminatingEntriesWithinRect(before, rect));
+ const afterInRect = contentKeyed(discriminatingEntriesWithinRect(after, rect));
+ for (const [content, entry] of beforeInRect) {
+ const other = afterInRect.get(content);
if (!other) return true;
if (!rectsWithinTolerance(entry, other)) return true;
- afterByKey.delete(entry.key);
+ afterInRect.delete(content);
+ }
+ return afterInRect.size > 0;
+}
+
+/** Entries by what they are rather than the state they are in; repeated content is told apart by document order. */
+function contentKeyed(
+ entries: InteractionSurfaceSignature,
+): Map {
+ const occurrences = new Map();
+ const keyed = new Map();
+ for (const entry of entries) {
+ const occurrence = occurrences.get(entry.content) ?? 0;
+ occurrences.set(entry.content, occurrence + 1);
+ keyed.set(`${entry.content}|#${occurrence}`, entry);
}
- return afterByKey.size > 0;
+ return keyed;
}
function discriminatingEntriesWithinRect(
@@ -541,6 +557,7 @@ function buildInteractionSurfaceEntry(
return {
key: `${semanticKey}|#${occurrence}`,
...(identity ? { identity } : {}),
+ content: interactionSurfaceContent(node, identity),
x: Math.round(node.rect.x),
y: Math.round(node.rect.y),
width: Math.round(node.rect.width),
@@ -549,10 +566,15 @@ function buildInteractionSurfaceEntry(
};
}
+/** What the element is without the state it is in: its identity, else the type and role of an anonymous node. */
+function interactionSurfaceContent(node: SnapshotNode, identity: string | undefined): string {
+ return identity ?? `${node.type ?? ''}|${node.role ?? ''}`;
+}
+
/**
* What the element IS — never where it sits, and never volatile state a gesture
* is expected to change. `interactionSurfaceSemanticKey` deliberately folds in
- * `hittable`/`enabled`/`selected` and an occurrence index, which is right for
+ * the states `stateMarkers` prints, `hittable`, and an occurrence index, which is right for
* "did these two back-to-back captures agree" and wrong for "is this the same
* element as before the gesture": scrolling flips `hittable` the moment a
* node's centre leaves the viewport, so keying on it evicts precisely the
@@ -584,6 +606,11 @@ function isNonDiscriminatingSurfaceNode(
return isViewportRootNode(node) || (node.ref !== undefined && keyboardChromeRefs.has(node.ref));
}
+/**
+ * What the element is and the state it is in. The states are the ones `stateMarkers` prints, so the
+ * outcome lane, the unchanged-snapshot comparison, and the diff weigh one list: a tap whose only
+ * effect is a toggle is a change here, not a no-op to retry.
+ */
function interactionSurfaceSemanticKey(node: SnapshotNode): string | undefined {
const semanticKey = [
node.identifier,
@@ -591,8 +618,7 @@ function interactionSurfaceSemanticKey(node: SnapshotNode): string | undefined {
node.value,
node.type,
node.role,
- node.enabled === false ? 'disabled' : 'enabled',
- node.selected === true ? 'selected' : 'unselected',
+ ...stateMarkers(node),
node.hittable === true ? 'hittable' : 'not-hittable',
]
.map((value) => (typeof value === 'string' ? value.trim() : ''))
diff --git a/src/daemon/response-views.ts b/src/daemon/response-views.ts
index 2454c8c59a..ee18e4a8c7 100644
--- a/src/daemon/response-views.ts
+++ b/src/daemon/response-views.ts
@@ -114,6 +114,7 @@ const SELECTOR_DIGEST_NODE_FIELDS = [
'identifier',
'enabled',
'selected',
+ 'checked',
'focused',
'heading',
'roleDescription',
diff --git a/src/daemon/session-state.ts b/src/daemon/session-state.ts
index d7ef7e626d..354b928636 100644
--- a/src/daemon/session-state.ts
+++ b/src/daemon/session-state.ts
@@ -53,6 +53,13 @@ export type InteractionSurfaceEntry = {
* comparison entirely.
*/
identity?: string;
+ /**
+ * What the element is without the state it is in: `identity` where the node
+ * has one, else its type and role. A within-container movement check compares
+ * entries on this, in document order when repeated, because a state flip
+ * changes `key` at the same rect and is not movement.
+ */
+ content: string;
x: number;
y: number;
width: number;
diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md
index 8a74a11cbb..d9eb54e337 100644
--- a/website/docs/docs/commands.md
+++ b/website/docs/docs/commands.md
@@ -391,8 +391,8 @@ agent-device get attrs @e1
act on stale React Native screens. API 23 cannot report sibling `drawing-order`, so this scan fails
conservative and `androidSnapshot.occlusionScanUnavailable: true` discloses the difference.
Android `--raw` is the acquired tree: it also keeps nodes Android marks invisible and stale
- application windows. The helper does not report `checked`/`checkable` state, and it caps
- captures at 5000 nodes before any `--scope` applies (`truncated: true`).
+ application windows. The helper caps captures at 5000 nodes before any `--scope` applies
+ (`truncated: true`).
- `truncated: true` means the backend cut the capture at one of its limits — the Android helper
and the iOS Simulator AX bridge at 5000 nodes, the XCTest runner and the web provider at their
own bounds. Every backend walks the tree in document order, so what falls off is what comes
diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md
index 99b08b5bf1..f07adc2541 100644
--- a/website/docs/docs/snapshots.md
+++ b/website/docs/docs/snapshots.md
@@ -117,16 +117,21 @@ the strategy owns which tiers it may use.
## Android node metadata
Android snapshot nodes and `get attrs` (including the digest response) carry the native
-`selected`, `heading`, `roleDescription`, `editable`, `password`, `hintShowing`, `placeholder`,
-`selectionStart`, and `selectionEnd` facts whenever the accessibility tree reports them. Explicit
-`false` and `0` are kept; an absent field means the fact was unavailable, not false. `hintShowing`
-and `placeholder` need Android API 26 or later, `heading` API 28 or later.
+`selected`, `checked`, `heading`, `roleDescription`, `editable`, `password`, `hintShowing`,
+`placeholder`, `selectionStart`, and `selectionEnd` facts whenever the accessibility tree reports
+them. Explicit `false` and `0` are kept; an absent field means the fact was unavailable, not false.
+`hintShowing` and `placeholder` need Android API 26 or later, `heading` API 28 or later.
- `selected` is the accessibility selected state an app sets on a control — the active bottom-tab or
segmented-control item, or the chosen row of a list. Android reports it explicitly as `true` or
`false`; an older helper APK omits the field, which means the answer is unavailable rather than
unselected. Snapshot text marks the node `[selected]`, and `is selected`, a `selected=true`
selector, and a Maestro `selected:` qualifier all match on it.
+- `checked` is the checked state of a checkable control — a switch, a checkbox, a radio button, or a
+ view an app marked checkable. Android reports it as `true` or `false` on those nodes only; a node
+ that cannot be checked, or an older helper APK, omits the field. Snapshot text marks the node
+ `[checked]` or `[unchecked]`, so a toggle that reads as plain text is one Android did not report as
+ checkable.
- `heading` is the accessibility heading flag an app sets on a node, the way React Native's
`accessibilityRole="header"` does on a plain `View`; it is present only as `true`.
- `roleDescription` is the localized role description an app sets beside the native class, verbatim