From 926d4f9f72193f30e0d06e1823afcd59e2825413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Thu, 24 Sep 2026 16:46:14 +0200 Subject: [PATCH] fix(android): carry the heading flag and the role description on the snapshot node React Native puts a header, a tab, a tab list, a radio group, a link, or a menu on a plain android.view.View and tells the accessibility tree what it is through two facts: the heading flag and the role description androidx stores in the node extras. The helper serialized neither, and the host reads only the helper's XML, so every one of them was a nameless View to an agent and to `get attrs`. The helper now writes `heading` when the node reports it (API 28 or later) and `role-description` when the app set one; the parser, the Android hierarchy node, and the published snapshot node carry them to `get attrs`, the unchanged-snapshot comparison, and the selector digest. The class stays the type, and snapshot text is unchanged: mapping a description to a role is the consumer's call. --- CHANGELOG.md | 7 +++ .../snapshothelper/AccessibilityTreeXml.java | 15 +++++ packages/kernel/src/snapshot.ts | 4 ++ .../__tests__/ui-hierarchy-role-facts.test.ts | 60 +++++++++++++++++++ .../src/ui-hierarchy-builder.ts | 2 + .../platform-android/src/ui-hierarchy-node.ts | 2 + packages/platform-android/src/ui-hierarchy.ts | 17 ++++++ src/__tests__/android-ui-hierarchy.test.ts | 36 +++++++++++ .../capture/runtime/snapshot-unchanged.ts | 2 + src/daemon/response-views.ts | 2 + website/docs/docs/snapshots.md | 12 +++- 11 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 packages/platform-android/src/__tests__/ui-hierarchy-role-facts.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a238a9932..7ee85fc461 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## Unreleased +- Fixed (android): snapshot nodes and `get attrs` carry the accessibility `heading` flag and the + `roleDescription` an app set on a node. React Native puts a header, a tab, a tab list, a link, or a + menu on a plain `android.view.View` and tells the accessibility tree what it is through these two + facts; the helper never serialized either, so every one of them was a nameless `View` to an agent. + The helper now writes `heading` when the node reports it (API 28 or later) and `role-description` + when the app set one, and the parser, the Android hierarchy node, and the published snapshot node + carry them to `get attrs` and the selector digest. The class stays the `type`. - Fixed (ios): `perf cpu profile report --kind xctrace` on Xcode 27 no longer fails with `Apple xctrace CPU report contained no samples` on a trace that holds thousands of samples. Xcode 27 exports each `time-profile` sample stack as `` instead of ``, and diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java index 0753f242d8..802cd38bb1 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java @@ -53,6 +53,8 @@ static void appendNode( appendAttribute(xml, "class", node.getClassName()); appendNonEmptyAttribute(xml, "package", node.getPackageName()); appendNonEmptyAttribute(xml, "content-desc", node.getContentDescription()); + appendNonEmptyAttribute(xml, "role-description", roleDescription(node)); + appendTrueAttribute(xml, "heading", isHeading(node)); appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser())); appendDrawingOrderAttribute(xml, node); appendTrueAttribute(xml, "clickable", node.isClickable()); @@ -146,6 +148,19 @@ private static void appendTrueAttribute(StringBuilder xml, String name, boolean } } + // The platform node has no role description getter: androidx writes the value an app set + // (AccessibilityNodeInfoCompat.setRoleDescription) into the node extras under this key, and + // TalkBack reads it from there. + private static CharSequence roleDescription(AccessibilityNodeInfo node) { + return node.getExtras().getCharSequence("AccessibilityNodeInfo.roleDescription"); + } + + // isHeading() arrived in API 28. Older releases keep the compat flag in an extras bit this + // helper does not read, so a heading on API 23-27 reports nothing. + private static boolean isHeading(AccessibilityNodeInfo node) { + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && node.isHeading(); + } + // Declared residue (agent-device #1832): checked / checkable / long-clickable are not serialized, // so toggle state is invisible to agents. Adding them is a helper protocol change (new attributes // + host parser + fields on the wire node), tracked there. diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index b0f38a3233..eace66eba8 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -238,6 +238,10 @@ export type RawSnapshotNode = { enabled?: boolean; selected?: boolean; focused?: boolean; + /** Accessibility heading flag an app set on the node; absent means not a heading or unavailable. */ + heading?: boolean; + /** Localized role description an app set beside the native class, verbatim (`Tab`, `Tab List`, `Link`). */ + roleDescription?: string; /** Native accessibility facts; absent means unavailable, not false. */ editable?: boolean; password?: boolean; diff --git a/packages/platform-android/src/__tests__/ui-hierarchy-role-facts.test.ts b/packages/platform-android/src/__tests__/ui-hierarchy-role-facts.test.ts new file mode 100644 index 0000000000..494920afa9 --- /dev/null +++ b/packages/platform-android/src/__tests__/ui-hierarchy-role-facts.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from 'vitest'; +import { buildUiHierarchySnapshot, parseUiHierarchyTree } from '../ui-hierarchy.ts'; + +// A React Native screen: `accessibilityRole="header"` is a plain View the helper flags as a heading, +// a tab bar is a View with the role description the app set, and a label carries neither. +const ROLE_FACTS_XML = ` + + + + + + + +`; + +function nodesById(raw: boolean, interactiveOnly = false) { + const { nodes } = buildUiHierarchySnapshot(parseUiHierarchyTree(ROLE_FACTS_XML), undefined, { + raw, + interactiveOnly, + }); + return (identifier: string) => nodes.find((node) => node.identifier === identifier); +} + +test.each([ + { raw: false, interactiveOnly: false }, + { raw: true, interactiveOnly: false }, + { raw: true, interactiveOnly: true }, +])( + 'the heading flag and the role description reach snapshot nodes (raw=$raw, -i=$interactiveOnly)', + ({ raw, interactiveOnly }) => { + const byId = nodesById(raw, interactiveOnly); + expect(byId('inventory-header')?.heading).toBe(true); + expect(byId('section-tabs')?.roleDescription).toBe('tab list'); + expect(byId('tab-fields')?.roleDescription).toBe('tab'); + }, +); + +test('a node without either fact carries neither key once serialized', () => { + const byId = nodesById(false); + const label = JSON.parse(JSON.stringify(byId('plain-label'))); + expect(label).not.toHaveProperty('heading'); + expect(label).not.toHaveProperty('roleDescription'); + // A heading is not a role description and a role description is not a heading. + expect(JSON.parse(JSON.stringify(byId('inventory-header')))).not.toHaveProperty( + 'roleDescription', + ); + expect(JSON.parse(JSON.stringify(byId('tab-fields')))).not.toHaveProperty('heading'); +}); + +test('the class stays the type: a role description refines nothing on its own', () => { + const byId = nodesById(false); + expect(byId('tab-fields')?.type).toBe('android.view.View'); + expect(byId('inventory-header')?.label).toBe('Inventory'); +}); diff --git a/packages/platform-android/src/ui-hierarchy-builder.ts b/packages/platform-android/src/ui-hierarchy-builder.ts index ea608a2c5e..a3aed983f0 100644 --- a/packages/platform-android/src/ui-hierarchy-builder.ts +++ b/packages/platform-android/src/ui-hierarchy-builder.ts @@ -354,6 +354,8 @@ function createAndroidRawSnapshotNode( enabled: node.enabled, focused: node.focused, selected: node.selected, + heading: node.heading, + roleDescription: node.roleDescription, editable: node.editable, password: node.password, hintShowing: node.hintShowing, diff --git a/packages/platform-android/src/ui-hierarchy-node.ts b/packages/platform-android/src/ui-hierarchy-node.ts index c6e3138c32..4ac4a47a1f 100644 --- a/packages/platform-android/src/ui-hierarchy-node.ts +++ b/packages/platform-android/src/ui-hierarchy-node.ts @@ -17,6 +17,8 @@ export type AndroidUiHierarchy = { visibleToUser?: boolean; focused?: boolean; selected?: boolean; + heading?: boolean; + roleDescription?: string; editable?: boolean; password?: boolean; hintShowing?: boolean; diff --git a/packages/platform-android/src/ui-hierarchy.ts b/packages/platform-android/src/ui-hierarchy.ts index cf5933cec7..006502d9b3 100644 --- a/packages/platform-android/src/ui-hierarchy.ts +++ b/packages/platform-android/src/ui-hierarchy.ts @@ -35,6 +35,10 @@ export type AndroidUiNodeMetadata = { focusable?: boolean; focused?: boolean; selected?: boolean; + /** Helper-only: the accessibility heading flag an app set on the node (API 28 or later). */ + heading?: boolean; + /** Helper-only: the localized role description an app set beside the class, verbatim. */ + roleDescription?: string; password?: boolean; editable?: boolean; selectionStart?: number; @@ -132,6 +136,15 @@ function readNodeAttributes(node: string): Omit { const value = parseBounds(getAttr(name)); return value === undefined ? {} : ({ [key]: value } as Pick); }; + const optionalStringAttr = ( + key: Key, + name: string, + ): Partial> => { + const value = getAttr(name); + return value === null || value === '' + ? {} + : ({ [key]: value } as Pick); + }; const optionalBoolAttr = ( key: Key, name: string, @@ -157,6 +170,8 @@ function readNodeAttributes(node: string): Omit { ...optionalBoolAttr('hintShowing', 'hint-showing'), ...optionalBoolAttr('visibleToUser', 'visible-to-user'), ...optionalBoolAttr('selected', 'selected'), + ...optionalBoolAttr('heading', 'heading'), + ...optionalStringAttr('roleDescription', 'role-description'), ...optionalNumberAttr('drawingOrder', 'drawing-order'), ...optionalBoolAttr('scrollable', 'scrollable'), ...optionalBoolAttr('canScrollForward', 'can-scroll-forward'), @@ -311,6 +326,8 @@ function normalizeAndroidUiHierarchyNode( enabled: attrs.enabled, focused: attrs.focused, selected: attrs.selected, + heading: attrs.heading, + roleDescription: attrs.roleDescription, editable: attrs.editable, password: attrs.password, hintShowing: attrs.hintShowing, diff --git a/src/__tests__/android-ui-hierarchy.test.ts b/src/__tests__/android-ui-hierarchy.test.ts index 62a41f90c9..d9edc4296d 100644 --- a/src/__tests__/android-ui-hierarchy.test.ts +++ b/src/__tests__/android-ui-hierarchy.test.ts @@ -236,6 +236,42 @@ test('a published Android snapshot answers a selected-qualified read (#2462)', a ); }); +// A React Native screen: the header role is a View flagged as a heading, the tab bar and its tabs +// are Views with the role description the app set, and the helper writes neither on a plain label. +const ANDROID_ROLE_FACTS_XML = ` + + + + + + + +`; + +test('a published Android snapshot carries the heading flag and the role description', () => { + const nodes = publishUiHierarchy(ANDROID_ROLE_FACTS_XML).nodes; + const byId = (identifier: string) => nodes.find((node) => node.identifier === identifier)!; + + assert.equal(byId('com.example.app:id/header').heading, true); + assert.equal(byId('com.example.app:id/tabs').roleDescription, 'tab list'); + assert.equal(byId('com.example.app:id/tab-fields').roleDescription, 'tab'); + assert.equal(byId('com.example.app:id/label').heading, undefined); + assert.equal(byId('com.example.app:id/label').roleDescription, undefined); + assert.deepEqual( + Array.from(androidUiNodes(ANDROID_ROLE_FACTS_XML)).map((node) => [ + node.heading, + node.roleDescription, + ]), + [ + [undefined, undefined], + [true, undefined], + [undefined, 'tab list'], + [undefined, 'tab'], + [undefined, undefined], + ], + ); +}); + test('parseUiHierarchy discards stale inactive Android application windows', () => { const xml = ` diff --git a/src/commands/capture/runtime/snapshot-unchanged.ts b/src/commands/capture/runtime/snapshot-unchanged.ts index 61929117be..1ec07b70ee 100644 --- a/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/src/commands/capture/runtime/snapshot-unchanged.ts @@ -112,6 +112,8 @@ const PRESENTATION_SCALAR_FIELDS = { enabled: true, selected: true, focused: true, + heading: true, + roleDescription: true, hittable: true, bundleId: true, appName: true, diff --git a/src/daemon/response-views.ts b/src/daemon/response-views.ts index 62ad99e599..2a21a34a12 100644 --- a/src/daemon/response-views.ts +++ b/src/daemon/response-views.ts @@ -115,6 +115,8 @@ const SELECTOR_DIGEST_NODE_FIELDS = [ 'enabled', 'selected', 'focused', + 'heading', + 'roleDescription', 'editable', 'password', 'hintShowing', diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 9a4a1a2492..f9ef3f34e0 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -117,15 +117,21 @@ the strategy owns which tiers it may use. ## Android node metadata Android snapshot nodes and `get attrs` (including the digest response) carry the native -`selected`, `editable`, `password`, `hintShowing`, `selectionStart`, and `selectionEnd` facts -whenever the accessibility tree reports them. Explicit `false` and `0` are kept; an absent field -means the fact was unavailable, not false. `hintShowing` needs Android API 26 or later. +`selected`, `heading`, `roleDescription`, `editable`, `password`, `hintShowing`, `selectionStart`, +and `selectionEnd` facts whenever the accessibility tree reports them. Explicit `false` and `0` are +kept; an absent field means the fact was unavailable, not false. `hintShowing` needs Android API 26 +or later, `heading` API 28 or later. - `selected` is the accessibility selected state an app sets on a control — the active bottom-tab or segmented-control item, or the chosen row of a list. Android reports it explicitly as `true` or `false`; an older helper APK omits the field, which means the answer is unavailable rather than unselected. Snapshot text marks the node `[selected]`, and `is selected`, a `selected=true` selector, and a Maestro `selected:` qualifier all match on it. +- `heading` is the accessibility heading flag an app sets on a node, the way React Native's + `accessibilityRole="header"` does on a plain `View`; it is present only as `true`. +- `roleDescription` is the localized role description an app sets beside the native class, verbatim + (React Native writes `Tab`, `Tab List`, `Radio Group`, `Link`, `Menu`), when the class alone would + not say what the control is. The `type` stays the class; a consumer maps the description to a role. - `value: ""` is an explicitly empty accessibility text; a missing `value` means no text was reported. The text of an empty field is its hint on modern Android, so check `hintShowing` before reading `value` as the entered contents.