fix(android): carry the heading flag and the role description on the snapshot node (#2915)

React Native puts a header, a tab, a tab list, a radio group, a link, or a menu on a plain android.view.View and tells the accessibility tree what it is through two facts: the heading flag and the role description androidx stores in the node extras. The helper serialized neither, and the host reads only the helper's XML, so every one of them was a nameless View to an agent and to `get attrs`. The helper now writes `heading` when the node reports it (API 28 or later) and `role-description` when the app set one; the parser, the Android hierarchy node, and the published snapshot node carry them to `get attrs`, the unchanged-snapshot comparison, and the selector digest. The class stays the type, and snapshot text is unchanged: mapping a description to a role is the consumer's call.

Author: okwasniewski

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## Unreleased
 
+- Fixed (android): snapshot nodes and `get attrs` carry the accessibility `heading` flag and the
+  `roleDescription` an app set on a node. React Native puts a header, a tab, a tab list, a link, or a
+  menu on a plain `android.view.View` and tells the accessibility tree what it is through these two
+  facts; the helper never serialized either, so every one of them was a nameless `View` to an agent.
+  The helper now writes `heading` when the node reports it (API 28 or later) and `role-description`
+  when the app set one, and the parser, the Android hierarchy node, and the published snapshot node
+  carry them to `get attrs` and the selector digest. The class stays the `type`.
 - Fixed (ios): `perf cpu profile report --kind xctrace` on Xcode 27 no longer fails with
   `Apple xctrace CPU report contained no samples` on a trace that holds thousands of samples. Xcode
   27 exports each `time-profile` sample stack as `<tagged-backtrace>` instead of `<backtrace>`, and

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java

@@ -53,6 +53,8 @@ static void appendNode(
     appendAttribute(xml, "class", node.getClassName());
     appendNonEmptyAttribute(xml, "package", node.getPackageName());
     appendNonEmptyAttribute(xml, "content-desc", node.getContentDescription());
+    appendNonEmptyAttribute(xml, "role-description", roleDescription(node));
+    appendTrueAttribute(xml, "heading", isHeading(node));
     appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser()));
     appendDrawingOrderAttribute(xml, node);
     appendTrueAttribute(xml, "clickable", node.isClickable());
@@ -146,6 +148,19 @@ private static void appendTrueAttribute(StringBuilder xml, String name, boolean
     }
   }
 
+  // The platform node has no role description getter: androidx writes the value an app set
+  // (AccessibilityNodeInfoCompat.setRoleDescription) into the node extras under this key, and
+  // TalkBack reads it from there.
+  private static CharSequence roleDescription(AccessibilityNodeInfo node) {
+    return node.getExtras().getCharSequence("AccessibilityNodeInfo.roleDescription");
+  }
+
+  // isHeading() arrived in API 28. Older releases keep the compat flag in an extras bit this
+  // helper does not read, so a heading on API 23-27 reports nothing.
+  private static boolean isHeading(AccessibilityNodeInfo node) {
+    return Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && node.isHeading();
+  }
+
   // Declared residue (agent-device #1832): checked / checkable / long-clickable are not serialized,
   // so toggle state is invisible to agents. Adding them is a helper protocol change (new attributes
   // + host parser + fields on the wire node), tracked there.

packages/kernel/src/snapshot.ts

@@ -238,6 +238,10 @@ export type RawSnapshotNode = {
   enabled?: boolean;
   selected?: boolean;
   focused?: boolean;
+  /** Accessibility heading flag an app set on the node; absent means not a heading or unavailable. */
+  heading?: boolean;
+  /** Localized role description an app set beside the native class, verbatim (`Tab`, `Tab List`, `Link`). */
+  roleDescription?: string;
   /** Native accessibility facts; absent means unavailable, not false. */
   editable?: boolean;
   password?: boolean;

packages/platform-android/src/__tests__/ui-hierarchy-role-facts.test.ts

@@ -0,0 +1,60 @@
+import { expect, test } from 'vitest';
+import { buildUiHierarchySnapshot, parseUiHierarchyTree } from '../ui-hierarchy.ts';
+
+// A React Native screen: `accessibilityRole="header"` is a plain View the helper flags as a heading,
+// a tab bar is a View with the role description the app set, and a label carries neither.
+const ROLE_FACTS_XML = `<hierarchy>
+  <node class="android.widget.FrameLayout" resource-id="root" bounds="[0,0][400,800]"
+    window-index="0" window-type="1" window-layer="1" window-active="true" window-focused="true"
+    window-bounds="[0,0][400,800]" visible-to-user="true" enabled="true">
+    <node class="android.view.View" resource-id="inventory-header" text="Inventory" heading="true"
+      bounds="[0,0][400,60]" enabled="true" visible-to-user="true" />
+    <node class="android.view.View" resource-id="section-tabs" role-description="tab list"
+      bounds="[0,60][400,120]" enabled="true" visible-to-user="true">
+      <node class="android.view.View" resource-id="tab-fields" text="Fields" role-description="tab"
+        bounds="[0,60][200,120]" clickable="true" enabled="true" visible-to-user="true" />
+    </node>
+    <node class="android.widget.TextView" resource-id="plain-label" text="Wi-Fi"
+      bounds="[0,120][400,180]" enabled="true" visible-to-user="true" />
+  </node>
+</hierarchy>`;
+
+function nodesById(raw: boolean, interactiveOnly = false) {
+  const { nodes } = buildUiHierarchySnapshot(parseUiHierarchyTree(ROLE_FACTS_XML), undefined, {
+    raw,
+    interactiveOnly,
+  });
+  return (identifier: string) => nodes.find((node) => node.identifier === identifier);
+}
+
+test.each([
+  { raw: false, interactiveOnly: false },
+  { raw: true, interactiveOnly: false },
+  { raw: true, interactiveOnly: true },
+])(
+  'the heading flag and the role description reach snapshot nodes (raw=$raw, -i=$interactiveOnly)',
+  ({ raw, interactiveOnly }) => {
+    const byId = nodesById(raw, interactiveOnly);
+    expect(byId('inventory-header')?.heading).toBe(true);
+    expect(byId('section-tabs')?.roleDescription).toBe('tab list');
+    expect(byId('tab-fields')?.roleDescription).toBe('tab');
+  },
+);
+
+test('a node without either fact carries neither key once serialized', () => {
+  const byId = nodesById(false);
+  const label = JSON.parse(JSON.stringify(byId('plain-label')));
+  expect(label).not.toHaveProperty('heading');
+  expect(label).not.toHaveProperty('roleDescription');
+  // A heading is not a role description and a role description is not a heading.
+  expect(JSON.parse(JSON.stringify(byId('inventory-header')))).not.toHaveProperty(
+    'roleDescription',
+  );
+  expect(JSON.parse(JSON.stringify(byId('tab-fields')))).not.toHaveProperty('heading');
+});
+
+test('the class stays the type: a role description refines nothing on its own', () => {
+  const byId = nodesById(false);
+  expect(byId('tab-fields')?.type).toBe('android.view.View');
+  expect(byId('inventory-header')?.label).toBe('Inventory');
+});

packages/platform-android/src/ui-hierarchy-builder.ts

@@ -354,6 +354,8 @@ function createAndroidRawSnapshotNode(
     enabled: node.enabled,
     focused: node.focused,
     selected: node.selected,
+    heading: node.heading,
+    roleDescription: node.roleDescription,
     editable: node.editable,
     password: node.password,
     hintShowing: node.hintShowing,

packages/platform-android/src/ui-hierarchy-node.ts

@@ -17,6 +17,8 @@ export type AndroidUiHierarchy = {
   visibleToUser?: boolean;
   focused?: boolean;
   selected?: boolean;
+  heading?: boolean;
+  roleDescription?: string;
   editable?: boolean;
   password?: boolean;
   hintShowing?: boolean;

packages/platform-android/src/ui-hierarchy.ts

@@ -35,6 +35,10 @@ export type AndroidUiNodeMetadata = {
   focusable?: boolean;
   focused?: boolean;
   selected?: boolean;
+  /** Helper-only: the accessibility heading flag an app set on the node (API 28 or later). */
+  heading?: boolean;
+  /** Helper-only: the localized role description an app set beside the class, verbatim. */
+  roleDescription?: string;
   password?: boolean;
   editable?: boolean;
   selectionStart?: number;
@@ -132,6 +136,15 @@ function readNodeAttributes(node: string): Omit<AndroidUiNodeMetadata, 'rect'> {
     const value = parseBounds(getAttr(name));
     return value === undefined ? {} : ({ [key]: value } as Pick<AndroidUiNodeMetadata, Key>);
   };
+  const optionalStringAttr = <Key extends keyof AndroidUiNodeMetadata>(
+    key: Key,
+    name: string,
+  ): Partial<Pick<AndroidUiNodeMetadata, Key>> => {
+    const value = getAttr(name);
+    return value === null || value === ''
+      ? {}
+      : ({ [key]: value } as Pick<AndroidUiNodeMetadata, Key>);
+  };
   const optionalBoolAttr = <Key extends keyof AndroidUiNodeMetadata>(
     key: Key,
     name: string,
@@ -157,6 +170,8 @@ function readNodeAttributes(node: string): Omit<AndroidUiNodeMetadata, 'rect'> {
     ...optionalBoolAttr('hintShowing', 'hint-showing'),
     ...optionalBoolAttr('visibleToUser', 'visible-to-user'),
     ...optionalBoolAttr('selected', 'selected'),
+    ...optionalBoolAttr('heading', 'heading'),
+    ...optionalStringAttr('roleDescription', 'role-description'),
     ...optionalNumberAttr('drawingOrder', 'drawing-order'),
     ...optionalBoolAttr('scrollable', 'scrollable'),
     ...optionalBoolAttr('canScrollForward', 'can-scroll-forward'),
@@ -311,6 +326,8 @@ function normalizeAndroidUiHierarchyNode(
       enabled: attrs.enabled,
       focused: attrs.focused,
       selected: attrs.selected,
+      heading: attrs.heading,
+      roleDescription: attrs.roleDescription,
       editable: attrs.editable,
       password: attrs.password,
       hintShowing: attrs.hintShowing,

src/__tests__/android-ui-hierarchy.test.ts

@@ -236,6 +236,42 @@ test('a published Android snapshot answers a selected-qualified read (#2462)', a
   );
 });
 
+// A React Native screen: the header role is a View flagged as a heading, the tab bar and its tabs
+// are Views with the role description the app set, and the helper writes neither on a plain label.
+const ANDROID_ROLE_FACTS_XML = `<hierarchy>
+  <node class="android.widget.FrameLayout" resource-id="com.example.app:id/root" bounds="[0,0][390,300]" enabled="true" visible-to-user="true">
+    <node class="android.view.View" resource-id="com.example.app:id/header" text="Inventory" heading="true" bounds="[0,0][390,60]" enabled="true" visible-to-user="true"/>
+    <node class="android.view.View" resource-id="com.example.app:id/tabs" role-description="tab list" bounds="[0,60][390,120]" enabled="true" visible-to-user="true">
+      <node class="android.view.View" resource-id="com.example.app:id/tab-fields" text="Fields" role-description="tab" bounds="[0,60][195,120]" clickable="true" enabled="true" visible-to-user="true"/>
+    </node>
+    <node class="android.widget.TextView" resource-id="com.example.app:id/label" text="Wi-Fi" bounds="[0,120][390,180]" enabled="true" visible-to-user="true"/>
+  </node>
+</hierarchy>`;
+
+test('a published Android snapshot carries the heading flag and the role description', () => {
+  const nodes = publishUiHierarchy(ANDROID_ROLE_FACTS_XML).nodes;
+  const byId = (identifier: string) => nodes.find((node) => node.identifier === identifier)!;
+
+  assert.equal(byId('com.example.app:id/header').heading, true);
+  assert.equal(byId('com.example.app:id/tabs').roleDescription, 'tab list');
+  assert.equal(byId('com.example.app:id/tab-fields').roleDescription, 'tab');
+  assert.equal(byId('com.example.app:id/label').heading, undefined);
+  assert.equal(byId('com.example.app:id/label').roleDescription, undefined);
+  assert.deepEqual(
+    Array.from(androidUiNodes(ANDROID_ROLE_FACTS_XML)).map((node) => [
+      node.heading,
+      node.roleDescription,
+    ]),
+    [
+      [undefined, undefined],
+      [true, undefined],
+      [undefined, 'tab list'],
+      [undefined, 'tab'],
+      [undefined, undefined],
+    ],
+  );
+});
+
 test('parseUiHierarchy discards stale inactive Android application windows', () => {
   const xml = `<hierarchy>
   <node class="android.widget.FrameLayout" package="com.example.app" bounds="[0,0][390,844]" window-index="0" window-type="1" window-layer="10" window-active="true" window-focused="true" window-bounds="[0,0][390,844]">

src/commands/capture/runtime/snapshot-unchanged.ts

@@ -112,6 +112,8 @@ const PRESENTATION_SCALAR_FIELDS = {
   enabled: true,
   selected: true,
   focused: true,
+  heading: true,
+  roleDescription: true,
   hittable: true,
   bundleId: true,
   appName: true,

src/daemon/response-views.ts

@@ -115,6 +115,8 @@ const SELECTOR_DIGEST_NODE_FIELDS = [
   'enabled',
   'selected',
   'focused',
+  'heading',
+  'roleDescription',
   'editable',
   'password',
   'hintShowing',