From e3c606e33162496bfe4e8fff8efcaecd478b9227 Mon Sep 17 00:00:00 2001 From: Prajwal Vedula Date: Mon, 5 Oct 2026 11:35:40 -0700 Subject: [PATCH 01/10] feat(android): report the keyboard band on snapshot captures Android captures now carry the same keyboard fact the Apple runner publishes. The snapshot helper already lists every accessibility window with its type and screen bounds, so the band is read from the input method window in the captured tree with no extra adb call. A capture that could not list every window reports unmeasurable instead of absent. --- .../contracts/src/tap-keyboard-occlusion.ts | 7 +- packages/kernel/src/snapshot.ts | 9 ++- .../src/__tests__/snapshot-capture.test.ts | 20 +++++ .../src/__tests__/snapshot-keyboard.test.ts | 74 +++++++++++++++++++ .../platform-android/src/snapshot-capture.ts | 3 + .../platform-android/src/snapshot-keyboard.ts | 47 ++++++++++++ packages/platform-android/src/snapshot.ts | 7 ++ 7 files changed, 160 insertions(+), 7 deletions(-) create mode 100644 packages/platform-android/src/__tests__/snapshot-keyboard.test.ts create mode 100644 packages/platform-android/src/snapshot-keyboard.ts diff --git a/packages/contracts/src/tap-keyboard-occlusion.ts b/packages/contracts/src/tap-keyboard-occlusion.ts index 7df8b81106..6ea6504eca 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.ts @@ -21,11 +21,12 @@ import { normalizeType } from './snapshot-text.ts'; * A capture whose producer measured the band directly publishes a {@link SnapshotKeyboardBandFact} * beside its tree, and the guard measures the tap point against that band — a point-in-rect check * with no geometry to believe. The Apple runner does this from `app.keyboards.firstMatch`, which - * answers in the app's own orientation space (#2660). + * answers in the app's own orientation space (#2660); the Android helper does it from the input + * method window's screen bounds. * * Otherwise the band is derived from the captured tree every acting path already holds, so the guard - * costs no round trip. This is the path for Android's input method nodes and for the producers that - * never see the app's windows (`appium-source`, `limrun-ios-tree`, the runner's own query-sweep tier). + * costs no round trip. This is the path for Android captures that could not list every window and for + * the producers that never see the app's windows (`appium-source`, `limrun-ios-tree`, the runner's own query-sweep tier). * Derivation, the rules that decide whether reported geometry may be measured at all, and the verdict * on a point are proven against `contracts/fixtures/tap-keyboard-occlusion-policy.json`; change a rule * only through that table. diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 6691445f25..684e9e55d1 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -316,7 +316,8 @@ export type HiddenContentHint = { * A keyboard is its own system surface, so it never reaches the tree as a covering sibling of app * content, and a consumer that wants to refuse a tap behind it has to learn where it is from * somewhere (#2589). A producer that can measure the band directly — the Apple runner, from its - * `app.keyboards` query — publishes one fact per capture and says nothing else about it. A consumer therefore gets three + * `app.keyboards` query, and the Android helper, from the input method window in the window list it + * captures — publishes one fact per capture and says nothing else about it. A consumer therefore gets three * answers and no fourth: a band in the same space as every node rect, a proven absence, or a * producer that could not look. * @@ -669,9 +670,9 @@ export type SnapshotState = { */ iosSystemSurfaceBundleId?: string; /** - * iOS: the keyboard band this capture's producer measured, when it measured one. The tap-path - * keyboard guard prefers this over the band it would otherwise derive from `nodes`, because a - * producer that can query the keyboard directly answers in the app's own orientation space and + * iOS and Android: the keyboard band this capture's producer measured, when it measured one. The + * tap-path keyboard guard prefers this over the band it would otherwise derive from `nodes`, + * because a producer that can query the keyboard directly answers in the tree's own space and * needs no geometry to be plausible (#2660). Absent means the guard measures the tree as before. */ keyboard?: SnapshotKeyboardBandFact; diff --git a/packages/platform-android/src/__tests__/snapshot-capture.test.ts b/packages/platform-android/src/__tests__/snapshot-capture.test.ts index 3853338b2f..3e4b8e7848 100644 --- a/packages/platform-android/src/__tests__/snapshot-capture.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-capture.test.ts @@ -174,3 +174,23 @@ test('an Android capture with geometry-free nodes still publishes the display it 'the tree really carries no geometry', ); }); + +test('the Android keyboard band survives the publication adapter into the daemon capture', async () => { + const capture = await snapshotAndroid(device, { + helperAdb: helperAdbServing( + { width: 1080, height: 2400 }, + [ + '', + '', + '', + '', + ].join(''), + ), + helperArtifact, + }); + + assert.deepEqual(androidSnapshotPublicationInput(capture).keyboard, { + kind: 'visible', + frame: { x: 0, y: 1500, width: 1080, height: 900 }, + }); +}); diff --git a/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts new file mode 100644 index 0000000000..3a75f0629d --- /dev/null +++ b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts @@ -0,0 +1,74 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { androidSnapshotKeyboardFromTree } from '../snapshot-keyboard.ts'; +import { parseUiHierarchyTree } from '../ui-hierarchy.ts'; + +const APP_WINDOW = + ''; +const GBOARD_WINDOW = + ''; + +const WINDOW_LIST = { captureMode: 'interactive-windows' } as const; + +function tree(...windows: string[]) { + return parseUiHierarchyTree(`${windows.join('')}`); +} + +test('an input method window on screen is the visible keyboard band', () => { + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(GBOARD_WINDOW, APP_WINDOW), WINDOW_LIST), { + kind: 'visible', + frame: { x: 0, y: 1500, width: 1080, height: 900 }, + }); +}); + +test('separate input method windows publish one band covering all of them', () => { + const composer = + ''; + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(composer, GBOARD_WINDOW, APP_WINDOW), WINDOW_LIST), + { kind: 'visible', frame: { x: 0, y: 1380, width: 1080, height: 1020 } }, + ); +}); + +test('a full window list without an input method window proves the keyboard absent', () => { + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(APP_WINDOW), WINDOW_LIST), { + kind: 'absent', + }); +}); + +test('an input method window with empty bounds draws nothing and reads as absent', () => { + const empty = + ''; + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(empty, APP_WINDOW), WINDOW_LIST), { + kind: 'absent', + }); +}); + +test('a capture that never listed the windows cannot prove absence', () => { + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(APP_WINDOW), { captureMode: 'active-window' }), + { kind: 'unmeasurable', reason: 'window-list-unavailable' }, + ); + assert.deepEqual( + androidSnapshotKeyboardFromTree( + tree(''), + WINDOW_LIST, + ), + { kind: 'unmeasurable', reason: 'window-list-unavailable' }, + ); +}); + +test('a truncated capture without an input method window cannot prove absence', () => { + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(APP_WINDOW), { ...WINDOW_LIST, helperTruncated: true }), + { kind: 'unmeasurable', reason: 'capture-truncated' }, + ); + assert.equal( + androidSnapshotKeyboardFromTree(tree(GBOARD_WINDOW, APP_WINDOW), { + ...WINDOW_LIST, + helperTruncated: true, + }).kind, + 'visible', + 'a window the capture did reach is still measured', + ); +}); diff --git a/packages/platform-android/src/snapshot-capture.ts b/packages/platform-android/src/snapshot-capture.ts index 141d83768e..e1b1255a88 100644 --- a/packages/platform-android/src/snapshot-capture.ts +++ b/packages/platform-android/src/snapshot-capture.ts @@ -8,6 +8,7 @@ import { import type { RawSnapshotNode, SnapshotBackend, + SnapshotKeyboardBandFact, SnapshotQualityVerdict, SnapshotViewportSize, } from '@agent-device/kernel/snapshot'; @@ -28,6 +29,8 @@ type AndroidSnapshotCaptureData = Readonly<{ analysis: AndroidSnapshotAnalysis; androidSnapshot: AndroidSnapshotBackendMetadata; quality?: SnapshotQualityVerdict; + /** The input method window the helper captured, if any (see `androidSnapshotKeyboardFromTree`). */ + keyboard?: SnapshotKeyboardBandFact; /** * The screen the captured bounds are measured in (#3182): the helper's own display read, so a * capture of an empty screen still reports it. Derived, never stored, so this carrier cannot drift diff --git a/packages/platform-android/src/snapshot-keyboard.ts b/packages/platform-android/src/snapshot-keyboard.ts new file mode 100644 index 0000000000..e142ee34bd --- /dev/null +++ b/packages/platform-android/src/snapshot-keyboard.ts @@ -0,0 +1,47 @@ +import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import type { Rect, SnapshotKeyboardBandFact } from '@agent-device/kernel/snapshot'; +import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; +import type { AndroidUiHierarchy } from './ui-hierarchy-node.ts'; + +/** `AccessibilityWindowInfo.TYPE_INPUT_METHOD`. */ +const ANDROID_WINDOW_TYPE_INPUT_METHOD = 2; + +/** + * The keyboard band an Android capture measured, read from the window roots the helper already + * captured: each root carries its `AccessibilityWindowInfo` type and screen bounds, so this costs no + * adb call. Bounds are screen pixels, the same space as every node rect. + * + * An input method that draws nothing (agent-device's test IME) puts no window on screen and reads as + * `absent`. Only a capture that listed every window can prove absence: the active-window fallback + * never saw the window list, a root without window metadata cannot be ruled out as the input method, + * and a truncated capture may have stopped before it. + */ +export function androidSnapshotKeyboardFromTree( + tree: AndroidUiHierarchy, + metadata: Pick, +): SnapshotKeyboardBandFact { + const windows = tree.children; + const inputMethodRects = windows + .filter((window) => window.windowType === ANDROID_WINDOW_TYPE_INPUT_METHOD) + .map((window) => window.windowRect) + .filter(isPositiveFiniteRect); + if (inputMethodRects.length > 0) return { kind: 'visible', frame: unionRects(inputMethodRects) }; + if ( + metadata.captureMode !== 'interactive-windows' || + windows.some((window) => window.windowType === undefined) + ) { + return { kind: 'unmeasurable', reason: 'window-list-unavailable' }; + } + if (metadata.helperTruncated === true) { + return { kind: 'unmeasurable', reason: 'capture-truncated' }; + } + return { kind: 'absent' }; +} + +function unionRects(rects: readonly Rect[]): Rect { + const left = Math.min(...rects.map((rect) => rect.x)); + const top = Math.min(...rects.map((rect) => rect.y)); + const right = Math.max(...rects.map((rect) => rect.x + rect.width)); + const bottom = Math.max(...rects.map((rect) => rect.y + rect.height)); + return { x: left, y: top, width: right - left, height: bottom - top }; +} diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index d2593771a6..dfda26476e 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -12,6 +12,7 @@ import { attachRefs, type HiddenContentHint, type RawSnapshotNode, + type SnapshotKeyboardBandFact, type SnapshotOptions, type SnapshotViewportSize, } from '@agent-device/kernel/snapshot'; @@ -77,6 +78,7 @@ import { type AndroidSnapshotPresentationOptions, } from './snapshot-presentation.ts'; import { readAndroidSiblingOrder } from './ui-hierarchy-node.ts'; +import { androidSnapshotKeyboardFromTree } from './snapshot-keyboard.ts'; import { androidSnapshotViewportFromHelperMetadata, createAndroidSnapshotCapture, @@ -133,6 +135,7 @@ export async function snapshotAndroid( const xml = capture.xml; const tree = parseUiHierarchyTree(xml); const androidSnapshot = withOcclusionScanDisclosure(capture.metadata, tree); + const keyboard = androidSnapshotKeyboardFromTree(tree, capture.metadata); const presentationOptions: AndroidUiHierarchySnapshotOptions = { ...options, androidPresentation: { @@ -160,6 +163,7 @@ export async function snapshotAndroid( ...androidSnapshotTruncationFields(truncated), androidSnapshot, quality: { state: 'healthy', backend: 'android-helper' } as const, + keyboard, ...(viewport ? { viewport } : {}), }; return createAndroidSnapshotCapture(result, { @@ -171,6 +175,7 @@ export async function snapshotAndroid( return attachAndroidPresentationFailureEvidence({ failure: error, androidSnapshot, + keyboard, ...(viewport ? { viewport } : {}), }); } @@ -179,6 +184,7 @@ export async function snapshotAndroid( function attachAndroidPresentationFailureEvidence(params: { failure: AndroidSnapshotPresentationFailure; androidSnapshot: AndroidSnapshotBackendMetadata; + keyboard: SnapshotKeyboardBandFact; viewport?: SnapshotViewportSize; }): AndroidSnapshotCapture { return createAndroidSnapshotCapture( @@ -202,6 +208,7 @@ function attachAndroidPresentationFailureEvidence(params: { reason: params.failure.message, reasonCode: params.failure.qualityReasonCode, }, + keyboard: params.keyboard, ...(params.viewport ? { viewport: params.viewport } : {}), }, { From 6c9fe5294916f90df5a10c91fe175ce1c721dab9 Mon Sep 17 00:00:00 2001 From: Prajwal Vedula Date: Mon, 5 Oct 2026 11:54:02 -0700 Subject: [PATCH 02/10] fix(android): report an unmeasured input method window as unmeasurable An input method window whose window-bounds did not parse, or parsed to non-finite numbers, used to fall through to absent. The capture saw the window but never measured it, so it cannot prove there is no keyboard. An input method window with parsed empty bounds still reads as absent. --- .../src/__tests__/snapshot-keyboard.test.ts | 16 +++++++++++++ .../platform-android/src/snapshot-keyboard.ts | 23 +++++++++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts index 3a75f0629d..030ee9a847 100644 --- a/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts @@ -44,6 +44,22 @@ test('an input method window with empty bounds draws nothing and reads as absent }); }); +test('an input method window whose bounds did not measure cannot prove absence', () => { + const huge = '9'.repeat(400); + for (const windowBounds of [ + '', + ' window-bounds="unknown"', + ` window-bounds="[0,1500][${huge},2400]"`, + ]) { + const unmeasured = ``; + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(unmeasured, APP_WINDOW), WINDOW_LIST), + { kind: 'unmeasurable', reason: 'window-bounds-unavailable' }, + windowBounds, + ); + } +}); + test('a capture that never listed the windows cannot prove absence', () => { assert.deepEqual( androidSnapshotKeyboardFromTree(tree(APP_WINDOW), { captureMode: 'active-window' }), diff --git a/packages/platform-android/src/snapshot-keyboard.ts b/packages/platform-android/src/snapshot-keyboard.ts index e142ee34bd..18d7dc8c98 100644 --- a/packages/platform-android/src/snapshot-keyboard.ts +++ b/packages/platform-android/src/snapshot-keyboard.ts @@ -12,20 +12,27 @@ const ANDROID_WINDOW_TYPE_INPUT_METHOD = 2; * adb call. Bounds are screen pixels, the same space as every node rect. * * An input method that draws nothing (agent-device's test IME) puts no window on screen and reads as - * `absent`. Only a capture that listed every window can prove absence: the active-window fallback - * never saw the window list, a root without window metadata cannot be ruled out as the input method, - * and a truncated capture may have stopped before it. + * `absent`, and so does an input method window whose bounds parsed to an empty box. One whose bounds + * did not parse, or parsed to non-finite numbers, was seen but not measured. Only a capture that + * listed every window can prove absence: the active-window fallback never saw the window list, a + * root without window metadata cannot be ruled out as the input method, and a truncated capture may + * have stopped before it. */ export function androidSnapshotKeyboardFromTree( tree: AndroidUiHierarchy, metadata: Pick, ): SnapshotKeyboardBandFact { const windows = tree.children; - const inputMethodRects = windows - .filter((window) => window.windowType === ANDROID_WINDOW_TYPE_INPUT_METHOD) + const inputMethodWindows = windows.filter( + (window) => window.windowType === ANDROID_WINDOW_TYPE_INPUT_METHOD, + ); + const inputMethodRects = inputMethodWindows .map((window) => window.windowRect) .filter(isPositiveFiniteRect); if (inputMethodRects.length > 0) return { kind: 'visible', frame: unionRects(inputMethodRects) }; + if (!inputMethodWindows.every((window) => isEmptyFiniteRect(window.windowRect))) { + return { kind: 'unmeasurable', reason: 'window-bounds-unavailable' }; + } if ( metadata.captureMode !== 'interactive-windows' || windows.some((window) => window.windowType === undefined) @@ -38,6 +45,12 @@ export function androidSnapshotKeyboardFromTree( return { kind: 'absent' }; } +function isEmptyFiniteRect(rect: Rect | undefined): boolean { + if (!rect) return false; + if (![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite)) return false; + return rect.width === 0 || rect.height === 0; +} + function unionRects(rects: readonly Rect[]): Rect { const left = Math.min(...rects.map((rect) => rect.x)); const top = Math.min(...rects.map((rect) => rect.y)); From d2e09817842a9aefe07137a2a729e4604fd4476d Mon Sep 17 00:00:00 2001 From: Prajwal Vedula Date: Mon, 5 Oct 2026 11:54:02 -0700 Subject: [PATCH 03/10] test(android): match helper window and node counts to the keyboard fixture The keyboard capture fixture has two window roots and four nodes, so the helper metadata served with it now says so. Other captures keep the single-window defaults. --- .../platform-android/src/__tests__/snapshot-capture.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/platform-android/src/__tests__/snapshot-capture.test.ts b/packages/platform-android/src/__tests__/snapshot-capture.test.ts index 3e4b8e7848..3b76faad1c 100644 --- a/packages/platform-android/src/__tests__/snapshot-capture.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-capture.test.ts @@ -28,6 +28,7 @@ afterEach(async () => { function helperAdbServing( display: { width?: number; height?: number } = {}, xml: string = SCREEN_XML, + counts: { windowCount: number; nodeCount: number } = { windowCount: 1, nodeCount: 1 }, ): AndroidAdbExecutor { const displayKeys = display.width !== undefined && display.height !== undefined @@ -54,8 +55,8 @@ function helperAdbServing( 'INSTRUMENTATION_RESULT: maxNodes=5000', 'INSTRUMENTATION_RESULT: rootPresent=true', 'INSTRUMENTATION_RESULT: captureMode=interactive-windows', - 'INSTRUMENTATION_RESULT: windowCount=1', - 'INSTRUMENTATION_RESULT: nodeCount=1', + `INSTRUMENTATION_RESULT: windowCount=${counts.windowCount}`, + `INSTRUMENTATION_RESULT: nodeCount=${counts.nodeCount}`, 'INSTRUMENTATION_RESULT: truncated=false', 'INSTRUMENTATION_RESULT: elapsedMs=12', 'INSTRUMENTATION_RESULT: pixelDensity=2.625', @@ -185,6 +186,7 @@ test('the Android keyboard band survives the publication adapter into the daemon '', '', ].join(''), + { windowCount: 2, nodeCount: 4 }, ), helperArtifact, }); From f64d86cac3e7aa987dc912deafbe7121f52b2ec9 Mon Sep 17 00:00:00 2001 From: Prajwal Vedula Date: Mon, 5 Oct 2026 12:27:59 -0700 Subject: [PATCH 04/10] refactor(android): keep the keyboard band in the snapshot module Reading the band from its own module added one module to what platform-android/src/mechanics.ts evaluates on import (177 against 176 at the merge-base). It now lives in snapshot.ts, which that closure already loads, so the import graph is the same as on main. --- .../src/__tests__/snapshot-keyboard.test.ts | 2 +- .../platform-android/src/snapshot-keyboard.ts | 60 ------------------- packages/platform-android/src/snapshot.ts | 59 +++++++++++++++++- 3 files changed, 59 insertions(+), 62 deletions(-) delete mode 100644 packages/platform-android/src/snapshot-keyboard.ts diff --git a/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts index 030ee9a847..b88fa49142 100644 --- a/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { androidSnapshotKeyboardFromTree } from '../snapshot-keyboard.ts'; +import { androidSnapshotKeyboardFromTree } from '../snapshot.ts'; import { parseUiHierarchyTree } from '../ui-hierarchy.ts'; const APP_WINDOW = diff --git a/packages/platform-android/src/snapshot-keyboard.ts b/packages/platform-android/src/snapshot-keyboard.ts deleted file mode 100644 index 18d7dc8c98..0000000000 --- a/packages/platform-android/src/snapshot-keyboard.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; -import type { Rect, SnapshotKeyboardBandFact } from '@agent-device/kernel/snapshot'; -import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; -import type { AndroidUiHierarchy } from './ui-hierarchy-node.ts'; - -/** `AccessibilityWindowInfo.TYPE_INPUT_METHOD`. */ -const ANDROID_WINDOW_TYPE_INPUT_METHOD = 2; - -/** - * The keyboard band an Android capture measured, read from the window roots the helper already - * captured: each root carries its `AccessibilityWindowInfo` type and screen bounds, so this costs no - * adb call. Bounds are screen pixels, the same space as every node rect. - * - * An input method that draws nothing (agent-device's test IME) puts no window on screen and reads as - * `absent`, and so does an input method window whose bounds parsed to an empty box. One whose bounds - * did not parse, or parsed to non-finite numbers, was seen but not measured. Only a capture that - * listed every window can prove absence: the active-window fallback never saw the window list, a - * root without window metadata cannot be ruled out as the input method, and a truncated capture may - * have stopped before it. - */ -export function androidSnapshotKeyboardFromTree( - tree: AndroidUiHierarchy, - metadata: Pick, -): SnapshotKeyboardBandFact { - const windows = tree.children; - const inputMethodWindows = windows.filter( - (window) => window.windowType === ANDROID_WINDOW_TYPE_INPUT_METHOD, - ); - const inputMethodRects = inputMethodWindows - .map((window) => window.windowRect) - .filter(isPositiveFiniteRect); - if (inputMethodRects.length > 0) return { kind: 'visible', frame: unionRects(inputMethodRects) }; - if (!inputMethodWindows.every((window) => isEmptyFiniteRect(window.windowRect))) { - return { kind: 'unmeasurable', reason: 'window-bounds-unavailable' }; - } - if ( - metadata.captureMode !== 'interactive-windows' || - windows.some((window) => window.windowType === undefined) - ) { - return { kind: 'unmeasurable', reason: 'window-list-unavailable' }; - } - if (metadata.helperTruncated === true) { - return { kind: 'unmeasurable', reason: 'capture-truncated' }; - } - return { kind: 'absent' }; -} - -function isEmptyFiniteRect(rect: Rect | undefined): boolean { - if (!rect) return false; - if (![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite)) return false; - return rect.width === 0 || rect.height === 0; -} - -function unionRects(rects: readonly Rect[]): Rect { - const left = Math.min(...rects.map((rect) => rect.x)); - const top = Math.min(...rects.map((rect) => rect.y)); - const right = Math.max(...rects.map((rect) => rect.x + rect.width)); - const bottom = Math.max(...rects.map((rect) => rect.y + rect.height)); - return { x: left, y: top, width: right - left, height: bottom - top }; -} diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index dfda26476e..9fe935a4eb 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -5,6 +5,7 @@ import { type NormalizedError, } from '@agent-device/kernel/errors'; import path from 'node:path'; +import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import { emitDiagnostic, withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; import type { SnapshotOptions as InteractorSnapshotOptions } from '@agent-device/contracts/interactor-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -12,6 +13,7 @@ import { attachRefs, type HiddenContentHint, type RawSnapshotNode, + type Rect, type SnapshotKeyboardBandFact, type SnapshotOptions, type SnapshotViewportSize, @@ -78,7 +80,6 @@ import { type AndroidSnapshotPresentationOptions, } from './snapshot-presentation.ts'; import { readAndroidSiblingOrder } from './ui-hierarchy-node.ts'; -import { androidSnapshotKeyboardFromTree } from './snapshot-keyboard.ts'; import { androidSnapshotViewportFromHelperMetadata, createAndroidSnapshotCapture, @@ -865,3 +866,59 @@ function applyHiddenContentHintsToInteractiveNodes( } } } + +/** `AccessibilityWindowInfo.TYPE_INPUT_METHOD`. */ +const ANDROID_WINDOW_TYPE_INPUT_METHOD = 2; + +/** + * The keyboard band an Android capture measured, read from the window roots the helper already + * captured: each root carries its `AccessibilityWindowInfo` type and screen bounds, so this costs no + * adb call. Bounds are screen pixels, the same space as every node rect. + * + * An input method that draws nothing (agent-device's test IME) puts no window on screen and reads as + * `absent`, and so does an input method window whose bounds parsed to an empty box. One whose bounds + * did not parse, or parsed to non-finite numbers, was seen but not measured. Only a capture that + * listed every window can prove absence: the active-window fallback never saw the window list, a + * root without window metadata cannot be ruled out as the input method, and a truncated capture may + * have stopped before it. + */ +export function androidSnapshotKeyboardFromTree( + tree: AndroidUiHierarchy, + metadata: Pick, +): SnapshotKeyboardBandFact { + const windows = tree.children; + const inputMethodWindows = windows.filter( + (window) => window.windowType === ANDROID_WINDOW_TYPE_INPUT_METHOD, + ); + const inputMethodRects = inputMethodWindows + .map((window) => window.windowRect) + .filter(isPositiveFiniteRect); + if (inputMethodRects.length > 0) return { kind: 'visible', frame: unionRects(inputMethodRects) }; + if (!inputMethodWindows.every((window) => isEmptyFiniteRect(window.windowRect))) { + return { kind: 'unmeasurable', reason: 'window-bounds-unavailable' }; + } + if ( + metadata.captureMode !== 'interactive-windows' || + windows.some((window) => window.windowType === undefined) + ) { + return { kind: 'unmeasurable', reason: 'window-list-unavailable' }; + } + if (metadata.helperTruncated === true) { + return { kind: 'unmeasurable', reason: 'capture-truncated' }; + } + return { kind: 'absent' }; +} + +function isEmptyFiniteRect(rect: Rect | undefined): boolean { + if (!rect) return false; + if (![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite)) return false; + return rect.width === 0 || rect.height === 0; +} + +function unionRects(rects: readonly Rect[]): Rect { + const left = Math.min(...rects.map((rect) => rect.x)); + const top = Math.min(...rects.map((rect) => rect.y)); + const right = Math.max(...rects.map((rect) => rect.x + rect.width)); + const bottom = Math.max(...rects.map((rect) => rect.y + rect.height)); + return { x: left, y: top, width: right - left, height: bottom - top }; +} From b4420a74d3b65f3fb4081426426c33f2f85f58ee Mon Sep 17 00:00:00 2001 From: Prajwal Vedula Date: Mon, 5 Oct 2026 13:54:57 -0700 Subject: [PATCH 05/10] fix(android): report the keyboard band unmeasurable when any input method window is unmeasured --- .../src/__tests__/snapshot-keyboard.test.ts | 18 +++++++++++++++ packages/platform-android/src/snapshot.ts | 23 +++++++++++-------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts index b88fa49142..30a7ac1e46 100644 --- a/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts @@ -60,6 +60,24 @@ test('an input method window whose bounds did not measure cannot prove absence', } }); +test('a measured input method window does not hide another one that did not measure', () => { + const unmeasured = + ''; + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(unmeasured, GBOARD_WINDOW, APP_WINDOW), WINDOW_LIST), + { kind: 'unmeasurable', reason: 'window-bounds-unavailable' }, + ); +}); + +test('an empty input method window beside a measured one leaves the measured band', () => { + const empty = + ''; + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(empty, GBOARD_WINDOW, APP_WINDOW), WINDOW_LIST), + { kind: 'visible', frame: { x: 0, y: 1500, width: 1080, height: 900 } }, + ); +}); + test('a capture that never listed the windows cannot prove absence', () => { assert.deepEqual( androidSnapshotKeyboardFromTree(tree(APP_WINDOW), { captureMode: 'active-window' }), diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index 9fe935a4eb..a131ad6a06 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -877,10 +877,12 @@ const ANDROID_WINDOW_TYPE_INPUT_METHOD = 2; * * An input method that draws nothing (agent-device's test IME) puts no window on screen and reads as * `absent`, and so does an input method window whose bounds parsed to an empty box. One whose bounds - * did not parse, or parsed to non-finite numbers, was seen but not measured. Only a capture that - * listed every window can prove absence: the active-window fallback never saw the window list, a - * root without window metadata cannot be ruled out as the input method, and a truncated capture may - * have stopped before it. + * did not parse, or parsed to non-finite numbers, was seen but not measured, so the band is + * unmeasurable even when another input method window did measure. Absence is only read from a + * window-list capture: the active-window fallback never saw the window list, a root without window + * metadata cannot be ruled out as the input method, and a truncated capture may have stopped before + * it. Even then `absent` means no input method window among the roots the helper serialized: the + * helper skips a window whose root is null without a record, which this cannot detect. */ export function androidSnapshotKeyboardFromTree( tree: AndroidUiHierarchy, @@ -890,13 +892,14 @@ export function androidSnapshotKeyboardFromTree( const inputMethodWindows = windows.filter( (window) => window.windowType === ANDROID_WINDOW_TYPE_INPUT_METHOD, ); - const inputMethodRects = inputMethodWindows - .map((window) => window.windowRect) - .filter(isPositiveFiniteRect); - if (inputMethodRects.length > 0) return { kind: 'visible', frame: unionRects(inputMethodRects) }; - if (!inputMethodWindows.every((window) => isEmptyFiniteRect(window.windowRect))) { - return { kind: 'unmeasurable', reason: 'window-bounds-unavailable' }; + const inputMethodRects: Rect[] = []; + for (const { windowRect } of inputMethodWindows) { + if (isPositiveFiniteRect(windowRect)) inputMethodRects.push(windowRect); + else if (!isEmptyFiniteRect(windowRect)) { + return { kind: 'unmeasurable', reason: 'window-bounds-unavailable' }; + } } + if (inputMethodRects.length > 0) return { kind: 'visible', frame: unionRects(inputMethodRects) }; if ( metadata.captureMode !== 'interactive-windows' || windows.some((window) => window.windowType === undefined) From 907ed8ca5da1f2c1cf4f0a877f90ed042803132a Mon Sep 17 00:00:00 2001 From: Prajwal Vedula Date: Mon, 5 Oct 2026 14:11:08 -0700 Subject: [PATCH 06/10] fix(android): report the keyboard band unmeasurable when the helper skipped an input method window The helper drops a listed window whose root reads null or throws without a record, and windowCount counts only the roots it serialized, so the host could not tell an input method window it failed to read from one that was not on screen, and reported the keyboard absent. The helper now publishes the AccessibilityWindowInfo types of those windows as missingRootWindowTypes (empty when every listed window was read), and the host publishes it as androidSnapshot.missingRootWindowTypes. When it names an input method window (type 2), the keyboard band is unmeasurable with reason window-root-unavailable. An older helper omits the field and is trusted on the roots it serialized. --- .../AccessibilityTreeCapture.java | 23 +++++++++++--- .../SnapshotInstrumentation.java | 3 ++ packages/contracts/src/snapshot-types.ts | 6 ++++ .../src/__tests__/snapshot-capture.test.ts | 25 +++++++++++++++ .../__tests__/snapshot-helper-capture.test.ts | 11 +++++++ .../snapshot-helper-session-protocol.test.ts | 16 ++++++++++ .../src/__tests__/snapshot-helper.test.ts | 1 + .../src/__tests__/snapshot-keyboard.test.ts | 31 +++++++++++++++++++ .../src/instrumentation-helper.ts | 10 ++++++ .../src/snapshot-helper-capture.ts | 4 +++ .../src/snapshot-helper-session-protocol.ts | 2 ++ .../src/snapshot-helper-types.ts | 6 ++++ .../platform-android/src/snapshot-types.ts | 6 ++++ packages/platform-android/src/snapshot.ts | 20 ++++++++---- website/docs/docs/snapshots.md | 6 ++++ 15 files changed, 160 insertions(+), 10 deletions(-) diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java index 2904fd6732..4ac7be3db5 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java @@ -5,6 +5,7 @@ import android.os.Build; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityWindowInfo; +import java.util.ArrayList; import java.util.List; /** Captures and serializes the Android accessibility window tree. */ @@ -25,7 +26,10 @@ private static Result captureOnce(UiAutomation automation, int maxDepth, int max StringBuilder xml = new StringBuilder(); xml.append(""); xml.append(""); - int windowCount = appendInteractiveWindowRoots(xml, automation, maxDepth, maxNodes, stats); + List missingRootWindowTypes = new ArrayList<>(); + int windowCount = + appendInteractiveWindowRoots( + xml, automation, maxDepth, maxNodes, stats, missingRootWindowTypes); String captureMode = "interactive-windows"; if (AccessibilityCaptureStabilizer.requiresActiveWindowFallback( windowCount, stats.activeWindowRootMissing)) { @@ -61,7 +65,8 @@ private static Result captureOnce(UiAutomation automation, int maxDepth, int max captureMode, windowCount, stats.nodeCount, - stats.truncated); + stats.truncated, + missingRootWindowTypes); } private static void clearAccessibilityCache(UiAutomation automation) { @@ -118,7 +123,8 @@ private static int appendInteractiveWindowRoots( UiAutomation automation, int maxDepth, int maxNodes, - AccessibilityTreeXml.Stats stats) { + AccessibilityTreeXml.Stats stats, + List missingRootWindowTypes) { List windows; try { windows = automation.getWindows(); @@ -140,6 +146,7 @@ private static int appendInteractiveWindowRoots( windowMetadata = AccessibilityTreeXml.readWindowMetadata(window, windowCount); root = window.getRoot(); if (root == null) { + missingRootWindowTypes.add(windowMetadata.type); stats.activeWindowRootMissing |= activeWindow; stats.focusedNonActiveWindowRootMissing |= focusedNonActiveWindow; if (activeWindow) { @@ -165,6 +172,9 @@ private static int appendInteractiveWindowRoots( // Accessibility windows can disappear while traversing; keep the rest of the snapshot. stats.activeWindowRootMissing |= activeWindow; stats.focusedNonActiveWindowRootMissing |= focusedNonActiveWindow; + if (windowMetadata != null) { + missingRootWindowTypes.add(windowMetadata.type); + } if (activeWindow && windowMetadata != null) { stats.activeWindowMetadata = windowMetadata; } @@ -202,6 +212,9 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture { final int windowCount; final int nodeCount; final boolean truncated; + // AccessibilityWindowInfo types of listed windows whose root read null or threw, so the host + // can tell a window it could not read (an input method, say) from one that was not on screen. + final List missingRootWindowTypes; Result( String xml, @@ -210,7 +223,8 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture { String captureMode, int windowCount, int nodeCount, - boolean truncated) { + boolean truncated, + List missingRootWindowTypes) { this.xml = xml; this.rootPresent = rootPresent; this.foregroundWindowRootsPresent = foregroundWindowRootsPresent; @@ -218,6 +232,7 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture { this.windowCount = windowCount; this.nodeCount = nodeCount; this.truncated = truncated; + this.missingRootWindowTypes = missingRootWindowTypes; } @Override diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index 6026f881e0..1a66f0e139 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -4,6 +4,7 @@ import android.app.UiAutomation; import android.content.res.Resources; import android.os.Bundle; +import android.text.TextUtils; import android.util.Base64; import android.util.DisplayMetrics; import java.io.BufferedReader; @@ -125,6 +126,8 @@ private static void putCaptureMetadata( result.putString("rootPresent", Boolean.toString(capture.rootPresent)); result.putString("captureMode", capture.captureMode); result.putString("windowCount", Integer.toString(capture.windowCount)); + result.putString( + "missingRootWindowTypes", TextUtils.join(",", capture.missingRootWindowTypes)); result.putString("nodeCount", Integer.toString(capture.nodeCount)); result.putString("truncated", Boolean.toString(capture.truncated)); result.putString("elapsedMs", Long.toString(elapsedMs)); diff --git a/packages/contracts/src/snapshot-types.ts b/packages/contracts/src/snapshot-types.ts index 087c3a53b0..bb0f7ba345 100644 --- a/packages/contracts/src/snapshot-types.ts +++ b/packages/contracts/src/snapshot-types.ts @@ -73,6 +73,12 @@ export type AndroidSnapshotBackendMetadata = { captureMode?: string; systemSurfaceOnly?: boolean; windowCount?: number; + /** + * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because + * reading their root returned null or threw (`2` is an input method window); `windowCount` counts + * only the roots it serialized. Empty when every listed window was read. Absent on an older helper. + */ + missingRootWindowTypes?: number[]; nodeCount?: number; helperTruncated?: boolean; elapsedMs?: number; diff --git a/packages/platform-android/src/__tests__/snapshot-capture.test.ts b/packages/platform-android/src/__tests__/snapshot-capture.test.ts index 3b76faad1c..c9f5d9bee2 100644 --- a/packages/platform-android/src/__tests__/snapshot-capture.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-capture.test.ts @@ -29,6 +29,7 @@ function helperAdbServing( display: { width?: number; height?: number } = {}, xml: string = SCREEN_XML, counts: { windowCount: number; nodeCount: number } = { windowCount: 1, nodeCount: 1 }, + resultLines: readonly string[] = [], ): AndroidAdbExecutor { const displayKeys = display.width !== undefined && display.height !== undefined @@ -61,6 +62,7 @@ function helperAdbServing( 'INSTRUMENTATION_RESULT: elapsedMs=12', 'INSTRUMENTATION_RESULT: pixelDensity=2.625', ...displayKeys, + ...resultLines, 'INSTRUMENTATION_CODE: 0', ].join('\n'); return async (args) => { @@ -196,3 +198,26 @@ test('the Android keyboard band survives the publication adapter into the daemon frame: { x: 0, y: 1500, width: 1080, height: 900 }, }); }); + +test('an input method window the helper could not read makes the published keyboard unmeasurable', async () => { + const capture = await snapshotAndroid(device, { + helperAdb: helperAdbServing( + { width: 1080, height: 2400 }, + [ + '', + '', + '', + ].join(''), + { windowCount: 1, nodeCount: 3 }, + ['INSTRUMENTATION_RESULT: missingRootWindowTypes=2'], + ), + helperArtifact, + }); + + const published = androidSnapshotPublicationInput(capture); + assert.deepEqual(published.androidSnapshot?.missingRootWindowTypes, [2]); + assert.deepEqual(published.keyboard, { + kind: 'unmeasurable', + reason: 'window-root-unavailable', + }); +}); diff --git a/packages/platform-android/src/__tests__/snapshot-helper-capture.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-capture.test.ts index 635f28ff43..99d8db46b7 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-capture.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-capture.test.ts @@ -196,6 +196,17 @@ test('parseAndroidSnapshotHelperOutput carries the helper display extent beside assert.equal(parsed.metadata.displayHeight, 2400); }); +test('parseAndroidSnapshotHelperOutput reads the window types the helper could not serialize', () => { + const read = (resultLines: string[]) => + parseAndroidSnapshotHelperOutput( + helperOutput('', resultLines), + ).metadata.missingRootWindowTypes; + + assert.deepEqual(read(['INSTRUMENTATION_RESULT: missingRootWindowTypes=2,1']), [2, 1]); + assert.deepEqual(read(['INSTRUMENTATION_RESULT: missingRootWindowTypes=']), []); + assert.equal(read([]), undefined, 'an older helper omits the field'); +}); + test('parseAndroidSnapshotHelperOutput leaves the display extent absent when the helper omits it (#3182)', () => { const parsed = parseAndroidSnapshotHelperOutput( helperOutput('', [ diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts index 4dbcfb7cd9..f31eb75823 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts @@ -15,6 +15,7 @@ test('parses the session envelope and snapshot metadata', () => { metadata: { captureMode: 'interactive-windows', windowCount: '2', + missingRootWindowTypes: '2', nodeCount: '1', pixelDensity: '2.625', }, @@ -27,6 +28,7 @@ test('parses the session envelope and snapshot metadata', () => { outputFormat: 'uiautomator-xml', captureMode: 'interactive-windows', windowCount: 2, + missingRootWindowTypes: [2], nodeCount: 1, pixelDensity: 2.625, waitForIdleTimeoutMs: undefined, @@ -69,6 +71,20 @@ test('parses the session display extent beside its density and absent headers st assert.equal(sparse.metadata.displayHeight, undefined); }); +test('reads the session window types the helper could not serialize, absent from an older helper', () => { + const xml = ''; + const read = (metadata: Record) => + parseAndroidSnapshotHelperSessionSnapshotResponse( + sessionResponse({ requestId: 'snapshot-1', xml, metadata }), + 'snapshot-1', + ).metadata.missingRootWindowTypes; + + assert.deepEqual(read({ missingRootWindowTypes: '1,2' }), [1, 2]); + assert.deepEqual(read({ missingRootWindowTypes: '' }), []); + assert.equal(read({ missingRootWindowTypes: '1,x' }), undefined); + assert.equal(read({}), undefined); +}); + test('rejects stale and truncated session snapshot responses', () => { const response = sessionResponse({ requestId: 'snapshot-old', diff --git a/packages/platform-android/src/__tests__/snapshot-helper.test.ts b/packages/platform-android/src/__tests__/snapshot-helper.test.ts index 802a5a36aa..0d385912c8 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper.test.ts @@ -78,6 +78,7 @@ test('parseAndroidSnapshotHelperOutput reconstructs XML chunks and metadata', () rootPresent: true, captureMode: 'interactive-windows', windowCount: 2, + missingRootWindowTypes: undefined, nodeCount: 1, truncated: false, elapsedMs: 42, diff --git a/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts index 30a7ac1e46..9dc7cb8ebe 100644 --- a/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts @@ -106,3 +106,34 @@ test('a truncated capture without an input method window cannot prove absence', 'a window the capture did reach is still measured', ); }); + +test('an input method window the helper could not read cannot prove absence', () => { + const skippedInputMethod = { ...WINDOW_LIST, missingRootWindowTypes: [1, 2] }; + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(APP_WINDOW), skippedInputMethod), { + kind: 'unmeasurable', + reason: 'window-root-unavailable', + }); + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(GBOARD_WINDOW, APP_WINDOW), skippedInputMethod), + { kind: 'unmeasurable', reason: 'window-root-unavailable' }, + 'a measured input method window does not hide one the helper could not read', + ); +}); + +test('a skipped window that is not an input method leaves the keyboard answer alone', () => { + const skippedApp = { ...WINDOW_LIST, missingRootWindowTypes: [1, 3] }; + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(APP_WINDOW), skippedApp), { + kind: 'absent', + }); + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(GBOARD_WINDOW, APP_WINDOW), skippedApp), { + kind: 'visible', + frame: { x: 0, y: 1500, width: 1080, height: 900 }, + }); + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(APP_WINDOW), { + ...WINDOW_LIST, + missingRootWindowTypes: [], + }), + { kind: 'absent' }, + ); +}); diff --git a/packages/platform-android/src/instrumentation-helper.ts b/packages/platform-android/src/instrumentation-helper.ts index 50a21b611e..9838c2869f 100644 --- a/packages/platform-android/src/instrumentation-helper.ts +++ b/packages/platform-android/src/instrumentation-helper.ts @@ -92,6 +92,16 @@ export function readInstrumentationResultNumber(value: string | undefined): numb return Number.isFinite(parsed) ? parsed : undefined; } +/** A comma-separated integer list; an empty value is an empty list, a malformed one is absent. */ +export function readInstrumentationResultIntegerList( + value: string | undefined, +): number[] | undefined { + if (value === undefined) return undefined; + if (value === '') return []; + const entries = value.split(',').map(Number); + return entries.every(Number.isInteger) ? entries : undefined; +} + export function readInstrumentationResultBoolean(value: string | undefined): boolean | undefined { if (value === 'true') { return true; diff --git a/packages/platform-android/src/snapshot-helper-capture.ts b/packages/platform-android/src/snapshot-helper-capture.ts index b4ed920a5d..2aecb82e1c 100644 --- a/packages/platform-android/src/snapshot-helper-capture.ts +++ b/packages/platform-android/src/snapshot-helper-capture.ts @@ -9,6 +9,7 @@ import { execFailureDetails } from '@agent-device/host-kit/command'; import { parseInstrumentationRecords, readInstrumentationResultBoolean, + readInstrumentationResultIntegerList, readInstrumentationResultNumber, } from './instrumentation-helper.ts'; import { @@ -487,6 +488,9 @@ function readHelperMetadata(finalResult: Record): AndroidSnapsho rootPresent: readOptionalBoolean(finalResult.rootPresent), captureMode: readOptionalCaptureMode(finalResult.captureMode), windowCount: readOptionalNumber(finalResult.windowCount), + missingRootWindowTypes: readInstrumentationResultIntegerList( + finalResult.missingRootWindowTypes, + ), nodeCount: readOptionalNumber(finalResult.nodeCount), truncated: readOptionalBoolean(finalResult.truncated), elapsedMs: readOptionalNumber(finalResult.elapsedMs), diff --git a/packages/platform-android/src/snapshot-helper-session-protocol.ts b/packages/platform-android/src/snapshot-helper-session-protocol.ts index 8fff3e894b..f7b874ef37 100644 --- a/packages/platform-android/src/snapshot-helper-session-protocol.ts +++ b/packages/platform-android/src/snapshot-helper-session-protocol.ts @@ -6,6 +6,7 @@ import { } from './snapshot-capture-failure-reason.ts'; import { readInstrumentationResultBoolean, + readInstrumentationResultIntegerList, readInstrumentationResultNumber, } from './instrumentation-helper.ts'; import { @@ -300,6 +301,7 @@ function readSessionMetadata(headers: Record): AndroidSnapshotHe ? headers.captureMode : undefined, windowCount: readInstrumentationResultNumber(headers.windowCount), + missingRootWindowTypes: readInstrumentationResultIntegerList(headers.missingRootWindowTypes), nodeCount: readInstrumentationResultNumber(headers.nodeCount), truncated: readInstrumentationResultBoolean(headers.truncated), elapsedMs: readInstrumentationResultNumber(headers.elapsedMs), diff --git a/packages/platform-android/src/snapshot-helper-types.ts b/packages/platform-android/src/snapshot-helper-types.ts index e392c7c2ab..6d31199abd 100644 --- a/packages/platform-android/src/snapshot-helper-types.ts +++ b/packages/platform-android/src/snapshot-helper-types.ts @@ -92,6 +92,12 @@ export type AndroidSnapshotHelperMetadata = { rootPresent?: boolean; captureMode?: AndroidSnapshotCaptureMode; windowCount?: number; + /** + * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because + * reading their root returned null or threw; `windowCount` counts only the roots it did serialize. + * Absent on an older helper. + */ + missingRootWindowTypes?: number[]; nodeCount?: number; truncated?: boolean; elapsedMs?: number; diff --git a/packages/platform-android/src/snapshot-types.ts b/packages/platform-android/src/snapshot-types.ts index 4f086a5b5c..33dada3395 100644 --- a/packages/platform-android/src/snapshot-types.ts +++ b/packages/platform-android/src/snapshot-types.ts @@ -41,6 +41,12 @@ export type AndroidSnapshotBackendMetadata = { captureMode?: AndroidSnapshotCaptureMode; systemSurfaceOnly?: boolean; windowCount?: number; + /** + * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because + * reading their root returned null or threw (`2` is an input method window); `windowCount` counts + * only the roots it serialized. Empty when every listed window was read. Absent on an older helper. + */ + missingRootWindowTypes?: number[]; nodeCount?: number; helperTruncated?: boolean; elapsedMs?: number; diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index a131ad6a06..4d541fb18a 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -515,6 +515,7 @@ function formatAndroidHelperCaptureResult( rootPresent: capture.metadata.rootPresent, captureMode: capture.metadata.captureMode, windowCount: capture.metadata.windowCount, + missingRootWindowTypes: capture.metadata.missingRootWindowTypes, nodeCount: capture.metadata.nodeCount, helperTruncated: capture.metadata.truncated, elapsedMs: capture.metadata.elapsedMs, @@ -878,15 +879,19 @@ const ANDROID_WINDOW_TYPE_INPUT_METHOD = 2; * An input method that draws nothing (agent-device's test IME) puts no window on screen and reads as * `absent`, and so does an input method window whose bounds parsed to an empty box. One whose bounds * did not parse, or parsed to non-finite numbers, was seen but not measured, so the band is - * unmeasurable even when another input method window did measure. Absence is only read from a - * window-list capture: the active-window fallback never saw the window list, a root without window - * metadata cannot be ruled out as the input method, and a truncated capture may have stopped before - * it. Even then `absent` means no input method window among the roots the helper serialized: the - * helper skips a window whose root is null without a record, which this cannot detect. + * unmeasurable even when another input method window did measure. The same holds for an input method + * window the helper listed but could not serialize because its root read null or threw: the helper + * names those windows' types in `missingRootWindowTypes`. Absence is only read from a window-list + * capture: the active-window fallback never saw the window list, a root without window metadata + * cannot be ruled out as the input method, and a truncated capture may have stopped before it. A + * helper too old to report `missingRootWindowTypes` is trusted on the roots it serialized. */ export function androidSnapshotKeyboardFromTree( tree: AndroidUiHierarchy, - metadata: Pick, + metadata: Pick< + AndroidSnapshotBackendMetadata, + 'captureMode' | 'helperTruncated' | 'missingRootWindowTypes' + >, ): SnapshotKeyboardBandFact { const windows = tree.children; const inputMethodWindows = windows.filter( @@ -899,6 +904,9 @@ export function androidSnapshotKeyboardFromTree( return { kind: 'unmeasurable', reason: 'window-bounds-unavailable' }; } } + if (metadata.missingRootWindowTypes?.includes(ANDROID_WINDOW_TYPE_INPUT_METHOD)) { + return { kind: 'unmeasurable', reason: 'window-root-unavailable' }; + } if (inputMethodRects.length > 0) return { kind: 'visible', frame: unionRects(inputMethodRects) }; if ( metadata.captureMode !== 'interactive-windows' || diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index baa3c7b3fb..277bb76cd1 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -148,6 +148,12 @@ report it (a 420 dpi phone reports `2.625`, a `wm density` override included); a in dp divides rects by it and multiplies its points. An older helper omits it. iOS reports points already, so it carries no such factor. +`androidSnapshot.missingRootWindowTypes` lists the `AccessibilityWindowInfo` types of windows the +helper listed but could not serialize, because reading the window's root returned nothing or failed +(`2` is an input method window); `windowCount` counts only the windows it did serialize. It is empty +when every listed window was read. When it names an input method window, the capture reports the +keyboard band as unmeasurable rather than absent. An older helper omits it. + Android snapshot nodes and `get attrs` (including the digest response) carry the native `selected`, `checked`, `heading`, `roleDescription`, `editable`, `password`, `hintShowing`, `placeholder`, `selectionStart`, and `selectionEnd` facts whenever the accessibility tree reports From f028f3ecf6ece3a491b21b9f1ed5a5f0929a4a43 Mon Sep 17 00:00:00 2001 From: Prajwal Vedula Date: Mon, 5 Oct 2026 14:40:35 -0700 Subject: [PATCH 07/10] fix(android): accept only decimal entries in the helper's window type list Number('') is 0 and Number('0x2') is 2, so a list like "1,,2" or "0x2" passed as integers and could put a window type the helper never sent into the keyboard decision. Each entry must now be plain decimal digits, or the list is absent. The docs now describe the field as windows the helper could not serialize, with a null or failing root as examples, since a failure while reading the window's tree also lands there. --- .../__tests__/snapshot-helper-session-protocol.test.ts | 5 +++++ packages/platform-android/src/instrumentation-helper.ts | 9 ++++++--- website/docs/docs/snapshots.md | 4 ++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts index f31eb75823..6358b5f154 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts @@ -82,6 +82,11 @@ test('reads the session window types the helper could not serialize, absent from assert.deepEqual(read({ missingRootWindowTypes: '1,2' }), [1, 2]); assert.deepEqual(read({ missingRootWindowTypes: '' }), []); assert.equal(read({ missingRootWindowTypes: '1,x' }), undefined); + // Number() would read these as 0 or 2; only plain decimal digits count. + assert.equal(read({ missingRootWindowTypes: '1,,2' }), undefined); + assert.equal(read({ missingRootWindowTypes: '0x2' }), undefined); + assert.equal(read({ missingRootWindowTypes: ' 2' }), undefined); + assert.equal(read({ missingRootWindowTypes: '-2' }), undefined); assert.equal(read({}), undefined); }); diff --git a/packages/platform-android/src/instrumentation-helper.ts b/packages/platform-android/src/instrumentation-helper.ts index 9838c2869f..daaf07a5df 100644 --- a/packages/platform-android/src/instrumentation-helper.ts +++ b/packages/platform-android/src/instrumentation-helper.ts @@ -92,14 +92,17 @@ export function readInstrumentationResultNumber(value: string | undefined): numb return Number.isFinite(parsed) ? parsed : undefined; } -/** A comma-separated integer list; an empty value is an empty list, a malformed one is absent. */ +/** + * A comma-separated list of non-negative decimal integers; an empty value is an empty list, and any + * entry that is not plain decimal digits (an empty entry, hex, a sign, a fraction) makes it absent. + */ export function readInstrumentationResultIntegerList( value: string | undefined, ): number[] | undefined { if (value === undefined) return undefined; if (value === '') return []; - const entries = value.split(',').map(Number); - return entries.every(Number.isInteger) ? entries : undefined; + const entries = value.split(','); + return entries.every((entry) => /^\d+$/.test(entry)) ? entries.map(Number) : undefined; } export function readInstrumentationResultBoolean(value: string | undefined): boolean | undefined { diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 277bb76cd1..389f545447 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -149,8 +149,8 @@ in dp divides rects by it and multiplies its points. An older helper omits it. i already, so it carries no such factor. `androidSnapshot.missingRootWindowTypes` lists the `AccessibilityWindowInfo` types of windows the -helper listed but could not serialize, because reading the window's root returned nothing or failed -(`2` is an input method window); `windowCount` counts only the windows it did serialize. It is empty +helper listed but could not serialize, for example because the window's root was null or reading its +tree failed (`2` is an input method window); `windowCount` counts only the windows it did serialize. It is empty when every listed window was read. When it names an input method window, the capture reports the keyboard band as unmeasurable rather than absent. An older helper omits it. From 3f61a4c9c32949a8277800480985ab8837d154fa Mon Sep 17 00:00:00 2001 From: Prajwal Vedula Date: Tue, 6 Oct 2026 09:30:26 -0700 Subject: [PATCH 08/10] refactor(kernel): share unionRects from the rect module --- .../src/ios-snapshot-engine/transitions.ts | 10 +------- .../src/screenshot-overlay-android.ts | 3 ++- .../src/screenshot-overlay-rects.ts | 23 ------------------ packages/kernel/src/rect.test.ts | 12 ++++++++++ packages/kernel/src/rect.ts | 24 +++++++++++++++++++ packages/platform-android/src/snapshot.ts | 10 +------- 6 files changed, 40 insertions(+), 42 deletions(-) diff --git a/packages/capture-kit/src/ios-snapshot-engine/transitions.ts b/packages/capture-kit/src/ios-snapshot-engine/transitions.ts index 9ac2f2c1b8..9adf93f17b 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/transitions.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/transitions.ts @@ -1,5 +1,5 @@ import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; -import { rectContains } from '@agent-device/kernel/rect'; +import { rectContains, unionRects } from '@agent-device/kernel/rect'; import { extractNodeText, normalizeType } from '@agent-device/contracts/snapshot'; import { collectChildrenByParent, mergeReplacement, type SnapshotTreeRuleContext } from './tree.ts'; import { collectIosReplacedActionShelves } from './action-shelf.ts'; @@ -116,11 +116,3 @@ function formsNavigationTitleAffordance(image: Rect, field: Rect, title: Rect, b function verticallyOverlaps(left: Rect, right: Rect): boolean { return Math.max(left.y, right.y) <= Math.min(left.y + left.height, right.y + right.height); } - -function unionRects(rects: Rect[]): Rect { - const x = Math.min(...rects.map((rect) => rect.x)); - const y = Math.min(...rects.map((rect) => rect.y)); - const right = Math.max(...rects.map((rect) => rect.x + rect.width)); - const bottom = Math.max(...rects.map((rect) => rect.y + rect.height)); - return { x, y, width: right - x, height: bottom - y }; -} diff --git a/packages/capture-kit/src/screenshot-overlay-android.ts b/packages/capture-kit/src/screenshot-overlay-android.ts index 76663699db..8e086857bb 100644 --- a/packages/capture-kit/src/screenshot-overlay-android.ts +++ b/packages/capture-kit/src/screenshot-overlay-android.ts @@ -1,6 +1,7 @@ import type { Rect, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; import { isViewportRootNode, normalizeType } from '@agent-device/contracts/snapshot'; -import { hasPositiveRect, rectArea, rectContains, unionRects } from './screenshot-overlay-rects.ts'; +import { unionRects } from '@agent-device/kernel/rect'; +import { hasPositiveRect, rectArea, rectContains } from './screenshot-overlay-rects.ts'; /** * Android overlay policy (#1983): which Android nodes earn an overlay ref, and what rectangle diff --git a/packages/capture-kit/src/screenshot-overlay-rects.ts b/packages/capture-kit/src/screenshot-overlay-rects.ts index a5590689ee..cf5e24e581 100644 --- a/packages/capture-kit/src/screenshot-overlay-rects.ts +++ b/packages/capture-kit/src/screenshot-overlay-rects.ts @@ -15,26 +15,3 @@ export function clamp(value: number, min: number, max: number): number { export function rectArea(rect: Rect): number { return rect.width * rect.height; } - -export function unionRects(rects: Rect[]): Rect { - const firstRect = rects[0]; - if (firstRect === undefined) { - throw new Error('unionRects requires at least one rect'); - } - let minX = firstRect.x; - let minY = firstRect.y; - let maxRight = firstRect.x + firstRect.width; - let maxBottom = firstRect.y + firstRect.height; - for (const rect of rects.slice(1)) { - minX = Math.min(minX, rect.x); - minY = Math.min(minY, rect.y); - maxRight = Math.max(maxRight, rect.x + rect.width); - maxBottom = Math.max(maxBottom, rect.y + rect.height); - } - return { - x: minX, - y: minY, - width: maxRight - minX, - height: maxBottom - minY, - }; -} diff --git a/packages/kernel/src/rect.test.ts b/packages/kernel/src/rect.test.ts index 4979592b3b..63362aee31 100644 --- a/packages/kernel/src/rect.test.ts +++ b/packages/kernel/src/rect.test.ts @@ -9,6 +9,7 @@ import { pickLargestRect, readSnapshotViewportSize, snapshotViewportSizeFrom, + unionRects, } from './rect.ts'; const VIEWPORT: Rect = { x: 0, y: 0, width: 300, height: 500 }; @@ -158,6 +159,17 @@ test('pickLargestRect selects by area and returns null for an empty list', () => assert.equal(pickLargestRect([]), null); }); +test('unionRects spans every rect and refuses an empty list', () => { + assert.deepEqual( + unionRects([ + { x: 10, y: 40, width: 20, height: 10 }, + { x: 0, y: 60, width: 5, height: 30 }, + ]), + { x: 0, y: 40, width: 30, height: 50 }, + ); + assert.throws(() => unionRects([]), /at least one rect/); +}); + // These rows are the TypeScript twin of the runner's Swift `SnapshotGeometry.isGeometricallyActionable` // (asserted over randomized rects by the snapshot differential and over authored rects by // CoordinateSpaceTests.swift). They pin it so the host AX bridge cannot drift from the XCTest runner. diff --git a/packages/kernel/src/rect.ts b/packages/kernel/src/rect.ts index 3b48ab6185..deaf8fa2a9 100644 --- a/packages/kernel/src/rect.ts +++ b/packages/kernel/src/rect.ts @@ -95,6 +95,30 @@ export function rectArea(rect: Rect): number { return rect.width * rect.height; } +/** The smallest rect that contains every rect in a non-empty list. */ +export function unionRects(rects: readonly Rect[]): Rect { + const firstRect = rects[0]; + if (firstRect === undefined) { + throw new Error('unionRects requires at least one rect'); + } + let minX = firstRect.x; + let minY = firstRect.y; + let maxRight = firstRect.x + firstRect.width; + let maxBottom = firstRect.y + firstRect.height; + for (const rect of rects.slice(1)) { + minX = Math.min(minX, rect.x); + minY = Math.min(minY, rect.y); + maxRight = Math.max(maxRight, rect.x + rect.width); + maxBottom = Math.max(maxBottom, rect.y + rect.height); + } + return { + x: minX, + y: minY, + width: maxRight - minX, + height: maxBottom - minY, + }; +} + /** Point-in-rect with inclusive edges on all four bounds. */ export function containsPoint(rect: Rect, x: number, y: number): boolean { return x >= rect.x && x <= rect.x + rect.width && y >= rect.y && y <= rect.y + rect.height; diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index 4d541fb18a..431397a121 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -5,7 +5,7 @@ import { type NormalizedError, } from '@agent-device/kernel/errors'; import path from 'node:path'; -import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import { isPositiveFiniteRect, unionRects } from '@agent-device/kernel/rect'; import { emitDiagnostic, withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; import type { SnapshotOptions as InteractorSnapshotOptions } from '@agent-device/contracts/interactor-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -925,11 +925,3 @@ function isEmptyFiniteRect(rect: Rect | undefined): boolean { if (![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite)) return false; return rect.width === 0 || rect.height === 0; } - -function unionRects(rects: readonly Rect[]): Rect { - const left = Math.min(...rects.map((rect) => rect.x)); - const top = Math.min(...rects.map((rect) => rect.y)); - const right = Math.max(...rects.map((rect) => rect.x + rect.width)); - const bottom = Math.max(...rects.map((rect) => rect.y + rect.height)); - return { x: left, y: top, width: right - left, height: bottom - top }; -} From 423def21c605ed90dcd5d741175cdbfb11615613 Mon Sep 17 00:00:00 2001 From: Prajwal Vedula Date: Tue, 6 Oct 2026 09:31:12 -0700 Subject: [PATCH 09/10] refactor(android): share the window type constants from ui-hierarchy-node --- .../platform-android/src/snapshot-content-recovery.ts | 2 +- packages/platform-android/src/snapshot.ts | 5 +---- packages/platform-android/src/ui-hierarchy-node.ts | 5 +++++ packages/platform-android/src/ui-hierarchy-visibility.ts | 8 +++++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/platform-android/src/snapshot-content-recovery.ts b/packages/platform-android/src/snapshot-content-recovery.ts index 9adbf3fdb2..3c0c0b5e45 100644 --- a/packages/platform-android/src/snapshot-content-recovery.ts +++ b/packages/platform-android/src/snapshot-content-recovery.ts @@ -4,8 +4,8 @@ import { hasAndroidSystemChromeProvenance } from '@agent-device/contracts/androi import { classifyAndroidAlertIdentifier } from './alert-detection.ts'; import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; +import { ANDROID_WINDOW_TYPE_APPLICATION } from './ui-hierarchy-node.ts'; -const ANDROID_WINDOW_TYPE_APPLICATION = 1; const MAX_REPORTED_WINDOW_TYPES = 8; const MIN_FOREGROUND_APP_MEANINGFUL_NODES = 2; const MIN_INPUT_METHOD_MEANINGFUL_NODES = 2; diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index 431397a121..66909037ee 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -79,7 +79,7 @@ import { type AndroidSnapshotPresentationFailure, type AndroidSnapshotPresentationOptions, } from './snapshot-presentation.ts'; -import { readAndroidSiblingOrder } from './ui-hierarchy-node.ts'; +import { ANDROID_WINDOW_TYPE_INPUT_METHOD, readAndroidSiblingOrder } from './ui-hierarchy-node.ts'; import { androidSnapshotViewportFromHelperMetadata, createAndroidSnapshotCapture, @@ -868,9 +868,6 @@ function applyHiddenContentHintsToInteractiveNodes( } } -/** `AccessibilityWindowInfo.TYPE_INPUT_METHOD`. */ -const ANDROID_WINDOW_TYPE_INPUT_METHOD = 2; - /** * The keyboard band an Android capture measured, read from the window roots the helper already * captured: each root carries its `AccessibilityWindowInfo` type and screen bounds, so this costs no diff --git a/packages/platform-android/src/ui-hierarchy-node.ts b/packages/platform-android/src/ui-hierarchy-node.ts index 65f09e5f46..639b68f1db 100644 --- a/packages/platform-android/src/ui-hierarchy-node.ts +++ b/packages/platform-android/src/ui-hierarchy-node.ts @@ -52,6 +52,11 @@ export type AndroidUiHierarchy = { export type AndroidNode = AndroidUiHierarchy; +/** `AccessibilityWindowInfo.TYPE_APPLICATION`, as carried in `windowType`. */ +export const ANDROID_WINDOW_TYPE_APPLICATION = 1; +/** `AccessibilityWindowInfo.TYPE_INPUT_METHOD`, as carried in `windowType`. */ +export const ANDROID_WINDOW_TYPE_INPUT_METHOD = 2; + export type AndroidSiblingOrder = { parent: AndroidNode; order: number }; const siblingOrderByNode = new WeakMap(); diff --git a/packages/platform-android/src/ui-hierarchy-visibility.ts b/packages/platform-android/src/ui-hierarchy-visibility.ts index 98f90cbb2f..7907e6f72c 100644 --- a/packages/platform-android/src/ui-hierarchy-visibility.ts +++ b/packages/platform-android/src/ui-hierarchy-visibility.ts @@ -1,7 +1,9 @@ import type { AndroidSnapshotPresentationBudget } from './snapshot-presentation.ts'; -import type { AndroidNode, AndroidUiHierarchy } from './ui-hierarchy-node.ts'; - -const ANDROID_WINDOW_TYPE_APPLICATION = 1; +import { + ANDROID_WINDOW_TYPE_APPLICATION, + type AndroidNode, + type AndroidUiHierarchy, +} from './ui-hierarchy-node.ts'; /** * What the regular Android projection hides before daemon publication: nodes the acquisition marks From 69f5a58ce103942d9c1cec5be4f861764f3ff8a0 Mon Sep 17 00:00:00 2001 From: Prajwal Vedula Date: Tue, 6 Oct 2026 10:23:08 -0700 Subject: [PATCH 10/10] fix(android): report the keyboard band unmeasurable when the input method region is not one rect --- android/snapshot-helper/README.md | 3 ++ .../snapshothelper/AccessibilityTreeXml.java | 41 +++++++++++++++++-- .../src/__tests__/snapshot-keyboard.test.ts | 33 +++++++++++++++ packages/platform-android/src/snapshot.ts | 10 ++++- .../platform-android/src/ui-hierarchy-node.ts | 5 +++ packages/platform-android/src/ui-hierarchy.ts | 3 ++ website/docs/docs/snapshots.md | 6 +++ 7 files changed, 97 insertions(+), 4 deletions(-) diff --git a/android/snapshot-helper/README.md b/android/snapshot-helper/README.md index 85e270c5d2..536bdcf693 100644 --- a/android/snapshot-helper/README.md +++ b/android/snapshot-helper/README.md @@ -106,6 +106,9 @@ The XML node attributes intentionally mirror acquisition facts decoded by the ho metadata on window roots. The helper emits `drawing-order` on Android API 24+ and omits it on API 23, where the platform API is unavailable. The host keeps that fact as private capture evidence; the daemon uses it to annotate covered actions without adding it to normalized snapshot nodes. +On API 33+ an input method window root also carries `window-region-rect`, whether its touchable +region is a single rectangle (`window-bounds` is that region's bounding box). It is omitted on other +window types, on API 32 and lower, and when the region reads empty. Each XML chunk is sent with: diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java index 775d622f7b..8dd382c412 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java @@ -1,6 +1,7 @@ package com.callstack.agentdevice.snapshothelper; import android.graphics.Rect; +import android.graphics.Region; import android.os.Build; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction; @@ -133,8 +134,29 @@ static void appendNode( static WindowMetadata readWindowMetadata(AccessibilityWindowInfo window, int index) { Rect bounds = new Rect(); window.getBoundsInScreen(bounds); + int type = window.getType(); return new WindowMetadata( - index, window.getType(), window.getLayer(), window.isActive(), window.isFocused(), bounds); + index, + type, + window.getLayer(), + window.isActive(), + window.isFocused(), + bounds, + readInputMethodRegionRect(window, type)); + } + + // The window bounds are the box around the window's touchable region. A floating keyboard's region + // is several rects (the panel and the gesture strip), so the box claims app content between them; + // the host must not read such a box as the keyboard band. getRegionInScreen arrived in API 33, so + // older releases, and a region that reads empty, report nothing and keep the bounds-only answer. + private static Boolean readInputMethodRegionRect(AccessibilityWindowInfo window, int type) { + if (type != AccessibilityWindowInfo.TYPE_INPUT_METHOD + || Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) { + return null; + } + Region region = new Region(); + window.getRegionInScreen(region); + return region.isEmpty() ? null : region.isRect(); } private static void appendNonEmptyAttribute( @@ -194,6 +216,9 @@ private static void appendWindowMetadata(StringBuilder xml, WindowMetadata metad metadata.bounds.top, metadata.bounds.right, metadata.bounds.bottom)); + if (metadata.regionRect != null) { + appendAttribute(xml, "window-region-rect", Boolean.toString(metadata.regionRect)); + } } private static void appendAttribute(StringBuilder xml, String name, CharSequence value) { @@ -279,18 +304,28 @@ static final class WindowMetadata { final boolean active; final boolean focused; final Rect bounds; + // Null unless an input method window's touchable region was read (API 33+). + final Boolean regionRect; - WindowMetadata(int index, int type, int layer, boolean active, boolean focused, Rect bounds) { + WindowMetadata( + int index, + int type, + int layer, + boolean active, + boolean focused, + Rect bounds, + Boolean regionRect) { this.index = index; this.type = type; this.layer = layer; this.active = active; this.focused = focused; this.bounds = bounds; + this.regionRect = regionRect; } WindowMetadata withIndex(int nextIndex) { - return new WindowMetadata(nextIndex, type, layer, active, focused, bounds); + return new WindowMetadata(nextIndex, type, layer, active, focused, bounds, regionRect); } } } diff --git a/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts index 9dc7cb8ebe..ae3b264b44 100644 --- a/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts @@ -137,3 +137,36 @@ test('a skipped window that is not an input method leaves the keyboard answer al { kind: 'absent' }, ); }); + +const FLOATING_GBOARD_WINDOW = + ''; + +test('the helper region mark parses as marked, unmarked, or absent', () => { + const windowRegionRect = (attribute: string) => + parseUiHierarchyTree( + ``, + ).children[0]?.windowRegionRect; + assert.equal(windowRegionRect(' window-region-rect="false"'), false); + assert.equal(windowRegionRect(' window-region-rect="true"'), true); + assert.equal(windowRegionRect(''), undefined); +}); + +test('a floating keyboard whose touchable region is not one rect cannot be measured', () => { + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(FLOATING_GBOARD_WINDOW, APP_WINDOW), WINDOW_LIST), + { kind: 'unmeasurable', reason: 'window-region-not-rectangular' }, + ); +}); + +test('a docked keyboard whose region is one rect, or was not read, keeps its band', () => { + const docked = GBOARD_WINDOW.replace( + ' window-bounds=', + ' window-region-rect="true" window-bounds=', + ); + for (const window of [docked, GBOARD_WINDOW]) { + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(window, APP_WINDOW), WINDOW_LIST), { + kind: 'visible', + frame: { x: 0, y: 1500, width: 1080, height: 900 }, + }); + } +}); diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index 66909037ee..e7f559b0b4 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -882,6 +882,11 @@ function applyHiddenContentHintsToInteractiveNodes( * capture: the active-window fallback never saw the window list, a root without window metadata * cannot be ruled out as the input method, and a truncated capture may have stopped before it. A * helper too old to report `missingRootWindowTypes` is trusted on the roots it serialized. + * + * The bounds are the box around the window's touchable region, and a floating keyboard's region is + * several rects with app content between them, so an input method window whose region the helper + * read as not rectangular is unmeasurable too: the tap guard then falls back to the tree. Below API + * 33 the helper cannot read the region, and the bounds are trusted as the band. */ export function androidSnapshotKeyboardFromTree( tree: AndroidUiHierarchy, @@ -895,7 +900,10 @@ export function androidSnapshotKeyboardFromTree( (window) => window.windowType === ANDROID_WINDOW_TYPE_INPUT_METHOD, ); const inputMethodRects: Rect[] = []; - for (const { windowRect } of inputMethodWindows) { + for (const { windowRect, windowRegionRect } of inputMethodWindows) { + if (windowRegionRect === false) { + return { kind: 'unmeasurable', reason: 'window-region-not-rectangular' }; + } if (isPositiveFiniteRect(windowRect)) inputMethodRects.push(windowRect); else if (!isEmptyFiniteRect(windowRect)) { return { kind: 'unmeasurable', reason: 'window-bounds-unavailable' }; diff --git a/packages/platform-android/src/ui-hierarchy-node.ts b/packages/platform-android/src/ui-hierarchy-node.ts index 639b68f1db..1499f45034 100644 --- a/packages/platform-android/src/ui-hierarchy-node.ts +++ b/packages/platform-android/src/ui-hierarchy-node.ts @@ -47,6 +47,11 @@ export type AndroidUiHierarchy = { windowActive?: boolean; windowFocused?: boolean; windowRect?: Rect; + /** + * Whether this input method window's touchable region is one rect, so `windowRect` is exactly the + * area it takes touches in. Absent below API 33, on other window types, and on older helpers. + */ + windowRegionRect?: boolean; children: AndroidUiHierarchy[]; }; diff --git a/packages/platform-android/src/ui-hierarchy.ts b/packages/platform-android/src/ui-hierarchy.ts index ef2f38a20b..bea4512ebd 100644 --- a/packages/platform-android/src/ui-hierarchy.ts +++ b/packages/platform-android/src/ui-hierarchy.ts @@ -61,6 +61,7 @@ export type AndroidUiNodeMetadata = { windowActive?: boolean; windowFocused?: boolean; windowRect?: Rect; + windowRegionRect?: boolean; } & AndroidSystemChromeProvenance; /** @@ -188,6 +189,7 @@ function readNodeAttributes(node: string): Omit { ...optionalBoolAttr('windowActive', 'window-active'), ...optionalBoolAttr('windowFocused', 'window-focused'), ...optionalRectAttr('windowRect', 'window-bounds'), + ...optionalBoolAttr('windowRegionRect', 'window-region-rect'), }; } @@ -356,6 +358,7 @@ function normalizeAndroidUiHierarchyNode( windowActive: attrs.windowActive, windowFocused: attrs.windowFocused, windowRect: attrs.windowRect, + windowRegionRect: attrs.windowRegionRect, depth, parentIndex: undefined, children: [], diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 389f545447..45f7c23155 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -154,6 +154,12 @@ tree failed (`2` is an input method window); `windowCount` counts only the windo when every listed window was read. When it names an input method window, the capture reports the keyboard band as unmeasurable rather than absent. An older helper omits it. +The Android keyboard band is the input method window's bounds, the box around the area it takes +touches in. On Android 13 (API 33) and later the helper also checks that this area is one rectangle. +A floating keyboard's is not (the panel plus the gesture strip, with app content between them), so the +band is reported as unmeasurable and taps fall back to the tree's own keyboard check. Earlier releases +cannot report the area, so their bounds are used as they are. + Android snapshot nodes and `get attrs` (including the digest response) carry the native `selected`, `checked`, `heading`, `roleDescription`, `editable`, `password`, `hintShowing`, `placeholder`, `selectionStart`, and `selectionEnd` facts whenever the accessibility tree reports