diff --git a/android/snapshot-helper/README.md b/android/snapshot-helper/README.md index 85e270c5d2..536bdcf693 100644 --- a/android/snapshot-helper/README.md +++ b/android/snapshot-helper/README.md @@ -106,6 +106,9 @@ The XML node attributes intentionally mirror acquisition facts decoded by the ho metadata on window roots. The helper emits `drawing-order` on Android API 24+ and omits it on API 23, where the platform API is unavailable. The host keeps that fact as private capture evidence; the daemon uses it to annotate covered actions without adding it to normalized snapshot nodes. +On API 33+ an input method window root also carries `window-region-rect`, whether its touchable +region is a single rectangle (`window-bounds` is that region's bounding box). It is omitted on other +window types, on API 32 and lower, and when the region reads empty. Each XML chunk is sent with: diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java index 2904fd6732..4ac7be3db5 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java @@ -5,6 +5,7 @@ import android.os.Build; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityWindowInfo; +import java.util.ArrayList; import java.util.List; /** Captures and serializes the Android accessibility window tree. */ @@ -25,7 +26,10 @@ private static Result captureOnce(UiAutomation automation, int maxDepth, int max StringBuilder xml = new StringBuilder(); xml.append(""); xml.append(""); - int windowCount = appendInteractiveWindowRoots(xml, automation, maxDepth, maxNodes, stats); + List missingRootWindowTypes = new ArrayList<>(); + int windowCount = + appendInteractiveWindowRoots( + xml, automation, maxDepth, maxNodes, stats, missingRootWindowTypes); String captureMode = "interactive-windows"; if (AccessibilityCaptureStabilizer.requiresActiveWindowFallback( windowCount, stats.activeWindowRootMissing)) { @@ -61,7 +65,8 @@ private static Result captureOnce(UiAutomation automation, int maxDepth, int max captureMode, windowCount, stats.nodeCount, - stats.truncated); + stats.truncated, + missingRootWindowTypes); } private static void clearAccessibilityCache(UiAutomation automation) { @@ -118,7 +123,8 @@ private static int appendInteractiveWindowRoots( UiAutomation automation, int maxDepth, int maxNodes, - AccessibilityTreeXml.Stats stats) { + AccessibilityTreeXml.Stats stats, + List missingRootWindowTypes) { List windows; try { windows = automation.getWindows(); @@ -140,6 +146,7 @@ private static int appendInteractiveWindowRoots( windowMetadata = AccessibilityTreeXml.readWindowMetadata(window, windowCount); root = window.getRoot(); if (root == null) { + missingRootWindowTypes.add(windowMetadata.type); stats.activeWindowRootMissing |= activeWindow; stats.focusedNonActiveWindowRootMissing |= focusedNonActiveWindow; if (activeWindow) { @@ -165,6 +172,9 @@ private static int appendInteractiveWindowRoots( // Accessibility windows can disappear while traversing; keep the rest of the snapshot. stats.activeWindowRootMissing |= activeWindow; stats.focusedNonActiveWindowRootMissing |= focusedNonActiveWindow; + if (windowMetadata != null) { + missingRootWindowTypes.add(windowMetadata.type); + } if (activeWindow && windowMetadata != null) { stats.activeWindowMetadata = windowMetadata; } @@ -202,6 +212,9 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture { final int windowCount; final int nodeCount; final boolean truncated; + // AccessibilityWindowInfo types of listed windows whose root read null or threw, so the host + // can tell a window it could not read (an input method, say) from one that was not on screen. + final List missingRootWindowTypes; Result( String xml, @@ -210,7 +223,8 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture { String captureMode, int windowCount, int nodeCount, - boolean truncated) { + boolean truncated, + List missingRootWindowTypes) { this.xml = xml; this.rootPresent = rootPresent; this.foregroundWindowRootsPresent = foregroundWindowRootsPresent; @@ -218,6 +232,7 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture { this.windowCount = windowCount; this.nodeCount = nodeCount; this.truncated = truncated; + this.missingRootWindowTypes = missingRootWindowTypes; } @Override diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java index 775d622f7b..8dd382c412 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java @@ -1,6 +1,7 @@ package com.callstack.agentdevice.snapshothelper; import android.graphics.Rect; +import android.graphics.Region; import android.os.Build; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction; @@ -133,8 +134,29 @@ static void appendNode( static WindowMetadata readWindowMetadata(AccessibilityWindowInfo window, int index) { Rect bounds = new Rect(); window.getBoundsInScreen(bounds); + int type = window.getType(); return new WindowMetadata( - index, window.getType(), window.getLayer(), window.isActive(), window.isFocused(), bounds); + index, + type, + window.getLayer(), + window.isActive(), + window.isFocused(), + bounds, + readInputMethodRegionRect(window, type)); + } + + // The window bounds are the box around the window's touchable region. A floating keyboard's region + // is several rects (the panel and the gesture strip), so the box claims app content between them; + // the host must not read such a box as the keyboard band. getRegionInScreen arrived in API 33, so + // older releases, and a region that reads empty, report nothing and keep the bounds-only answer. + private static Boolean readInputMethodRegionRect(AccessibilityWindowInfo window, int type) { + if (type != AccessibilityWindowInfo.TYPE_INPUT_METHOD + || Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) { + return null; + } + Region region = new Region(); + window.getRegionInScreen(region); + return region.isEmpty() ? null : region.isRect(); } private static void appendNonEmptyAttribute( @@ -194,6 +216,9 @@ private static void appendWindowMetadata(StringBuilder xml, WindowMetadata metad metadata.bounds.top, metadata.bounds.right, metadata.bounds.bottom)); + if (metadata.regionRect != null) { + appendAttribute(xml, "window-region-rect", Boolean.toString(metadata.regionRect)); + } } private static void appendAttribute(StringBuilder xml, String name, CharSequence value) { @@ -279,18 +304,28 @@ static final class WindowMetadata { final boolean active; final boolean focused; final Rect bounds; + // Null unless an input method window's touchable region was read (API 33+). + final Boolean regionRect; - WindowMetadata(int index, int type, int layer, boolean active, boolean focused, Rect bounds) { + WindowMetadata( + int index, + int type, + int layer, + boolean active, + boolean focused, + Rect bounds, + Boolean regionRect) { this.index = index; this.type = type; this.layer = layer; this.active = active; this.focused = focused; this.bounds = bounds; + this.regionRect = regionRect; } WindowMetadata withIndex(int nextIndex) { - return new WindowMetadata(nextIndex, type, layer, active, focused, bounds); + return new WindowMetadata(nextIndex, type, layer, active, focused, bounds, regionRect); } } } diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index 6026f881e0..1a66f0e139 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -4,6 +4,7 @@ import android.app.UiAutomation; import android.content.res.Resources; import android.os.Bundle; +import android.text.TextUtils; import android.util.Base64; import android.util.DisplayMetrics; import java.io.BufferedReader; @@ -125,6 +126,8 @@ private static void putCaptureMetadata( result.putString("rootPresent", Boolean.toString(capture.rootPresent)); result.putString("captureMode", capture.captureMode); result.putString("windowCount", Integer.toString(capture.windowCount)); + result.putString( + "missingRootWindowTypes", TextUtils.join(",", capture.missingRootWindowTypes)); result.putString("nodeCount", Integer.toString(capture.nodeCount)); result.putString("truncated", Boolean.toString(capture.truncated)); result.putString("elapsedMs", Long.toString(elapsedMs)); diff --git a/packages/capture-kit/src/ios-snapshot-engine/transitions.ts b/packages/capture-kit/src/ios-snapshot-engine/transitions.ts index 9ac2f2c1b8..9adf93f17b 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/transitions.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/transitions.ts @@ -1,5 +1,5 @@ import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; -import { rectContains } from '@agent-device/kernel/rect'; +import { rectContains, unionRects } from '@agent-device/kernel/rect'; import { extractNodeText, normalizeType } from '@agent-device/contracts/snapshot'; import { collectChildrenByParent, mergeReplacement, type SnapshotTreeRuleContext } from './tree.ts'; import { collectIosReplacedActionShelves } from './action-shelf.ts'; @@ -116,11 +116,3 @@ function formsNavigationTitleAffordance(image: Rect, field: Rect, title: Rect, b function verticallyOverlaps(left: Rect, right: Rect): boolean { return Math.max(left.y, right.y) <= Math.min(left.y + left.height, right.y + right.height); } - -function unionRects(rects: Rect[]): Rect { - const x = Math.min(...rects.map((rect) => rect.x)); - const y = Math.min(...rects.map((rect) => rect.y)); - const right = Math.max(...rects.map((rect) => rect.x + rect.width)); - const bottom = Math.max(...rects.map((rect) => rect.y + rect.height)); - return { x, y, width: right - x, height: bottom - y }; -} diff --git a/packages/capture-kit/src/screenshot-overlay-android.ts b/packages/capture-kit/src/screenshot-overlay-android.ts index 76663699db..8e086857bb 100644 --- a/packages/capture-kit/src/screenshot-overlay-android.ts +++ b/packages/capture-kit/src/screenshot-overlay-android.ts @@ -1,6 +1,7 @@ import type { Rect, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; import { isViewportRootNode, normalizeType } from '@agent-device/contracts/snapshot'; -import { hasPositiveRect, rectArea, rectContains, unionRects } from './screenshot-overlay-rects.ts'; +import { unionRects } from '@agent-device/kernel/rect'; +import { hasPositiveRect, rectArea, rectContains } from './screenshot-overlay-rects.ts'; /** * Android overlay policy (#1983): which Android nodes earn an overlay ref, and what rectangle diff --git a/packages/capture-kit/src/screenshot-overlay-rects.ts b/packages/capture-kit/src/screenshot-overlay-rects.ts index a5590689ee..cf5e24e581 100644 --- a/packages/capture-kit/src/screenshot-overlay-rects.ts +++ b/packages/capture-kit/src/screenshot-overlay-rects.ts @@ -15,26 +15,3 @@ export function clamp(value: number, min: number, max: number): number { export function rectArea(rect: Rect): number { return rect.width * rect.height; } - -export function unionRects(rects: Rect[]): Rect { - const firstRect = rects[0]; - if (firstRect === undefined) { - throw new Error('unionRects requires at least one rect'); - } - let minX = firstRect.x; - let minY = firstRect.y; - let maxRight = firstRect.x + firstRect.width; - let maxBottom = firstRect.y + firstRect.height; - for (const rect of rects.slice(1)) { - minX = Math.min(minX, rect.x); - minY = Math.min(minY, rect.y); - maxRight = Math.max(maxRight, rect.x + rect.width); - maxBottom = Math.max(maxBottom, rect.y + rect.height); - } - return { - x: minX, - y: minY, - width: maxRight - minX, - height: maxBottom - minY, - }; -} diff --git a/packages/contracts/src/snapshot-types.ts b/packages/contracts/src/snapshot-types.ts index 087c3a53b0..bb0f7ba345 100644 --- a/packages/contracts/src/snapshot-types.ts +++ b/packages/contracts/src/snapshot-types.ts @@ -73,6 +73,12 @@ export type AndroidSnapshotBackendMetadata = { captureMode?: string; systemSurfaceOnly?: boolean; windowCount?: number; + /** + * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because + * reading their root returned null or threw (`2` is an input method window); `windowCount` counts + * only the roots it serialized. Empty when every listed window was read. Absent on an older helper. + */ + missingRootWindowTypes?: number[]; nodeCount?: number; helperTruncated?: boolean; elapsedMs?: number; diff --git a/packages/contracts/src/tap-keyboard-occlusion.ts b/packages/contracts/src/tap-keyboard-occlusion.ts index 7df8b81106..6ea6504eca 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.ts @@ -21,11 +21,12 @@ import { normalizeType } from './snapshot-text.ts'; * A capture whose producer measured the band directly publishes a {@link SnapshotKeyboardBandFact} * beside its tree, and the guard measures the tap point against that band — a point-in-rect check * with no geometry to believe. The Apple runner does this from `app.keyboards.firstMatch`, which - * answers in the app's own orientation space (#2660). + * answers in the app's own orientation space (#2660); the Android helper does it from the input + * method window's screen bounds. * * Otherwise the band is derived from the captured tree every acting path already holds, so the guard - * costs no round trip. This is the path for Android's input method nodes and for the producers that - * never see the app's windows (`appium-source`, `limrun-ios-tree`, the runner's own query-sweep tier). + * costs no round trip. This is the path for Android captures that could not list every window and for + * the producers that never see the app's windows (`appium-source`, `limrun-ios-tree`, the runner's own query-sweep tier). * Derivation, the rules that decide whether reported geometry may be measured at all, and the verdict * on a point are proven against `contracts/fixtures/tap-keyboard-occlusion-policy.json`; change a rule * only through that table. diff --git a/packages/kernel/src/rect.test.ts b/packages/kernel/src/rect.test.ts index 4979592b3b..63362aee31 100644 --- a/packages/kernel/src/rect.test.ts +++ b/packages/kernel/src/rect.test.ts @@ -9,6 +9,7 @@ import { pickLargestRect, readSnapshotViewportSize, snapshotViewportSizeFrom, + unionRects, } from './rect.ts'; const VIEWPORT: Rect = { x: 0, y: 0, width: 300, height: 500 }; @@ -158,6 +159,17 @@ test('pickLargestRect selects by area and returns null for an empty list', () => assert.equal(pickLargestRect([]), null); }); +test('unionRects spans every rect and refuses an empty list', () => { + assert.deepEqual( + unionRects([ + { x: 10, y: 40, width: 20, height: 10 }, + { x: 0, y: 60, width: 5, height: 30 }, + ]), + { x: 0, y: 40, width: 30, height: 50 }, + ); + assert.throws(() => unionRects([]), /at least one rect/); +}); + // These rows are the TypeScript twin of the runner's Swift `SnapshotGeometry.isGeometricallyActionable` // (asserted over randomized rects by the snapshot differential and over authored rects by // CoordinateSpaceTests.swift). They pin it so the host AX bridge cannot drift from the XCTest runner. diff --git a/packages/kernel/src/rect.ts b/packages/kernel/src/rect.ts index 3b48ab6185..deaf8fa2a9 100644 --- a/packages/kernel/src/rect.ts +++ b/packages/kernel/src/rect.ts @@ -95,6 +95,30 @@ export function rectArea(rect: Rect): number { return rect.width * rect.height; } +/** The smallest rect that contains every rect in a non-empty list. */ +export function unionRects(rects: readonly Rect[]): Rect { + const firstRect = rects[0]; + if (firstRect === undefined) { + throw new Error('unionRects requires at least one rect'); + } + let minX = firstRect.x; + let minY = firstRect.y; + let maxRight = firstRect.x + firstRect.width; + let maxBottom = firstRect.y + firstRect.height; + for (const rect of rects.slice(1)) { + minX = Math.min(minX, rect.x); + minY = Math.min(minY, rect.y); + maxRight = Math.max(maxRight, rect.x + rect.width); + maxBottom = Math.max(maxBottom, rect.y + rect.height); + } + return { + x: minX, + y: minY, + width: maxRight - minX, + height: maxBottom - minY, + }; +} + /** Point-in-rect with inclusive edges on all four bounds. */ export function containsPoint(rect: Rect, x: number, y: number): boolean { return x >= rect.x && x <= rect.x + rect.width && y >= rect.y && y <= rect.y + rect.height; diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 6691445f25..684e9e55d1 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -316,7 +316,8 @@ export type HiddenContentHint = { * A keyboard is its own system surface, so it never reaches the tree as a covering sibling of app * content, and a consumer that wants to refuse a tap behind it has to learn where it is from * somewhere (#2589). A producer that can measure the band directly — the Apple runner, from its - * `app.keyboards` query — publishes one fact per capture and says nothing else about it. A consumer therefore gets three + * `app.keyboards` query, and the Android helper, from the input method window in the window list it + * captures — publishes one fact per capture and says nothing else about it. A consumer therefore gets three * answers and no fourth: a band in the same space as every node rect, a proven absence, or a * producer that could not look. * @@ -669,9 +670,9 @@ export type SnapshotState = { */ iosSystemSurfaceBundleId?: string; /** - * iOS: the keyboard band this capture's producer measured, when it measured one. The tap-path - * keyboard guard prefers this over the band it would otherwise derive from `nodes`, because a - * producer that can query the keyboard directly answers in the app's own orientation space and + * iOS and Android: the keyboard band this capture's producer measured, when it measured one. The + * tap-path keyboard guard prefers this over the band it would otherwise derive from `nodes`, + * because a producer that can query the keyboard directly answers in the tree's own space and * needs no geometry to be plausible (#2660). Absent means the guard measures the tree as before. */ keyboard?: SnapshotKeyboardBandFact; diff --git a/packages/platform-android/src/__tests__/snapshot-capture.test.ts b/packages/platform-android/src/__tests__/snapshot-capture.test.ts index 3853338b2f..c9f5d9bee2 100644 --- a/packages/platform-android/src/__tests__/snapshot-capture.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-capture.test.ts @@ -28,6 +28,8 @@ afterEach(async () => { function helperAdbServing( display: { width?: number; height?: number } = {}, xml: string = SCREEN_XML, + counts: { windowCount: number; nodeCount: number } = { windowCount: 1, nodeCount: 1 }, + resultLines: readonly string[] = [], ): AndroidAdbExecutor { const displayKeys = display.width !== undefined && display.height !== undefined @@ -54,12 +56,13 @@ function helperAdbServing( 'INSTRUMENTATION_RESULT: maxNodes=5000', 'INSTRUMENTATION_RESULT: rootPresent=true', 'INSTRUMENTATION_RESULT: captureMode=interactive-windows', - 'INSTRUMENTATION_RESULT: windowCount=1', - 'INSTRUMENTATION_RESULT: nodeCount=1', + `INSTRUMENTATION_RESULT: windowCount=${counts.windowCount}`, + `INSTRUMENTATION_RESULT: nodeCount=${counts.nodeCount}`, 'INSTRUMENTATION_RESULT: truncated=false', 'INSTRUMENTATION_RESULT: elapsedMs=12', 'INSTRUMENTATION_RESULT: pixelDensity=2.625', ...displayKeys, + ...resultLines, 'INSTRUMENTATION_CODE: 0', ].join('\n'); return async (args) => { @@ -174,3 +177,47 @@ test('an Android capture with geometry-free nodes still publishes the display it 'the tree really carries no geometry', ); }); + +test('the Android keyboard band survives the publication adapter into the daemon capture', async () => { + const capture = await snapshotAndroid(device, { + helperAdb: helperAdbServing( + { width: 1080, height: 2400 }, + [ + '', + '', + '', + '', + ].join(''), + { windowCount: 2, nodeCount: 4 }, + ), + helperArtifact, + }); + + assert.deepEqual(androidSnapshotPublicationInput(capture).keyboard, { + kind: 'visible', + frame: { x: 0, y: 1500, width: 1080, height: 900 }, + }); +}); + +test('an input method window the helper could not read makes the published keyboard unmeasurable', async () => { + const capture = await snapshotAndroid(device, { + helperAdb: helperAdbServing( + { width: 1080, height: 2400 }, + [ + '', + '', + '', + ].join(''), + { windowCount: 1, nodeCount: 3 }, + ['INSTRUMENTATION_RESULT: missingRootWindowTypes=2'], + ), + helperArtifact, + }); + + const published = androidSnapshotPublicationInput(capture); + assert.deepEqual(published.androidSnapshot?.missingRootWindowTypes, [2]); + assert.deepEqual(published.keyboard, { + kind: 'unmeasurable', + reason: 'window-root-unavailable', + }); +}); diff --git a/packages/platform-android/src/__tests__/snapshot-helper-capture.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-capture.test.ts index 635f28ff43..99d8db46b7 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-capture.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-capture.test.ts @@ -196,6 +196,17 @@ test('parseAndroidSnapshotHelperOutput carries the helper display extent beside assert.equal(parsed.metadata.displayHeight, 2400); }); +test('parseAndroidSnapshotHelperOutput reads the window types the helper could not serialize', () => { + const read = (resultLines: string[]) => + parseAndroidSnapshotHelperOutput( + helperOutput('', resultLines), + ).metadata.missingRootWindowTypes; + + assert.deepEqual(read(['INSTRUMENTATION_RESULT: missingRootWindowTypes=2,1']), [2, 1]); + assert.deepEqual(read(['INSTRUMENTATION_RESULT: missingRootWindowTypes=']), []); + assert.equal(read([]), undefined, 'an older helper omits the field'); +}); + test('parseAndroidSnapshotHelperOutput leaves the display extent absent when the helper omits it (#3182)', () => { const parsed = parseAndroidSnapshotHelperOutput( helperOutput('', [ diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts index 4dbcfb7cd9..6358b5f154 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts @@ -15,6 +15,7 @@ test('parses the session envelope and snapshot metadata', () => { metadata: { captureMode: 'interactive-windows', windowCount: '2', + missingRootWindowTypes: '2', nodeCount: '1', pixelDensity: '2.625', }, @@ -27,6 +28,7 @@ test('parses the session envelope and snapshot metadata', () => { outputFormat: 'uiautomator-xml', captureMode: 'interactive-windows', windowCount: 2, + missingRootWindowTypes: [2], nodeCount: 1, pixelDensity: 2.625, waitForIdleTimeoutMs: undefined, @@ -69,6 +71,25 @@ test('parses the session display extent beside its density and absent headers st assert.equal(sparse.metadata.displayHeight, undefined); }); +test('reads the session window types the helper could not serialize, absent from an older helper', () => { + const xml = ''; + const read = (metadata: Record) => + parseAndroidSnapshotHelperSessionSnapshotResponse( + sessionResponse({ requestId: 'snapshot-1', xml, metadata }), + 'snapshot-1', + ).metadata.missingRootWindowTypes; + + assert.deepEqual(read({ missingRootWindowTypes: '1,2' }), [1, 2]); + assert.deepEqual(read({ missingRootWindowTypes: '' }), []); + assert.equal(read({ missingRootWindowTypes: '1,x' }), undefined); + // Number() would read these as 0 or 2; only plain decimal digits count. + assert.equal(read({ missingRootWindowTypes: '1,,2' }), undefined); + assert.equal(read({ missingRootWindowTypes: '0x2' }), undefined); + assert.equal(read({ missingRootWindowTypes: ' 2' }), undefined); + assert.equal(read({ missingRootWindowTypes: '-2' }), undefined); + assert.equal(read({}), undefined); +}); + test('rejects stale and truncated session snapshot responses', () => { const response = sessionResponse({ requestId: 'snapshot-old', diff --git a/packages/platform-android/src/__tests__/snapshot-helper.test.ts b/packages/platform-android/src/__tests__/snapshot-helper.test.ts index 802a5a36aa..0d385912c8 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper.test.ts @@ -78,6 +78,7 @@ test('parseAndroidSnapshotHelperOutput reconstructs XML chunks and metadata', () rootPresent: true, captureMode: 'interactive-windows', windowCount: 2, + missingRootWindowTypes: undefined, nodeCount: 1, truncated: false, elapsedMs: 42, diff --git a/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts new file mode 100644 index 0000000000..ae3b264b44 --- /dev/null +++ b/packages/platform-android/src/__tests__/snapshot-keyboard.test.ts @@ -0,0 +1,172 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { androidSnapshotKeyboardFromTree } from '../snapshot.ts'; +import { parseUiHierarchyTree } from '../ui-hierarchy.ts'; + +const APP_WINDOW = + ''; +const GBOARD_WINDOW = + ''; + +const WINDOW_LIST = { captureMode: 'interactive-windows' } as const; + +function tree(...windows: string[]) { + return parseUiHierarchyTree(`${windows.join('')}`); +} + +test('an input method window on screen is the visible keyboard band', () => { + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(GBOARD_WINDOW, APP_WINDOW), WINDOW_LIST), { + kind: 'visible', + frame: { x: 0, y: 1500, width: 1080, height: 900 }, + }); +}); + +test('separate input method windows publish one band covering all of them', () => { + const composer = + ''; + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(composer, GBOARD_WINDOW, APP_WINDOW), WINDOW_LIST), + { kind: 'visible', frame: { x: 0, y: 1380, width: 1080, height: 1020 } }, + ); +}); + +test('a full window list without an input method window proves the keyboard absent', () => { + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(APP_WINDOW), WINDOW_LIST), { + kind: 'absent', + }); +}); + +test('an input method window with empty bounds draws nothing and reads as absent', () => { + const empty = + ''; + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(empty, APP_WINDOW), WINDOW_LIST), { + kind: 'absent', + }); +}); + +test('an input method window whose bounds did not measure cannot prove absence', () => { + const huge = '9'.repeat(400); + for (const windowBounds of [ + '', + ' window-bounds="unknown"', + ` window-bounds="[0,1500][${huge},2400]"`, + ]) { + const unmeasured = ``; + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(unmeasured, APP_WINDOW), WINDOW_LIST), + { kind: 'unmeasurable', reason: 'window-bounds-unavailable' }, + windowBounds, + ); + } +}); + +test('a measured input method window does not hide another one that did not measure', () => { + const unmeasured = + ''; + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(unmeasured, GBOARD_WINDOW, APP_WINDOW), WINDOW_LIST), + { kind: 'unmeasurable', reason: 'window-bounds-unavailable' }, + ); +}); + +test('an empty input method window beside a measured one leaves the measured band', () => { + const empty = + ''; + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(empty, GBOARD_WINDOW, APP_WINDOW), WINDOW_LIST), + { kind: 'visible', frame: { x: 0, y: 1500, width: 1080, height: 900 } }, + ); +}); + +test('a capture that never listed the windows cannot prove absence', () => { + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(APP_WINDOW), { captureMode: 'active-window' }), + { kind: 'unmeasurable', reason: 'window-list-unavailable' }, + ); + assert.deepEqual( + androidSnapshotKeyboardFromTree( + tree(''), + WINDOW_LIST, + ), + { kind: 'unmeasurable', reason: 'window-list-unavailable' }, + ); +}); + +test('a truncated capture without an input method window cannot prove absence', () => { + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(APP_WINDOW), { ...WINDOW_LIST, helperTruncated: true }), + { kind: 'unmeasurable', reason: 'capture-truncated' }, + ); + assert.equal( + androidSnapshotKeyboardFromTree(tree(GBOARD_WINDOW, APP_WINDOW), { + ...WINDOW_LIST, + helperTruncated: true, + }).kind, + 'visible', + 'a window the capture did reach is still measured', + ); +}); + +test('an input method window the helper could not read cannot prove absence', () => { + const skippedInputMethod = { ...WINDOW_LIST, missingRootWindowTypes: [1, 2] }; + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(APP_WINDOW), skippedInputMethod), { + kind: 'unmeasurable', + reason: 'window-root-unavailable', + }); + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(GBOARD_WINDOW, APP_WINDOW), skippedInputMethod), + { kind: 'unmeasurable', reason: 'window-root-unavailable' }, + 'a measured input method window does not hide one the helper could not read', + ); +}); + +test('a skipped window that is not an input method leaves the keyboard answer alone', () => { + const skippedApp = { ...WINDOW_LIST, missingRootWindowTypes: [1, 3] }; + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(APP_WINDOW), skippedApp), { + kind: 'absent', + }); + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(GBOARD_WINDOW, APP_WINDOW), skippedApp), { + kind: 'visible', + frame: { x: 0, y: 1500, width: 1080, height: 900 }, + }); + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(APP_WINDOW), { + ...WINDOW_LIST, + missingRootWindowTypes: [], + }), + { kind: 'absent' }, + ); +}); + +const FLOATING_GBOARD_WINDOW = + ''; + +test('the helper region mark parses as marked, unmarked, or absent', () => { + const windowRegionRect = (attribute: string) => + parseUiHierarchyTree( + ``, + ).children[0]?.windowRegionRect; + assert.equal(windowRegionRect(' window-region-rect="false"'), false); + assert.equal(windowRegionRect(' window-region-rect="true"'), true); + assert.equal(windowRegionRect(''), undefined); +}); + +test('a floating keyboard whose touchable region is not one rect cannot be measured', () => { + assert.deepEqual( + androidSnapshotKeyboardFromTree(tree(FLOATING_GBOARD_WINDOW, APP_WINDOW), WINDOW_LIST), + { kind: 'unmeasurable', reason: 'window-region-not-rectangular' }, + ); +}); + +test('a docked keyboard whose region is one rect, or was not read, keeps its band', () => { + const docked = GBOARD_WINDOW.replace( + ' window-bounds=', + ' window-region-rect="true" window-bounds=', + ); + for (const window of [docked, GBOARD_WINDOW]) { + assert.deepEqual(androidSnapshotKeyboardFromTree(tree(window, APP_WINDOW), WINDOW_LIST), { + kind: 'visible', + frame: { x: 0, y: 1500, width: 1080, height: 900 }, + }); + } +}); diff --git a/packages/platform-android/src/instrumentation-helper.ts b/packages/platform-android/src/instrumentation-helper.ts index 50a21b611e..daaf07a5df 100644 --- a/packages/platform-android/src/instrumentation-helper.ts +++ b/packages/platform-android/src/instrumentation-helper.ts @@ -92,6 +92,19 @@ export function readInstrumentationResultNumber(value: string | undefined): numb return Number.isFinite(parsed) ? parsed : undefined; } +/** + * A comma-separated list of non-negative decimal integers; an empty value is an empty list, and any + * entry that is not plain decimal digits (an empty entry, hex, a sign, a fraction) makes it absent. + */ +export function readInstrumentationResultIntegerList( + value: string | undefined, +): number[] | undefined { + if (value === undefined) return undefined; + if (value === '') return []; + const entries = value.split(','); + return entries.every((entry) => /^\d+$/.test(entry)) ? entries.map(Number) : undefined; +} + export function readInstrumentationResultBoolean(value: string | undefined): boolean | undefined { if (value === 'true') { return true; diff --git a/packages/platform-android/src/snapshot-capture.ts b/packages/platform-android/src/snapshot-capture.ts index 141d83768e..e1b1255a88 100644 --- a/packages/platform-android/src/snapshot-capture.ts +++ b/packages/platform-android/src/snapshot-capture.ts @@ -8,6 +8,7 @@ import { import type { RawSnapshotNode, SnapshotBackend, + SnapshotKeyboardBandFact, SnapshotQualityVerdict, SnapshotViewportSize, } from '@agent-device/kernel/snapshot'; @@ -28,6 +29,8 @@ type AndroidSnapshotCaptureData = Readonly<{ analysis: AndroidSnapshotAnalysis; androidSnapshot: AndroidSnapshotBackendMetadata; quality?: SnapshotQualityVerdict; + /** The input method window the helper captured, if any (see `androidSnapshotKeyboardFromTree`). */ + keyboard?: SnapshotKeyboardBandFact; /** * The screen the captured bounds are measured in (#3182): the helper's own display read, so a * capture of an empty screen still reports it. Derived, never stored, so this carrier cannot drift diff --git a/packages/platform-android/src/snapshot-content-recovery.ts b/packages/platform-android/src/snapshot-content-recovery.ts index 9adbf3fdb2..3c0c0b5e45 100644 --- a/packages/platform-android/src/snapshot-content-recovery.ts +++ b/packages/platform-android/src/snapshot-content-recovery.ts @@ -4,8 +4,8 @@ import { hasAndroidSystemChromeProvenance } from '@agent-device/contracts/androi import { classifyAndroidAlertIdentifier } from './alert-detection.ts'; import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; +import { ANDROID_WINDOW_TYPE_APPLICATION } from './ui-hierarchy-node.ts'; -const ANDROID_WINDOW_TYPE_APPLICATION = 1; const MAX_REPORTED_WINDOW_TYPES = 8; const MIN_FOREGROUND_APP_MEANINGFUL_NODES = 2; const MIN_INPUT_METHOD_MEANINGFUL_NODES = 2; diff --git a/packages/platform-android/src/snapshot-helper-capture.ts b/packages/platform-android/src/snapshot-helper-capture.ts index b4ed920a5d..2aecb82e1c 100644 --- a/packages/platform-android/src/snapshot-helper-capture.ts +++ b/packages/platform-android/src/snapshot-helper-capture.ts @@ -9,6 +9,7 @@ import { execFailureDetails } from '@agent-device/host-kit/command'; import { parseInstrumentationRecords, readInstrumentationResultBoolean, + readInstrumentationResultIntegerList, readInstrumentationResultNumber, } from './instrumentation-helper.ts'; import { @@ -487,6 +488,9 @@ function readHelperMetadata(finalResult: Record): AndroidSnapsho rootPresent: readOptionalBoolean(finalResult.rootPresent), captureMode: readOptionalCaptureMode(finalResult.captureMode), windowCount: readOptionalNumber(finalResult.windowCount), + missingRootWindowTypes: readInstrumentationResultIntegerList( + finalResult.missingRootWindowTypes, + ), nodeCount: readOptionalNumber(finalResult.nodeCount), truncated: readOptionalBoolean(finalResult.truncated), elapsedMs: readOptionalNumber(finalResult.elapsedMs), diff --git a/packages/platform-android/src/snapshot-helper-session-protocol.ts b/packages/platform-android/src/snapshot-helper-session-protocol.ts index 8fff3e894b..f7b874ef37 100644 --- a/packages/platform-android/src/snapshot-helper-session-protocol.ts +++ b/packages/platform-android/src/snapshot-helper-session-protocol.ts @@ -6,6 +6,7 @@ import { } from './snapshot-capture-failure-reason.ts'; import { readInstrumentationResultBoolean, + readInstrumentationResultIntegerList, readInstrumentationResultNumber, } from './instrumentation-helper.ts'; import { @@ -300,6 +301,7 @@ function readSessionMetadata(headers: Record): AndroidSnapshotHe ? headers.captureMode : undefined, windowCount: readInstrumentationResultNumber(headers.windowCount), + missingRootWindowTypes: readInstrumentationResultIntegerList(headers.missingRootWindowTypes), nodeCount: readInstrumentationResultNumber(headers.nodeCount), truncated: readInstrumentationResultBoolean(headers.truncated), elapsedMs: readInstrumentationResultNumber(headers.elapsedMs), diff --git a/packages/platform-android/src/snapshot-helper-types.ts b/packages/platform-android/src/snapshot-helper-types.ts index e392c7c2ab..6d31199abd 100644 --- a/packages/platform-android/src/snapshot-helper-types.ts +++ b/packages/platform-android/src/snapshot-helper-types.ts @@ -92,6 +92,12 @@ export type AndroidSnapshotHelperMetadata = { rootPresent?: boolean; captureMode?: AndroidSnapshotCaptureMode; windowCount?: number; + /** + * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because + * reading their root returned null or threw; `windowCount` counts only the roots it did serialize. + * Absent on an older helper. + */ + missingRootWindowTypes?: number[]; nodeCount?: number; truncated?: boolean; elapsedMs?: number; diff --git a/packages/platform-android/src/snapshot-types.ts b/packages/platform-android/src/snapshot-types.ts index 4f086a5b5c..33dada3395 100644 --- a/packages/platform-android/src/snapshot-types.ts +++ b/packages/platform-android/src/snapshot-types.ts @@ -41,6 +41,12 @@ export type AndroidSnapshotBackendMetadata = { captureMode?: AndroidSnapshotCaptureMode; systemSurfaceOnly?: boolean; windowCount?: number; + /** + * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because + * reading their root returned null or threw (`2` is an input method window); `windowCount` counts + * only the roots it serialized. Empty when every listed window was read. Absent on an older helper. + */ + missingRootWindowTypes?: number[]; nodeCount?: number; helperTruncated?: boolean; elapsedMs?: number; diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index d2593771a6..e7f559b0b4 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -5,6 +5,7 @@ import { type NormalizedError, } from '@agent-device/kernel/errors'; import path from 'node:path'; +import { isPositiveFiniteRect, unionRects } from '@agent-device/kernel/rect'; import { emitDiagnostic, withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; import type { SnapshotOptions as InteractorSnapshotOptions } from '@agent-device/contracts/interactor-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -12,6 +13,8 @@ import { attachRefs, type HiddenContentHint, type RawSnapshotNode, + type Rect, + type SnapshotKeyboardBandFact, type SnapshotOptions, type SnapshotViewportSize, } from '@agent-device/kernel/snapshot'; @@ -76,7 +79,7 @@ import { type AndroidSnapshotPresentationFailure, type AndroidSnapshotPresentationOptions, } from './snapshot-presentation.ts'; -import { readAndroidSiblingOrder } from './ui-hierarchy-node.ts'; +import { ANDROID_WINDOW_TYPE_INPUT_METHOD, readAndroidSiblingOrder } from './ui-hierarchy-node.ts'; import { androidSnapshotViewportFromHelperMetadata, createAndroidSnapshotCapture, @@ -133,6 +136,7 @@ export async function snapshotAndroid( const xml = capture.xml; const tree = parseUiHierarchyTree(xml); const androidSnapshot = withOcclusionScanDisclosure(capture.metadata, tree); + const keyboard = androidSnapshotKeyboardFromTree(tree, capture.metadata); const presentationOptions: AndroidUiHierarchySnapshotOptions = { ...options, androidPresentation: { @@ -160,6 +164,7 @@ export async function snapshotAndroid( ...androidSnapshotTruncationFields(truncated), androidSnapshot, quality: { state: 'healthy', backend: 'android-helper' } as const, + keyboard, ...(viewport ? { viewport } : {}), }; return createAndroidSnapshotCapture(result, { @@ -171,6 +176,7 @@ export async function snapshotAndroid( return attachAndroidPresentationFailureEvidence({ failure: error, androidSnapshot, + keyboard, ...(viewport ? { viewport } : {}), }); } @@ -179,6 +185,7 @@ export async function snapshotAndroid( function attachAndroidPresentationFailureEvidence(params: { failure: AndroidSnapshotPresentationFailure; androidSnapshot: AndroidSnapshotBackendMetadata; + keyboard: SnapshotKeyboardBandFact; viewport?: SnapshotViewportSize; }): AndroidSnapshotCapture { return createAndroidSnapshotCapture( @@ -202,6 +209,7 @@ function attachAndroidPresentationFailureEvidence(params: { reason: params.failure.message, reasonCode: params.failure.qualityReasonCode, }, + keyboard: params.keyboard, ...(params.viewport ? { viewport: params.viewport } : {}), }, { @@ -507,6 +515,7 @@ function formatAndroidHelperCaptureResult( rootPresent: capture.metadata.rootPresent, captureMode: capture.metadata.captureMode, windowCount: capture.metadata.windowCount, + missingRootWindowTypes: capture.metadata.missingRootWindowTypes, nodeCount: capture.metadata.nodeCount, helperTruncated: capture.metadata.truncated, elapsedMs: capture.metadata.elapsedMs, @@ -858,3 +867,66 @@ function applyHiddenContentHintsToInteractiveNodes( } } } + +/** + * The keyboard band an Android capture measured, read from the window roots the helper already + * captured: each root carries its `AccessibilityWindowInfo` type and screen bounds, so this costs no + * adb call. Bounds are screen pixels, the same space as every node rect. + * + * An input method that draws nothing (agent-device's test IME) puts no window on screen and reads as + * `absent`, and so does an input method window whose bounds parsed to an empty box. One whose bounds + * did not parse, or parsed to non-finite numbers, was seen but not measured, so the band is + * unmeasurable even when another input method window did measure. The same holds for an input method + * window the helper listed but could not serialize because its root read null or threw: the helper + * names those windows' types in `missingRootWindowTypes`. Absence is only read from a window-list + * capture: the active-window fallback never saw the window list, a root without window metadata + * cannot be ruled out as the input method, and a truncated capture may have stopped before it. A + * helper too old to report `missingRootWindowTypes` is trusted on the roots it serialized. + * + * The bounds are the box around the window's touchable region, and a floating keyboard's region is + * several rects with app content between them, so an input method window whose region the helper + * read as not rectangular is unmeasurable too: the tap guard then falls back to the tree. Below API + * 33 the helper cannot read the region, and the bounds are trusted as the band. + */ +export function androidSnapshotKeyboardFromTree( + tree: AndroidUiHierarchy, + metadata: Pick< + AndroidSnapshotBackendMetadata, + 'captureMode' | 'helperTruncated' | 'missingRootWindowTypes' + >, +): SnapshotKeyboardBandFact { + const windows = tree.children; + const inputMethodWindows = windows.filter( + (window) => window.windowType === ANDROID_WINDOW_TYPE_INPUT_METHOD, + ); + const inputMethodRects: Rect[] = []; + for (const { windowRect, windowRegionRect } of inputMethodWindows) { + if (windowRegionRect === false) { + return { kind: 'unmeasurable', reason: 'window-region-not-rectangular' }; + } + if (isPositiveFiniteRect(windowRect)) inputMethodRects.push(windowRect); + else if (!isEmptyFiniteRect(windowRect)) { + return { kind: 'unmeasurable', reason: 'window-bounds-unavailable' }; + } + } + if (metadata.missingRootWindowTypes?.includes(ANDROID_WINDOW_TYPE_INPUT_METHOD)) { + return { kind: 'unmeasurable', reason: 'window-root-unavailable' }; + } + if (inputMethodRects.length > 0) return { kind: 'visible', frame: unionRects(inputMethodRects) }; + if ( + metadata.captureMode !== 'interactive-windows' || + windows.some((window) => window.windowType === undefined) + ) { + return { kind: 'unmeasurable', reason: 'window-list-unavailable' }; + } + if (metadata.helperTruncated === true) { + return { kind: 'unmeasurable', reason: 'capture-truncated' }; + } + return { kind: 'absent' }; +} + +function isEmptyFiniteRect(rect: Rect | undefined): boolean { + if (!rect) return false; + if (![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite)) return false; + return rect.width === 0 || rect.height === 0; +} diff --git a/packages/platform-android/src/ui-hierarchy-node.ts b/packages/platform-android/src/ui-hierarchy-node.ts index 65f09e5f46..1499f45034 100644 --- a/packages/platform-android/src/ui-hierarchy-node.ts +++ b/packages/platform-android/src/ui-hierarchy-node.ts @@ -47,11 +47,21 @@ export type AndroidUiHierarchy = { windowActive?: boolean; windowFocused?: boolean; windowRect?: Rect; + /** + * Whether this input method window's touchable region is one rect, so `windowRect` is exactly the + * area it takes touches in. Absent below API 33, on other window types, and on older helpers. + */ + windowRegionRect?: boolean; children: AndroidUiHierarchy[]; }; export type AndroidNode = AndroidUiHierarchy; +/** `AccessibilityWindowInfo.TYPE_APPLICATION`, as carried in `windowType`. */ +export const ANDROID_WINDOW_TYPE_APPLICATION = 1; +/** `AccessibilityWindowInfo.TYPE_INPUT_METHOD`, as carried in `windowType`. */ +export const ANDROID_WINDOW_TYPE_INPUT_METHOD = 2; + export type AndroidSiblingOrder = { parent: AndroidNode; order: number }; const siblingOrderByNode = new WeakMap(); diff --git a/packages/platform-android/src/ui-hierarchy-visibility.ts b/packages/platform-android/src/ui-hierarchy-visibility.ts index 98f90cbb2f..7907e6f72c 100644 --- a/packages/platform-android/src/ui-hierarchy-visibility.ts +++ b/packages/platform-android/src/ui-hierarchy-visibility.ts @@ -1,7 +1,9 @@ import type { AndroidSnapshotPresentationBudget } from './snapshot-presentation.ts'; -import type { AndroidNode, AndroidUiHierarchy } from './ui-hierarchy-node.ts'; - -const ANDROID_WINDOW_TYPE_APPLICATION = 1; +import { + ANDROID_WINDOW_TYPE_APPLICATION, + type AndroidNode, + type AndroidUiHierarchy, +} from './ui-hierarchy-node.ts'; /** * What the regular Android projection hides before daemon publication: nodes the acquisition marks diff --git a/packages/platform-android/src/ui-hierarchy.ts b/packages/platform-android/src/ui-hierarchy.ts index ef2f38a20b..bea4512ebd 100644 --- a/packages/platform-android/src/ui-hierarchy.ts +++ b/packages/platform-android/src/ui-hierarchy.ts @@ -61,6 +61,7 @@ export type AndroidUiNodeMetadata = { windowActive?: boolean; windowFocused?: boolean; windowRect?: Rect; + windowRegionRect?: boolean; } & AndroidSystemChromeProvenance; /** @@ -188,6 +189,7 @@ function readNodeAttributes(node: string): Omit { ...optionalBoolAttr('windowActive', 'window-active'), ...optionalBoolAttr('windowFocused', 'window-focused'), ...optionalRectAttr('windowRect', 'window-bounds'), + ...optionalBoolAttr('windowRegionRect', 'window-region-rect'), }; } @@ -356,6 +358,7 @@ function normalizeAndroidUiHierarchyNode( windowActive: attrs.windowActive, windowFocused: attrs.windowFocused, windowRect: attrs.windowRect, + windowRegionRect: attrs.windowRegionRect, depth, parentIndex: undefined, children: [], diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index baa3c7b3fb..45f7c23155 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -148,6 +148,18 @@ report it (a 420 dpi phone reports `2.625`, a `wm density` override included); a in dp divides rects by it and multiplies its points. An older helper omits it. iOS reports points already, so it carries no such factor. +`androidSnapshot.missingRootWindowTypes` lists the `AccessibilityWindowInfo` types of windows the +helper listed but could not serialize, for example because the window's root was null or reading its +tree failed (`2` is an input method window); `windowCount` counts only the windows it did serialize. It is empty +when every listed window was read. When it names an input method window, the capture reports the +keyboard band as unmeasurable rather than absent. An older helper omits it. + +The Android keyboard band is the input method window's bounds, the box around the area it takes +touches in. On Android 13 (API 33) and later the helper also checks that this area is one rectangle. +A floating keyboard's is not (the panel plus the gesture strip, with app content between them), so the +band is reported as unmeasurable and taps fall back to the tree's own keyboard check. Earlier releases +cannot report the area, so their bounds are used as they are. + Android snapshot nodes and `get attrs` (including the digest response) carry the native `selected`, `checked`, `heading`, `roleDescription`, `editable`, `password`, `hintShowing`, `placeholder`, `selectionStart`, and `selectionEnd` facts whenever the accessibility tree reports