feat(android): report the keyboard band on snapshot captures

android/snapshot-helper/README.md

@@ -106,6 +106,9 @@ The XML node attributes intentionally mirror acquisition facts decoded by the ho
 metadata on window roots. The helper emits `drawing-order` on Android API 24+ and omits it on API
 23, where the platform API is unavailable. The host keeps that fact as private capture evidence;
 the daemon uses it to annotate covered actions without adding it to normalized snapshot nodes.
+On API 33+ an input method window root also carries `window-region-rect`, whether its touchable
+region is a single rectangle (`window-bounds` is that region's bounding box). It is omitted on other
+window types, on API 32 and lower, and when the region reads empty.
 
 Each XML chunk is sent with:
 

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java

@@ -5,6 +5,7 @@
 import android.os.Build;
 import android.view.accessibility.AccessibilityNodeInfo;
 import android.view.accessibility.AccessibilityWindowInfo;
+import java.util.ArrayList;
 import java.util.List;
 
 /** Captures and serializes the Android accessibility window tree. */
@@ -25,7 +26,10 @@ private static Result captureOnce(UiAutomation automation, int maxDepth, int max
     StringBuilder xml = new StringBuilder();
     xml.append("<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>");
     xml.append("<hierarchy rotation=\"0\">");
-    int windowCount = appendInteractiveWindowRoots(xml, automation, maxDepth, maxNodes, stats);
+    List<Integer> missingRootWindowTypes = new ArrayList<>();
+    int windowCount =
+        appendInteractiveWindowRoots(
+            xml, automation, maxDepth, maxNodes, stats, missingRootWindowTypes);
     String captureMode = "interactive-windows";
     if (AccessibilityCaptureStabilizer.requiresActiveWindowFallback(
         windowCount, stats.activeWindowRootMissing)) {
@@ -61,7 +65,8 @@ private static Result captureOnce(UiAutomation automation, int maxDepth, int max
         captureMode,
         windowCount,
         stats.nodeCount,
-        stats.truncated);
+        stats.truncated,
+        missingRootWindowTypes);
   }
 
   private static void clearAccessibilityCache(UiAutomation automation) {
@@ -118,7 +123,8 @@ private static int appendInteractiveWindowRoots(
       UiAutomation automation,
       int maxDepth,
       int maxNodes,
-      AccessibilityTreeXml.Stats stats) {
+      AccessibilityTreeXml.Stats stats,
+      List<Integer> missingRootWindowTypes) {
     List<AccessibilityWindowInfo> windows;
     try {
       windows = automation.getWindows();
@@ -140,6 +146,7 @@ private static int appendInteractiveWindowRoots(
         windowMetadata = AccessibilityTreeXml.readWindowMetadata(window, windowCount);
         root = window.getRoot();
         if (root == null) {
+          missingRootWindowTypes.add(windowMetadata.type);
           stats.activeWindowRootMissing |= activeWindow;
           stats.focusedNonActiveWindowRootMissing |= focusedNonActiveWindow;
           if (activeWindow) {
@@ -165,6 +172,9 @@ private static int appendInteractiveWindowRoots(
         // Accessibility windows can disappear while traversing; keep the rest of the snapshot.
         stats.activeWindowRootMissing |= activeWindow;
         stats.focusedNonActiveWindowRootMissing |= focusedNonActiveWindow;
+        if (windowMetadata != null) {
+          missingRootWindowTypes.add(windowMetadata.type);
+        }
         if (activeWindow && windowMetadata != null) {
           stats.activeWindowMetadata = windowMetadata;
         }
@@ -202,6 +212,9 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture {
     final int windowCount;
     final int nodeCount;
     final boolean truncated;
+    // AccessibilityWindowInfo types of listed windows whose root read null or threw, so the host
+    // can tell a window it could not read (an input method, say) from one that was not on screen.
+    final List<Integer> missingRootWindowTypes;
 
     Result(
         String xml,
@@ -210,14 +223,16 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture {
         String captureMode,
         int windowCount,
         int nodeCount,
-        boolean truncated) {
+        boolean truncated,
+        List<Integer> missingRootWindowTypes) {
       this.xml = xml;
       this.rootPresent = rootPresent;
       this.foregroundWindowRootsPresent = foregroundWindowRootsPresent;
       this.captureMode = captureMode;
       this.windowCount = windowCount;
       this.nodeCount = nodeCount;
       this.truncated = truncated;
+      this.missingRootWindowTypes = missingRootWindowTypes;
     }
 
     @Override

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java

@@ -1,6 +1,7 @@
 package com.callstack.agentdevice.snapshothelper;
 
 import android.graphics.Rect;
+import android.graphics.Region;
 import android.os.Build;
 import android.view.accessibility.AccessibilityNodeInfo;
 import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction;
@@ -133,8 +134,29 @@ static void appendNode(
   static WindowMetadata readWindowMetadata(AccessibilityWindowInfo window, int index) {
     Rect bounds = new Rect();
     window.getBoundsInScreen(bounds);
+    int type = window.getType();
     return new WindowMetadata(
-        index, window.getType(), window.getLayer(), window.isActive(), window.isFocused(), bounds);
+        index,
+        type,
+        window.getLayer(),
+        window.isActive(),
+        window.isFocused(),
+        bounds,
+        readInputMethodRegionRect(window, type));
+  }
+
+  // The window bounds are the box around the window's touchable region. A floating keyboard's region
+  // is several rects (the panel and the gesture strip), so the box claims app content between them;
+  // the host must not read such a box as the keyboard band. getRegionInScreen arrived in API 33, so
+  // older releases, and a region that reads empty, report nothing and keep the bounds-only answer.
+  private static Boolean readInputMethodRegionRect(AccessibilityWindowInfo window, int type) {
+    if (type != AccessibilityWindowInfo.TYPE_INPUT_METHOD
+        || Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) {
+      return null;
+    }
+    Region region = new Region();
+    window.getRegionInScreen(region);
+    return region.isEmpty() ? null : region.isRect();
   }
 
   private static void appendNonEmptyAttribute(
@@ -194,6 +216,9 @@ private static void appendWindowMetadata(StringBuilder xml, WindowMetadata metad
             metadata.bounds.top,
             metadata.bounds.right,
             metadata.bounds.bottom));
+    if (metadata.regionRect != null) {
+      appendAttribute(xml, "window-region-rect", Boolean.toString(metadata.regionRect));
+    }
   }
 
   private static void appendAttribute(StringBuilder xml, String name, CharSequence value) {
@@ -279,18 +304,28 @@ static final class WindowMetadata {
     final boolean active;
     final boolean focused;
     final Rect bounds;
+    // Null unless an input method window's touchable region was read (API 33+).
+    final Boolean regionRect;
 
-    WindowMetadata(int index, int type, int layer, boolean active, boolean focused, Rect bounds) {
+    WindowMetadata(
+        int index,
+        int type,
+        int layer,
+        boolean active,
+        boolean focused,
+        Rect bounds,
+        Boolean regionRect) {
       this.index = index;
       this.type = type;
       this.layer = layer;
       this.active = active;
       this.focused = focused;
       this.bounds = bounds;
+      this.regionRect = regionRect;
     }
 
     WindowMetadata withIndex(int nextIndex) {
-      return new WindowMetadata(nextIndex, type, layer, active, focused, bounds);
+      return new WindowMetadata(nextIndex, type, layer, active, focused, bounds, regionRect);
     }
   }
 }

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java

@@ -4,6 +4,7 @@
 import android.app.UiAutomation;
 import android.content.res.Resources;
 import android.os.Bundle;
+import android.text.TextUtils;
 import android.util.Base64;
 import android.util.DisplayMetrics;
 import java.io.BufferedReader;
@@ -125,6 +126,8 @@ private static void putCaptureMetadata(
     result.putString("rootPresent", Boolean.toString(capture.rootPresent));
     result.putString("captureMode", capture.captureMode);
     result.putString("windowCount", Integer.toString(capture.windowCount));
+    result.putString(
+        "missingRootWindowTypes", TextUtils.join(",", capture.missingRootWindowTypes));
     result.putString("nodeCount", Integer.toString(capture.nodeCount));
     result.putString("truncated", Boolean.toString(capture.truncated));
     result.putString("elapsedMs", Long.toString(elapsedMs));

packages/capture-kit/src/ios-snapshot-engine/transitions.ts

@@ -1,5 +1,5 @@
 import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot';
-import { rectContains } from '@agent-device/kernel/rect';
+import { rectContains, unionRects } from '@agent-device/kernel/rect';
 import { extractNodeText, normalizeType } from '@agent-device/contracts/snapshot';
 import { collectChildrenByParent, mergeReplacement, type SnapshotTreeRuleContext } from './tree.ts';
 import { collectIosReplacedActionShelves } from './action-shelf.ts';
@@ -116,11 +116,3 @@ function formsNavigationTitleAffordance(image: Rect, field: Rect, title: Rect, b
 function verticallyOverlaps(left: Rect, right: Rect): boolean {
   return Math.max(left.y, right.y) <= Math.min(left.y + left.height, right.y + right.height);
 }
-
-function unionRects(rects: Rect[]): Rect {
-  const x = Math.min(...rects.map((rect) => rect.x));
-  const y = Math.min(...rects.map((rect) => rect.y));
-  const right = Math.max(...rects.map((rect) => rect.x + rect.width));
-  const bottom = Math.max(...rects.map((rect) => rect.y + rect.height));
-  return { x, y, width: right - x, height: bottom - y };
-}

packages/capture-kit/src/screenshot-overlay-android.ts

@@ -1,6 +1,7 @@
 import type { Rect, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot';
 import { isViewportRootNode, normalizeType } from '@agent-device/contracts/snapshot';
-import { hasPositiveRect, rectArea, rectContains, unionRects } from './screenshot-overlay-rects.ts';
+import { unionRects } from '@agent-device/kernel/rect';
+import { hasPositiveRect, rectArea, rectContains } from './screenshot-overlay-rects.ts';
 
 /**
  * Android overlay policy (#1983): which Android nodes earn an overlay ref, and what rectangle

packages/capture-kit/src/screenshot-overlay-rects.ts

@@ -15,26 +15,3 @@ export function clamp(value: number, min: number, max: number): number {
 export function rectArea(rect: Rect): number {
   return rect.width * rect.height;
 }
-
-export function unionRects(rects: Rect[]): Rect {
-  const firstRect = rects[0];
-  if (firstRect === undefined) {
-    throw new Error('unionRects requires at least one rect');
-  }
-  let minX = firstRect.x;
-  let minY = firstRect.y;
-  let maxRight = firstRect.x + firstRect.width;
-  let maxBottom = firstRect.y + firstRect.height;
-  for (const rect of rects.slice(1)) {
-    minX = Math.min(minX, rect.x);
-    minY = Math.min(minY, rect.y);
-    maxRight = Math.max(maxRight, rect.x + rect.width);
-    maxBottom = Math.max(maxBottom, rect.y + rect.height);
-  }
-  return {
-    x: minX,
-    y: minY,
-    width: maxRight - minX,
-    height: maxBottom - minY,
-  };
-}

packages/contracts/src/snapshot-types.ts

@@ -73,6 +73,12 @@ export type AndroidSnapshotBackendMetadata = {
   captureMode?: string;
   systemSurfaceOnly?: boolean;
   windowCount?: number;
+  /**
+   * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because
+   * reading their root returned null or threw (`2` is an input method window); `windowCount` counts
+   * only the roots it serialized. Empty when every listed window was read. Absent on an older helper.
+   */
+  missingRootWindowTypes?: number[];
   nodeCount?: number;
   helperTruncated?: boolean;
   elapsedMs?: number;

packages/contracts/src/tap-keyboard-occlusion.ts

@@ -21,11 +21,12 @@ import { normalizeType } from './snapshot-text.ts';
  * A capture whose producer measured the band directly publishes a {@link SnapshotKeyboardBandFact}
  * beside its tree, and the guard measures the tap point against that band — a point-in-rect check
  * with no geometry to believe. The Apple runner does this from `app.keyboards.firstMatch`, which
- * answers in the app's own orientation space (#2660).
+ * answers in the app's own orientation space (#2660); the Android helper does it from the input
+ * method window's screen bounds.
  *
  * Otherwise the band is derived from the captured tree every acting path already holds, so the guard
- * costs no round trip. This is the path for Android's input method nodes and for the producers that
- * never see the app's windows (`appium-source`, `limrun-ios-tree`, the runner's own query-sweep tier).
+ * costs no round trip. This is the path for Android captures that could not list every window and for
+ * the producers that never see the app's windows (`appium-source`, `limrun-ios-tree`, the runner's own query-sweep tier).
  * Derivation, the rules that decide whether reported geometry may be measured at all, and the verdict
  * on a point are proven against `contracts/fixtures/tap-keyboard-occlusion-policy.json`; change a rule
  * only through that table.

packages/kernel/src/rect.test.ts

@@ -9,6 +9,7 @@ import {
   pickLargestRect,
   readSnapshotViewportSize,
   snapshotViewportSizeFrom,
+  unionRects,
 } from './rect.ts';
 
 const VIEWPORT: Rect = { x: 0, y: 0, width: 300, height: 500 };
@@ -158,6 +159,17 @@ test('pickLargestRect selects by area and returns null for an empty list', () =>
   assert.equal(pickLargestRect([]), null);
 });
 
+test('unionRects spans every rect and refuses an empty list', () => {
+  assert.deepEqual(
+    unionRects([
+      { x: 10, y: 40, width: 20, height: 10 },
+      { x: 0, y: 60, width: 5, height: 30 },
+    ]),
+    { x: 0, y: 40, width: 30, height: 50 },
+  );
+  assert.throws(() => unionRects([]), /at least one rect/);
+});
+
 // These rows are the TypeScript twin of the runner's Swift `SnapshotGeometry.isGeometricallyActionable`
 // (asserted over randomized rects by the snapshot differential and over authored rects by
 // CoordinateSpaceTests.swift). They pin it so the host AX bridge cannot drift from the XCTest runner.

packages/kernel/src/rect.ts

@@ -95,6 +95,30 @@ export function rectArea(rect: Rect): number {
   return rect.width * rect.height;
 }
 
+/** The smallest rect that contains every rect in a non-empty list. */
+export function unionRects(rects: readonly Rect[]): Rect {
+  const firstRect = rects[0];
+  if (firstRect === undefined) {
+    throw new Error('unionRects requires at least one rect');
+  }
+  let minX = firstRect.x;
+  let minY = firstRect.y;
+  let maxRight = firstRect.x + firstRect.width;
+  let maxBottom = firstRect.y + firstRect.height;
+  for (const rect of rects.slice(1)) {
+    minX = Math.min(minX, rect.x);
+    minY = Math.min(minY, rect.y);
+    maxRight = Math.max(maxRight, rect.x + rect.width);
+    maxBottom = Math.max(maxBottom, rect.y + rect.height);
+  }
+  return {
+    x: minX,
+    y: minY,
+    width: maxRight - minX,
+    height: maxBottom - minY,
+  };
+}
+
 /** Point-in-rect with inclusive edges on all four bounds. */
 export function containsPoint(rect: Rect, x: number, y: number): boolean {
   return x >= rect.x && x <= rect.x + rect.width && y >= rect.y && y <= rect.y + rect.height;

packages/kernel/src/snapshot.ts

@@ -316,7 +316,8 @@ export type HiddenContentHint = {
  * A keyboard is its own system surface, so it never reaches the tree as a covering sibling of app
  * content, and a consumer that wants to refuse a tap behind it has to learn where it is from
  * somewhere (#2589). A producer that can measure the band directly — the Apple runner, from its
- * `app.keyboards` query — publishes one fact per capture and says nothing else about it. A consumer therefore gets three
+ * `app.keyboards` query, and the Android helper, from the input method window in the window list it
+ * captures — publishes one fact per capture and says nothing else about it. A consumer therefore gets three
  * answers and no fourth: a band in the same space as every node rect, a proven absence, or a
  * producer that could not look.
  *
@@ -669,9 +670,9 @@ export type SnapshotState = {
    */
   iosSystemSurfaceBundleId?: string;
   /**
-   * iOS: the keyboard band this capture's producer measured, when it measured one. The tap-path
-   * keyboard guard prefers this over the band it would otherwise derive from `nodes`, because a
-   * producer that can query the keyboard directly answers in the app's own orientation space and
+   * iOS and Android: the keyboard band this capture's producer measured, when it measured one. The
+   * tap-path keyboard guard prefers this over the band it would otherwise derive from `nodes`,
+   * because a producer that can query the keyboard directly answers in the tree's own space and
    * needs no geometry to be plausible (#2660). Absent means the guard measures the tree as before.
    */
   keyboard?: SnapshotKeyboardBandFact;