feat(engine): pass the measured keyboard state to executors

.changeset/keyboard-state-observation.md

@@ -0,0 +1,8 @@
+---
+"e2e": patch
+"@e2e-dev/mobile": patch
+"@e2e-dev/web": patch
+"@e2e-dev/decision": patch
+---
+
+Executor observations carry `keyboardVisible`, the engine's measured on-screen keyboard state, absent when it can't tell. The mobile engine sets it, and the decision executor uses it to offer `dismiss_keyboard`.

docs/decision-models.mdx

@@ -113,7 +113,8 @@ not controls, such as a card that opens a menu on hover, and a drag can drop
 onto any of them. `tap` also reaches text fields, to open a date picker or a
 suggestion list. While an on-screen
 keyboard shows on an engine that can dismiss it, such as the mobile engine,
-the executor also offers `dismiss_keyboard`.
+the executor also offers `dismiss_keyboard`, even when the keyboard is
+missing from the tree.
 
 Every operation that acts on an element asks for its target, even when only
 one element fits it.

docs/executors.mdx

@@ -144,6 +144,9 @@ without acting. Engines with a keyboard add `typeText`, `pressKey`, and
 path outside the project root or a hidden one with `POLICY_DENIED`, and a
 missing file with `INVALID_ARGUMENT`.
 
+The observation's `keyboardVisible` says whether an on-screen keyboard is
+showing, and is absent when the engine can't tell.
+
 When semantic capture fails, the observation sets `treeUnavailable: true`,
 has no `tree`, and includes a screenshot if policy allows. Earlier node ids
 are invalid, and a missing tree doesn't prove a control is absent.

docs/reference/engine.mdx

@@ -629,6 +629,12 @@ export interface EngineSnapshot {
    * read whole.
    */
   readonly truncated?: boolean;
+  /**
+   * Whether an on-screen keyboard is showing, as the engine measured it apart
+   * from `root`. Set `false` when it measured none, and omit it only when the
+   * engine did not measure it.
+   */
+  readonly keyboardVisible?: boolean;
   /** Semantic capture failed; only fresh, independently masked pixels are evidence. */
   readonly treeUnavailable?: true;
   /** Masked pixels, when requested and producible; omitted otherwise. */

packages/decision/src/elements.ts

@@ -136,6 +136,8 @@ interface SpaceObservation {
   readonly pixels?: ExecutorPixels;
   /** Whether the step can locate a drawn control on the pixels: the decision model scores and a text model names it. */
   readonly locates?: boolean;
+  /** The engine's measured keyboard state; undefined leaves it to the tree. */
+  readonly keyboardVisible?: boolean | undefined;
 }
 
 /** Builds the element table and bound targets from the newest observation. */
@@ -294,7 +296,7 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat
     controls.add('scroll_down');
   }
   if (verbs.has('back')) controls.add('back');
-  if (verbs.has('dismissKeyboard') && keyboardShowing(observation.tree)) controls.add('dismiss_keyboard');
+  if (verbs.has('dismissKeyboard') && (observation.keyboardVisible === undefined ? keyboardShowing(observation.tree) : observation.keyboardVisible)) controls.add('dismiss_keyboard');
   return {
     elements,
     element: (key) => byIndex.get(targetKeyIndex(key)),
@@ -309,7 +311,7 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat
     omitted,
     pageText: clip(pageText.join('\n'), 6000),
     statuses,
-    fingerprint: fingerprint(observation.path ?? '', observation.tree, observation.viewport, observation.pixels),
+    fingerprint: fingerprint(observation),
   };
 }
 /**
@@ -347,8 +349,9 @@ export function targetKeyIndex(key: string): string {
 }
 
 /**
- * Whether the tree lists an on-screen keyboard. Offering the control only
- * then keeps a choice that does nothing out of the operation question.
+ * Whether the tree lists an on-screen keyboard, for an engine that does not
+ * measure it. Offering the control only then keeps a choice that does
+ * nothing out of the operation question.
  */
 function keyboardShowing(node: ExecutorNode): boolean {
   if (node.states?.hidden === true) return false;
@@ -386,17 +389,19 @@ function clip(text: string, limit: number): string {
 }
 
 /**
- * Stable hash of the path, the tree content with node ids removed, whether
- * each node meets the viewport, and the pixels when the step sees them. Ids
- * change on every capture, so they stay out. In-view membership lets a
- * scroll that brings other nodes into view count as progress, while one
- * that moves nothing (the page bottom) still reads as unchanged; raw
- * coordinates stay out so small layout shifts do not count. Pixels are in
- * because a drawn control changes nothing in the tree: a keypad digit
- * entered on a canvas is progress only the screenshot shows.
+ * Stable hash of the path, the measured keyboard state, the tree content
+ * with node ids removed, whether each node meets the viewport, and the
+ * pixels when the step sees them. Ids change on every capture, so they stay
+ * out. In-view membership lets a scroll that brings other nodes into view
+ * count as progress, while one that moves nothing (the page bottom) still
+ * reads as unchanged; raw coordinates stay out so small layout shifts do not
+ * count. Pixels are in because a drawn control changes nothing in the tree:
+ * a keypad digit entered on a canvas is progress only the screenshot shows.
+ * The keyboard state is in because a dismissal can leave the tree as it was,
+ * and an unmeasured state hashes apart from a measured `false`.
  */
-function fingerprint(path: string, tree: ExecutorNode, viewport: { width: number; height: number }, pixels?: ExecutorPixels): string {
-  const parts: (string | Uint8Array)[] = [path];
+function fingerprint({ path, tree, viewport, keyboardVisible, pixels }: SpaceObservation): string {
+  const parts: (string | Uint8Array)[] = [path ?? '', keyboardVisible === undefined ? 'unmeasured' : String(keyboardVisible)];
   if (pixels !== undefined) parts.push(pixels.data);
   const visit = (node: ExecutorNode): void => {
     parts.push(node.role ?? '', node.name ?? '', node.text ?? '', node.value ?? '', JSON.stringify(node.states ?? null), JSON.stringify(node.attributes ?? null), intersects(node, viewport) ? 'v' : '-');

packages/decision/src/executor.ts

@@ -165,7 +165,7 @@ class Step {
     const screenshot = this.pixelsAllowed ? observation.pixels : undefined;
     const tree = observation.tree;
     if (observation.treeUnavailable || tree === undefined || emptyTree(observation)) return { path, space: undefined, screenshot };
-    const base = { path, viewport: observation.viewport, tree };
+    const base = { path, viewport: observation.viewport, tree, keyboardVisible: observation.keyboardVisible };
     const seen = screenshot === undefined ? base : { ...base, pixels: screenshot, locates: this.locates };
     return { path, space: actionSpace(this.ctx, seen, typing), screenshot };
   }

packages/decision/tests/elements.test.ts

@@ -6,9 +6,9 @@ import { context } from './helpers.ts';
 function tree(children: ExecutorNode[]): ExecutorNode {
   return { id: 'root', children };
 }
-function spaceFor(children: ExecutorNode[], verbs?: (keyof ExecutorActions)[]) {
+function spaceFor(children: ExecutorNode[], verbs?: (keyof ExecutorActions)[], keyboardVisible?: boolean) {
   const fixture = context(verbs === undefined ? {} : { verbs });
-  return actionSpace(fixture.ctx, { path: '/form', viewport: { width: 800, height: 600 }, tree: tree(children) }, true);
+  return actionSpace(fixture.ctx, { path: '/form', viewport: { width: 800, height: 600 }, tree: tree(children), keyboardVisible }, true);
 }
 describe('element table', () => {
   it('offers tap on buttons and links with indexed keys', () => {
@@ -228,15 +228,24 @@ describe('element table', () => {
     expect(spaceFor([hiddenPad]).controls.has('dismiss_keyboard')).toBe(false);
     expect(spaceFor(numberPad, ['tap', 'type', 'scroll']).controls.has('dismiss_keyboard')).toBe(false);
   });
+  it('takes the keyboard state the engine measured over the tree, and falls back to the tree without one', () => {
+    const field: ExecutorNode = { id: 'reps', role: 'textbox', name: 'Reps', value: '8' };
+    // A keyboard the tree does not list, measured by the engine.
+    expect(spaceFor([field], undefined, true).controls.has('dismiss_keyboard')).toBe(true);
+    // Key nodes left in the tree after the engine measured the keyboard gone.
+    expect(spaceFor([field, { id: 'k1', role: 'key', name: '1' }], undefined, false).controls.has('dismiss_keyboard')).toBe(false);
+    expect(spaceFor([field, { id: 'k1', role: 'key', name: '1' }]).controls.has('dismiss_keyboard')).toBe(true);
+    expect(spaceFor([field], ['tap', 'type', 'scroll'], true).controls.has('dismiss_keyboard')).toBe(false);
+  });
 });
 describe('fingerprint', () => {
   const page = (): ExecutorNode => ({ id: 'root', children: [
     { id: 'a', role: 'textbox', name: 'Name', value: '' },
     { id: 'b', role: 'button', name: 'Add' },
   ] });
-  function prints(path: string, root: ExecutorNode): string {
+  function prints(path: string, root: ExecutorNode, keyboardVisible?: boolean): string {
     const fixture = context();
-    return actionSpace(fixture.ctx, { path, viewport: { width: 800, height: 600 }, tree: root }, true).fingerprint;
+    return actionSpace(fixture.ctx, { path, viewport: { width: 800, height: 600 }, tree: root, keyboardVisible }, true).fingerprint;
   }
   it('is stable when only node ids change', () => {
     const first = prints('/form', page());
@@ -252,6 +261,14 @@ describe('fingerprint', () => {
     expect(prints('/form', edited)).not.toBe(base);
     expect(prints('/other', page())).not.toBe(base);
   });
+  it('moves when only the measured keyboard state changes, and tells unmeasured from hidden', () => {
+    const shown = prints('/form', page(), true);
+    const hidden = prints('/form', page(), false);
+    const unmeasured = prints('/form', page());
+    expect(hidden).not.toBe(shown);
+    expect(unmeasured).not.toBe(shown);
+    expect(unmeasured).not.toBe(hidden);
+  });
   it('moves when other nodes come into view, and holds for a small shift', () => {
     const rows = (offset: number): ExecutorNode => ({ id: 'root', children: Array.from({ length: 6 }, (_, index) => ({
       id: `r${index}`, role: 'button', name: `Row ${index}`,

packages/decision/tests/executor.test.ts

@@ -137,6 +137,18 @@ describe('keyboard', () => {
     expect(Object.keys(requests[0]?.questions.operation?.criteria ?? {})).toContain('dismiss_keyboard');
     expect(fixture.turns[0]?.calls).toEqual(['dismiss the keyboard']);
   });
+  it('offers dismiss_keyboard for a keyboard the engine measured and the tree does not list', async () => {
+    const { model, requests } = scriptedDecision((id, keys, call) => ({
+      choice: id === 'operation' ? (call === 0 ? 'dismiss_keyboard' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''),
+    }));
+    const fixture = context({
+      tree: { id: 'root', children: [{ id: 'reps', role: 'textbox', name: 'Reps', value: '8' }] },
+      observation: { keyboardVisible: true },
+    });
+    await decisionExecutor({ model }).runStep(fixture.ctx);
+    expect(Object.keys(requests[0]?.questions.operation?.criteria ?? {})).toContain('dismiss_keyboard');
+    expect(fixture.actions.dismissKeyboard).toHaveBeenCalledTimes(1);
+  });
 });
 
 describe('secrets', () => {

packages/e2e/src/agent/executor.ts

@@ -129,6 +129,13 @@ export interface ExecutorObservation {
   readonly text: string;
   readonly truncated: boolean;
   readonly viewport: ViewportSize;
+  /**
+   * Whether an on-screen keyboard is showing, as the engine measured it apart
+   * from the tree; `false` when it measured none. Absent only when the engine
+   * did not measure it; the tree's `keyboard` and `key` nodes, or a
+   * screenshot, are then the evidence.
+   */
+  readonly keyboardVisible?: boolean;
   /**
    * The current location as path and query, redacted, when the engine
    * reports one. Absent on engines without a location (a device screen).

packages/e2e/src/agent/observation-feed.ts

@@ -180,6 +180,7 @@ export class ObservationFeed {
       ...metadata,
       text: observation.text,
       truncated: observation.truncated,
+      ...(observation.keyboardVisible === undefined ? {} : { keyboardVisible: observation.keyboardVisible }),
       ...(options.tree === true ? { tree: projectTree(observation.tree, this.runtime.app.base?.origin) } : {}),
       ...pixels,
     };

packages/e2e/src/agent/observation.ts

@@ -28,6 +28,8 @@ export type AgentObservation = AgentObservationMetadata & (
       readonly parents: ReadonlyMap<string, string>;
       readonly tree: RedactedNode;
       readonly truncated: boolean;
+      /** The engine's measured keyboard state; `false` when it measured none, absent only when it did not measure. */
+      readonly keyboardVisible?: boolean | undefined;
       readonly pixels?: ExecutorPixels | undefined;
       readonly pixelsWithheld?: PixelsWithheld | undefined;
     }
@@ -147,6 +149,7 @@ export function prepareObservation(
     parents,
     tree,
     truncated,
+    ...(observation.keyboardVisible === undefined ? {} : { keyboardVisible: observation.keyboardVisible }),
     ...(pixels.cleared === undefined ? {} : { pixels: pixels.cleared }),
     ...(pixels.withheld === undefined ? {} : { pixelsWithheld: pixels.withheld }),
   };

packages/e2e/src/engine/index.ts

@@ -637,6 +637,13 @@ export interface EngineSnapshot {
    * read whole.
    */
   readonly truncated?: boolean;
+  /**
+   * Whether an on-screen keyboard is showing, as the engine measured it apart
+   * from `root`: a platform can raise a keyboard the tree lists only in part,
+   * or not at all. Set `false` when it measured none, and omit it only when
+   * the engine did not measure it; consumers then fall back to the tree.
+   */
+  readonly keyboardVisible?: boolean;
   /**
    * Semantic capture timed out. Return only the stable root reference, with no other fields,
    * and fresh pixels whose masking was proven independently of the failed

packages/e2e/src/engine/session.ts

@@ -395,6 +395,7 @@ export function createEngineSession(options: EngineSessionOptions): TargetSessio
         kind: 'semantic',
         tree: stampRevision(snapshot.root, minted),
         truncated: snapshot.truncated === true,
+        ...(snapshot.keyboardVisible === undefined ? {} : { keyboardVisible: snapshot.keyboardVisible }),
         ...(snapshot.pixels === undefined ? {} : { pixels: snapshot.pixels }),
       };
       showScreen(observation);

packages/e2e/src/engine/surface.ts

@@ -69,6 +69,8 @@ export type Observation = ObservationMetadata & (
       readonly kind: 'semantic';
       readonly tree: SemanticNode;
       readonly truncated: boolean;
+      /** The engine's measured keyboard state (`EngineSnapshot.keyboardVisible`); `false` when it measured none, absent only when it did not measure. */
+      readonly keyboardVisible?: boolean;
       readonly pixels?: ObservationPixels;
     }
   | {

packages/e2e/tests/types/sdk-types.ts

@@ -119,6 +119,11 @@ declare const engineSnapshot: EngineSnapshot;
 engineSnapshot.treeUnavailable satisfies true | undefined;
 // @ts-expect-error unavailable semantics are explicitly true or absent, never a separate false state.
 ({ ...engineSnapshot, treeUnavailable: false }) satisfies EngineSnapshot;
+// The measured keyboard is a yes or no, or absent when the engine did not measure it.
+engineSnapshot.keyboardVisible satisfies boolean | undefined;
+customExecutorObservation.keyboardVisible satisfies boolean | undefined;
+// @ts-expect-error the keyboard state carries no band or reason, only whether one is showing.
+({ ...engineSnapshot, keyboardVisible: { kind: 'visible' } }) satisfies EngineSnapshot;
 
 // There is no implicit target: a config names at least one, and the type says so before the loader does.
 // @ts-expect-error targets is required

packages/e2e/tests/unit/fixture-runtime.test.ts

@@ -501,6 +501,24 @@ describe('project tool dispatch', () => {
     expect(path).toBe('/settings/general');
   });
 
+  it("hands an executor the engine's measured keyboard state, and nothing when the engine measured none", async () => {
+    const measured: (boolean | undefined)[] = [];
+    let reported: boolean | undefined;
+    const engine = defineEngine({ name: 'fake', version: '1', spiVersion: 1,
+      observe: async () => snapshot([], reported === undefined ? {} : { keyboardVisible: reported }),
+    });
+    const executor: StepExecutor = { name: 'test', async runStep(context) {
+      measured.push((await context.observe()).keyboardVisible);
+      return { status: 'passed', summary: 'looked' };
+    } };
+    const { fixtures } = runtime(engine, { agents: { default: { executor } } });
+    for (const state of [undefined, true, false]) {
+      reported = state;
+      await fixtures.agent.act('look');
+    }
+    expect(measured).toEqual([undefined, true, false]);
+  });
+
 });
 
 describe('press key grammar', () => {

packages/mobile/src/surface.ts

@@ -145,6 +145,16 @@ const KEYBOARD_TYPES: ReadonlySet<string> = new Set(['keyboard', 'key']);
  */
 const KEYBOARD_DRAG_RATIO = 0.03;
 
+/**
+ * Whether the capture's producer measured the soft keyboard on screen:
+ * undefined when it did not look or could not.
+ */
+function measuredKeyboard(raw: RawSnapshot): boolean | undefined {
+  if (raw.keyboard?.kind === 'visible') return true;
+  if (raw.keyboard?.kind === 'absent') return false;
+  return undefined;
+}
+
 /**
  * Whether a capture shows the soft keyboard: the band its producer measured
  * when it measured one, else a keyboard or key element in the tree, the probe
@@ -153,9 +163,7 @@ const KEYBOARD_DRAG_RATIO = 0.03;
  * tap refuses against by name.
  */
 function keyboardShowing(raw: RawSnapshot): boolean {
-  if (raw.keyboard?.kind === 'visible') return true;
-  if (raw.keyboard?.kind === 'absent') return false;
-  return (raw.nodes ?? []).some((node) => KEYBOARD_TYPES.has(normalizeKind(node.type ?? '')));
+  return measuredKeyboard(raw) ?? (raw.nodes ?? []).some((node) => KEYBOARD_TYPES.has(normalizeKind(node.type ?? '')));
 }
 
 /**
@@ -945,10 +953,12 @@ export class AgentDeviceSurface {
     const viewport = await this.viewportFor(projected, operation.signal);
     const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected));
     const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }) : undefined;
+    const keyboardVisible = measuredKeyboard(raw);
     return {
       root: screenRoot(projected.roots, viewport),
       viewport,
       ...(isTruncated(raw) ? { truncated: true } : {}),
+      ...(keyboardVisible === undefined ? {} : { keyboardVisible }),
       ...(location === undefined ? {} : { location }),
       ...(capture === undefined ? {} : { pixels: capture.pixels, maskedRegionCount: capture.masked }),
     };

packages/mobile/tests/unit/engine.test.ts

@@ -679,6 +679,22 @@ describe('observation', () => {
     }
   });
 
+  it('reports the keyboard state the capture measured, and nothing when it measured none', async () => {
+    const h = harness();
+    await openAttempt(h);
+    expect((await h.engine.observe!(operation())).keyboardVisible).toBeUndefined();
+
+    const frame = { x: 0, y: 583, width: 390, height: 261 };
+    h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'visible', frame } }));
+    expect((await h.engine.observe!(operation())).keyboardVisible).toBe(true);
+
+    h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'absent' } }));
+    expect((await h.engine.observe!(operation())).keyboardVisible).toBe(false);
+
+    h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'unmeasurable', reason: 'keyboard-frame-query-timeout' } }));
+    expect((await h.engine.observe!(operation())).keyboardVisible).toBeUndefined();
+  });
+
   it('measures the screen in the viewport the snapshot publishes, and in its tree when it publishes none', async () => {
     const h = harness();
     // The application node reads 390x844; a published viewport outranks it.