From a48e14cd55cc98f937af7f8bf6cdff186e27ce21 Mon Sep 17 00:00:00 2001
From: Prajwal Vedula <prajwalvepicx@gmail.com>
Date: Fri, 9 Oct 2026 12:18:57 -0700
Subject: [PATCH 1/4] feat(engine): carry a measured keyboard state from the
 engine to executors

EngineSnapshot and ExecutorObservation gain an optional keyboardVisible:
whether an on-screen keyboard is showing, as the engine measured it apart
from the tree. false means it measured none; the field is omitted only
when the engine did not measure it. The session copies it onto the
semantic observation and the observation feed hands it to the executor, so
an executor no longer has to infer the keyboard from keyboard and key
nodes that a platform may list only in part.

It is a boolean, not the producer's band: dismissing the keyboard only
needs to know whether one is up, and the band's frame is in the producer's
own coordinate space.
---
 .changeset/keyboard-state-observation.md       |  7 +++++++
 docs/executors.mdx                             |  6 ++++++
 docs/reference/engine.mdx                      | 15 +++++++++++++++
 packages/e2e/src/agent/executor.ts             |  7 +++++++
 packages/e2e/src/agent/observation-feed.ts     |  1 +
 packages/e2e/src/agent/observation.ts          |  3 +++
 packages/e2e/src/engine/index.ts               |  7 +++++++
 packages/e2e/src/engine/session.ts             |  1 +
 packages/e2e/src/engine/surface.ts             |  2 ++
 packages/e2e/tests/types/sdk-types.ts          |  5 +++++
 .../e2e/tests/unit/fixture-runtime.test.ts     | 18 ++++++++++++++++++
 11 files changed, 72 insertions(+)
 create mode 100644 .changeset/keyboard-state-observation.md

diff --git a/.changeset/keyboard-state-observation.md b/.changeset/keyboard-state-observation.md
new file mode 100644
index 000000000..2ae3d185d
--- /dev/null
+++ b/.changeset/keyboard-state-observation.md
@@ -0,0 +1,7 @@
+---
+"e2e": minor
+"@e2e-dev/mobile": minor
+"@e2e-dev/web": patch
+---
+
+`EngineSnapshot` and `ExecutorObservation` carry an optional `keyboardVisible`: whether an on-screen keyboard is showing, as the engine measured it apart from the tree, and absent when it did not measure it. The mobile engine sets it from the keyboard band agent-device's iOS runner measures with each capture. The web engine does not set it.
diff --git a/docs/executors.mdx b/docs/executors.mdx
index d43695375..5d41a0f0a 100644
--- a/docs/executors.mdx
+++ b/docs/executors.mdx
@@ -142,6 +142,12 @@ without acting. Engines with a keyboard add `typeText`, `pressKey`, and
 path outside the project root or a hidden one with `POLICY_DENIED`, and a
 missing file with `INVALID_ARGUMENT`.
 
+The observation has `keyboardVisible` when the engine measured whether an
+on-screen keyboard shows (`true` or `false`), and omits it when it didn't.
+Without it, the [decision executor](/decision-models) falls back to a
+`keyboard` or `key` node in the tree, and your own executor can check the
+screenshot from `ctx.observe({ pixels: true })`.
+
 When semantic capture fails, the observation sets `treeUnavailable: true`,
 has no `tree`, and includes a screenshot if policy allows. Earlier node ids
 are invalid, and a missing tree doesn't prove a control is absent.
diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx
index 650432dec..0c995ab7f 100644
--- a/docs/reference/engine.mdx
+++ b/docs/reference/engine.mdx
@@ -624,6 +624,12 @@ export interface EngineSnapshot {
    * read whole.
    */
   readonly truncated?: boolean;
+  /**
+   * Whether an on-screen keyboard is showing, as the engine measured it apart
+   * from `root`. Set `false` when it measured none, and omit it only when the
+   * engine did not measure it.
+   */
+  readonly keyboardVisible?: boolean;
   /** Semantic capture failed; only fresh, independently masked pixels are evidence. */
   readonly treeUnavailable?: true;
   /** Masked pixels, when requested and producible; omitted otherwise. */
@@ -651,6 +657,15 @@ Set `truncated` when a node limit or inaccessible subtree leaves content out.
 The runner warns the model that the tree is incomplete and never treats a
 truncated observation as unchanged. Omit it for a complete tree.
 
+Set `keyboardVisible` when the platform tells the engine whether a soft
+keyboard is up, separately from the tree: `true` while one is showing,
+`false` when it measured none. A keyboard can be missing from the tree, or
+listed only as a few keys, so the [decision executor](/decision-models)
+reads this first and falls back to the tree's `keyboard` and `key` nodes
+when it is absent. The mobile engine sets it on iOS from the keyboard band
+agent-device's runner measures with each capture; Android captures carry no
+such measurement, so there it is absent.
+
 A valid empty tree may have a childless root and include `pixels`.
 
 If semantic capture times out, an engine may recover only when `pixelFallback`
diff --git a/packages/e2e/src/agent/executor.ts b/packages/e2e/src/agent/executor.ts
index 23902075a..e694f8b12 100644
--- a/packages/e2e/src/agent/executor.ts
+++ b/packages/e2e/src/agent/executor.ts
@@ -129,6 +129,13 @@ export interface ExecutorObservation {
   readonly text: string;
   readonly truncated: boolean;
   readonly viewport: ViewportSize;
+  /**
+   * Whether an on-screen keyboard is showing, as the engine measured it apart
+   * from the tree; `false` when it measured none. Absent only when the engine
+   * did not measure it; the tree's `keyboard` and `key` nodes, or a
+   * screenshot, are then the evidence.
+   */
+  readonly keyboardVisible?: boolean;
   /**
    * The current location as path and query, redacted, when the engine
    * reports one. Absent on engines without a location (a device screen).
diff --git a/packages/e2e/src/agent/observation-feed.ts b/packages/e2e/src/agent/observation-feed.ts
index c0a0db517..ade1cf8d3 100644
--- a/packages/e2e/src/agent/observation-feed.ts
+++ b/packages/e2e/src/agent/observation-feed.ts
@@ -178,6 +178,7 @@ export class ObservationFeed {
       ...metadata,
       text: observation.text,
       truncated: observation.truncated,
+      ...(observation.keyboardVisible === undefined ? {} : { keyboardVisible: observation.keyboardVisible }),
       ...(options.tree === true ? { tree: projectTree(observation.tree, this.runtime.app.base?.origin) } : {}),
       ...pixels,
     };
diff --git a/packages/e2e/src/agent/observation.ts b/packages/e2e/src/agent/observation.ts
index 3a08c47f3..049085f27 100644
--- a/packages/e2e/src/agent/observation.ts
+++ b/packages/e2e/src/agent/observation.ts
@@ -28,6 +28,8 @@ export type AgentObservation = AgentObservationMetadata & (
       readonly parents: ReadonlyMap<string, string>;
       readonly tree: RedactedNode;
       readonly truncated: boolean;
+      /** The engine's measured keyboard state; `false` when it measured none, absent only when it did not measure. */
+      readonly keyboardVisible?: boolean | undefined;
       readonly pixels?: ExecutorPixels | undefined;
       readonly pixelsWithheld?: PixelsWithheld | undefined;
     }
@@ -147,6 +149,7 @@ export function prepareObservation(
     parents,
     tree,
     truncated,
+    ...(observation.keyboardVisible === undefined ? {} : { keyboardVisible: observation.keyboardVisible }),
     ...(pixels.cleared === undefined ? {} : { pixels: pixels.cleared }),
     ...(pixels.withheld === undefined ? {} : { pixelsWithheld: pixels.withheld }),
   };
diff --git a/packages/e2e/src/engine/index.ts b/packages/e2e/src/engine/index.ts
index 9185b5549..9b1478421 100644
--- a/packages/e2e/src/engine/index.ts
+++ b/packages/e2e/src/engine/index.ts
@@ -637,6 +637,13 @@ export interface EngineSnapshot {
    * read whole.
    */
   readonly truncated?: boolean;
+  /**
+   * Whether an on-screen keyboard is showing, as the engine measured it apart
+   * from `root`: a platform can raise a keyboard the tree lists only in part,
+   * or not at all. Set `false` when it measured none, and omit it only when
+   * the engine did not measure it; consumers then fall back to the tree.
+   */
+  readonly keyboardVisible?: boolean;
   /**
    * Semantic capture timed out. Return only the stable root reference, with no other fields,
    * and fresh pixels whose masking was proven independently of the failed
diff --git a/packages/e2e/src/engine/session.ts b/packages/e2e/src/engine/session.ts
index e248a8a4e..8bbbf082c 100644
--- a/packages/e2e/src/engine/session.ts
+++ b/packages/e2e/src/engine/session.ts
@@ -395,6 +395,7 @@ export function createEngineSession(options: EngineSessionOptions): TargetSessio
         kind: 'semantic',
         tree: stampRevision(snapshot.root, minted),
         truncated: snapshot.truncated === true,
+        ...(typeof snapshot.keyboardVisible === 'boolean' ? { keyboardVisible: snapshot.keyboardVisible } : {}),
         ...(snapshot.pixels === undefined ? {} : { pixels: snapshot.pixels }),
       };
       showScreen(observation);
diff --git a/packages/e2e/src/engine/surface.ts b/packages/e2e/src/engine/surface.ts
index fe5815c9e..1614a9622 100644
--- a/packages/e2e/src/engine/surface.ts
+++ b/packages/e2e/src/engine/surface.ts
@@ -69,6 +69,8 @@ export type Observation = ObservationMetadata & (
       readonly kind: 'semantic';
       readonly tree: SemanticNode;
       readonly truncated: boolean;
+      /** The engine's measured keyboard state (`EngineSnapshot.keyboardVisible`); `false` when it measured none, absent only when it did not measure. */
+      readonly keyboardVisible?: boolean;
       readonly pixels?: ObservationPixels;
     }
   | {
diff --git a/packages/e2e/tests/types/sdk-types.ts b/packages/e2e/tests/types/sdk-types.ts
index 2b6f13b7e..41643108a 100644
--- a/packages/e2e/tests/types/sdk-types.ts
+++ b/packages/e2e/tests/types/sdk-types.ts
@@ -95,6 +95,11 @@ declare const engineSnapshot: EngineSnapshot;
 engineSnapshot.treeUnavailable satisfies true | undefined;
 // @ts-expect-error unavailable semantics are explicitly true or absent, never a separate false state.
 ({ ...engineSnapshot, treeUnavailable: false }) satisfies EngineSnapshot;
+// The measured keyboard is a yes or no, or absent when the engine did not measure it.
+engineSnapshot.keyboardVisible satisfies boolean | undefined;
+customExecutorObservation.keyboardVisible satisfies boolean | undefined;
+// @ts-expect-error the keyboard state carries no band or reason, only whether one is showing.
+({ ...engineSnapshot, keyboardVisible: { kind: 'visible' } }) satisfies EngineSnapshot;
 
 // There is no implicit target: a config names at least one, and the type says so before the loader does.
 // @ts-expect-error targets is required
diff --git a/packages/e2e/tests/unit/fixture-runtime.test.ts b/packages/e2e/tests/unit/fixture-runtime.test.ts
index 820ab7d05..0b3046d57 100644
--- a/packages/e2e/tests/unit/fixture-runtime.test.ts
+++ b/packages/e2e/tests/unit/fixture-runtime.test.ts
@@ -501,6 +501,24 @@ describe('project tool dispatch', () => {
     expect(path).toBe('/settings/general');
   });
 
+  it("hands an executor the engine's measured keyboard state, and nothing when the engine measured none", async () => {
+    const measured: (boolean | undefined)[] = [];
+    let reported: boolean | undefined;
+    const engine = defineEngine({ name: 'fake', version: '1', spiVersion: 1,
+      observe: async () => snapshot([], reported === undefined ? {} : { keyboardVisible: reported }),
+    });
+    const executor: StepExecutor = { name: 'test', async runStep(context) {
+      measured.push((await context.observe()).keyboardVisible);
+      return { status: 'passed', summary: 'looked' };
+    } };
+    const { fixtures } = runtime(engine, { agents: { default: { executor } } });
+    for (const state of [undefined, true, false]) {
+      reported = state;
+      await fixtures.agent.act('look');
+    }
+    expect(measured).toEqual([undefined, true, false]);
+  });
+
 });
 
 describe('press key grammar', () => {

From 44b27fd9a895a61240ffcfd2b71771e2b2f89a4c Mon Sep 17 00:00:00 2001
From: Prajwal Vedula <prajwalvepicx@gmail.com>
Date: Fri, 9 Oct 2026 12:18:57 -0700
Subject: [PATCH 2/4] feat(mobile): report the keyboard band agent-device
 measured on each observation

observe() sets keyboardVisible from the capture's keyboard fact: true for a
visible band, false for a proven absence, and nothing when the producer
could not measure it or published nothing. agent-device's iOS runner
publishes the fact with every capture; Android captures carry none, so
Android observations leave the field out.

The gesture fallback for dismissing the keyboard reads the same fact
through the same helper.
---
 packages/mobile/src/surface.ts            | 17 ++++++++++++++---
 packages/mobile/tests/unit/engine.test.ts | 16 ++++++++++++++++
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts
index 506fc8e28..762351998 100644
--- a/packages/mobile/src/surface.ts
+++ b/packages/mobile/src/surface.ts
@@ -143,6 +143,17 @@ const KEYBOARD_TYPES: ReadonlySet<string> = new Set(['keyboard', 'key']);
  */
 const KEYBOARD_DRAG_RATIO = 0.03;
 
+/**
+ * Whether the capture's producer measured the soft keyboard on screen:
+ * undefined when it did not look or could not (only the iOS runner measures
+ * it today).
+ */
+function measuredKeyboard(raw: RawSnapshot): boolean | undefined {
+  if (raw.keyboard?.kind === 'visible') return true;
+  if (raw.keyboard?.kind === 'absent') return false;
+  return undefined;
+}
+
 /**
  * Whether a capture shows the soft keyboard: the band its producer measured
  * when it measured one, else a keyboard or key element in the tree, the probe
@@ -151,9 +162,7 @@ const KEYBOARD_DRAG_RATIO = 0.03;
  * tap refuses against by name.
  */
 function keyboardShowing(raw: RawSnapshot): boolean {
-  if (raw.keyboard?.kind === 'visible') return true;
-  if (raw.keyboard?.kind === 'absent') return false;
-  return (raw.nodes ?? []).some((node) => KEYBOARD_TYPES.has((node.type ?? '').toLowerCase()));
+  return measuredKeyboard(raw) ?? (raw.nodes ?? []).some((node) => KEYBOARD_TYPES.has((node.type ?? '').toLowerCase()));
 }
 
 /**
@@ -977,10 +986,12 @@ export class AgentDeviceSurface {
     const viewport = await this.viewportFor(projected, operation.signal);
     const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected));
     const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }) : undefined;
+    const keyboardVisible = measuredKeyboard(raw);
     return {
       root: screenRoot(projected.roots, viewport),
       viewport,
       ...(isTruncated(raw) ? { truncated: true } : {}),
+      ...(keyboardVisible === undefined ? {} : { keyboardVisible }),
       ...(location === undefined ? {} : { location }),
       ...(capture === undefined ? {} : { pixels: capture.pixels, maskedRegionCount: capture.masked }),
     };
diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts
index 75cc68a35..b0326ede3 100644
--- a/packages/mobile/tests/unit/engine.test.ts
+++ b/packages/mobile/tests/unit/engine.test.ts
@@ -680,6 +680,22 @@ describe('observation', () => {
     }
   });
 
+  it('reports the keyboard state the capture measured, and nothing when it measured none', async () => {
+    const h = harness();
+    await openAttempt(h);
+    expect((await h.engine.observe!(operation())).keyboardVisible).toBeUndefined();
+
+    const frame = { x: 0, y: 583, width: 390, height: 261 };
+    h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'visible', frame } }));
+    expect((await h.engine.observe!(operation())).keyboardVisible).toBe(true);
+
+    h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'absent' } }));
+    expect((await h.engine.observe!(operation())).keyboardVisible).toBe(false);
+
+    h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'unmeasurable', reason: 'keyboard-frame-query-timeout' } }));
+    expect((await h.engine.observe!(operation())).keyboardVisible).toBeUndefined();
+  });
+
   it('is an empty screen before any app is open when no app is pinned, and a fault when one is', async () => {
     const free = harness({}, false);
     free.fake.respond('capture.snapshot', () => {

From 641a61d4af36d8071212d8b5d9dddb4252c41668 Mon Sep 17 00:00:00 2001
From: Prajwal Vedula <prajwalvepicx@gmail.com>
Date: Fri, 9 Oct 2026 12:18:57 -0700
Subject: [PATCH 3/4] feat(decision): offer dismiss_keyboard from the measured
 keyboard state

The action space reads the observation's keyboardVisible when the engine
measured one and falls back to the tree's keyboard and key nodes when it
did not. A keyboard the tree leaves out still offers the control, and key
nodes left behind after the keyboard closed no longer do.

The page fingerprint hashes the measured state beside the pixels, so a
dismissal that closes the keyboard without changing the tree no longer
reads as unchanged and cannot complete the three-action stall streak. An
unmeasured state hashes apart from a measured false.
---
 .changeset/decision-measured-keyboard.md |  5 +++
 docs/decision-models.mdx                 |  4 ++-
 packages/decision/src/elements.ts        | 39 +++++++++++++++---------
 packages/decision/src/executor.ts        |  2 +-
 packages/decision/tests/elements.test.ts | 25 ++++++++++++---
 packages/decision/tests/executor.test.ts | 12 ++++++++
 6 files changed, 67 insertions(+), 20 deletions(-)
 create mode 100644 .changeset/decision-measured-keyboard.md

diff --git a/.changeset/decision-measured-keyboard.md b/.changeset/decision-measured-keyboard.md
new file mode 100644
index 000000000..95935c1f1
--- /dev/null
+++ b/.changeset/decision-measured-keyboard.md
@@ -0,0 +1,5 @@
+---
+"@e2e-dev/decision": patch
+---
+
+The decision executor offers `dismiss_keyboard` from the observation's `keyboardVisible` when the engine measured the keyboard, and falls back to the tree's `keyboard` and `key` nodes when it did not. A keyboard the tree leaves out no longer hides the control, and key nodes left behind after the keyboard closed no longer offer it. A dismissal that changes only the measured state, not the tree, counts as a change on screen.
diff --git a/docs/decision-models.mdx b/docs/decision-models.mdx
index 2995fb7c3..2139de023 100644
--- a/docs/decision-models.mdx
+++ b/docs/decision-models.mdx
@@ -109,7 +109,9 @@ newest screen: `tap`, `type`, `typeSecret`, `submit`, `select`, `check`,
 Hover, right-click, drag, and scroll-to also reach labeled nodes that are
 not controls, such as a card that opens a menu on hover. While an on-screen
 keyboard shows on an engine that can dismiss it, such as the mobile engine,
-the executor also offers `dismiss_keyboard`.
+the executor also offers `dismiss_keyboard`. It uses the observation's
+measured `keyboardVisible` when the engine reports one, and otherwise looks
+for a `keyboard` or `key` node in the tree.
 
 Every target question has a `none` option, so a provider with no fitting
 target answers `none` instead of refusing. `none` blocks the step.
diff --git a/packages/decision/src/elements.ts b/packages/decision/src/elements.ts
index 5cd1ab7d7..3873f84d4 100644
--- a/packages/decision/src/elements.ts
+++ b/packages/decision/src/elements.ts
@@ -134,6 +134,8 @@ interface SpaceObservation {
   readonly pixels?: ExecutorPixels;
   /** Whether the step can locate a drawn control on the pixels: the decision model scores and a text model names it. */
   readonly locates?: boolean;
+  /** The engine's measured keyboard state; undefined leaves it to the tree. */
+  readonly keyboardVisible?: boolean | undefined;
 }
 
 /** Builds the element table and bound targets from the newest observation. */
@@ -285,7 +287,7 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat
     controls.add('scroll_down');
   }
   if (verbs.has('back')) controls.add('back');
-  if (verbs.has('dismissKeyboard') && keyboardShowing(observation.tree)) controls.add('dismiss_keyboard');
+  if (verbs.has('dismissKeyboard') && (observation.keyboardVisible ?? keyboardShowing(observation.tree))) controls.add('dismiss_keyboard');
   return {
     elements,
     element: (key) => byIndex.get(targetKeyIndex(key)),
@@ -298,7 +300,7 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat
     omitted,
     pageText: clip(pageText.join('\n'), 6000),
     statuses,
-    fingerprint: fingerprint(observation.path ?? '', observation.tree, observation.viewport, observation.pixels),
+    fingerprint: fingerprint(observation.path ?? '', observation.tree, observation.viewport, observation.keyboardVisible, observation.pixels),
   };
 }
 /**
@@ -335,8 +337,9 @@ export function targetKeyIndex(key: string): string {
 }
 
 /**
- * Whether the tree lists an on-screen keyboard. Offering the control only
- * then keeps a choice that does nothing out of the operation question.
+ * Whether the tree lists an on-screen keyboard, for an engine that does not
+ * measure it. Offering the control only then keeps a choice that does
+ * nothing out of the operation question.
  */
 function keyboardShowing(node: ExecutorNode): boolean {
   if (node.states?.hidden === true) return false;
@@ -374,17 +377,25 @@ function clip(text: string, limit: number): string {
 }
 
 /**
- * Stable hash of the path, the tree content with node ids removed, whether
- * each node meets the viewport, and the pixels when the step sees them. Ids
- * change on every capture, so they stay out. In-view membership lets a
- * scroll that brings other nodes into view count as progress, while one
- * that moves nothing (the page bottom) still reads as unchanged; raw
- * coordinates stay out so small layout shifts do not count. Pixels are in
- * because a drawn control changes nothing in the tree: a keypad digit
- * entered on a canvas is progress only the screenshot shows.
+ * Stable hash of the path, the measured keyboard state, the tree content
+ * with node ids removed, whether each node meets the viewport, and the
+ * pixels when the step sees them. Ids change on every capture, so they stay
+ * out. In-view membership lets a scroll that brings other nodes into view
+ * count as progress, while one that moves nothing (the page bottom) still
+ * reads as unchanged; raw coordinates stay out so small layout shifts do not
+ * count. Pixels are in because a drawn control changes nothing in the tree:
+ * a keypad digit entered on a canvas is progress only the screenshot shows.
+ * The keyboard state is in because a dismissal can leave the tree as it was,
+ * and an unmeasured state hashes apart from a measured `false`.
  */
-function fingerprint(path: string, tree: ExecutorNode, viewport: { width: number; height: number }, pixels?: ExecutorPixels): string {
-  const parts: (string | Uint8Array)[] = [path];
+function fingerprint(
+  path: string,
+  tree: ExecutorNode,
+  viewport: { width: number; height: number },
+  keyboardVisible: boolean | undefined,
+  pixels?: ExecutorPixels,
+): string {
+  const parts: (string | Uint8Array)[] = [path, JSON.stringify(keyboardVisible ?? null)];
   if (pixels !== undefined) parts.push(pixels.data);
   const visit = (node: ExecutorNode): void => {
     parts.push(node.role ?? '', node.name ?? '', node.text ?? '', node.value ?? '', JSON.stringify(node.states ?? null), JSON.stringify(node.attributes ?? null), intersects(node, viewport) ? 'v' : '-');
diff --git a/packages/decision/src/executor.ts b/packages/decision/src/executor.ts
index 1a1777c5c..69a96848d 100644
--- a/packages/decision/src/executor.ts
+++ b/packages/decision/src/executor.ts
@@ -164,7 +164,7 @@ class Step {
     const screenshot = this.pixelsAllowed ? observation.pixels : undefined;
     const tree = observation.tree;
     if (observation.treeUnavailable || tree === undefined || emptyTree(observation)) return { path, space: undefined, screenshot };
-    const base = { path, viewport: observation.viewport, tree };
+    const base = { path, viewport: observation.viewport, tree, keyboardVisible: observation.keyboardVisible };
     const seen = screenshot === undefined ? base : { ...base, pixels: screenshot, locates: this.locates };
     return { path, space: actionSpace(this.ctx, seen, typing), screenshot };
   }
diff --git a/packages/decision/tests/elements.test.ts b/packages/decision/tests/elements.test.ts
index 4d15aeebf..63738eb20 100644
--- a/packages/decision/tests/elements.test.ts
+++ b/packages/decision/tests/elements.test.ts
@@ -6,9 +6,9 @@ import { context } from './helpers.ts';
 function tree(children: ExecutorNode[]): ExecutorNode {
   return { id: 'root', children };
 }
-function spaceFor(children: ExecutorNode[], verbs?: (keyof ExecutorActions)[]) {
+function spaceFor(children: ExecutorNode[], verbs?: (keyof ExecutorActions)[], keyboardVisible?: boolean) {
   const fixture = context(verbs === undefined ? {} : { verbs });
-  return actionSpace(fixture.ctx, { path: '/form', viewport: { width: 800, height: 600 }, tree: tree(children) }, true);
+  return actionSpace(fixture.ctx, { path: '/form', viewport: { width: 800, height: 600 }, tree: tree(children), keyboardVisible }, true);
 }
 describe('element table', () => {
   it('offers tap on buttons and links with indexed keys', () => {
@@ -218,15 +218,24 @@ describe('element table', () => {
     expect(spaceFor([hiddenPad]).controls.has('dismiss_keyboard')).toBe(false);
     expect(spaceFor(numberPad, ['tap', 'type', 'scroll']).controls.has('dismiss_keyboard')).toBe(false);
   });
+  it('takes the keyboard state the engine measured over the tree, and falls back to the tree without one', () => {
+    const field: ExecutorNode = { id: 'reps', role: 'textbox', name: 'Reps', value: '8' };
+    // A keyboard the tree does not list, measured by the engine.
+    expect(spaceFor([field], undefined, true).controls.has('dismiss_keyboard')).toBe(true);
+    // Key nodes left in the tree after the engine measured the keyboard gone.
+    expect(spaceFor([field, { id: 'k1', role: 'key', name: '1' }], undefined, false).controls.has('dismiss_keyboard')).toBe(false);
+    expect(spaceFor([field, { id: 'k1', role: 'key', name: '1' }]).controls.has('dismiss_keyboard')).toBe(true);
+    expect(spaceFor([field], ['tap', 'type', 'scroll'], true).controls.has('dismiss_keyboard')).toBe(false);
+  });
 });
 describe('fingerprint', () => {
   const page = (): ExecutorNode => ({ id: 'root', children: [
     { id: 'a', role: 'textbox', name: 'Name', value: '' },
     { id: 'b', role: 'button', name: 'Add' },
   ] });
-  function prints(path: string, root: ExecutorNode): string {
+  function prints(path: string, root: ExecutorNode, keyboardVisible?: boolean): string {
     const fixture = context();
-    return actionSpace(fixture.ctx, { path, viewport: { width: 800, height: 600 }, tree: root }, true).fingerprint;
+    return actionSpace(fixture.ctx, { path, viewport: { width: 800, height: 600 }, tree: root, keyboardVisible }, true).fingerprint;
   }
   it('is stable when only node ids change', () => {
     const first = prints('/form', page());
@@ -242,6 +251,14 @@ describe('fingerprint', () => {
     expect(prints('/form', edited)).not.toBe(base);
     expect(prints('/other', page())).not.toBe(base);
   });
+  it('moves when only the measured keyboard state changes, and tells unmeasured from hidden', () => {
+    const shown = prints('/form', page(), true);
+    const hidden = prints('/form', page(), false);
+    const unmeasured = prints('/form', page());
+    expect(hidden).not.toBe(shown);
+    expect(unmeasured).not.toBe(shown);
+    expect(unmeasured).not.toBe(hidden);
+  });
   it('moves when other nodes come into view, and holds for a small shift', () => {
     const rows = (offset: number): ExecutorNode => ({ id: 'root', children: Array.from({ length: 6 }, (_, index) => ({
       id: `r${index}`, role: 'button', name: `Row ${index}`,
diff --git a/packages/decision/tests/executor.test.ts b/packages/decision/tests/executor.test.ts
index 85d18aae3..16f925f19 100644
--- a/packages/decision/tests/executor.test.ts
+++ b/packages/decision/tests/executor.test.ts
@@ -137,6 +137,18 @@ describe('keyboard', () => {
     expect(Object.keys(requests[0]?.questions.operation?.criteria ?? {})).toContain('dismiss_keyboard');
     expect(fixture.turns[0]?.calls).toEqual(['dismiss the keyboard']);
   });
+  it('offers dismiss_keyboard for a keyboard the engine measured and the tree does not list', async () => {
+    const { model, requests } = scriptedDecision((id, keys, call) => ({
+      choice: id === 'operation' ? (call === 0 ? 'dismiss_keyboard' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''),
+    }));
+    const fixture = context({
+      tree: { id: 'root', children: [{ id: 'reps', role: 'textbox', name: 'Reps', value: '8' }] },
+      observation: { keyboardVisible: true },
+    });
+    await decisionExecutor({ model }).runStep(fixture.ctx);
+    expect(Object.keys(requests[0]?.questions.operation?.criteria ?? {})).toContain('dismiss_keyboard');
+    expect(fixture.actions.dismissKeyboard).toHaveBeenCalledTimes(1);
+  });
 });
 
 describe('secrets', () => {

From e3cb38f17776b1c8e1b20080e242ad67d323e49d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= <oskar@okwasniewski.com>
Date: Thu, 15 Oct 2026 13:20:51 +0200
Subject: [PATCH 4/4] refactor: address review on measured keyboard state

- fingerprint takes the observation
- keyboardVisible checks use === undefined in every layer
- trim docs to one user-facing sentence, drop engine.mdx paragraph
- merge changesets into one patch changeset
---
 .changeset/decision-measured-keyboard.md |  5 -----
 .changeset/keyboard-state-observation.md |  7 ++++---
 docs/decision-models.mdx                 |  5 ++---
 docs/executors.mdx                       |  7 ++-----
 docs/reference/engine.mdx                |  9 ---------
 packages/decision/src/elements.ts        | 14 ++++----------
 packages/e2e/src/engine/session.ts       |  2 +-
 packages/mobile/src/surface.ts           |  3 +--
 8 files changed, 14 insertions(+), 38 deletions(-)
 delete mode 100644 .changeset/decision-measured-keyboard.md

diff --git a/.changeset/decision-measured-keyboard.md b/.changeset/decision-measured-keyboard.md
deleted file mode 100644
index 95935c1f1..000000000
--- a/.changeset/decision-measured-keyboard.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-"@e2e-dev/decision": patch
----
-
-The decision executor offers `dismiss_keyboard` from the observation's `keyboardVisible` when the engine measured the keyboard, and falls back to the tree's `keyboard` and `key` nodes when it did not. A keyboard the tree leaves out no longer hides the control, and key nodes left behind after the keyboard closed no longer offer it. A dismissal that changes only the measured state, not the tree, counts as a change on screen.
diff --git a/.changeset/keyboard-state-observation.md b/.changeset/keyboard-state-observation.md
index 2ae3d185d..c3cf3b211 100644
--- a/.changeset/keyboard-state-observation.md
+++ b/.changeset/keyboard-state-observation.md
@@ -1,7 +1,8 @@
 ---
-"e2e": minor
-"@e2e-dev/mobile": minor
+"e2e": patch
+"@e2e-dev/mobile": patch
 "@e2e-dev/web": patch
+"@e2e-dev/decision": patch
 ---
 
-`EngineSnapshot` and `ExecutorObservation` carry an optional `keyboardVisible`: whether an on-screen keyboard is showing, as the engine measured it apart from the tree, and absent when it did not measure it. The mobile engine sets it from the keyboard band agent-device's iOS runner measures with each capture. The web engine does not set it.
+Executor observations carry `keyboardVisible`, the engine's measured on-screen keyboard state, absent when it can't tell. The mobile engine sets it, and the decision executor uses it to offer `dismiss_keyboard`.
diff --git a/docs/decision-models.mdx b/docs/decision-models.mdx
index a891ff25b..036afde41 100644
--- a/docs/decision-models.mdx
+++ b/docs/decision-models.mdx
@@ -113,9 +113,8 @@ not controls, such as a card that opens a menu on hover, and a drag can drop
 onto any of them. `tap` also reaches text fields, to open a date picker or a
 suggestion list. While an on-screen
 keyboard shows on an engine that can dismiss it, such as the mobile engine,
-the executor also offers `dismiss_keyboard`. It uses the observation's
-measured `keyboardVisible` when the engine reports one, and otherwise looks
-for a `keyboard` or `key` node in the tree.
+the executor also offers `dismiss_keyboard`, even when the keyboard is
+missing from the tree.
 
 Every operation that acts on an element asks for its target, even when only
 one element fits it.
diff --git a/docs/executors.mdx b/docs/executors.mdx
index c6e3ac1cb..577c29781 100644
--- a/docs/executors.mdx
+++ b/docs/executors.mdx
@@ -144,11 +144,8 @@ without acting. Engines with a keyboard add `typeText`, `pressKey`, and
 path outside the project root or a hidden one with `POLICY_DENIED`, and a
 missing file with `INVALID_ARGUMENT`.
 
-The observation has `keyboardVisible` when the engine measured whether an
-on-screen keyboard shows (`true` or `false`), and omits it when it didn't.
-Without it, the [decision executor](/decision-models) falls back to a
-`keyboard` or `key` node in the tree, and your own executor can check the
-screenshot from `ctx.observe({ pixels: true })`.
+The observation's `keyboardVisible` says whether an on-screen keyboard is
+showing, and is absent when the engine can't tell.
 
 When semantic capture fails, the observation sets `treeUnavailable: true`,
 has no `tree`, and includes a screenshot if policy allows. Earlier node ids
diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx
index 3956d935f..282906db2 100644
--- a/docs/reference/engine.mdx
+++ b/docs/reference/engine.mdx
@@ -662,15 +662,6 @@ Set `truncated` when a node limit or inaccessible subtree leaves content out.
 The runner warns the model that the tree is incomplete and never treats a
 truncated observation as unchanged. Omit it for a complete tree.
 
-Set `keyboardVisible` when the platform tells the engine whether a soft
-keyboard is up, separately from the tree: `true` while one is showing,
-`false` when it measured none. A keyboard can be missing from the tree, or
-listed only as a few keys, so the [decision executor](/decision-models)
-reads this first and falls back to the tree's `keyboard` and `key` nodes
-when it is absent. The mobile engine sets it on iOS from the keyboard band
-agent-device's runner measures with each capture; Android captures carry no
-such measurement, so there it is absent.
-
 A valid empty tree may have a childless root and include `pixels`.
 
 If semantic capture times out, an engine may recover only when `pixelFallback`
diff --git a/packages/decision/src/elements.ts b/packages/decision/src/elements.ts
index d9e52ab4c..d71ce37ae 100644
--- a/packages/decision/src/elements.ts
+++ b/packages/decision/src/elements.ts
@@ -296,7 +296,7 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat
     controls.add('scroll_down');
   }
   if (verbs.has('back')) controls.add('back');
-  if (verbs.has('dismissKeyboard') && (observation.keyboardVisible ?? keyboardShowing(observation.tree))) controls.add('dismiss_keyboard');
+  if (verbs.has('dismissKeyboard') && (observation.keyboardVisible === undefined ? keyboardShowing(observation.tree) : observation.keyboardVisible)) controls.add('dismiss_keyboard');
   return {
     elements,
     element: (key) => byIndex.get(targetKeyIndex(key)),
@@ -311,7 +311,7 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat
     omitted,
     pageText: clip(pageText.join('\n'), 6000),
     statuses,
-    fingerprint: fingerprint(observation.path ?? '', observation.tree, observation.viewport, observation.keyboardVisible, observation.pixels),
+    fingerprint: fingerprint(observation),
   };
 }
 /**
@@ -400,14 +400,8 @@ function clip(text: string, limit: number): string {
  * The keyboard state is in because a dismissal can leave the tree as it was,
  * and an unmeasured state hashes apart from a measured `false`.
  */
-function fingerprint(
-  path: string,
-  tree: ExecutorNode,
-  viewport: { width: number; height: number },
-  keyboardVisible: boolean | undefined,
-  pixels?: ExecutorPixels,
-): string {
-  const parts: (string | Uint8Array)[] = [path, JSON.stringify(keyboardVisible ?? null)];
+function fingerprint({ path, tree, viewport, keyboardVisible, pixels }: SpaceObservation): string {
+  const parts: (string | Uint8Array)[] = [path ?? '', keyboardVisible === undefined ? 'unmeasured' : String(keyboardVisible)];
   if (pixels !== undefined) parts.push(pixels.data);
   const visit = (node: ExecutorNode): void => {
     parts.push(node.role ?? '', node.name ?? '', node.text ?? '', node.value ?? '', JSON.stringify(node.states ?? null), JSON.stringify(node.attributes ?? null), intersects(node, viewport) ? 'v' : '-');
diff --git a/packages/e2e/src/engine/session.ts b/packages/e2e/src/engine/session.ts
index 8bbbf082c..3f965b43d 100644
--- a/packages/e2e/src/engine/session.ts
+++ b/packages/e2e/src/engine/session.ts
@@ -395,7 +395,7 @@ export function createEngineSession(options: EngineSessionOptions): TargetSessio
         kind: 'semantic',
         tree: stampRevision(snapshot.root, minted),
         truncated: snapshot.truncated === true,
-        ...(typeof snapshot.keyboardVisible === 'boolean' ? { keyboardVisible: snapshot.keyboardVisible } : {}),
+        ...(snapshot.keyboardVisible === undefined ? {} : { keyboardVisible: snapshot.keyboardVisible }),
         ...(snapshot.pixels === undefined ? {} : { pixels: snapshot.pixels }),
       };
       showScreen(observation);
diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts
index 11deeaaa5..2f8e5aa02 100644
--- a/packages/mobile/src/surface.ts
+++ b/packages/mobile/src/surface.ts
@@ -147,8 +147,7 @@ const KEYBOARD_DRAG_RATIO = 0.03;
 
 /**
  * Whether the capture's producer measured the soft keyboard on screen:
- * undefined when it did not look or could not (only the iOS runner measures
- * it today).
+ * undefined when it did not look or could not.
  */
 function measuredKeyboard(raw: RawSnapshot): boolean | undefined {
   if (raw.keyboard?.kind === 'visible') return true;