diff --git a/.changeset/keyboard-state-observation.md b/.changeset/keyboard-state-observation.md new file mode 100644 index 000000000..c3cf3b211 --- /dev/null +++ b/.changeset/keyboard-state-observation.md @@ -0,0 +1,8 @@ +--- +"e2e": patch +"@e2e-dev/mobile": patch +"@e2e-dev/web": patch +"@e2e-dev/decision": patch +--- + +Executor observations carry `keyboardVisible`, the engine's measured on-screen keyboard state, absent when it can't tell. The mobile engine sets it, and the decision executor uses it to offer `dismiss_keyboard`. diff --git a/docs/decision-models.mdx b/docs/decision-models.mdx index 59eeda475..036afde41 100644 --- a/docs/decision-models.mdx +++ b/docs/decision-models.mdx @@ -113,7 +113,8 @@ not controls, such as a card that opens a menu on hover, and a drag can drop onto any of them. `tap` also reaches text fields, to open a date picker or a suggestion list. While an on-screen keyboard shows on an engine that can dismiss it, such as the mobile engine, -the executor also offers `dismiss_keyboard`. +the executor also offers `dismiss_keyboard`, even when the keyboard is +missing from the tree. Every operation that acts on an element asks for its target, even when only one element fits it. diff --git a/docs/executors.mdx b/docs/executors.mdx index 0ce36ff25..577c29781 100644 --- a/docs/executors.mdx +++ b/docs/executors.mdx @@ -144,6 +144,9 @@ without acting. Engines with a keyboard add `typeText`, `pressKey`, and path outside the project root or a hidden one with `POLICY_DENIED`, and a missing file with `INVALID_ARGUMENT`. +The observation's `keyboardVisible` says whether an on-screen keyboard is +showing, and is absent when the engine can't tell. + When semantic capture fails, the observation sets `treeUnavailable: true`, has no `tree`, and includes a screenshot if policy allows. Earlier node ids are invalid, and a missing tree doesn't prove a control is absent. diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index 7bc204f0f..282906db2 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -629,6 +629,12 @@ export interface EngineSnapshot { * read whole. */ readonly truncated?: boolean; + /** + * Whether an on-screen keyboard is showing, as the engine measured it apart + * from `root`. Set `false` when it measured none, and omit it only when the + * engine did not measure it. + */ + readonly keyboardVisible?: boolean; /** Semantic capture failed; only fresh, independently masked pixels are evidence. */ readonly treeUnavailable?: true; /** Masked pixels, when requested and producible; omitted otherwise. */ diff --git a/packages/decision/src/elements.ts b/packages/decision/src/elements.ts index fe83408a3..d71ce37ae 100644 --- a/packages/decision/src/elements.ts +++ b/packages/decision/src/elements.ts @@ -136,6 +136,8 @@ interface SpaceObservation { readonly pixels?: ExecutorPixels; /** Whether the step can locate a drawn control on the pixels: the decision model scores and a text model names it. */ readonly locates?: boolean; + /** The engine's measured keyboard state; undefined leaves it to the tree. */ + readonly keyboardVisible?: boolean | undefined; } /** Builds the element table and bound targets from the newest observation. */ @@ -294,7 +296,7 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat controls.add('scroll_down'); } if (verbs.has('back')) controls.add('back'); - if (verbs.has('dismissKeyboard') && keyboardShowing(observation.tree)) controls.add('dismiss_keyboard'); + if (verbs.has('dismissKeyboard') && (observation.keyboardVisible === undefined ? keyboardShowing(observation.tree) : observation.keyboardVisible)) controls.add('dismiss_keyboard'); return { elements, element: (key) => byIndex.get(targetKeyIndex(key)), @@ -309,7 +311,7 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat omitted, pageText: clip(pageText.join('\n'), 6000), statuses, - fingerprint: fingerprint(observation.path ?? '', observation.tree, observation.viewport, observation.pixels), + fingerprint: fingerprint(observation), }; } /** @@ -347,8 +349,9 @@ export function targetKeyIndex(key: string): string { } /** - * Whether the tree lists an on-screen keyboard. Offering the control only - * then keeps a choice that does nothing out of the operation question. + * Whether the tree lists an on-screen keyboard, for an engine that does not + * measure it. Offering the control only then keeps a choice that does + * nothing out of the operation question. */ function keyboardShowing(node: ExecutorNode): boolean { if (node.states?.hidden === true) return false; @@ -386,17 +389,19 @@ function clip(text: string, limit: number): string { } /** - * Stable hash of the path, the tree content with node ids removed, whether - * each node meets the viewport, and the pixels when the step sees them. Ids - * change on every capture, so they stay out. In-view membership lets a - * scroll that brings other nodes into view count as progress, while one - * that moves nothing (the page bottom) still reads as unchanged; raw - * coordinates stay out so small layout shifts do not count. Pixels are in - * because a drawn control changes nothing in the tree: a keypad digit - * entered on a canvas is progress only the screenshot shows. + * Stable hash of the path, the measured keyboard state, the tree content + * with node ids removed, whether each node meets the viewport, and the + * pixels when the step sees them. Ids change on every capture, so they stay + * out. In-view membership lets a scroll that brings other nodes into view + * count as progress, while one that moves nothing (the page bottom) still + * reads as unchanged; raw coordinates stay out so small layout shifts do not + * count. Pixels are in because a drawn control changes nothing in the tree: + * a keypad digit entered on a canvas is progress only the screenshot shows. + * The keyboard state is in because a dismissal can leave the tree as it was, + * and an unmeasured state hashes apart from a measured `false`. */ -function fingerprint(path: string, tree: ExecutorNode, viewport: { width: number; height: number }, pixels?: ExecutorPixels): string { - const parts: (string | Uint8Array)[] = [path]; +function fingerprint({ path, tree, viewport, keyboardVisible, pixels }: SpaceObservation): string { + const parts: (string | Uint8Array)[] = [path ?? '', keyboardVisible === undefined ? 'unmeasured' : String(keyboardVisible)]; if (pixels !== undefined) parts.push(pixels.data); const visit = (node: ExecutorNode): void => { parts.push(node.role ?? '', node.name ?? '', node.text ?? '', node.value ?? '', JSON.stringify(node.states ?? null), JSON.stringify(node.attributes ?? null), intersects(node, viewport) ? 'v' : '-'); diff --git a/packages/decision/src/executor.ts b/packages/decision/src/executor.ts index 7fc95e5d0..195d4f66a 100644 --- a/packages/decision/src/executor.ts +++ b/packages/decision/src/executor.ts @@ -165,7 +165,7 @@ class Step { const screenshot = this.pixelsAllowed ? observation.pixels : undefined; const tree = observation.tree; if (observation.treeUnavailable || tree === undefined || emptyTree(observation)) return { path, space: undefined, screenshot }; - const base = { path, viewport: observation.viewport, tree }; + const base = { path, viewport: observation.viewport, tree, keyboardVisible: observation.keyboardVisible }; const seen = screenshot === undefined ? base : { ...base, pixels: screenshot, locates: this.locates }; return { path, space: actionSpace(this.ctx, seen, typing), screenshot }; } diff --git a/packages/decision/tests/elements.test.ts b/packages/decision/tests/elements.test.ts index 801d36471..6933ec979 100644 --- a/packages/decision/tests/elements.test.ts +++ b/packages/decision/tests/elements.test.ts @@ -6,9 +6,9 @@ import { context } from './helpers.ts'; function tree(children: ExecutorNode[]): ExecutorNode { return { id: 'root', children }; } -function spaceFor(children: ExecutorNode[], verbs?: (keyof ExecutorActions)[]) { +function spaceFor(children: ExecutorNode[], verbs?: (keyof ExecutorActions)[], keyboardVisible?: boolean) { const fixture = context(verbs === undefined ? {} : { verbs }); - return actionSpace(fixture.ctx, { path: '/form', viewport: { width: 800, height: 600 }, tree: tree(children) }, true); + return actionSpace(fixture.ctx, { path: '/form', viewport: { width: 800, height: 600 }, tree: tree(children), keyboardVisible }, true); } describe('element table', () => { it('offers tap on buttons and links with indexed keys', () => { @@ -228,15 +228,24 @@ describe('element table', () => { expect(spaceFor([hiddenPad]).controls.has('dismiss_keyboard')).toBe(false); expect(spaceFor(numberPad, ['tap', 'type', 'scroll']).controls.has('dismiss_keyboard')).toBe(false); }); + it('takes the keyboard state the engine measured over the tree, and falls back to the tree without one', () => { + const field: ExecutorNode = { id: 'reps', role: 'textbox', name: 'Reps', value: '8' }; + // A keyboard the tree does not list, measured by the engine. + expect(spaceFor([field], undefined, true).controls.has('dismiss_keyboard')).toBe(true); + // Key nodes left in the tree after the engine measured the keyboard gone. + expect(spaceFor([field, { id: 'k1', role: 'key', name: '1' }], undefined, false).controls.has('dismiss_keyboard')).toBe(false); + expect(spaceFor([field, { id: 'k1', role: 'key', name: '1' }]).controls.has('dismiss_keyboard')).toBe(true); + expect(spaceFor([field], ['tap', 'type', 'scroll'], true).controls.has('dismiss_keyboard')).toBe(false); + }); }); describe('fingerprint', () => { const page = (): ExecutorNode => ({ id: 'root', children: [ { id: 'a', role: 'textbox', name: 'Name', value: '' }, { id: 'b', role: 'button', name: 'Add' }, ] }); - function prints(path: string, root: ExecutorNode): string { + function prints(path: string, root: ExecutorNode, keyboardVisible?: boolean): string { const fixture = context(); - return actionSpace(fixture.ctx, { path, viewport: { width: 800, height: 600 }, tree: root }, true).fingerprint; + return actionSpace(fixture.ctx, { path, viewport: { width: 800, height: 600 }, tree: root, keyboardVisible }, true).fingerprint; } it('is stable when only node ids change', () => { const first = prints('/form', page()); @@ -252,6 +261,14 @@ describe('fingerprint', () => { expect(prints('/form', edited)).not.toBe(base); expect(prints('/other', page())).not.toBe(base); }); + it('moves when only the measured keyboard state changes, and tells unmeasured from hidden', () => { + const shown = prints('/form', page(), true); + const hidden = prints('/form', page(), false); + const unmeasured = prints('/form', page()); + expect(hidden).not.toBe(shown); + expect(unmeasured).not.toBe(shown); + expect(unmeasured).not.toBe(hidden); + }); it('moves when other nodes come into view, and holds for a small shift', () => { const rows = (offset: number): ExecutorNode => ({ id: 'root', children: Array.from({ length: 6 }, (_, index) => ({ id: `r${index}`, role: 'button', name: `Row ${index}`, diff --git a/packages/decision/tests/executor.test.ts b/packages/decision/tests/executor.test.ts index e5fe48d48..b3f9ff8c0 100644 --- a/packages/decision/tests/executor.test.ts +++ b/packages/decision/tests/executor.test.ts @@ -137,6 +137,18 @@ describe('keyboard', () => { expect(Object.keys(requests[0]?.questions.operation?.criteria ?? {})).toContain('dismiss_keyboard'); expect(fixture.turns[0]?.calls).toEqual(['dismiss the keyboard']); }); + it('offers dismiss_keyboard for a keyboard the engine measured and the tree does not list', async () => { + const { model, requests } = scriptedDecision((id, keys, call) => ({ + choice: id === 'operation' ? (call === 0 ? 'dismiss_keyboard' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), + })); + const fixture = context({ + tree: { id: 'root', children: [{ id: 'reps', role: 'textbox', name: 'Reps', value: '8' }] }, + observation: { keyboardVisible: true }, + }); + await decisionExecutor({ model }).runStep(fixture.ctx); + expect(Object.keys(requests[0]?.questions.operation?.criteria ?? {})).toContain('dismiss_keyboard'); + expect(fixture.actions.dismissKeyboard).toHaveBeenCalledTimes(1); + }); }); describe('secrets', () => { diff --git a/packages/e2e/src/agent/executor.ts b/packages/e2e/src/agent/executor.ts index 23902075a..e694f8b12 100644 --- a/packages/e2e/src/agent/executor.ts +++ b/packages/e2e/src/agent/executor.ts @@ -129,6 +129,13 @@ export interface ExecutorObservation { readonly text: string; readonly truncated: boolean; readonly viewport: ViewportSize; + /** + * Whether an on-screen keyboard is showing, as the engine measured it apart + * from the tree; `false` when it measured none. Absent only when the engine + * did not measure it; the tree's `keyboard` and `key` nodes, or a + * screenshot, are then the evidence. + */ + readonly keyboardVisible?: boolean; /** * The current location as path and query, redacted, when the engine * reports one. Absent on engines without a location (a device screen). diff --git a/packages/e2e/src/agent/observation-feed.ts b/packages/e2e/src/agent/observation-feed.ts index 93d1e7f30..4dda61310 100644 --- a/packages/e2e/src/agent/observation-feed.ts +++ b/packages/e2e/src/agent/observation-feed.ts @@ -180,6 +180,7 @@ export class ObservationFeed { ...metadata, text: observation.text, truncated: observation.truncated, + ...(observation.keyboardVisible === undefined ? {} : { keyboardVisible: observation.keyboardVisible }), ...(options.tree === true ? { tree: projectTree(observation.tree, this.runtime.app.base?.origin) } : {}), ...pixels, }; diff --git a/packages/e2e/src/agent/observation.ts b/packages/e2e/src/agent/observation.ts index 3a08c47f3..049085f27 100644 --- a/packages/e2e/src/agent/observation.ts +++ b/packages/e2e/src/agent/observation.ts @@ -28,6 +28,8 @@ export type AgentObservation = AgentObservationMetadata & ( readonly parents: ReadonlyMap; readonly tree: RedactedNode; readonly truncated: boolean; + /** The engine's measured keyboard state; `false` when it measured none, absent only when it did not measure. */ + readonly keyboardVisible?: boolean | undefined; readonly pixels?: ExecutorPixels | undefined; readonly pixelsWithheld?: PixelsWithheld | undefined; } @@ -147,6 +149,7 @@ export function prepareObservation( parents, tree, truncated, + ...(observation.keyboardVisible === undefined ? {} : { keyboardVisible: observation.keyboardVisible }), ...(pixels.cleared === undefined ? {} : { pixels: pixels.cleared }), ...(pixels.withheld === undefined ? {} : { pixelsWithheld: pixels.withheld }), }; diff --git a/packages/e2e/src/engine/index.ts b/packages/e2e/src/engine/index.ts index 9185b5549..9b1478421 100644 --- a/packages/e2e/src/engine/index.ts +++ b/packages/e2e/src/engine/index.ts @@ -637,6 +637,13 @@ export interface EngineSnapshot { * read whole. */ readonly truncated?: boolean; + /** + * Whether an on-screen keyboard is showing, as the engine measured it apart + * from `root`: a platform can raise a keyboard the tree lists only in part, + * or not at all. Set `false` when it measured none, and omit it only when + * the engine did not measure it; consumers then fall back to the tree. + */ + readonly keyboardVisible?: boolean; /** * Semantic capture timed out. Return only the stable root reference, with no other fields, * and fresh pixels whose masking was proven independently of the failed diff --git a/packages/e2e/src/engine/session.ts b/packages/e2e/src/engine/session.ts index e248a8a4e..3f965b43d 100644 --- a/packages/e2e/src/engine/session.ts +++ b/packages/e2e/src/engine/session.ts @@ -395,6 +395,7 @@ export function createEngineSession(options: EngineSessionOptions): TargetSessio kind: 'semantic', tree: stampRevision(snapshot.root, minted), truncated: snapshot.truncated === true, + ...(snapshot.keyboardVisible === undefined ? {} : { keyboardVisible: snapshot.keyboardVisible }), ...(snapshot.pixels === undefined ? {} : { pixels: snapshot.pixels }), }; showScreen(observation); diff --git a/packages/e2e/src/engine/surface.ts b/packages/e2e/src/engine/surface.ts index fe5815c9e..1614a9622 100644 --- a/packages/e2e/src/engine/surface.ts +++ b/packages/e2e/src/engine/surface.ts @@ -69,6 +69,8 @@ export type Observation = ObservationMetadata & ( readonly kind: 'semantic'; readonly tree: SemanticNode; readonly truncated: boolean; + /** The engine's measured keyboard state (`EngineSnapshot.keyboardVisible`); `false` when it measured none, absent only when it did not measure. */ + readonly keyboardVisible?: boolean; readonly pixels?: ObservationPixels; } | { diff --git a/packages/e2e/tests/types/sdk-types.ts b/packages/e2e/tests/types/sdk-types.ts index 8dde9c2f2..19829bdc8 100644 --- a/packages/e2e/tests/types/sdk-types.ts +++ b/packages/e2e/tests/types/sdk-types.ts @@ -119,6 +119,11 @@ declare const engineSnapshot: EngineSnapshot; engineSnapshot.treeUnavailable satisfies true | undefined; // @ts-expect-error unavailable semantics are explicitly true or absent, never a separate false state. ({ ...engineSnapshot, treeUnavailable: false }) satisfies EngineSnapshot; +// The measured keyboard is a yes or no, or absent when the engine did not measure it. +engineSnapshot.keyboardVisible satisfies boolean | undefined; +customExecutorObservation.keyboardVisible satisfies boolean | undefined; +// @ts-expect-error the keyboard state carries no band or reason, only whether one is showing. +({ ...engineSnapshot, keyboardVisible: { kind: 'visible' } }) satisfies EngineSnapshot; // There is no implicit target: a config names at least one, and the type says so before the loader does. // @ts-expect-error targets is required diff --git a/packages/e2e/tests/unit/fixture-runtime.test.ts b/packages/e2e/tests/unit/fixture-runtime.test.ts index 820ab7d05..0b3046d57 100644 --- a/packages/e2e/tests/unit/fixture-runtime.test.ts +++ b/packages/e2e/tests/unit/fixture-runtime.test.ts @@ -501,6 +501,24 @@ describe('project tool dispatch', () => { expect(path).toBe('/settings/general'); }); + it("hands an executor the engine's measured keyboard state, and nothing when the engine measured none", async () => { + const measured: (boolean | undefined)[] = []; + let reported: boolean | undefined; + const engine = defineEngine({ name: 'fake', version: '1', spiVersion: 1, + observe: async () => snapshot([], reported === undefined ? {} : { keyboardVisible: reported }), + }); + const executor: StepExecutor = { name: 'test', async runStep(context) { + measured.push((await context.observe()).keyboardVisible); + return { status: 'passed', summary: 'looked' }; + } }; + const { fixtures } = runtime(engine, { agents: { default: { executor } } }); + for (const state of [undefined, true, false]) { + reported = state; + await fixtures.agent.act('look'); + } + expect(measured).toEqual([undefined, true, false]); + }); + }); describe('press key grammar', () => { diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 5005ee4c5..2f8e5aa02 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -145,6 +145,16 @@ const KEYBOARD_TYPES: ReadonlySet = new Set(['keyboard', 'key']); */ const KEYBOARD_DRAG_RATIO = 0.03; +/** + * Whether the capture's producer measured the soft keyboard on screen: + * undefined when it did not look or could not. + */ +function measuredKeyboard(raw: RawSnapshot): boolean | undefined { + if (raw.keyboard?.kind === 'visible') return true; + if (raw.keyboard?.kind === 'absent') return false; + return undefined; +} + /** * Whether a capture shows the soft keyboard: the band its producer measured * when it measured one, else a keyboard or key element in the tree, the probe @@ -153,9 +163,7 @@ const KEYBOARD_DRAG_RATIO = 0.03; * tap refuses against by name. */ function keyboardShowing(raw: RawSnapshot): boolean { - if (raw.keyboard?.kind === 'visible') return true; - if (raw.keyboard?.kind === 'absent') return false; - return (raw.nodes ?? []).some((node) => KEYBOARD_TYPES.has(normalizeKind(node.type ?? ''))); + return measuredKeyboard(raw) ?? (raw.nodes ?? []).some((node) => KEYBOARD_TYPES.has(normalizeKind(node.type ?? ''))); } /** @@ -945,10 +953,12 @@ export class AgentDeviceSurface { const viewport = await this.viewportFor(projected, operation.signal); const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected)); const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }) : undefined; + const keyboardVisible = measuredKeyboard(raw); return { root: screenRoot(projected.roots, viewport), viewport, ...(isTruncated(raw) ? { truncated: true } : {}), + ...(keyboardVisible === undefined ? {} : { keyboardVisible }), ...(location === undefined ? {} : { location }), ...(capture === undefined ? {} : { pixels: capture.pixels, maskedRegionCount: capture.masked }), }; diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 140804b1a..9b276a60b 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -679,6 +679,22 @@ describe('observation', () => { } }); + it('reports the keyboard state the capture measured, and nothing when it measured none', async () => { + const h = harness(); + await openAttempt(h); + expect((await h.engine.observe!(operation())).keyboardVisible).toBeUndefined(); + + const frame = { x: 0, y: 583, width: 390, height: 261 }; + h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'visible', frame } })); + expect((await h.engine.observe!(operation())).keyboardVisible).toBe(true); + + h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'absent' } })); + expect((await h.engine.observe!(operation())).keyboardVisible).toBe(false); + + h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'unmeasurable', reason: 'keyboard-frame-query-timeout' } })); + expect((await h.engine.observe!(operation())).keyboardVisible).toBeUndefined(); + }); + it('measures the screen in the viewport the snapshot publishes, and in its tree when it publishes none', async () => { const h = harness(); // The application node reads 390x844; a published viewport outranks it.