diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index 09698d2fd3..0308f90b4d 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -181,8 +181,18 @@ export type ResolvedInteractionTarget = resolution?: ResolutionDisclosure; recordingTarget?: RecordingTargetOverride; preAction?: SurfaceScopedNodes; + readiness?: ReadinessWaitEvidence; }; +/** + * The wait a successful press/click/longpress spent before its target appeared. Reported only when + * more than one capture was needed, so its presence means the step would have failed without the wait. + */ +export type ReadinessWaitEvidence = { + polls: number; + waitedMs: number; +}; + /** * A post-action capture that describes a DIFFERENT surface than the pre-action baseline (#2438): an * in-place iOS system surface (a web sign-in or Apple Pay sheet, hosted out of the app's process) @@ -354,6 +364,7 @@ type TouchResponseDataBase = { evidence?: InteractionEvidence; settle?: SettleObservation; resolution?: ResolutionDisclosure; + readiness?: ReadinessWaitEvidence; cost?: ResponseCost; /** Direct iOS Maestro coordinate-fallback signals. */ maestroNonHittableCoordinateFallbackAllowed?: boolean; diff --git a/src/__tests__/client-interactions-readiness.test.ts b/src/__tests__/client-interactions-readiness.test.ts index 9b77ccf024..bff493c383 100644 --- a/src/__tests__/client-interactions-readiness.test.ts +++ b/src/__tests__/client-interactions-readiness.test.ts @@ -17,3 +17,17 @@ test('readinessTimeoutMs on press/click/longpress reaches the request flags', as assert.equal(setup.calls[1]?.flags?.readinessTimeoutMs, 1_500); assert.equal(setup.calls[2]?.flags?.readinessTimeoutMs, 900); }); + +test('a press that waited returns the daemon data.readiness untouched', async () => { + const readiness = { polls: 2, waitedMs: 180 }; + const setup = createTransport(async () => ({ ok: true, data: { ref: '@e1', readiness } })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + const pressed = await client.interactions.press({ selector: 'label=Foo' }); + const clicked = await client.interactions.click({ selector: 'label=Foo' }); + const longPressed = await client.interactions.longPress({ selector: 'label=Foo' }); + + for (const result of [pressed, clicked, longPressed]) { + assert.deepEqual((result as { readiness?: unknown }).readiness, readiness); + } +}); diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index a141a486ee..24c3c5af3b 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -5,6 +5,7 @@ import { readinessScheduleFor } from '@agent-device/selectors/selector-pipeline- import { attemptSelectorResolution, pollForSelectorReadiness, + type SelectorReadinessWait, selectorInteractionFailure, } from './selector-readiness.ts'; import { @@ -138,6 +139,7 @@ async function resolveSelectorInteractionTarget( const selectorExpression = target.selector; let capture: InteractionSnapshot; let resolved: SelectorResolution | null; + let readiness: SelectorReadinessWait | undefined; const readinessSchedule = readinessScheduleFor(params.pipeline.poll, params.readinessTimeoutMs); if (!readinessSchedule) { const attempt = await attemptSelectorResolution(runtime, options, selectorExpression, params); @@ -162,6 +164,7 @@ async function resolveSelectorInteractionTarget( ); capture = ready.capture; resolved = ready.resolved; + readiness = ready.readiness; } // #1542: see the ref-target twin in ref-target-resolution.ts. const selected = resolved; @@ -188,6 +191,7 @@ async function resolveSelectorInteractionTarget( kind: 'selector', point, target: { kind: 'selector', selector: resolved.selector }, + ...(readiness ? { readiness } : {}), ...describeResolvedInteractionNode( runtime, visibleNode, diff --git a/src/commands/interaction/runtime/selector-readiness.test.ts b/src/commands/interaction/runtime/selector-readiness.test.ts index a1493b2247..dc3ade6d06 100644 --- a/src/commands/interaction/runtime/selector-readiness.test.ts +++ b/src/commands/interaction/runtime/selector-readiness.test.ts @@ -60,12 +60,28 @@ test('runtime press with readinessTimeoutMs polls until the target appears', asy assert.equal(result.kind, 'selector'); assert.equal(result.node?.label, 'Continue'); + assert.equal(result.kind === 'selector' ? result.readiness?.polls : undefined, 2); assert.ok( captures >= 4, `expected at least 4 captures before the target appeared, got ${captures}`, ); }); +test('runtime press that resolves on the first capture reports no readiness despite a budget', async () => { + const device = createInteractionDevice(makeSnapshotState([CONTINUE_BUTTON]), { + clock: createFakeClock(), + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + readinessTimeoutMs: 2_000, + }); + + assert.equal(result.kind, 'selector'); + assert.equal('readiness' in result, false); +}); + test('runtime press caps a readinessTimeoutMs larger than the row maxTimeoutMs at 2_000ms', async () => { const device = createInteractionDevice(makeSnapshotState([]), { clock: createFakeClock(), diff --git a/src/commands/interaction/runtime/selector-readiness.ts b/src/commands/interaction/runtime/selector-readiness.ts index cddb669138..758f09cf9d 100644 --- a/src/commands/interaction/runtime/selector-readiness.ts +++ b/src/commands/interaction/runtime/selector-readiness.ts @@ -44,8 +44,13 @@ export type SelectorResolutionAttempt = { export type ResolvedSelectorAttempt = { capture: InteractionSnapshot; resolved: SelectorResolution; + /** The wait that preceded the hit; absent when the first capture resolved. */ + readiness?: SelectorReadinessWait; }; +/** What a successful readiness wait reports: present only when more than one capture was needed. */ +export type SelectorReadinessWait = Pick; + /** The readiness poll's evidence, attached to a target-not-found failure only (never to a refusal). */ export type SelectorReadinessDetails = { polls: number; @@ -272,7 +277,14 @@ export async function pollForSelectorReadiness( ...(runtime.clock ? { clock: runtime.clock } : {}), phase: 'interaction_target_readiness', }); - if (observed.kind === 'done') return observed.result; + if (observed.kind === 'done') { + return observed.polls.length > 1 + ? { + ...observed.result, + readiness: { polls: observed.polls.length, waitedMs: observed.waitedMs }, + } + : observed.result; + } if (observed.kind === 'failed') throw withSparseReadiness(observed.error, observed); throw await readinessExhaustedFailure(runtime, selectorExpression, params, observed); } diff --git a/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts index f3bf5784b5..a471a83106 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts @@ -2,7 +2,11 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; import { handleInteractionCommands } from '../../index.ts'; -import { transformTouchResponseData } from '../interaction-touch-response.ts'; +import { + buildInteractionResponseData, + transformTouchResponseData, +} from '../interaction-touch-response.ts'; +import type { PressCommandResult } from '@agent-device/contracts/interaction'; import { getRuntimeBindings, mockFillPoint, @@ -82,6 +86,41 @@ test('commandless longpress response omits internal interaction diagnostics', () ).toEqual({ completedSteps: 1 }); }); +function selectorResult(readiness?: { polls: number; waitedMs: number }): PressCommandResult { + return { + kind: 'selector', + point: { x: 10, y: 20 }, + target: { kind: 'selector', selector: 'label=Continue' }, + node: { ref: 'e1', index: 0, type: 'Button', label: 'Continue' }, + selectorChain: ['label=Continue'], + warning: 'earlier warning', + ...(readiness ? { readiness } : {}), + }; +} + +const WAITED_SELECTOR_RESULT = selectorResult({ polls: 3, waitedMs: 400 }); + +test('the response builder reports the resolved wait and keeps the prior warning', () => { + const { responseData, result } = buildInteractionResponseData({ + source: { kind: 'runtime', result: WAITED_SELECTOR_RESULT }, + referenceFrame: undefined, + staleRefsWarning: 'stale refs', + }); + + expect(responseData.readiness).toEqual({ polls: 3, waitedMs: 400 }); + expect(responseData.warning).toBe('earlier warning stale refs'); + expect(result.readiness).toBeUndefined(); +}); + +test('the response builder omits readiness when nothing waited', () => { + const { responseData } = buildInteractionResponseData({ + source: { kind: 'runtime', result: selectorResult() }, + referenceFrame: undefined, + }); + + expect('readiness' in responseData).toBe(false); +}); + test('press @ref --verify surfaces evidence through the interactionResultExtra allowlist', async () => { const sessionStore = makeSessionStore(); const sessionName = 'verify-press'; diff --git a/src/daemon/interaction/internal/interaction-touch-response.ts b/src/daemon/interaction/internal/interaction-touch-response.ts index 7238a8677b..5924dca3c3 100644 --- a/src/daemon/interaction/internal/interaction-touch-response.ts +++ b/src/daemon/interaction/internal/interaction-touch-response.ts @@ -234,6 +234,7 @@ export function buildInteractionResponseData(params: { visualization.warning = warning; responseData.warning = warning; } + if ('readiness' in result && result.readiness) responseData.readiness = result.readiness; return { result: visualization, responseData, ...recordedTargetCapture(result) }; } diff --git a/src/mcp/__tests__/command-tools-readiness.test.ts b/src/mcp/__tests__/command-tools-readiness.test.ts new file mode 100644 index 0000000000..73ab2c4596 --- /dev/null +++ b/src/mcp/__tests__/command-tools-readiness.test.ts @@ -0,0 +1,20 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createAgentDeviceClient } from '../../agent-device-client.ts'; +import { createTransport } from '../../__tests__/client-transport-fixture.ts'; +import { createCommandToolExecutor } from '../command-tools.ts'; + +test('MCP press returns the daemon data.readiness in structuredContent', async () => { + const readiness = { polls: 2, waitedMs: 180 }; + const setup = createTransport(async () => ({ ok: true, data: { ref: '@e1', readiness } })); + const executor = createCommandToolExecutor({ + createClient: () => createAgentDeviceClient(setup.config, { transport: setup.transport }), + }); + + const result = await executor.execute('press', { + target: { kind: 'selector', selector: 'label=Continue' }, + }); + + assert.equal(result.isError, false); + assert.deepEqual(result.structuredContent?.readiness, readiness); +}); diff --git a/test/integration/provider-scenarios/press-target-readiness.test.ts b/test/integration/provider-scenarios/press-target-readiness.test.ts index d98e43766a..a9b0b16e9c 100644 --- a/test/integration/provider-scenarios/press-target-readiness.test.ts +++ b/test/integration/provider-scenarios/press-target-readiness.test.ts @@ -142,6 +142,41 @@ test('press waits for a selector missing on the first two captures, then taps on ); }); +test('press whose target appears on the third poll reports readiness with polls 3 on the success response', async () => { + await withPressReadinessDaemon( + [ + snapshotEntry(APPLICATION_ONLY_NODES), + snapshotEntry(APPLICATION_ONLY_NODES), + snapshotEntry(APPLICATION_ONLY_NODES), + snapshotEntry(APPLICATION_ONLY_NODES), + repeatSnapshotEntry(CONTINUE_BUTTON_NODES), + tapEntry(200, 322), + ], + async (daemon) => { + const press = await daemon.callCommand('press', ['label=Continue'], { + readinessTimeoutMs: 2_000, + }); + const data = assertRpcOk(press); + const readiness = data.readiness as { polls: number; waitedMs: number } | undefined; + assert.equal(readiness?.polls, 3); + assert.equal(typeof readiness?.waitedMs, 'number'); + }, + ); +}); + +test('press whose first capture hits carries no readiness field even with a readiness budget', async () => { + await withPressReadinessDaemon( + [snapshotEntry(CONTINUE_BUTTON_NODES), tapEntry(200, 322)], + async (daemon) => { + const press = await daemon.callCommand('press', ['label=Continue'], { + readinessTimeoutMs: 2_000, + }); + const data = assertRpcOk(press); + assert.equal('readiness' in data, false); + }, + ); +}); + test('press fails with the standard no-match error, carrying readiness evidence, when the target never appears', async () => { await withPressReadinessDaemon([repeatSnapshotEntry(APPLICATION_ONLY_NODES)], async (daemon) => { // No fake clock is wired into this daemon-composed runtime path (AgentDeviceRuntime.clock is diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 08cd4d31de..f5a707514d 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -297,7 +297,7 @@ await client.interactions.press({ }); ``` -The wait is capped at 2 seconds and covers only a target that has not appeared. When the target is still missing after the wait, the error carries `error.details.readiness` with `waitedMs`, `polls`, and `end` (`expired` or `stalled`). A capture that shows an empty accessibility tree ends the wait at once with `capture_sparse` and `readiness.end: sparse`. A covered, off-screen, or ambiguous target fails at once, and a screen that stays unreadable for the whole wait fails with its own error; neither carries `readiness`. `readinessTimeoutMs` is not an MCP tool argument and has no CLI flag. +The wait is capped at 2 seconds and covers only a target that has not appeared. When the target is still missing after the wait, the error carries `error.details.readiness` with `waitedMs`, `polls`, and `end` (`expired` or `stalled`). A capture that shows an empty accessibility tree ends the wait at once with `capture_sparse` and `readiness.end: sparse`. When the command had to wait and then succeeded, the result carries `data.readiness` with `polls` and `waitedMs`. A command that found its target on the first look has no `readiness` field. A covered, off-screen, or ambiguous target fails at once, and a screen that stays unreadable for the whole wait fails with its own error; neither carries `readiness`. `readinessTimeoutMs` is not an MCP tool argument and has no CLI flag. Vega OS client support is currently VVD-only and covers device discovery, app open/close, `back`, `home`, and `tvRemote`. Physical Fire TV, capture, selector, install, logging, and performance methods report unsupported for Vega targets. diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 3023fa48d1..13486bf252 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -62,6 +62,8 @@ agent-device replay ~/.agent-device/sessions/e2e-2026-02-09T12-00-00-000Z.ad --s they fail. A step recorded against a screen that was still loading passes on replay once the target shows up. The wait covers only a target that is not on screen yet: a target that is covered, off-screen, or matched by more than one element fails at once, as it does live. + A step that waited and then passed shows no trace of the wait in the replay output; run with + `--debug` to see it in the diagnostics. - When the target never appears, replay stops with `REPLAY_DIVERGENCE`, and `error.details.readiness` says how long the step waited and how many times it looked (`waitedMs`, `polls`, `end`). For a step recorded with a target annotation (the `# agent-device:target-v1`