From 47c2206e172894cffb884213abeb9734407fd5e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 30 Sep 2026 20:46:33 +0200 Subject: [PATCH 1/4] feat(interaction): report the readiness wait on a successful press, click, or longpress A step that waited for its target and then acted now returns data.readiness with polls and waitedMs, present only when more than one capture was needed. The runtime selector poll reports the wait on the resolved target, the shared interaction response builder attaches it without touching the warning, and a replay step's pre-dispatch gate hands its own wait to the dispatch through internal.replayReadinessWait so one step reports one combined readiness. Tests and the mutation that fails each: - press-target-readiness.test.ts "third poll" (provider scenario, polls 3): resolution.ts drops `readiness = ready.readiness`. - press-target-readiness.test.ts "first capture hits" and selector-readiness.test.ts "first capture ... no readiness": the `observed.polls.length > 1` guard becomes `> 0`. - selector-readiness.test.ts "polls until the target appears": same drop as the third-poll scenario. - interaction-touch-response.test.ts carried-wait cases: the builder stops combining `params.carriedReadiness`; the first case also pins the prior warning staying composed. - session-replay-target-verification-runtime.test.ts gate cases: the adapter stops passing gateWait to applyReplayReadinessWait; or the gate records a wait at `polls > 0`. --- packages/contracts/src/interaction.ts | 11 ++++ packages/contracts/src/replay.ts | 7 +++ .../session-replay-dispatch-narrowing.ts | 11 ++++ .../session-replay-runtime-engine-adapter.ts | 6 +- .../interaction/runtime/resolution.ts | 4 ++ .../runtime/selector-readiness.test.ts | 16 +++++ .../interaction/runtime/selector-readiness.ts | 14 ++++- ...replay-target-verification-runtime.test.ts | 28 +++++++++ .../interaction-touch-response.test.ts | 61 ++++++++++++++++++- .../internal/interaction-touch-response.ts | 20 ++++++ .../press-target-readiness.test.ts | 35 +++++++++++ 11 files changed, 210 insertions(+), 3 deletions(-) diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index 09698d2fd3..0308f90b4d 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -181,8 +181,18 @@ export type ResolvedInteractionTarget = resolution?: ResolutionDisclosure; recordingTarget?: RecordingTargetOverride; preAction?: SurfaceScopedNodes; + readiness?: ReadinessWaitEvidence; }; +/** + * The wait a successful press/click/longpress spent before its target appeared. Reported only when + * more than one capture was needed, so its presence means the step would have failed without the wait. + */ +export type ReadinessWaitEvidence = { + polls: number; + waitedMs: number; +}; + /** * A post-action capture that describes a DIFFERENT surface than the pre-action baseline (#2438): an * in-place iOS system surface (a web sign-in or Apple Pay sheet, hosted out of the app's process) @@ -354,6 +364,7 @@ type TouchResponseDataBase = { evidence?: InteractionEvidence; settle?: SettleObservation; resolution?: ResolutionDisclosure; + readiness?: ReadinessWaitEvidence; cost?: ResponseCost; /** Direct iOS Maestro coordinate-fallback signals. */ maestroNonHittableCoordinateFallbackAllowed?: boolean; diff --git a/packages/contracts/src/replay.ts b/packages/contracts/src/replay.ts index 64da7630c7..aaaf011fd3 100644 --- a/packages/contracts/src/replay.ts +++ b/packages/contracts/src/replay.ts @@ -1,6 +1,7 @@ import type { DaemonError } from '@agent-device/kernel/errors'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import type { Rect, SnapshotState } from '@agent-device/kernel/snapshot'; +import type { ReadinessWaitEvidence } from './interaction.ts'; import type { GestureExecutionProfile } from './gesture-plan-types.ts'; import type { SessionScope } from './session-scope.ts'; import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; @@ -260,6 +261,12 @@ export type ReplayDispatchOptions = Readonly<{ * (pre-action) when its winner differs in local identity OR structural position. */ replayTargetGuard?: ReplayTargetGuardDenotation; + /** + * The wait the replay step's pre-dispatch target gate already spent, set ONLY by the replay step + * loop when it polled more than once. The interaction response builder adds it to the dispatch's + * own wait so a successful step reports one `readiness` for the whole step. + */ + replayReadinessWait?: ReadinessWaitEvidence; /** Dual-endpoint counterpart of `replayTargetGuard` for target-authored drag. */ replayTargetGuards?: Readonly<{ source: ReplayTargetGuardDenotation; diff --git a/packages/replay-port/src/daemon-port/session-replay-dispatch-narrowing.ts b/packages/replay-port/src/daemon-port/session-replay-dispatch-narrowing.ts index 41393349fd..5bf37477e5 100644 --- a/packages/replay-port/src/daemon-port/session-replay-dispatch-narrowing.ts +++ b/packages/replay-port/src/daemon-port/session-replay-dispatch-narrowing.ts @@ -8,6 +8,7 @@ import type { } from '@agent-device/ad-replay'; import type { LocalIdentity } from '@agent-device/ad-script'; import type { TargetAncestryEntry } from '@agent-device/contracts/replay'; +import type { ReadinessWaitEvidence } from '@agent-device/contracts/interaction'; import { isReplayTargetGuardMismatchResponse, isWaitLandmarkMismatchResponse, @@ -62,6 +63,16 @@ export function applyReplayDispatchGuard( : replayReq; } +/** Carries the wait the pre-dispatch gate spent into the dispatch, so its success response reports it. */ +export function applyReplayReadinessWait( + replayReq: ReplayDispatchRequest, + wait: ReadinessWaitEvidence | undefined, +): ReplayDispatchRequest { + return wait + ? { ...replayReq, dispatch: { ...replayReq.dispatch, replayReadinessWait: wait } } + : replayReq; +} + function readGuardMismatchObservedIdentity(value: unknown): LocalIdentity | undefined { if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined; const record = value as Record; diff --git a/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts b/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts index d46a0cdf09..9101362241 100644 --- a/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts +++ b/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts @@ -31,6 +31,7 @@ import type { ObservationEvidence } from '@agent-device/capture-kit/observe-unti import { collectReplayActionArtifactPaths } from '@agent-device/replay-port/session-replay-runtime-artifacts'; import { applyReplayDispatchGuard, + applyReplayReadinessWait, classifyReplayDispatchFailure, toAdReplayStepFailure, } from './session-replay-dispatch-narrowing.ts'; @@ -270,7 +271,10 @@ export function createAdReplayStepRuntime(params: { readinessTimeoutMs: gateWait.remainingBudgetMs, } : {}), - req: applyReplayDispatchGuard(ctx.replayReq, guard), + req: applyReplayReadinessWait( + applyReplayDispatchGuard(ctx.replayReq, guard), + gateWait && { polls: gateWait.readiness.polls, waitedMs: gateWait.readiness.waitedMs }, + ), sessionName: ctx.sessionName, action, resolved: resolvedAction, diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index a141a486ee..24c3c5af3b 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -5,6 +5,7 @@ import { readinessScheduleFor } from '@agent-device/selectors/selector-pipeline- import { attemptSelectorResolution, pollForSelectorReadiness, + type SelectorReadinessWait, selectorInteractionFailure, } from './selector-readiness.ts'; import { @@ -138,6 +139,7 @@ async function resolveSelectorInteractionTarget( const selectorExpression = target.selector; let capture: InteractionSnapshot; let resolved: SelectorResolution | null; + let readiness: SelectorReadinessWait | undefined; const readinessSchedule = readinessScheduleFor(params.pipeline.poll, params.readinessTimeoutMs); if (!readinessSchedule) { const attempt = await attemptSelectorResolution(runtime, options, selectorExpression, params); @@ -162,6 +164,7 @@ async function resolveSelectorInteractionTarget( ); capture = ready.capture; resolved = ready.resolved; + readiness = ready.readiness; } // #1542: see the ref-target twin in ref-target-resolution.ts. const selected = resolved; @@ -188,6 +191,7 @@ async function resolveSelectorInteractionTarget( kind: 'selector', point, target: { kind: 'selector', selector: resolved.selector }, + ...(readiness ? { readiness } : {}), ...describeResolvedInteractionNode( runtime, visibleNode, diff --git a/src/commands/interaction/runtime/selector-readiness.test.ts b/src/commands/interaction/runtime/selector-readiness.test.ts index a1493b2247..e6cb1a88ee 100644 --- a/src/commands/interaction/runtime/selector-readiness.test.ts +++ b/src/commands/interaction/runtime/selector-readiness.test.ts @@ -60,12 +60,28 @@ test('runtime press with readinessTimeoutMs polls until the target appears', asy assert.equal(result.kind, 'selector'); assert.equal(result.node?.label, 'Continue'); + assert.ok(result.kind === 'selector' && result.readiness && result.readiness.polls > 1); assert.ok( captures >= 4, `expected at least 4 captures before the target appeared, got ${captures}`, ); }); +test('runtime press that resolves on the first capture reports no readiness despite a budget', async () => { + const device = createInteractionDevice(makeSnapshotState([CONTINUE_BUTTON]), { + clock: createFakeClock(), + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + readinessTimeoutMs: 2_000, + }); + + assert.equal(result.kind, 'selector'); + assert.equal('readiness' in result, false); +}); + test('runtime press caps a readinessTimeoutMs larger than the row maxTimeoutMs at 2_000ms', async () => { const device = createInteractionDevice(makeSnapshotState([]), { clock: createFakeClock(), diff --git a/src/commands/interaction/runtime/selector-readiness.ts b/src/commands/interaction/runtime/selector-readiness.ts index cddb669138..758f09cf9d 100644 --- a/src/commands/interaction/runtime/selector-readiness.ts +++ b/src/commands/interaction/runtime/selector-readiness.ts @@ -44,8 +44,13 @@ export type SelectorResolutionAttempt = { export type ResolvedSelectorAttempt = { capture: InteractionSnapshot; resolved: SelectorResolution; + /** The wait that preceded the hit; absent when the first capture resolved. */ + readiness?: SelectorReadinessWait; }; +/** What a successful readiness wait reports: present only when more than one capture was needed. */ +export type SelectorReadinessWait = Pick; + /** The readiness poll's evidence, attached to a target-not-found failure only (never to a refusal). */ export type SelectorReadinessDetails = { polls: number; @@ -272,7 +277,14 @@ export async function pollForSelectorReadiness( ...(runtime.clock ? { clock: runtime.clock } : {}), phase: 'interaction_target_readiness', }); - if (observed.kind === 'done') return observed.result; + if (observed.kind === 'done') { + return observed.polls.length > 1 + ? { + ...observed.result, + readiness: { polls: observed.polls.length, waitedMs: observed.waitedMs }, + } + : observed.result; + } if (observed.kind === 'failed') throw withSparseReadiness(observed.error, observed); throw await readinessExhaustedFailure(runtime, selectorExpression, params, observed); } diff --git a/src/daemon/__tests__/replay-target/session-replay-target-verification-runtime.test.ts b/src/daemon/__tests__/replay-target/session-replay-target-verification-runtime.test.ts index ddffb39004..012c2382d2 100644 --- a/src/daemon/__tests__/replay-target/session-replay-target-verification-runtime.test.ts +++ b/src/daemon/__tests__/replay-target/session-replay-target-verification-runtime.test.ts @@ -225,6 +225,34 @@ test('an annotated click whose target renders on the second capture waits for it ]); }); +test('a step that waited in the gate hands its wait to the dispatch for the success response', async () => { + const scene = replayScriptScene('agent-device-replay-target-verify-wait-evidence-', [ + SAVE_ANNOTATION, + 'click id="save"', + ]); + + mockDispatchCommand.mockResolvedValueOnce(emptyCapture()).mockResolvedValue(saveButtonCapture()); + + const response = await scene.replay(); + + expect(response.ok).toBe(true); + expect(scene.invoked[0]?.internal?.replayReadinessWait).toEqual({ polls: 2, waitedMs: 200 }); +}); + +test('a step whose gate hit on the first capture hands no wait to the dispatch', async () => { + const scene = replayScriptScene('agent-device-replay-target-verify-no-wait-evidence-', [ + SAVE_ANNOTATION, + 'click id="save"', + ]); + + mockDispatchCommand.mockResolvedValue(saveButtonCapture()); + + const response = await scene.replay(); + + expect(response.ok).toBe(true); + expect(scene.invoked[0]?.internal?.replayReadinessWait).toBeUndefined(); +}); + test('the dispatch gets only the readiness budget the gate left', async () => { const scene = replayScriptScene('agent-device-replay-target-verify-shared-budget-', [ SAVE_ANNOTATION, diff --git a/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts index f3bf5784b5..02a12c7650 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts @@ -2,7 +2,11 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; import { handleInteractionCommands } from '../../index.ts'; -import { transformTouchResponseData } from '../interaction-touch-response.ts'; +import { + buildInteractionResponseData, + transformTouchResponseData, +} from '../interaction-touch-response.ts'; +import type { PressCommandResult } from '@agent-device/contracts/interaction'; import { getRuntimeBindings, mockFillPoint, @@ -82,6 +86,61 @@ test('commandless longpress response omits internal interaction diagnostics', () ).toEqual({ completedSteps: 1 }); }); +function selectorResult(readiness?: { polls: number; waitedMs: number }): PressCommandResult { + return { + kind: 'selector', + point: { x: 10, y: 20 }, + target: { kind: 'selector', selector: 'label=Continue' }, + node: { ref: 'e1', index: 0, type: 'Button', label: 'Continue' }, + selectorChain: ['label=Continue'], + warning: 'earlier warning', + ...(readiness ? { readiness } : {}), + }; +} + +const WAITED_SELECTOR_RESULT = selectorResult({ polls: 3, waitedMs: 400 }); + +test('the response builder reports the resolved wait and keeps the prior warning', () => { + const { responseData, result } = buildInteractionResponseData({ + source: { kind: 'runtime', result: WAITED_SELECTOR_RESULT }, + referenceFrame: undefined, + staleRefsWarning: 'stale refs', + }); + + expect(responseData.readiness).toEqual({ polls: 3, waitedMs: 400 }); + expect(responseData.warning).toBe('earlier warning stale refs'); + expect(result.readiness).toBeUndefined(); +}); + +test('the response builder adds the wait the request carried to the dispatch wait', () => { + const { responseData } = buildInteractionResponseData({ + source: { kind: 'runtime', result: WAITED_SELECTOR_RESULT }, + referenceFrame: undefined, + carriedReadiness: { polls: 2, waitedMs: 200 }, + }); + + expect(responseData.readiness).toEqual({ polls: 5, waitedMs: 600 }); +}); + +test('the response builder reports the carried wait when the dispatch resolved on its first capture', () => { + const { responseData } = buildInteractionResponseData({ + source: { kind: 'runtime', result: selectorResult() }, + referenceFrame: undefined, + carriedReadiness: { polls: 2, waitedMs: 200 }, + }); + + expect(responseData.readiness).toEqual({ polls: 2, waitedMs: 200 }); +}); + +test('the response builder omits readiness when nothing waited', () => { + const { responseData } = buildInteractionResponseData({ + source: { kind: 'runtime', result: selectorResult() }, + referenceFrame: undefined, + }); + + expect('readiness' in responseData).toBe(false); +}); + test('press @ref --verify surfaces evidence through the interactionResultExtra allowlist', async () => { const sessionStore = makeSessionStore(); const sessionName = 'verify-press'; diff --git a/src/daemon/interaction/internal/interaction-touch-response.ts b/src/daemon/interaction/internal/interaction-touch-response.ts index 7238a8677b..57746977af 100644 --- a/src/daemon/interaction/internal/interaction-touch-response.ts +++ b/src/daemon/interaction/internal/interaction-touch-response.ts @@ -4,6 +4,7 @@ import type { HoverCommandResult, LongPressCommandResult, PressCommandResult, + ReadinessWaitEvidence, RecordingTargetOverride, ResolutionDisclosure, SettleObservation, @@ -172,6 +173,8 @@ export function buildInteractionResponseData(params: { * Only attached when the settle observation actually carries a diff. */ settleRefsGeneration?: number; + /** A wait the request already spent before this dispatch (the replay step's pre-dispatch gate). */ + carriedReadiness?: ReadinessWaitEvidence; }): InteractionResponsePayloads { const { source, referenceFrame, extra } = params; if (source.kind === 'runner-payload' || source.kind === 'corroborated-tap') { @@ -234,9 +237,25 @@ export function buildInteractionResponseData(params: { visualization.warning = warning; responseData.warning = warning; } + const readiness = combineReadinessWaits( + 'readiness' in result ? result.readiness : undefined, + params.carriedReadiness, + ); + if (readiness) responseData.readiness = readiness; return { result: visualization, responseData, ...recordedTargetCapture(result) }; } +function combineReadinessWaits( + ...waits: Array +): ReadinessWaitEvidence | undefined { + const spent = waits.filter((wait): wait is ReadinessWaitEvidence => wait !== undefined); + if (spent.length === 0) return undefined; + return { + polls: spent.reduce((total, wait) => total + wait.polls, 0), + waitedMs: spent.reduce((total, wait) => total + wait.waitedMs, 0), + }; +} + function recordedTargetCapture( result: InteractionRuntimeResult, ): Pick { @@ -311,6 +330,7 @@ export async function buildTargetedTouchResponsePayloads(params: { extra, staleRefsWarning: params.staleRefsWarning, settleRefsGeneration: issueSettleRefs(session, result.settle), + carriedReadiness: handlerParams.req.internal?.replayReadinessWait, }); } diff --git a/test/integration/provider-scenarios/press-target-readiness.test.ts b/test/integration/provider-scenarios/press-target-readiness.test.ts index d98e43766a..a9b0b16e9c 100644 --- a/test/integration/provider-scenarios/press-target-readiness.test.ts +++ b/test/integration/provider-scenarios/press-target-readiness.test.ts @@ -142,6 +142,41 @@ test('press waits for a selector missing on the first two captures, then taps on ); }); +test('press whose target appears on the third poll reports readiness with polls 3 on the success response', async () => { + await withPressReadinessDaemon( + [ + snapshotEntry(APPLICATION_ONLY_NODES), + snapshotEntry(APPLICATION_ONLY_NODES), + snapshotEntry(APPLICATION_ONLY_NODES), + snapshotEntry(APPLICATION_ONLY_NODES), + repeatSnapshotEntry(CONTINUE_BUTTON_NODES), + tapEntry(200, 322), + ], + async (daemon) => { + const press = await daemon.callCommand('press', ['label=Continue'], { + readinessTimeoutMs: 2_000, + }); + const data = assertRpcOk(press); + const readiness = data.readiness as { polls: number; waitedMs: number } | undefined; + assert.equal(readiness?.polls, 3); + assert.equal(typeof readiness?.waitedMs, 'number'); + }, + ); +}); + +test('press whose first capture hits carries no readiness field even with a readiness budget', async () => { + await withPressReadinessDaemon( + [snapshotEntry(CONTINUE_BUTTON_NODES), tapEntry(200, 322)], + async (daemon) => { + const press = await daemon.callCommand('press', ['label=Continue'], { + readinessTimeoutMs: 2_000, + }); + const data = assertRpcOk(press); + assert.equal('readiness' in data, false); + }, + ); +}); + test('press fails with the standard no-match error, carrying readiness evidence, when the target never appears', async () => { await withPressReadinessDaemon([repeatSnapshotEntry(APPLICATION_ONLY_NODES)], async (daemon) => { // No fake clock is wired into this daemon-composed runtime path (AgentDeviceRuntime.clock is From abc995e3daf0d6b53d95fe5a02988fdc348cbb92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 30 Sep 2026 20:46:34 +0200 Subject: [PATCH 2/4] docs: document data.readiness on a press that had to wait --- website/docs/docs/client-api.md | 2 +- website/docs/docs/replay-e2e.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 08cd4d31de..f5a707514d 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -297,7 +297,7 @@ await client.interactions.press({ }); ``` -The wait is capped at 2 seconds and covers only a target that has not appeared. When the target is still missing after the wait, the error carries `error.details.readiness` with `waitedMs`, `polls`, and `end` (`expired` or `stalled`). A capture that shows an empty accessibility tree ends the wait at once with `capture_sparse` and `readiness.end: sparse`. A covered, off-screen, or ambiguous target fails at once, and a screen that stays unreadable for the whole wait fails with its own error; neither carries `readiness`. `readinessTimeoutMs` is not an MCP tool argument and has no CLI flag. +The wait is capped at 2 seconds and covers only a target that has not appeared. When the target is still missing after the wait, the error carries `error.details.readiness` with `waitedMs`, `polls`, and `end` (`expired` or `stalled`). A capture that shows an empty accessibility tree ends the wait at once with `capture_sparse` and `readiness.end: sparse`. When the command had to wait and then succeeded, the result carries `data.readiness` with `polls` and `waitedMs`. A command that found its target on the first look has no `readiness` field. A covered, off-screen, or ambiguous target fails at once, and a screen that stays unreadable for the whole wait fails with its own error; neither carries `readiness`. `readinessTimeoutMs` is not an MCP tool argument and has no CLI flag. Vega OS client support is currently VVD-only and covers device discovery, app open/close, `back`, `home`, and `tvRemote`. Physical Fire TV, capture, selector, install, logging, and performance methods report unsupported for Vega targets. diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 3023fa48d1..b4a5fb5806 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -62,6 +62,9 @@ agent-device replay ~/.agent-device/sessions/e2e-2026-02-09T12-00-00-000Z.ad --s they fail. A step recorded against a screen that was still loading passes on replay once the target shows up. The wait covers only a target that is not on screen yet: a target that is covered, off-screen, or matched by more than one element fails at once, as it does live. + A step that had to wait and then passed reports it in `data.readiness` (`polls`, `waitedMs`). + A step that found its target on the first look has no `readiness` field, so its presence marks a + step that would have failed without the wait. - When the target never appears, replay stops with `REPLAY_DIVERGENCE`, and `error.details.readiness` says how long the step waited and how many times it looked (`waitedMs`, `polls`, `end`). For a step recorded with a target annotation (the `# agent-device:target-v1` From 9794df64a1d4564aab398219b7f217c469058074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 30 Sep 2026 20:54:28 +0200 Subject: [PATCH 3/4] refactor(replay): drop the carried gate wait; replay evidence stays the readiness diagnostic --- packages/contracts/src/replay.ts | 7 ----- .../session-replay-dispatch-narrowing.ts | 11 -------- .../session-replay-runtime-engine-adapter.ts | 6 +--- ...replay-target-verification-runtime.test.ts | 28 ------------------- .../interaction-touch-response.test.ts | 20 ------------- .../internal/interaction-touch-response.ts | 21 +------------- website/docs/docs/replay-e2e.md | 4 +-- 7 files changed, 3 insertions(+), 94 deletions(-) diff --git a/packages/contracts/src/replay.ts b/packages/contracts/src/replay.ts index aaaf011fd3..64da7630c7 100644 --- a/packages/contracts/src/replay.ts +++ b/packages/contracts/src/replay.ts @@ -1,7 +1,6 @@ import type { DaemonError } from '@agent-device/kernel/errors'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import type { Rect, SnapshotState } from '@agent-device/kernel/snapshot'; -import type { ReadinessWaitEvidence } from './interaction.ts'; import type { GestureExecutionProfile } from './gesture-plan-types.ts'; import type { SessionScope } from './session-scope.ts'; import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; @@ -261,12 +260,6 @@ export type ReplayDispatchOptions = Readonly<{ * (pre-action) when its winner differs in local identity OR structural position. */ replayTargetGuard?: ReplayTargetGuardDenotation; - /** - * The wait the replay step's pre-dispatch target gate already spent, set ONLY by the replay step - * loop when it polled more than once. The interaction response builder adds it to the dispatch's - * own wait so a successful step reports one `readiness` for the whole step. - */ - replayReadinessWait?: ReadinessWaitEvidence; /** Dual-endpoint counterpart of `replayTargetGuard` for target-authored drag. */ replayTargetGuards?: Readonly<{ source: ReplayTargetGuardDenotation; diff --git a/packages/replay-port/src/daemon-port/session-replay-dispatch-narrowing.ts b/packages/replay-port/src/daemon-port/session-replay-dispatch-narrowing.ts index 5bf37477e5..41393349fd 100644 --- a/packages/replay-port/src/daemon-port/session-replay-dispatch-narrowing.ts +++ b/packages/replay-port/src/daemon-port/session-replay-dispatch-narrowing.ts @@ -8,7 +8,6 @@ import type { } from '@agent-device/ad-replay'; import type { LocalIdentity } from '@agent-device/ad-script'; import type { TargetAncestryEntry } from '@agent-device/contracts/replay'; -import type { ReadinessWaitEvidence } from '@agent-device/contracts/interaction'; import { isReplayTargetGuardMismatchResponse, isWaitLandmarkMismatchResponse, @@ -63,16 +62,6 @@ export function applyReplayDispatchGuard( : replayReq; } -/** Carries the wait the pre-dispatch gate spent into the dispatch, so its success response reports it. */ -export function applyReplayReadinessWait( - replayReq: ReplayDispatchRequest, - wait: ReadinessWaitEvidence | undefined, -): ReplayDispatchRequest { - return wait - ? { ...replayReq, dispatch: { ...replayReq.dispatch, replayReadinessWait: wait } } - : replayReq; -} - function readGuardMismatchObservedIdentity(value: unknown): LocalIdentity | undefined { if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined; const record = value as Record; diff --git a/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts b/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts index 9101362241..d46a0cdf09 100644 --- a/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts +++ b/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts @@ -31,7 +31,6 @@ import type { ObservationEvidence } from '@agent-device/capture-kit/observe-unti import { collectReplayActionArtifactPaths } from '@agent-device/replay-port/session-replay-runtime-artifacts'; import { applyReplayDispatchGuard, - applyReplayReadinessWait, classifyReplayDispatchFailure, toAdReplayStepFailure, } from './session-replay-dispatch-narrowing.ts'; @@ -271,10 +270,7 @@ export function createAdReplayStepRuntime(params: { readinessTimeoutMs: gateWait.remainingBudgetMs, } : {}), - req: applyReplayReadinessWait( - applyReplayDispatchGuard(ctx.replayReq, guard), - gateWait && { polls: gateWait.readiness.polls, waitedMs: gateWait.readiness.waitedMs }, - ), + req: applyReplayDispatchGuard(ctx.replayReq, guard), sessionName: ctx.sessionName, action, resolved: resolvedAction, diff --git a/src/daemon/__tests__/replay-target/session-replay-target-verification-runtime.test.ts b/src/daemon/__tests__/replay-target/session-replay-target-verification-runtime.test.ts index 012c2382d2..ddffb39004 100644 --- a/src/daemon/__tests__/replay-target/session-replay-target-verification-runtime.test.ts +++ b/src/daemon/__tests__/replay-target/session-replay-target-verification-runtime.test.ts @@ -225,34 +225,6 @@ test('an annotated click whose target renders on the second capture waits for it ]); }); -test('a step that waited in the gate hands its wait to the dispatch for the success response', async () => { - const scene = replayScriptScene('agent-device-replay-target-verify-wait-evidence-', [ - SAVE_ANNOTATION, - 'click id="save"', - ]); - - mockDispatchCommand.mockResolvedValueOnce(emptyCapture()).mockResolvedValue(saveButtonCapture()); - - const response = await scene.replay(); - - expect(response.ok).toBe(true); - expect(scene.invoked[0]?.internal?.replayReadinessWait).toEqual({ polls: 2, waitedMs: 200 }); -}); - -test('a step whose gate hit on the first capture hands no wait to the dispatch', async () => { - const scene = replayScriptScene('agent-device-replay-target-verify-no-wait-evidence-', [ - SAVE_ANNOTATION, - 'click id="save"', - ]); - - mockDispatchCommand.mockResolvedValue(saveButtonCapture()); - - const response = await scene.replay(); - - expect(response.ok).toBe(true); - expect(scene.invoked[0]?.internal?.replayReadinessWait).toBeUndefined(); -}); - test('the dispatch gets only the readiness budget the gate left', async () => { const scene = replayScriptScene('agent-device-replay-target-verify-shared-budget-', [ SAVE_ANNOTATION, diff --git a/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts index 02a12c7650..a471a83106 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts @@ -112,26 +112,6 @@ test('the response builder reports the resolved wait and keeps the prior warning expect(result.readiness).toBeUndefined(); }); -test('the response builder adds the wait the request carried to the dispatch wait', () => { - const { responseData } = buildInteractionResponseData({ - source: { kind: 'runtime', result: WAITED_SELECTOR_RESULT }, - referenceFrame: undefined, - carriedReadiness: { polls: 2, waitedMs: 200 }, - }); - - expect(responseData.readiness).toEqual({ polls: 5, waitedMs: 600 }); -}); - -test('the response builder reports the carried wait when the dispatch resolved on its first capture', () => { - const { responseData } = buildInteractionResponseData({ - source: { kind: 'runtime', result: selectorResult() }, - referenceFrame: undefined, - carriedReadiness: { polls: 2, waitedMs: 200 }, - }); - - expect(responseData.readiness).toEqual({ polls: 2, waitedMs: 200 }); -}); - test('the response builder omits readiness when nothing waited', () => { const { responseData } = buildInteractionResponseData({ source: { kind: 'runtime', result: selectorResult() }, diff --git a/src/daemon/interaction/internal/interaction-touch-response.ts b/src/daemon/interaction/internal/interaction-touch-response.ts index 57746977af..5924dca3c3 100644 --- a/src/daemon/interaction/internal/interaction-touch-response.ts +++ b/src/daemon/interaction/internal/interaction-touch-response.ts @@ -4,7 +4,6 @@ import type { HoverCommandResult, LongPressCommandResult, PressCommandResult, - ReadinessWaitEvidence, RecordingTargetOverride, ResolutionDisclosure, SettleObservation, @@ -173,8 +172,6 @@ export function buildInteractionResponseData(params: { * Only attached when the settle observation actually carries a diff. */ settleRefsGeneration?: number; - /** A wait the request already spent before this dispatch (the replay step's pre-dispatch gate). */ - carriedReadiness?: ReadinessWaitEvidence; }): InteractionResponsePayloads { const { source, referenceFrame, extra } = params; if (source.kind === 'runner-payload' || source.kind === 'corroborated-tap') { @@ -237,25 +234,10 @@ export function buildInteractionResponseData(params: { visualization.warning = warning; responseData.warning = warning; } - const readiness = combineReadinessWaits( - 'readiness' in result ? result.readiness : undefined, - params.carriedReadiness, - ); - if (readiness) responseData.readiness = readiness; + if ('readiness' in result && result.readiness) responseData.readiness = result.readiness; return { result: visualization, responseData, ...recordedTargetCapture(result) }; } -function combineReadinessWaits( - ...waits: Array -): ReadinessWaitEvidence | undefined { - const spent = waits.filter((wait): wait is ReadinessWaitEvidence => wait !== undefined); - if (spent.length === 0) return undefined; - return { - polls: spent.reduce((total, wait) => total + wait.polls, 0), - waitedMs: spent.reduce((total, wait) => total + wait.waitedMs, 0), - }; -} - function recordedTargetCapture( result: InteractionRuntimeResult, ): Pick { @@ -330,7 +312,6 @@ export async function buildTargetedTouchResponsePayloads(params: { extra, staleRefsWarning: params.staleRefsWarning, settleRefsGeneration: issueSettleRefs(session, result.settle), - carriedReadiness: handlerParams.req.internal?.replayReadinessWait, }); } diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index b4a5fb5806..917d1c19a7 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -62,9 +62,7 @@ agent-device replay ~/.agent-device/sessions/e2e-2026-02-09T12-00-00-000Z.ad --s they fail. A step recorded against a screen that was still loading passes on replay once the target shows up. The wait covers only a target that is not on screen yet: a target that is covered, off-screen, or matched by more than one element fails at once, as it does live. - A step that had to wait and then passed reports it in `data.readiness` (`polls`, `waitedMs`). - A step that found its target on the first look has no `readiness` field, so its presence marks a - step that would have failed without the wait. + Under `--debug`, each wait shows up as an `interaction_target_readiness` diagnostic. - When the target never appears, replay stops with `REPLAY_DIVERGENCE`, and `error.details.readiness` says how long the step waited and how many times it looked (`waitedMs`, `polls`, `end`). For a step recorded with a target annotation (the `# agent-device:target-v1` From 8868fb544946320d687a8601183f43722c7a6502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 1 Oct 2026 01:26:21 +0200 Subject: [PATCH 4/4] test(interaction): pin data.readiness on the client and MCP success paths --- .../client-interactions-readiness.test.ts | 14 +++++++++++++ .../runtime/selector-readiness.test.ts | 2 +- .../__tests__/command-tools-readiness.test.ts | 20 +++++++++++++++++++ website/docs/docs/replay-e2e.md | 3 ++- 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 src/mcp/__tests__/command-tools-readiness.test.ts diff --git a/src/__tests__/client-interactions-readiness.test.ts b/src/__tests__/client-interactions-readiness.test.ts index 9b77ccf024..bff493c383 100644 --- a/src/__tests__/client-interactions-readiness.test.ts +++ b/src/__tests__/client-interactions-readiness.test.ts @@ -17,3 +17,17 @@ test('readinessTimeoutMs on press/click/longpress reaches the request flags', as assert.equal(setup.calls[1]?.flags?.readinessTimeoutMs, 1_500); assert.equal(setup.calls[2]?.flags?.readinessTimeoutMs, 900); }); + +test('a press that waited returns the daemon data.readiness untouched', async () => { + const readiness = { polls: 2, waitedMs: 180 }; + const setup = createTransport(async () => ({ ok: true, data: { ref: '@e1', readiness } })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + const pressed = await client.interactions.press({ selector: 'label=Foo' }); + const clicked = await client.interactions.click({ selector: 'label=Foo' }); + const longPressed = await client.interactions.longPress({ selector: 'label=Foo' }); + + for (const result of [pressed, clicked, longPressed]) { + assert.deepEqual((result as { readiness?: unknown }).readiness, readiness); + } +}); diff --git a/src/commands/interaction/runtime/selector-readiness.test.ts b/src/commands/interaction/runtime/selector-readiness.test.ts index e6cb1a88ee..dc3ade6d06 100644 --- a/src/commands/interaction/runtime/selector-readiness.test.ts +++ b/src/commands/interaction/runtime/selector-readiness.test.ts @@ -60,7 +60,7 @@ test('runtime press with readinessTimeoutMs polls until the target appears', asy assert.equal(result.kind, 'selector'); assert.equal(result.node?.label, 'Continue'); - assert.ok(result.kind === 'selector' && result.readiness && result.readiness.polls > 1); + assert.equal(result.kind === 'selector' ? result.readiness?.polls : undefined, 2); assert.ok( captures >= 4, `expected at least 4 captures before the target appeared, got ${captures}`, diff --git a/src/mcp/__tests__/command-tools-readiness.test.ts b/src/mcp/__tests__/command-tools-readiness.test.ts new file mode 100644 index 0000000000..73ab2c4596 --- /dev/null +++ b/src/mcp/__tests__/command-tools-readiness.test.ts @@ -0,0 +1,20 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createAgentDeviceClient } from '../../agent-device-client.ts'; +import { createTransport } from '../../__tests__/client-transport-fixture.ts'; +import { createCommandToolExecutor } from '../command-tools.ts'; + +test('MCP press returns the daemon data.readiness in structuredContent', async () => { + const readiness = { polls: 2, waitedMs: 180 }; + const setup = createTransport(async () => ({ ok: true, data: { ref: '@e1', readiness } })); + const executor = createCommandToolExecutor({ + createClient: () => createAgentDeviceClient(setup.config, { transport: setup.transport }), + }); + + const result = await executor.execute('press', { + target: { kind: 'selector', selector: 'label=Continue' }, + }); + + assert.equal(result.isError, false); + assert.deepEqual(result.structuredContent?.readiness, readiness); +}); diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 917d1c19a7..13486bf252 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -62,7 +62,8 @@ agent-device replay ~/.agent-device/sessions/e2e-2026-02-09T12-00-00-000Z.ad --s they fail. A step recorded against a screen that was still loading passes on replay once the target shows up. The wait covers only a target that is not on screen yet: a target that is covered, off-screen, or matched by more than one element fails at once, as it does live. - Under `--debug`, each wait shows up as an `interaction_target_readiness` diagnostic. + A step that waited and then passed shows no trace of the wait in the replay output; run with + `--debug` to see it in the diagnostics. - When the target never appears, replay stops with `REPLAY_DIVERGENCE`, and `error.details.readiness` says how long the step waited and how many times it looked (`waitedMs`, `polls`, `end`). For a step recorded with a target annotation (the `# agent-device:target-v1`