From 6194b26318e54bdbab06fa57de1cd2c237a5cc2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 13 Oct 2026 08:29:46 +0200 Subject: [PATCH 1/7] feat(interaction)!: remove target-readiness waits --- contracts/fixtures/dispatch-disclosure.json | 6 - .../src/internal/runtime-port-types.ts | 8 +- packages/command-registry/src/registry.ts | 12 +- .../command-registry/src/timeout-policy.ts | 28 +- packages/command-registry/src/types.ts | 17 - packages/contracts/src/client-gesture.ts | 17 +- packages/contracts/src/command-flags.ts | 5 - .../contracts/src/daemon-command-surface.ts | 3 - packages/contracts/src/facades/client.ts | 1 - .../contracts/src/interaction-guarantees.ts | 46 --- packages/contracts/src/interaction.ts | 11 - packages/contracts/src/request-envelope.ts | 6 - .../internal/interaction-touch-press.ts | 2 - .../replay-runtime/replay-command-fixture.ts | 18 +- .../session-replay-action-runtime.test.ts | 119 +------ .../session-replay-scenario.fixtures.ts | 13 +- ...replay-target-verification-runtime.test.ts | 119 +------ .../src/daemon-port/command-types.ts | 6 - .../session-replay-action-runtime.ts | 48 +-- .../session-replay-runtime-engine-adapter.ts | 130 ++------ packages/selectors/src/interaction-error.ts | 4 +- .../src/selector-pipeline-policy.test.ts | 30 -- .../selectors/src/selector-pipeline-policy.ts | 69 +--- .../selectors/src/selector-pipeline.test.ts | 17 +- packages/selectors/src/selector-pipeline.ts | 7 - ...lient-interactions-input-filtering.test.ts | 37 +++ .../client-interactions-readiness.test.ts | 33 -- .../command-descriptor-timeout-policy.test.ts | 58 ---- .../interaction-dispatch-disclosure.test.ts | 30 -- src/commands/command-flags.ts | 1 - src/commands/command-input.ts | 1 - src/commands/common-input-fields.ts | 25 +- src/commands/interaction/index.ts | 2 - src/commands/interaction/metadata.test.ts | 33 -- src/commands/interaction/metadata.ts | 28 -- .../__tests__/selector-read-policy.test.ts | 2 +- .../runtime/__tests__/test-utils/index.ts | 2 +- src/commands/interaction/runtime/gestures.ts | 6 - .../runtime/interaction-resolution-request.ts | 6 - .../runtime/interaction-snapshot-capture.ts | 2 +- .../interaction/runtime/interactions.ts | 6 - .../interaction/runtime/resolution.ts | 50 +-- .../runtime/selector-readiness.test.ts | 143 -------- .../interaction/runtime/selector-readiness.ts | 305 ------------------ .../selector-target-resolution.test.ts | 55 ++++ .../runtime/selector-target-resolution.ts | 106 ++++++ .../runtime/target-visibility-stages.ts | 2 +- src/commands/schema/cli-help.ts | 1 + .../command-tools-operator-inputs.test.ts | 27 -- .../runtime-selector.contract.test.ts | 29 -- .../runtime-selector.coverage.ts | 2 - .../press-selector-resolution.test.ts | 167 ++++++++++ .../press-target-readiness.test.ts | 284 ---------------- website/docs/docs/client-api.md | 10 +- website/docs/docs/replay-e2e.md | 27 +- 55 files changed, 460 insertions(+), 1762 deletions(-) delete mode 100644 packages/selectors/src/selector-pipeline-policy.test.ts create mode 100644 src/__tests__/client-interactions-input-filtering.test.ts delete mode 100644 src/__tests__/client-interactions-readiness.test.ts delete mode 100644 src/commands/interaction/runtime/selector-readiness.test.ts delete mode 100644 src/commands/interaction/runtime/selector-readiness.ts create mode 100644 src/commands/interaction/runtime/selector-target-resolution.test.ts create mode 100644 src/commands/interaction/runtime/selector-target-resolution.ts create mode 100644 test/integration/provider-scenarios/press-selector-resolution.test.ts delete mode 100644 test/integration/provider-scenarios/press-target-readiness.test.ts diff --git a/contracts/fixtures/dispatch-disclosure.json b/contracts/fixtures/dispatch-disclosure.json index b8a94edd42..dae5d97f60 100644 --- a/contracts/fixtures/dispatch-disclosure.json +++ b/contracts/fixtures/dispatch-disclosure.json @@ -35,12 +35,6 @@ "trigger": "Android blocking-dialog readiness tapped Close app and relaunched the app, then re-admission aborted the requested ref press", "dispatched": "unknown" }, - { - "id": "daemon.refusal.selector-readiness-exhausted", - "producer": "daemon", - "trigger": "press with readinessTimeoutMs whose selector target never appears: the readiness wait (pollForSelectorReadiness) spends its budget over two or more polls and refuses with selector_not_found before any backend touch", - "dispatched": "no" - }, { "id": "daemon.unclassified", "producer": "daemon", diff --git a/packages/ad-replay/src/internal/runtime-port-types.ts b/packages/ad-replay/src/internal/runtime-port-types.ts index 502c8ce457..9c1c2d6e87 100644 --- a/packages/ad-replay/src/internal/runtime-port-types.ts +++ b/packages/ad-replay/src/internal/runtime-port-types.ts @@ -223,11 +223,9 @@ export type AdReplayStepRuntime = Readonly<{ * Captures a fresh snapshot (with the #1385 launch-race retry) and resolves * the recorded target against it using the SAME lookup/matching a real * dispatch would — daemon authority (`SessionStore`, the capture pipeline, - * tree helpers and the selectors package engine). A step whose dispatch - * would wait for its target under a readiness budget re-captures while the - * target does not match yet, under that same budget, so this gate never - * refuses a target the dispatch itself would have waited for. The last - * capture is the divergence's `screen`. + * tree helpers and the selectors package engine). A missing target is + * refused after this observation; scripts wait explicitly for loading + * screens. The capture is the divergence's `screen`. */ observeTarget(params: { action: SessionAction; diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index a2f8c806e1..5893defd91 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -1334,7 +1334,6 @@ export const RAW_COMMAND_DESCRIPTORS = [ }, timeoutPolicy: postActionObservationTimeoutPolicy('click', PRESERVE_DAEMON_TIMEOUT_POLICY), postActionObservation: postActionObservation('click'), - targetReadiness: 'budgeted', responseDataTransform: TOUCH_INTERACTION_RESPONSE_DATA_TRANSFORM, batchable: true, platformExecution: { kind: 'device-runtime', uses: clickRuntimeUses }, @@ -1365,7 +1364,6 @@ export const RAW_COMMAND_DESCRIPTORS = [ envelopeMs: 210_000, }, postActionObservation: postActionObservation('longpress'), - targetReadiness: 'budgeted', batchable: true, platformExecution: { kind: 'device-runtime', uses: longPressRuntimeUses }, }, @@ -1395,7 +1393,6 @@ export const RAW_COMMAND_DESCRIPTORS = [ frameworkTier: 'core', timeoutPolicy: postActionObservationTimeoutPolicy('press', PRESERVE_DAEMON_TIMEOUT_POLICY), postActionObservation: postActionObservation('press'), - targetReadiness: 'budgeted', responseDataTransform: TOUCH_INTERACTION_RESPONSE_DATA_TRANSFORM, batchable: true, platformExecution: { kind: 'device-runtime', uses: pressRuntimeUses }, @@ -2028,9 +2025,7 @@ function readCatalogKey(descriptor: { const TIMEOUT_POLICY_BY_COMMAND: ReadonlyMap = new Map( Array.from(COMMAND_DESCRIPTOR_BY_NAME.values(), (descriptor) => [ descriptor.name, - descriptor.targetReadiness - ? { ...descriptor.timeoutPolicy, targetReadiness: descriptor.targetReadiness } - : descriptor.timeoutPolicy, + descriptor.timeoutPolicy, ]), ); @@ -2058,11 +2053,6 @@ export function commandSupportsSettleObservation(command: string | undefined): b return resolveCommandPostActionObservationSupport(command) !== undefined; } -export function commandAcceptsReadinessBudget(command: string | undefined): boolean { - if (command === undefined) return false; - return COMMAND_DESCRIPTOR_BY_NAME.get(command)?.targetReadiness === 'budgeted'; -} - export function commandSupportsVerifyEvidence(command: string | undefined): boolean { return resolveCommandPostActionObservationSupport(command) === 'settle-and-verify'; } diff --git a/packages/command-registry/src/timeout-policy.ts b/packages/command-registry/src/timeout-policy.ts index 587c59315c..2c2d950d22 100644 --- a/packages/command-registry/src/timeout-policy.ts +++ b/packages/command-registry/src/timeout-policy.ts @@ -68,7 +68,6 @@ type RequestTimeoutInput = Readonly<{ timeoutMs?: number; settle?: boolean; waitMs?: number; - readinessTimeoutMs?: number; }>; }>; @@ -79,32 +78,11 @@ export function resolveCommandRequestTimeoutMs( ): number | undefined { if (policy.envelopeMs === 'unbounded') return undefined; const boundedPolicy: BoundedTimeoutPolicy = { ...policy, envelopeMs: policy.envelopeMs }; - const envelopeMs = + return ( resolvePositionalBudgetTimeoutMs(boundedPolicy, input.positionals ?? []) ?? resolveFlagBudgetTimeoutMs(boundedPolicy, input.flags) ?? - boundedPolicy.envelopeMs; - return envelopeMs + readinessBudgetMs(policy, input.flags); -} - -/** - * The ceiling of a `targetReadiness: 'budgeted'` command's readiness budget: the promotedTarget - * selector row's poll ceiling, pinned to it by test. - */ -export const READINESS_BUDGET_MAX_MS = 2_000; - -/** - * A readiness budget is target-poll time spent before the action itself, so it extends whatever - * envelope the command otherwise has; the daemon's own poll must never outlive the client's clock. - * The poll never runs past the promotedTarget row's ceiling, so neither does the widening. - */ -function readinessBudgetMs( - policy: CommandTimeoutPolicy, - flags: RequestTimeoutInput['flags'], -): number { - if (policy.targetReadiness !== 'budgeted') return 0; - const budgetMs = flags?.readinessTimeoutMs; - if (typeof budgetMs !== 'number' || !Number.isInteger(budgetMs) || budgetMs <= 0) return 0; - return Math.min(budgetMs, READINESS_BUDGET_MAX_MS); + boundedPolicy.envelopeMs + ); } function resolvePositionalBudgetTimeoutMs( diff --git a/packages/command-registry/src/types.ts b/packages/command-registry/src/types.ts index 16911f71aa..934bd0bc38 100644 --- a/packages/command-registry/src/types.ts +++ b/packages/command-registry/src/types.ts @@ -75,21 +75,8 @@ export type CommandTimeoutPolicy = { budget: CommandTimeoutBudget; envelopeMs: number | 'unbounded'; onTimeout: 'preserve-daemon' | 'reset-daemon'; - /** - * The descriptor's {@link CommandTargetReadiness} trait, attached by `resolveCommandTimeoutPolicy` - * so the envelope can widen by the readiness budget. Descriptors declare it as `targetReadiness`, - * never on their `timeoutPolicy`. - */ - targetReadiness?: CommandTargetReadiness; }; -/** - * `budgeted`: the command's target resolution may poll for a target that does not exist yet, under - * a caller-supplied `readinessTimeoutMs`, capped at `READINESS_BUDGET_MAX_MS` (`timeout-policy.ts`). - * Must match the commands the `targetReadiness` interaction guarantee cells mark `runtime`. - */ -export type CommandTargetReadiness = 'budgeted'; - /** * #1320 "Command descriptor policy": what a command may do with the host-global * device claim store. REQUIRED on every descriptor (no default), and read by the @@ -212,9 +199,6 @@ export type TargetIdentityVerification = 'pre-dispatch' | 'post-resolution'; * commands that support `--settle`/`--verify`; consumed by * command surfaces and timeout policy instead of repeated * command-name lists. - * - `targetReadiness` — optional; the commands whose target resolution accepts a - * `readinessTimeoutMs` budget. Read by replay (which supplies a - * default budget) and by the request envelope (which widens by it). * - `responseDataTransform` — optional public response data shaping rules for * command-owned fields in daemon responses. This keeps * response shaping on the same descriptor surface as other @@ -237,7 +221,6 @@ type CommandDescriptorBase = { */ deviceClaimPolicy: DeviceClaimPolicy; postActionObservation?: PostActionObservationSupport; - targetReadiness?: CommandTargetReadiness; responseDataTransform?: CommandResponseDataTransform; catalog: CommandCatalogFacet; /** Required iff `catalog.group === 'public'`; see {@link CommandFrameworkTier}. */ diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index 75f2a7b769..e636d621a6 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -35,20 +35,11 @@ export type SettleCommandOptions = { timeoutMs?: number; }; -/** - * How long a tap-shaped interaction may poll for a target that does not exist yet, capped at the - * promotedTarget row's maxTimeoutMs. Never model- or CLI-writable; omitted means one attempt. - */ -export type ReadinessBudgetOptions = { - readinessTimeoutMs?: number; -}; - export type ClickOptions = DeviceCommandBaseOptions & SelectorSnapshotCommandOptions & InteractionTarget & RepeatedPressOptions & - SettleCommandOptions & - ReadinessBudgetOptions & { + SettleCommandOptions & { button?: ClickButton; /** * Opt-in (#1047): return cheap post-action evidence (AX digest, node counts, @@ -62,16 +53,14 @@ export type PressOptions = DeviceCommandBaseOptions & SelectorSnapshotCommandOptions & InteractionTarget & RepeatedPressOptions & - SettleCommandOptions & - ReadinessBudgetOptions & { + SettleCommandOptions & { verify?: boolean; }; export type LongPressOptions = DeviceCommandBaseOptions & SelectorSnapshotCommandOptions & InteractionTarget & - SettleCommandOptions & - ReadinessBudgetOptions & { + SettleCommandOptions & { durationMs?: number; }; diff --git a/packages/contracts/src/command-flags.ts b/packages/contracts/src/command-flags.ts index f429b8ed70..7b56cc0e6d 100644 --- a/packages/contracts/src/command-flags.ts +++ b/packages/contracts/src/command-flags.ts @@ -32,11 +32,6 @@ export type CommandFlags = Omit & { kind?: string; maestro?: MaestroRuntimeFlags; postGestureStabilization?: boolean; - /** - * Readiness budget for press/click/longpress, capped at the promotedTarget row's maxTimeoutMs. - * No CliFlags counterpart: never CLI- or model-writable. - */ - readinessTimeoutMs?: number; snapshotIncludeHiddenContentHints?: boolean; leaseProvider?: string; provider?: string; diff --git a/packages/contracts/src/daemon-command-surface.ts b/packages/contracts/src/daemon-command-surface.ts index 31f25c8d3e..2596dba14b 100644 --- a/packages/contracts/src/daemon-command-surface.ts +++ b/packages/contracts/src/daemon-command-surface.ts @@ -199,8 +199,6 @@ export type DaemonPressOptions = CommandContext & Readonly<{ target: InteractionTarget; button?: ClickButton; - /** Polls for the target to become actionable; absent takes one attempt. */ - readinessTimeoutMs?: number; expectedResolvedTarget?: ReplayTargetGuardDenotation; /** #1654: a mutating `find`'s already-resolved node. */ preresolvedTarget?: PreresolvedInteractionTarget; @@ -222,7 +220,6 @@ export type DaemonLongPressOptions = CommandContext & Readonly<{ target: InteractionTarget; durationMs?: number; - readinessTimeoutMs?: number; expectedResolvedTarget?: ReplayTargetGuardDenotation; settle?: SettleParams; }>; diff --git a/packages/contracts/src/facades/client.ts b/packages/contracts/src/facades/client.ts index f7ec417f78..8cd5d9dad2 100644 --- a/packages/contracts/src/facades/client.ts +++ b/packages/contracts/src/facades/client.ts @@ -55,7 +55,6 @@ export type { PanOptions, PinchOptions, PressOptions, - ReadinessBudgetOptions, RepeatedPressOptions, RotateGestureOptions, ScrollOptions, diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index f783faa40e..f4f7e4c9ae 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -73,11 +73,6 @@ export const INTERACTION_GUARANTEES = [ // unique/disambiguated/exact/label-fallback/not-observed provenance, // pre-action diagnostics only — never ref-issued or MCP-pinned. 'resolutionDisclosure', - // How the path waits for the target to exist and become actionable before acting: a declared - // budget under which a plain no-match keeps retrying, versus a target that must already exist at - // dispatch time. Distinct from occlusion/offscreen/nonHittable, which judge a target already - // found; this is about whether one is found at all. - 'targetReadiness', // What the path observes after dispatch to back its success claim, and whether that observation // is advisory (a hint the caller may ignore) or can change the response (a corroborated failure, // a settled diff). Distinct from the opt-in verifyEvidence/settleObservation features; this is the @@ -192,17 +187,6 @@ const TAP_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE, }; -// Both Maestro-compatible fast paths (packages/daemon/src/execution/interaction/internal/interaction-touch-direct-ios.ts) -// dispatch ONE fused XCTest runner request (`command: 'tap'` with a selectorKey/selectorValue) built -// in packages/platform-apple/src/interactions.ts#tapElementSelector — not the separate -// packages/maestro/ flow-script engine, which drives standalone `.yaml` Maestro flows and never -// participates in an ordinary daemon click. -const DIRECT_IOS_SINGLE_QUERY_READINESS: GuaranteeEnforcement = { - kind: 'waived', - reason: - "Intentional: the fused runner request performs a single XCTest selector/frame query per dispatch and does not itself retry for an as-yet-nonexistent target; a miss is a runner failure (see errorTaxonomy) rather than a wait, and delegation-on-error (ADR 0011) is a different path's guarantee, not a retry of this one.", -}; - // Shares the SAME reasoning as TAP_OUTCOME_NOT_OBSERVED_GAP: the tap outcome is not observed on the // success path here either. The shared ambiguous-XCTest-failure corroboration // (packages/daemon/src/execution/interaction/internal/interaction-ios-tap-outcome.ts#corroborateIosTapFailure) is @@ -316,14 +300,6 @@ export const INTERACTION_DISPATCH_PATHS: Record & durationMs?: number; holdMs?: number; jitterPx?: number; - /** - * The readiness budget a tap-shaped interaction (press/click/longpress) may spend polling for a - * target that does not exist yet, capped at the promotedTarget row's maxTimeoutMs. Never model- - * or CLI-writable; absent means one attempt. - */ - readinessTimeoutMs?: number; pixels?: number; /** Scroll: repeat passes until this selector is visible on screen. */ until?: string; diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts index e19a4384f1..24a2e7f247 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts @@ -179,7 +179,6 @@ async function runTargetedTouchInteraction(params: { return await runtime.interactions.longPress(target, { ...shared, durationMs: params.durationMs, - readinessTimeoutMs: flags?.readinessTimeoutMs, }); case 'hover': return await runtime.interactions.hover(target, shared); @@ -213,7 +212,6 @@ function pressRuntimeOptions( jitterPx: flags?.jitterPx, doubleTap: flags?.doubleTap, verify: flags?.verify, - readinessTimeoutMs: flags?.readinessTimeoutMs, // Only click/press take it: `find` dispatches click and fill, never // longpress or hover, so declaring it on their options would be an // unconsumed claim (#1649 review). diff --git a/packages/daemon/src/sessions/__tests__/replay-runtime/replay-command-fixture.ts b/packages/daemon/src/sessions/__tests__/replay-runtime/replay-command-fixture.ts index f742002b4e..fd2962ae8d 100644 --- a/packages/daemon/src/sessions/__tests__/replay-runtime/replay-command-fixture.ts +++ b/packages/daemon/src/sessions/__tests__/replay-runtime/replay-command-fixture.ts @@ -15,7 +15,6 @@ import { splitReplayCommandRequest, } from '@agent-device/replay-port/replay-dispatch-envelope'; import type { ReplayCommand } from '@agent-device/replay-port/command-types'; -import type { ObservationClock } from '@agent-device/capture-kit/observe-until'; export type ReplayCommandTestInput = Readonly<{ req: DaemonRequest; @@ -25,8 +24,6 @@ export type ReplayCommandTestInput = Readonly<{ invoke: DaemonInvokeFn; tracePath?: string; onStep?: ReplayTestAttemptStepSink; - /** Paces the target-readiness wait; default: `instantReplayClock`. */ - clock?: ObservationClock; }>; /** @@ -35,12 +32,12 @@ export type ReplayCommandTestInput = Readonly<{ * test's `invoke` sees the same `DaemonRequest` the daemon would. */ export function replayCommandForTest(params: ReplayCommandTestInput): ReplayCommand { - const { req, sessionName, logPath, sessionStore, invoke, tracePath, onStep, clock } = params; + const { req, sessionName, logPath, sessionStore, invoke, tracePath, onStep } = params; return { ...splitReplayCommandRequest(req), session: createReplaySession(sessionName, logPath, sessionStore), invoke: replayInvokeOverDispatch(invoke, req), - dependencies: { ...replayDaemonDependencies, clock: clock ?? instantReplayClock() }, + dependencies: replayDaemonDependencies, ...(tracePath === undefined ? {} : { tracePath }), ...(onStep === undefined ? {} : { onStep }), }; @@ -49,14 +46,3 @@ export function replayCommandForTest(params: ReplayCommandTestInput): ReplayComm export function runReplayForTest(params: ReplayCommandTestInput): Promise { return runReplayCommand(replayCommandForTest(params)); } - -/** Target-readiness waits advance this clock instead of sleeping, so a unit replay spends no wall time. */ -function instantReplayClock(): ObservationClock { - let nowMs = Date.now(); - return { - now: () => nowMs, - sleep: async (ms) => { - nowMs += ms; - }, - }; -} diff --git a/packages/daemon/src/sessions/__tests__/replay-runtime/session-replay-action-runtime.test.ts b/packages/daemon/src/sessions/__tests__/replay-runtime/session-replay-action-runtime.test.ts index 85ab5c0e37..cd99c6cf67 100644 --- a/packages/daemon/src/sessions/__tests__/replay-runtime/session-replay-action-runtime.test.ts +++ b/packages/daemon/src/sessions/__tests__/replay-runtime/session-replay-action-runtime.test.ts @@ -3,20 +3,9 @@ import { expect, test } from 'vitest'; import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; import { recordActionEntry } from '../../../core/session-action-recorder.ts'; import type { DaemonRequest } from '../../../core/daemon-request.ts'; -import { - invokeReplayAction, - replayStepReadinessSchedule, -} from '@agent-device/replay-port/session-replay-action-runtime'; -import { - SELECTOR_PIPELINE_POLICIES, - readinessScheduleFor, -} from '@agent-device/selectors/selector-pipeline-policy'; +import { invokeReplayAction } from '@agent-device/replay-port/session-replay-action-runtime'; import { replayDaemonDependencies } from '../../handlers/session-replay-command.ts'; import { resolveReplayAction } from '@agent-device/ad-script'; -import { - commandAcceptsReadinessBudget, - commandDescriptors, -} from '@agent-device/command-registry/registry'; const REPLAY_REQUEST: DaemonRequest = { token: 'token', @@ -69,14 +58,8 @@ test.each(['', ' '])( }, ); -const READINESS_BUDGETED_COMMANDS = commandDescriptors - .map((descriptor) => descriptor.name) - .filter((command) => commandAcceptsReadinessBudget(command)); - -// A replay step has no CLI flag to carry a readiness budget, so `buildReplayActionFlags` defaults -// one in for every readiness-budgeted command. -test.each(READINESS_BUDGETED_COMMANDS)( - 'replay defaults readinessTimeoutMs onto a dispatched %s step', +test.each(['click', 'press', 'longpress'])( + 'replay dispatches a %s step with only authored and inherited flags', async (command) => { const action: SessionAction = { ts: 0, command, positionals: ['label="Continue"'], flags: {} }; let dispatchedFlags: Record | undefined; @@ -97,48 +80,10 @@ test.each(READINESS_BUDGETED_COMMANDS)( }); expect(response.ok).toBe(true); - expect(dispatchedFlags?.readinessTimeoutMs).toBe(2_000); + expect(dispatchedFlags).toEqual({}); }, ); -// The dispatch resolves a press/click/longpress target under the promotedTarget row with the -// readinessTimeoutMs it receives; the pre-dispatch gate must poll under that same schedule. -test.each([ - ...READINESS_BUDGETED_COMMANDS.map((command) => ({ command, flags: {} })), - { command: 'click', flags: { readinessTimeoutMs: 700 } }, - { command: 'press', flags: { readinessTimeoutMs: 9_000 } }, -])('the replay gate and the dispatch poll a $command step under one schedule', async (step) => { - const action: SessionAction = { - ts: 0, - command: step.command, - positionals: ['label="Continue"'], - flags: step.flags, - }; - let dispatchedFlags: Record | undefined; - await invokeReplayAction({ - req: REPLAY_REQUEST, - sessionName: 'default', - action, - resolved: action, - filePath: 'flow.ad', - line: 1, - step: 1, - resolvedSessionScope: undefined, - dependencies: replayDaemonDependencies, - invoke: async (request) => { - dispatchedFlags = request.flags; - return { ok: true, data: {} }; - }, - }); - - const dispatchSchedule = readinessScheduleFor( - SELECTOR_PIPELINE_POLICIES.promotedTarget.poll, - dispatchedFlags?.readinessTimeoutMs as number | undefined, - ); - expect(dispatchSchedule).toBeDefined(); - expect(await replayStepReadinessSchedule(REPLAY_REQUEST.flags, action)).toEqual(dispatchSchedule); -}); - // #2997: the replay/test command's own --test-ime opt-in rides the parent flags onto the // open this step dispatches; an authored step flag wins because mergeParentFlags only // fills gaps. Without the inheritance the real-device flow open silently defaults off. @@ -196,62 +141,6 @@ test('replay keeps an authored open step testIme over the flow command opt-out', expect(dispatchedFlags?.testIme).toBe(true); }); -test('replay keeps a readinessTimeoutMs the step already carries instead of overwriting it', async () => { - const action: SessionAction = { - ts: 0, - command: 'press', - positionals: ['label="Continue"'], - flags: { readinessTimeoutMs: 500 }, - }; - let dispatchedFlags: Record | undefined; - const response = await invokeReplayAction({ - req: REPLAY_REQUEST, - sessionName: 'default', - action, - resolved: action, - filePath: 'flow.ad', - line: 1, - step: 1, - resolvedSessionScope: undefined, - dependencies: replayDaemonDependencies, - invoke: async (request) => { - dispatchedFlags = request.flags; - return { ok: true, data: {} }; - }, - }); - - expect(response.ok).toBe(true); - expect(dispatchedFlags?.readinessTimeoutMs).toBe(500); -}); - -test('replay never defaults readinessTimeoutMs onto a non-acting step', async () => { - const action: SessionAction = { - ts: 0, - command: 'wait', - positionals: ['label="Continue"'], - flags: {}, - }; - let dispatchedFlags: Record | undefined; - const response = await invokeReplayAction({ - req: REPLAY_REQUEST, - sessionName: 'default', - action, - resolved: action, - filePath: 'flow.ad', - line: 1, - step: 1, - resolvedSessionScope: undefined, - dependencies: replayDaemonDependencies, - invoke: async (request) => { - dispatchedFlags = request.flags; - return { ok: true, data: {} }; - }, - }); - - expect(response.ok).toBe(true); - expect(dispatchedFlags?.readinessTimeoutMs).toBeUndefined(); -}); - // #3197: a script line now parses `scroll down --until ` into a flag, and the // dispatch reads the stop condition off the request flags. If the replay dispatch dropped // action flags for this command, the hunt would degrade to one fixed gesture again. diff --git a/packages/daemon/src/sessions/__tests__/replay-target/session-replay-scenario.fixtures.ts b/packages/daemon/src/sessions/__tests__/replay-target/session-replay-scenario.fixtures.ts index 08c7da086e..487f47031c 100644 --- a/packages/daemon/src/sessions/__tests__/replay-target/session-replay-scenario.fixtures.ts +++ b/packages/daemon/src/sessions/__tests__/replay-target/session-replay-scenario.fixtures.ts @@ -2,7 +2,6 @@ import path from 'node:path'; import { mkdtempForTestSync } from '@agent-device/test-kit/tmp-dir'; import { makeIosAppSession } from '../../../__tests__/support/session-factories.ts'; import { SessionStore } from '../../../core/session-store.ts'; -import type { ObservationClock } from '@agent-device/capture-kit/observe-until'; import type { DaemonInvokeFn, DaemonRequest, @@ -38,14 +37,9 @@ export type ReplayScriptScene = ReplaySessionScene & /** * Runs the script through `runReplayCommand`. Each request is appended to * `invoked` before `invoke` answers it; without `invoke` every request succeeds - * with empty data. `clock` paces its target-readiness waits; `requestId` lets a test cancel the - * replay through the request registry. + * with empty data. `requestId` lets a test cancel the replay through the request registry. */ - replay(options?: { - invoke?: DaemonInvokeFn; - clock?: ObservationClock; - requestId?: string; - }): Promise; + replay(options?: { invoke?: DaemonInvokeFn; requestId?: string }): Promise; }>; /** An iOS app session plus a written `.ad` script, ready to replay. */ @@ -57,13 +51,12 @@ export function replayScriptScene(prefix: string, lines: string[]): ReplayScript ...scene, filePath, invoked, - replay: ({ invoke, clock, requestId } = {}) => + replay: ({ invoke, requestId } = {}) => runReplayForTest({ req: baseReplayRequest({ positionals: [filePath], ...(requestId === undefined ? {} : { meta: { requestId } }), }), - ...(clock === undefined ? {} : { clock }), sessionName: scene.sessionName, logPath: scene.logPath, sessionStore: scene.sessionStore, diff --git a/packages/daemon/src/sessions/__tests__/replay-target/session-replay-target-verification-runtime.test.ts b/packages/daemon/src/sessions/__tests__/replay-target/session-replay-target-verification-runtime.test.ts index 1f333aeafa..c9b5107f61 100644 --- a/packages/daemon/src/sessions/__tests__/replay-target/session-replay-target-verification-runtime.test.ts +++ b/packages/daemon/src/sessions/__tests__/replay-target/session-replay-target-verification-runtime.test.ts @@ -28,18 +28,7 @@ vi.mock('@agent-device/host-kit/retry', async (importOriginal) => { return { ...actual, sleep: vi.fn(async () => {}) }; }); -vi.mock('@agent-device/host-kit/diagnostics', async (importOriginal) => { - const actual = await importOriginal(); - return { ...actual, emitDiagnostic: vi.fn(actual.emitDiagnostic) }; -}); - import { AppError } from '@agent-device/kernel/errors'; -import { - clearRequestCanceled, - markRequestCanceled, - registerRequestAbort, -} from '@agent-device/host-kit/request'; -import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { legacyDispatchCapture, resetLegacySnapshotCapture, @@ -62,16 +51,8 @@ const mockCaptureSnapshotWithInteractor = vi.mocked(captureSnapshotWithInteracto beforeEach(() => { resetLegacySnapshotCapture(mockCaptureSnapshotWithInteractor); - vi.mocked(emitDiagnostic).mockClear(); }); -function readinessDiagnostics(): unknown[] { - return vi - .mocked(emitDiagnostic) - .mock.calls.filter(([event]) => event.phase === 'interaction_target_readiness') - .map(([event]) => event.data); -} - test('an unannotated action executes unchanged (old-script pass-through)', async () => { const scene = replayScriptScene('agent-device-replay-target-verify-passthrough-', [ 'click id="save"', @@ -99,9 +80,7 @@ test('a verified target proceeds to dispatch the action', async () => { expect(response.ok).toBe(true); expect(scene.invoked.map((req) => req.command)).toEqual(['click']); expect(scene.invoked[0]?.positionals).toEqual(['id="save"']); - // Present on the gate's first capture: the dispatch keeps the step's whole budget. - expect(scene.invoked[0]?.flags?.readinessTimeoutMs).toBe(2_000); - expect(readinessDiagnostics()).toEqual([]); + expect(scene.invoked[0]?.flags).toEqual({}); }); test('a verified drag guards both source and destination before dispatch', async () => { @@ -199,105 +178,27 @@ test('a selector-miss divergence blocks dispatch and never sends the action', as expect(targetBinding.matchCount).toBe(0); expect(targetBinding.observed).toBeUndefined(); expect(targetBinding.recorded).toEqual({ id: 'save', role: 'button', label: 'Save' }); - // click waits for its target, so the gate re-captured before refusing. - expect(mockDispatchCommand.mock.calls.length).toBeGreaterThan(1); - expect(response.error.details?.readiness).toMatchObject({ waitedMs: 2_000, end: 'expired' }); + expect(mockDispatchCommand).toHaveBeenCalledTimes(1); }); -test('an annotated click whose target renders on the second capture waits for it and dispatches', async () => { +test('an annotated click refuses the first selector miss even when a later capture would match', async () => { const scene = replayScriptScene('agent-device-replay-target-verify-late-render-', [ SAVE_ANNOTATION, 'click id="save"', ]); - mockDispatchCommand.mockResolvedValueOnce(emptyCapture()).mockResolvedValue(saveButtonCapture()); const response = await scene.replay(); - expect(response.ok).toBe(true); - expect(scene.invoked.map((req) => req.command)).toEqual(['click']); - expect(scene.invoked[0]?.internal?.replayTargetGuard).toMatchObject({ - identity: { id: 'save', role: 'button', label: 'Save' }, - }); - expect(mockDispatchCommand).toHaveBeenCalledTimes(2); - expect(readinessDiagnostics()).toEqual([ - { polls: 2, waitedMs: 200, end: 'done', command: 'click' }, - ]); -}); - -test('the dispatch gets only the readiness budget the gate left', async () => { - const scene = replayScriptScene('agent-device-replay-target-verify-shared-budget-', [ - SAVE_ANNOTATION, - 'click id="save"', - ]); - - // Eight misses, one interval apart, before the target renders 1.6 s into the 2 s budget. - for (let miss = 0; miss < 8; miss += 1) mockDispatchCommand.mockResolvedValueOnce(emptyCapture()); - mockDispatchCommand.mockResolvedValue(saveButtonCapture()); - - const response = await scene.replay(); - - expect(response.ok).toBe(true); - expect(scene.invoked[0]?.flags?.readinessTimeoutMs).toBe(400); -}); - -test('the budget the gate hands the dispatch counts from the end of its first capture', async () => { - const scene = replayScriptScene('agent-device-replay-target-verify-budget-origin-', [ - SAVE_ANNOTATION, - 'click id="save"', - ]); - let nowMs = 0; - const clock = { - now: () => nowMs, - sleep: async (ms: number) => { - nowMs += ms; - }, - }; - - // A 500 ms first capture, then seven more misses one interval apart: 1.4 s of the 2 s budget. - mockDispatchCommand.mockImplementationOnce(async () => { - nowMs += 500; - return emptyCapture(); - }); - for (let miss = 0; miss < 7; miss += 1) mockDispatchCommand.mockResolvedValueOnce(emptyCapture()); - mockDispatchCommand.mockResolvedValue(saveButtonCapture()); - - const response = await scene.replay({ clock }); - - expect(response.ok).toBe(true); - expect(readinessDiagnostics()).toEqual([ - { polls: 9, waitedMs: 2_100, end: 'done', command: 'click' }, - ]); - expect(scene.invoked[0]?.flags?.readinessTimeoutMs).toBe(400); -}); - -test('cancelling the request during the gate wait ends it at the next poll with no dispatch', async () => { - const scene = replayScriptScene('agent-device-replay-target-verify-gate-cancel-', [ - SAVE_ANNOTATION, - 'click id="save"', - ]); - const requestId = 'replay-gate-cancel'; - const registration = registerRequestAbort(requestId); - mockDispatchCommand.mockResolvedValueOnce(emptyCapture()).mockImplementationOnce(async () => { - markRequestCanceled(requestId); - return emptyCapture(); - }); - mockDispatchCommand.mockResolvedValue(saveButtonCapture()); - - try { - const response = await scene.replay({ requestId }); - - expect(response.ok).toBe(false); - if (response.ok) return; - expect(response.error.details?.reason).toBe('request_canceled'); - expect(mockDispatchCommand).toHaveBeenCalledTimes(2); - expect(scene.invoked).toEqual([]); - } finally { - clearRequestCanceled(requestId, registration); - } + expect(response.ok).toBe(false); + expect(scene.invoked).toEqual([]); + expect(mockDispatchCommand).toHaveBeenCalledTimes(1); + if (response.ok) return; + expect(response.error.code).toBe('REPLAY_DIVERGENCE'); + expect(response.error.details?.divergence).toMatchObject({ kind: 'selector-miss' }); }); -test('an annotated step whose command does not wait for its target refuses a selector miss on one capture', async () => { +test('an annotated fill refuses a selector miss on one capture', async () => { const scene = replayScriptScene('agent-device-replay-target-verify-no-wait-', [ SAVE_ANNOTATION, 'fill id="save" "hello"', diff --git a/packages/replay-port/src/daemon-port/command-types.ts b/packages/replay-port/src/daemon-port/command-types.ts index f9214bf21e..59e8633dcc 100644 --- a/packages/replay-port/src/daemon-port/command-types.ts +++ b/packages/replay-port/src/daemon-port/command-types.ts @@ -10,7 +10,6 @@ import type { ReplayTestAttemptStepSink } from '@agent-device/replay-test'; import type { DaemonResponse, SessionRuntimeHints } from '@agent-device/kernel/contracts'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { SnapshotState } from '@agent-device/kernel/snapshot'; -import type { ObservationClock } from '@agent-device/capture-kit/observe-until'; /** * The slice of the daemon's live session record replay reads. The daemon passes its full @@ -130,11 +129,6 @@ export type ReplayInvoke = (request: ReplayDispatchRequest) => Promise SessionScope; - /** - * The time source the pre-dispatch target-readiness wait paces itself by. Absent: wall clock. - * The daemon never sets it; tests inject one, as `AgentDeviceRuntime.clock` does for the dispatch. - */ - clock?: ObservationClock; }>; export type ReplayCommand = Readonly<{ diff --git a/packages/replay-port/src/daemon-port/session-replay-action-runtime.ts b/packages/replay-port/src/daemon-port/session-replay-action-runtime.ts index e826dc7cc5..80c688ee94 100644 --- a/packages/replay-port/src/daemon-port/session-replay-action-runtime.ts +++ b/packages/replay-port/src/daemon-port/session-replay-action-runtime.ts @@ -1,13 +1,10 @@ import type { SessionAction, SessionScope } from '@agent-device/contracts/session'; -import type { CommandFlags } from '@agent-device/contracts/command'; import type { ReplayDaemonDependencies, ReplayDispatchRequest, ReplayInvoke, } from '@agent-device/replay-port/command-types'; import { mergeParentFlags } from '@agent-device/command-registry/batch'; -import { commandAcceptsReadinessBudget } from '@agent-device/command-registry/registry'; -import type { ReadinessSchedule } from '@agent-device/selectors/selector-pipeline-policy'; import { AppError, normalizeError } from '@agent-device/kernel/errors'; import { gesturePayloadFromPositionals, @@ -46,11 +43,6 @@ export async function invokeReplayAction(params: { /** The isolation scope the daemon already resolved for the request, when it did. */ resolvedSessionScope: SessionScope | undefined; dependencies: ReplayDaemonDependencies; - /** - * What is left of the step's readiness budget after the pre-dispatch target gate waited; 0 makes - * the dispatch resolve its target once. Absent: the step's full budget. - */ - readinessTimeoutMs?: number; }): Promise { const { req, @@ -92,7 +84,6 @@ export async function invokeReplayAction(params: { invoke, resolvedSessionScope, dependencies, - readinessTimeoutMs: params.readinessTimeoutMs, }); } catch (error) { // Only an expected AppError dispatch failure (e.g. a selector-miss) gets @@ -150,12 +141,10 @@ async function invokeResolvedReplayAction(params: { invoke: ReplayInvoke; resolvedSessionScope: SessionScope | undefined; dependencies: ReplayDaemonDependencies; - readinessTimeoutMs: number | undefined; }): Promise { const { req, sessionName, resolved, sourceAction, invoke, resolvedSessionScope, dependencies } = params; - const flags = buildReplayActionFlags(req.flags, resolved.flags, resolved.command); - if (params.readinessTimeoutMs !== undefined) flags.readinessTimeoutMs = params.readinessTimeoutMs; + const flags = mergeParentFlags(req.flags, { ...(resolved.flags ?? {}) }); const recordedInputVariable = sourceAction.command === 'fill' ? readRecordedInputVariableName(inferFillText(sourceAction)) @@ -224,38 +213,3 @@ function readResponseTiming(data: unknown): Record | undefined }), ); } - -/** - * A replayed step of a readiness-budgeted command carries no budget of its own; replay supplies one - * so a step recorded against a loading screen can land. - */ -const REPLAY_DEFAULT_READINESS_TIMEOUT_MS = 2_000; - -/** - * The readiness schedule a replayed step's dispatch polls its target under, or undefined when the - * step's command does not wait for its target. The pre-dispatch target gate polls under this same - * schedule, built by the selector policy's `readinessScheduleFor` as the dispatch builds its own. - */ -export async function replayStepReadinessSchedule( - parentFlags: CommandFlags | undefined, - action: SessionAction, -): Promise { - const { readinessTimeoutMs } = buildReplayActionFlags(parentFlags, action.flags, action.command); - if (readinessTimeoutMs === undefined) return undefined; - // Loaded only by a step that waits, so the replay entry does not evaluate the selector policy. - const { SELECTOR_PIPELINE_POLICIES, readinessScheduleFor } = - await import('@agent-device/selectors/selector-pipeline-policy'); - return readinessScheduleFor(SELECTOR_PIPELINE_POLICIES.promotedTarget.poll, readinessTimeoutMs); -} - -function buildReplayActionFlags( - parentFlags: CommandFlags | undefined, - actionFlags: SessionAction['flags'] | undefined, - command: string, -): CommandFlags { - const flags = mergeParentFlags(parentFlags, { ...(actionFlags ?? {}) }); - if (commandAcceptsReadinessBudget(command) && flags.readinessTimeoutMs === undefined) { - flags.readinessTimeoutMs = REPLAY_DEFAULT_READINESS_TIMEOUT_MS; - } - return flags; -} diff --git a/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts b/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts index d46a0cdf09..d016a5f38b 100644 --- a/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts +++ b/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts @@ -17,17 +17,8 @@ import { import type { SnapshotTimingSample } from '@agent-device/contracts/capture'; import { withReplayFailureDiagnostics } from './session-replay-runtime-failure.ts'; -import { - invokeReplayAction, - replayStepReadinessSchedule, -} from './session-replay-action-runtime.ts'; -import type { - AdReplayStepFailure, - AdReplayStepRuntime, - AdReplayTargetObservation, -} from '@agent-device/ad-replay'; -import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; -import type { ObservationEvidence } from '@agent-device/capture-kit/observe-until'; +import { invokeReplayAction } from './session-replay-action-runtime.ts'; +import type { AdReplayStepFailure, AdReplayStepRuntime } from '@agent-device/ad-replay'; import { collectReplayActionArtifactPaths } from '@agent-device/replay-port/session-replay-runtime-artifacts'; import { applyReplayDispatchGuard, @@ -124,8 +115,6 @@ export function createAdReplayStepRuntime(params: { const { ctx, req, artifactPaths, onStep, armSaveScript } = params; let lastResponse: DaemonResponse | undefined; let lastObservation: DivergenceObservation | undefined; - /** This step's pre-dispatch readiness wait, when the gate polled more than once. */ - let gateWait: GateReadinessWait | undefined; /** * The `TargetBindingDivergenceContext` every wire-builder needs — built @@ -204,59 +193,24 @@ export function createAdReplayStepRuntime(params: { }, async observeTarget({ action, token }) { - const observeOnce = async (): Promise => { - const observation = await captureTargetObservation(action); - lastObservation = observation; - if (observation.state !== 'available') { - return { state: 'unavailable', reason: observation.reason, hint: observation.hint }; - } - const session = ctx.sessionStore.get(); - return { - state: 'classified', - classification: classifyPreDispatchTarget({ - // An available capture implies an active session, and the engine - // only calls this for an action carrying `targetEvidence`. - recorded: action.targetEvidence!, - token, - action, - nodes: [...observation.nodes], - platform: session!.device.platform, - }), - }; - }; - // The gate waits only where the dispatch would: a readiness-budgeted - // command resolving a selector (a `@ref` resolves without a wait). - const readiness = token.startsWith('@') - ? undefined - : await replayStepReadinessSchedule(ctx.replayReq.flags, action); - if (!readiness) return await observeOnce(); - const { observeUntil } = await import('@agent-device/capture-kit/observe-until'); - const observed = await observeUntil({ - capture: observeOnce, - verdict: (latest) => - isTargetNotRenderedYet(latest) ? { kind: 'continue' } : { kind: 'done', result: latest }, - schedule: readiness, - ...(ctx.signal ? { signal: ctx.signal } : {}), - ...(ctx.dependencies.clock ? { clock: ctx.dependencies.clock } : {}), - }); - if (observed.polls.length > 1) { - gateWait = { - remainingBudgetMs: Math.max(0, readiness.budgetMs - budgetSpentMs(observed)), - readiness: { - polls: observed.polls.length, - waitedMs: observed.waitedMs, - end: observed.kind, - }, - }; - emitDiagnostic({ - level: 'debug', - phase: 'interaction_target_readiness', - data: { ...gateWait.readiness, command: action.command }, - }); + const observation = await captureTargetObservation(action); + lastObservation = observation; + if (observation.state !== 'available') { + return { state: 'unavailable', reason: observation.reason, hint: observation.hint }; } - if (observed.kind === 'done') return observed.result; - if (observed.last !== undefined) return observed.last; - throw observed.kind === 'expired' ? observed.lastError : observed.error; + const session = ctx.sessionStore.get(); + return { + state: 'classified', + classification: classifyPreDispatchTarget({ + // An available capture implies an active session, and the engine + // only calls this for an action carrying `targetEvidence`. + recorded: action.targetEvidence!, + token, + action, + nodes: [...observation.nodes], + platform: session!.device.platform, + }), + }; }, // `_stepArtifactPaths` (the pre-step snapshot) is unused here — dispatch @@ -265,11 +219,6 @@ export function createAdReplayStepRuntime(params: { async dispatchStep(action, resolvedAction, index, _stepArtifactPaths, guard) { const sourceLine = ctx.actionLines[index] ?? 1; const response = await invokeReplayAction({ - ...(gateWait - ? { - readinessTimeoutMs: gateWait.remainingBudgetMs, - } - : {}), req: applyReplayDispatchGuard(ctx.replayReq, guard), sessionName: ctx.sessionName, action, @@ -318,11 +267,7 @@ export function createAdReplayStepRuntime(params: { evidence, observation, ); - return recordFailure( - evidence.kind === 'selector-miss' && gateWait - ? withReadinessDetail(response, gateWait.readiness) - : response, - ); + return recordFailure(response); }, async buildPostDispatchTargetBindingFailure( @@ -375,7 +320,6 @@ export function createAdReplayStepRuntime(params: { // capabilities — the natural per-step boundary to clear the previous // step's capture (see this factory's own header). lastObservation = undefined; - gateWait = undefined; armSaveScript(); }, isRepairArmed: () => ctx.coordinator.view()?.repairBoundary !== undefined, @@ -513,37 +457,3 @@ function readSessionSnapshotSamplesSince( ): SnapshotTimingSample[] { return sessionStore.get()?.snapshotDiagnostics?.samples.slice(start) ?? []; } - -/** A selector miss on a readiness-budgeted step: the target may still be rendering. */ -function isTargetNotRenderedYet(observation: AdReplayTargetObservation): boolean { - return ( - observation.state === 'classified' && - !observation.classification.verified && - observation.classification.kind === 'selector-miss' - ); -} - -type GateReadinessWait = { - /** The step's readiness budget the gate left for the dispatch. */ - remainingBudgetMs: number; - /** The same evidence a dispatched readiness wait reports (`SelectorReadinessDetails`). */ - readiness: { polls: number; waitedMs: number; end: string }; -}; - -/** The budget a `budgetFrom: 'first-capture'` wait spent: its time after the first capture ended. */ -function budgetSpentMs(observed: ObservationEvidence): number { - const first = observed.polls[0]; - return first ? observed.waitedMs - (first.startedMs + first.durationMs) : 0; -} - -/** Carries the gate's wait where a dispatched step's target-not-found failure carries its own. */ -function withReadinessDetail( - response: DaemonResponse, - readiness: GateReadinessWait['readiness'], -): DaemonResponse { - if (response.ok) return response; - return { - ...response, - error: { ...response.error, details: { ...response.error.details, readiness } }, - }; -} diff --git a/packages/selectors/src/interaction-error.ts b/packages/selectors/src/interaction-error.ts index d02455a144..ba09c40e56 100644 --- a/packages/selectors/src/interaction-error.ts +++ b/packages/selectors/src/interaction-error.ts @@ -22,8 +22,8 @@ export const INTERACTION_ERROR_REASONS = { /** The target names a node with no usable centre to touch: a missing, non-finite, or negative rect. */ targetBoundsInvalid: 'target_bounds_invalid', /** - * A readiness poll captured a tree the snapshot-quality verdict calls sparse and the selector did - * not resolve in it: the tree is untrustworthy, so absence proves nothing and the wait ends now. + * A capture's snapshot-quality verdict is sparse and the selector did not resolve in it: + * the tree is untrustworthy, so absence proves nothing and the interaction is refused. * `details.snapshotQuality` carries the verdict. */ captureSparse: 'capture_sparse', diff --git a/packages/selectors/src/selector-pipeline-policy.test.ts b/packages/selectors/src/selector-pipeline-policy.test.ts deleted file mode 100644 index c074c1a223..0000000000 --- a/packages/selectors/src/selector-pipeline-policy.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { describe, expect, test } from 'vitest'; -import { SELECTOR_PIPELINE_POLICIES, readinessScheduleFor } from './selector-pipeline-policy.ts'; - -const promotedPoll = SELECTOR_PIPELINE_POLICIES.promotedTarget.poll; - -describe('readinessScheduleFor', () => { - test('polls at the row cadence with the supplied budget, counted from the first capture', () => { - expect(readinessScheduleFor(promotedPoll, 800)).toEqual({ - intervalMs: promotedPoll.intervalMs, - budgetMs: 800, - budgetFrom: 'first-capture', - }); - }); - - test('caps the supplied budget at the row ceiling', () => { - expect(readinessScheduleFor(promotedPoll, promotedPoll.maxTimeoutMs + 5_000)?.budgetMs).toBe( - promotedPoll.maxTimeoutMs, - ); - }); - - test('a row that resolves against one capture has no schedule', () => { - expect(readinessScheduleFor(SELECTOR_PIPELINE_POLICIES.resolvedTarget.poll, 800)).toBe( - undefined, - ); - }); - - test.each([undefined, 0, -1, 1.5])('a %s budget takes the one-attempt path', (budget) => { - expect(readinessScheduleFor(promotedPoll, budget)).toBe(undefined); - }); -}); diff --git a/packages/selectors/src/selector-pipeline-policy.ts b/packages/selectors/src/selector-pipeline-policy.ts index aed75591b7..1cbcdb4efb 100644 --- a/packages/selectors/src/selector-pipeline-policy.ts +++ b/packages/selectors/src/selector-pipeline-policy.ts @@ -2,7 +2,6 @@ import { SELECTOR_RESOLUTION_POLICIES, type SelectorResolutionPolicy, } from '@agent-device/selectors'; -import type { ObservationSchedule } from '@agent-device/capture-kit/observe-until'; /** * The structural half of the per-caller selector policy (#1656), companion to @@ -17,7 +16,7 @@ import type { ObservationSchedule } from '@agent-device/capture-kit/observe-unti * * A row may only declare what these runners enforce (#1649 review). Every row, * skips included, is driven through every runner in - * selector-pipeline-policy.test.ts, so flipping any cell changes an assertion — + * selector-pipeline.test.ts, so flipping any cell changes an assertion — * add a column only once a runner reads it. */ @@ -76,31 +75,8 @@ export type WaitPollBudget = { intervalMs: number; }; -/** - * `promotedTarget`: the row states only a ceiling and a cadence, never a default — a miss with no - * caller-supplied budget takes the one-attempt path instead of polling (`resolveSelectorInteractionTarget`). - * When the caller does supply one (`readinessTimeoutMs`, never model- or - * CLI-writable), it is capped at `maxTimeoutMs` before it bounds the readiness loop. - */ -export type ReadinessPollBudget = { - /** The ceiling a caller-supplied readiness timeout may not exceed. */ - maxTimeoutMs: number; - /** Delay between polls, clamped to the remaining budget. */ - intervalMs: number; -}; - -/** The poll budget of a row that polls: `wait`/`findWait`'s own default, or `promotedTarget`'s cap. */ -export type SelectorPollBudget = WaitPollBudget | ReadinessPollBudget; - -/** - * Consumed by `selectorPollBudget` (wait-shaped rows only) and — for `promotedTarget` — by - * `resolveSelectorInteractionTarget`'s target-readiness loop. `'none'` is an answer, not an - * omission: a row that resolves against one capture has no polling contract, so asking for its - * budget is a caller bug — never a place to default one in. Acting rows are not all `'none'`: - * `promotedTarget` polls for the target to exist, while `resolvedTarget` and every read row still - * resolve against a single capture. - */ -export type SelectorPollStage = SelectorPollBudget | 'none'; +/** A row either resolves once or supplies the deadline and cadence for an explicit wait. */ +export type SelectorPollStage = WaitPollBudget | 'none'; /** * A row whose result is the candidate SET, not a target: `find list`. @@ -131,20 +107,13 @@ export type SelectorPipelinePolicy = SelectorListPolicy & { const WAIT_POLL_BUDGET: WaitPollBudget = { defaultTimeoutMs: 10_000, intervalMs: 300 }; export const SELECTOR_PIPELINE_POLICIES = { - /** - * `click`/`press`/`longpress`: the tap lands on the actionable owner of the - * match. When the caller supplies a readiness budget, polls for the target to - * appear and become resolvable (a rect), capped at this row's `maxTimeoutMs`, before refusing — - * resolution only; occlusion, off-screen, and promotion still run once, against the winning - * capture, after the loop ends. A miss with no caller-supplied budget takes the one-attempt path - * (`resolveSelectorInteractionTarget`). - */ + /** `click`/`press`/`longpress`: the tap lands on the actionable owner of the match. */ promotedTarget: { resolution: SELECTOR_RESOLUTION_POLICIES.act, occlusion: 'exclude-and-refuse', offscreen: 'refuse', promotion: 'hittable-ancestor', - poll: { maxTimeoutMs: 2_000, intervalMs: 200 }, + poll: 'none', }, /** * `fill`/`focus`/`scroll`/gesture endpoints, and the native-ref preflight — @@ -232,34 +201,6 @@ export const SELECTOR_PIPELINE_POLICIES = { export type SelectorPipelinePolicyName = keyof typeof SELECTOR_PIPELINE_POLICIES; -/** - * The schedule a readiness wait polls its target under. The budget counts from the end of the - * first capture: that capture is the one-attempt lookup a step pays without any wait, so the - * replay target gate and the dispatch spend the budget only on retries. - */ -export type ReadinessSchedule = Readonly< - Pick & { budgetFrom: 'first-capture' } ->; - -/** - * The one owner of a readiness schedule, for the replay target gate and the dispatch alike: - * `undefined` for a row that resolves against one capture or when no positive integer budget was - * supplied, otherwise the row's cadence and the supplied budget capped at the row's `maxTimeoutMs`. - */ -export function readinessScheduleFor( - poll: ReadinessPollBudget | 'none', - readinessTimeoutMs: number | undefined, -): ReadinessSchedule | undefined { - if (poll === 'none') return undefined; - if (readinessTimeoutMs === undefined || !Number.isInteger(readinessTimeoutMs)) return undefined; - if (readinessTimeoutMs <= 0) return undefined; - return { - intervalMs: poll.intervalMs, - budgetMs: Math.min(readinessTimeoutMs, poll.maxTimeoutMs), - budgetFrom: 'first-capture', - }; -} - /** * The two questions a row asks the engine, derived from its ambiguity contract * rather than from a hand-kept list of row names: `reject-candidates` rows go diff --git a/packages/selectors/src/selector-pipeline.test.ts b/packages/selectors/src/selector-pipeline.test.ts index cd7ac88f5b..e04d72ca69 100644 --- a/packages/selectors/src/selector-pipeline.test.ts +++ b/packages/selectors/src/selector-pipeline.test.ts @@ -418,24 +418,11 @@ test('the poll stage answers only for the rows that poll a caller-default budget intervalMs: 300, }); } - for (const row of NODE_STAGE_ROWS.filter( - (name) => name !== 'wait' && name !== 'findWait' && name !== 'promotedTarget', - )) { + for (const row of NODE_STAGE_ROWS.filter((name) => name !== 'wait' && name !== 'findWait')) { assert.throws(() => selectorPollBudget(SELECTOR_PIPELINE_POLICIES[row]), /no poll budget/, row); } }); -test('promotedTarget states a caller-capped readiness budget, not a caller-default one', () => { - assert.deepEqual(SELECTOR_PIPELINE_POLICIES.promotedTarget.poll, { - maxTimeoutMs: 2_000, - intervalMs: 200, - }); - assert.throws( - () => selectorPollBudget(SELECTOR_PIPELINE_POLICIES.promotedTarget), - /caller-capped readiness budget/, - ); -}); - test('a listing row cannot be handed the node stages it does not declare', () => { // The `@ts-expect-error` directives ARE the assertion: a listing has no // single target to retarget, keep on screen, or wait for, so widening @@ -498,7 +485,7 @@ test('the documented per-caller pipelines are the ones declared', () => { }), ), { - promotedTarget: ['exclude-and-refuse', 'refuse', 'hittable-ancestor', 'poll'], + promotedTarget: ['exclude-and-refuse', 'refuse', 'hittable-ancestor', 'no-poll'], resolvedTarget: ['exclude-and-refuse', 'refuse', 'none', 'no-poll'], coveredDiagnosis: ['refuse', 'ignore', 'none', 'no-poll'], readText: ['ignore', 'ignore', 'none', 'no-poll'], diff --git a/packages/selectors/src/selector-pipeline.ts b/packages/selectors/src/selector-pipeline.ts index 1309af9e28..dc7921ea1e 100644 --- a/packages/selectors/src/selector-pipeline.ts +++ b/packages/selectors/src/selector-pipeline.ts @@ -271,8 +271,6 @@ async function guardOffscreen( /** * The poll stage: the caller-default budget `createWaitPolling` runs a wait-shaped row under. - * `promotedTarget`'s row polls too, but under a caller-supplied, row-capped budget rather than a - * row-owned default — it has no `defaultTimeoutMs` to hand back, so it is not a legal input here. */ export function selectorPollBudget(policy: SelectorPipelinePolicy): WaitPollBudget { if (policy.poll === 'none') { @@ -280,10 +278,5 @@ export function selectorPollBudget(policy: SelectorPipelinePolicy): WaitPollBudg 'selector pipeline row resolves against one capture and declares no poll budget', ); } - if (!('defaultTimeoutMs' in policy.poll)) { - throw new Error( - 'selector pipeline row polls under a caller-capped readiness budget, not a caller-default one', - ); - } return policy.poll; } diff --git a/src/__tests__/client-interactions-input-filtering.test.ts b/src/__tests__/client-interactions-input-filtering.test.ts new file mode 100644 index 0000000000..415bbbdc0d --- /dev/null +++ b/src/__tests__/client-interactions-input-filtering.test.ts @@ -0,0 +1,37 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createAgentDeviceClient } from '../agent-device-client.ts'; +import { createTransport } from './client-transport-fixture.ts'; + +test('tap commands filter undeclared input without losing post-action observation', async () => { + const setup = createTransport(async () => ({ ok: true, data: {} })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await client.interactions.press({ + selector: 'label=Foo', + settle: true, + // @ts-expect-error readinessTimeoutMs is not a client option. + readinessTimeoutMs: 2_000, + }); + await client.interactions.click({ + selector: 'label=Foo', + settle: true, + // @ts-expect-error readinessTimeoutMs is not a client option. + readinessTimeoutMs: 1_500, + }); + await client.interactions.longPress({ + selector: 'label=Foo', + settle: true, + // @ts-expect-error readinessTimeoutMs is not a client option. + readinessTimeoutMs: 900, + }); + + assert.deepEqual( + setup.calls.map((call) => call.command), + ['press', 'click', 'longpress'], + ); + for (const call of setup.calls) { + assert.equal(Object.hasOwn(call.flags ?? {}, 'readinessTimeoutMs'), false); + assert.equal(call.flags?.settle, true); + } +}); diff --git a/src/__tests__/client-interactions-readiness.test.ts b/src/__tests__/client-interactions-readiness.test.ts deleted file mode 100644 index bff493c383..0000000000 --- a/src/__tests__/client-interactions-readiness.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { createAgentDeviceClient } from '../agent-device-client.ts'; -import { createTransport } from './client-transport-fixture.ts'; - -// The readiness budget must reach `req.flags` for press/click/longpress exactly like -// `button`/`durationMs` do. It is never CLI- or model-writable; the SDK client option is its route. -test('readinessTimeoutMs on press/click/longpress reaches the request flags', async () => { - const setup = createTransport(async () => ({ ok: true, data: {} })); - const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); - - await client.interactions.press({ selector: 'label=Foo', readinessTimeoutMs: 2_000 }); - await client.interactions.click({ selector: 'label=Foo', readinessTimeoutMs: 1_500 }); - await client.interactions.longPress({ selector: 'label=Foo', readinessTimeoutMs: 900 }); - - assert.equal(setup.calls[0]?.flags?.readinessTimeoutMs, 2_000); - assert.equal(setup.calls[1]?.flags?.readinessTimeoutMs, 1_500); - assert.equal(setup.calls[2]?.flags?.readinessTimeoutMs, 900); -}); - -test('a press that waited returns the daemon data.readiness untouched', async () => { - const readiness = { polls: 2, waitedMs: 180 }; - const setup = createTransport(async () => ({ ok: true, data: { ref: '@e1', readiness } })); - const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); - - const pressed = await client.interactions.press({ selector: 'label=Foo' }); - const clicked = await client.interactions.click({ selector: 'label=Foo' }); - const longPressed = await client.interactions.longPress({ selector: 'label=Foo' }); - - for (const result of [pressed, clicked, longPressed]) { - assert.deepEqual((result as { readiness?: unknown }).readiness, readiness); - } -}); diff --git a/src/__tests__/command-descriptor-timeout-policy.test.ts b/src/__tests__/command-descriptor-timeout-policy.test.ts index f824d75ac7..0de631a9ae 100644 --- a/src/__tests__/command-descriptor-timeout-policy.test.ts +++ b/src/__tests__/command-descriptor-timeout-policy.test.ts @@ -2,17 +2,13 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { - commandAcceptsReadinessBudget, commandDescriptors, resolveCommandPostActionObservationSupport, resolveCommandTimeoutPolicy, } from '@agent-device/command-registry/registry'; -import { INTERACTION_DISPATCH_PATHS } from '@agent-device/contracts/interaction-guarantees'; -import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; import { OVERLAY_BUDGET_MS } from '@agent-device/capture-kit/recording-overlay'; import { DEFAULT_TIMEOUT_POLICY, - READINESS_BUDGET_MAX_MS, resolveCommandRequestTimeoutMs, } from '@agent-device/command-registry/timeout-policy'; import { DEFAULT_STABLE_TIMEOUT_MS } from '../commands/interaction/runtime/stable-capture.ts'; @@ -424,57 +420,3 @@ test('open and prepare startup budgets keep a client-envelope margin over the da 90_000, ); }); - -test('a readiness budget widens the request envelope on top of the settle envelope', () => { - const press = resolveCommandTimeoutPolicy('press'); - const withoutReadiness = resolveCommandRequestTimeoutMs(press, { flags: {} }); - assert.equal(withoutReadiness, 90_000); - assert.equal( - resolveCommandRequestTimeoutMs(press, { flags: { readinessTimeoutMs: 2_000 } }), - 92_000, - ); - assert.equal( - resolveCommandRequestTimeoutMs(press, { flags: { settle: true, readinessTimeoutMs: 2_000 } }), - 90_000 + 10_000 + 30_000 + 2_000, - ); - assert.equal( - resolveCommandRequestTimeoutMs(resolveCommandTimeoutPolicy('longpress'), { - flags: { readinessTimeoutMs: 2_000 }, - }), - 212_000, - ); - assert.equal(resolveCommandRequestTimeoutMs(press, { flags: { readinessTimeoutMs: 0 } }), 90_000); - assert.equal( - resolveCommandRequestTimeoutMs(press, { flags: { readinessTimeoutMs: 999_000 } }), - 90_000 + 2_000, - ); -}); - -test('the envelope readiness cap is the promotedTarget row poll ceiling', () => { - assert.equal( - READINESS_BUDGET_MAX_MS, - SELECTOR_PIPELINE_POLICIES.promotedTarget.poll.maxTimeoutMs, - ); - assert.equal( - resolveCommandRequestTimeoutMs(resolveCommandTimeoutPolicy('fill'), { - flags: { readinessTimeoutMs: 2_000 }, - }), - 90_000, - ); -}); - -test('the readiness-budgeted commands are the ones a runtime targetReadiness cell enforces', () => { - const declared = commandDescriptors - .map((descriptor) => descriptor.name) - .filter((command) => commandAcceptsReadinessBudget(command)) - .sort(); - const enforced = [ - ...new Set( - Object.values(INTERACTION_DISPATCH_PATHS).flatMap((path) => { - const cell = path.guarantees.targetReadiness; - return cell.kind === 'runtime' ? (cell.appliesTo ?? path.commands) : []; - }), - ), - ].sort(); - assert.deepEqual(declared, enforced); -}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts b/src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts index 038cf9a770..5721eca17c 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts +++ b/src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts @@ -126,35 +126,6 @@ async function refusedFill(positionals: string[]): Promise { } } -/** A press whose readiness wait polls a tree that never lists the selector's target. */ -async function pressWhoseTargetNeverAppears(): Promise { - const capture = vi.mocked(captureSnapshotWithInteractor); - capture.mockResolvedValue({ - nodes: [ - { - index: 0, - type: 'Application', - label: 'Example', - rect: { x: 0, y: 0, width: 400, height: 800 }, - }, - ], - backend: 'xctest', - producer: 'apple-runner', - }); - try { - await refusedPress(['label="Missing"'], { readinessTimeoutMs: 300 }); - } catch (error) { - assert.ok(error instanceof AppError); - assert.equal(error.details?.reason, 'selector_not_found'); - const readiness = error.details?.readiness as { polls: number } | undefined; - assert.ok(readiness && readiness.polls >= 2, `expected >=2 polls, got ${readiness?.polls}`); - throw error; - } finally { - capture.mockReset(); - } - assert.fail('expected the press to be refused'); -} - async function pressAfterUnclassifiedTouchFailure( details?: Record, ): Promise { @@ -363,7 +334,6 @@ const DRIVERS: Record Promise> = { 'daemon.refusal.ref-not-found': () => refusedPress(['@e9']), 'daemon.refusal.admission': () => refusedPress([]), 'daemon.refusal.fill-admission': () => refusedFill(['@e1']), - 'daemon.refusal.selector-readiness-exhausted': pressWhoseTargetNeverAppears, 'daemon.unclassified': () => pressAfterUnclassifiedTouchFailure(), 'daemon.series.swipe-later-repetition-refused': swipeRefusedOnSecondRepetition, 'daemon.refusal.gesture-drag-admission': gestureDragRefusedBeforeDispatch, diff --git a/src/commands/command-flags.ts b/src/commands/command-flags.ts index fa04c2d2ef..e2c103d857 100644 --- a/src/commands/command-flags.ts +++ b/src/commands/command-flags.ts @@ -88,7 +88,6 @@ function buildFlags(options: InternalRequestOptions): CommandFlags { durationMs: options.durationMs, holdMs: options.holdMs, jitterPx: options.jitterPx, - readinessTimeoutMs: options.readinessTimeoutMs, pixels: options.pixels, until: options.until, doubleTap: options.doubleTap, diff --git a/src/commands/command-input.ts b/src/commands/command-input.ts index 675e1b9d84..1fa4b59ebb 100644 --- a/src/commands/command-input.ts +++ b/src/commands/command-input.ts @@ -412,7 +412,6 @@ export function readFieldInput( ); const commonInput = readCommonInput(record, { readTargetAlias: !Object.hasOwn(fields, 'target'), - readinessBudgetDeclared: Object.hasOwn(fields, 'readinessTimeoutMs'), }); return compactRecord({ ...commonInput, diff --git a/src/commands/common-input-fields.ts b/src/commands/common-input-fields.ts index f608d87b79..b172c1109c 100644 --- a/src/commands/common-input-fields.ts +++ b/src/commands/common-input-fields.ts @@ -34,8 +34,6 @@ export type CommonCommandInput = Pick< export type CommonInputReadOptions = { readTargetAlias?: boolean; - /** The command's own fields declare `readinessTimeoutMs` (`interaction/metadata.ts`). */ - readinessBudgetDeclared?: boolean; }; /** @@ -217,11 +215,6 @@ const COMMON_INPUT_FIELDS = { flagKey: 'noRecord', flagIn: ['input', 'selection'], }, - readinessTimeoutMs: { - // No schema and no value: a command whose descriptor declares `targetReadiness: 'budgeted'` - // carries and reads the key through its own fields; every other command refuses it here. - read: refuseUndeclaredReadinessBudget, - }, daemonBaseUrl: { schema: { type: 'string', description: 'Remote daemon base URL.' }, read: (record) => optionalString(record, 'daemonBaseUrl'), @@ -256,10 +249,7 @@ const COMMON_INPUT_FIELDS = { schema: { type: 'boolean', description: 'Enable debug diagnostics.' }, read: (record) => optionalBoolean(record, 'debug'), }, -} as const satisfies Record< - keyof CommonCommandInput | 'target' | 'readinessTimeoutMs', - CommonInputFieldSpec ->; +} as const satisfies Record; const COMMON_INPUT_ROWS: ReadonlyArray = Object.entries(COMMON_INPUT_FIELDS); @@ -337,16 +327,3 @@ function readDeviceTarget( } return deviceTarget ?? targetAlias; } - -function refuseUndeclaredReadinessBudget( - record: Record, - options: CommonInputReadOptions, -): undefined { - if (options.readinessBudgetDeclared === true || !Object.hasOwn(record, 'readinessTimeoutMs')) { - return undefined; - } - throw new AppError( - 'INVALID_ARGS', - 'readinessTimeoutMs applies only to commands that wait for their target: press, click, and longpress.', - ); -} diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 6baab3a1c8..0ebaf14cc6 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -678,7 +678,6 @@ function toPressOptions(input: PressInput): PressOptions { ...toSelectorSnapshotOptions(input), ...toRepeatedOptions(input), verify: input.verify, - readinessTimeoutMs: input.readinessTimeoutMs, ...toSettleOptions(input), }; } @@ -703,7 +702,6 @@ function toLongPressOptions(input: LongPressInput): LongPressOptions { ...toClientInteractionTarget(input.target), ...toSelectorSnapshotOptions(input), durationMs: input.durationMs, - readinessTimeoutMs: input.readinessTimeoutMs, ...toSettleOptions(input), }; } diff --git a/src/commands/interaction/metadata.test.ts b/src/commands/interaction/metadata.test.ts index 7d108661e5..182265e17b 100644 --- a/src/commands/interaction/metadata.test.ts +++ b/src/commands/interaction/metadata.test.ts @@ -1,7 +1,5 @@ import { expect, test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; -import { commandAcceptsReadinessBudget } from '@agent-device/command-registry/registry'; -import { findCommandMetadata, listCommandMetadata } from '../command-metadata.ts'; import { interactionCommandMetadata } from './metadata.ts'; // `fill ""` is the clear-field primitive (#2063): before it existed, emptying an input @@ -52,34 +50,3 @@ test('type keeps refusing an empty text: appending nothing is not a clear', () = expect((error as AppError).code).toBe('INVALID_ARGS'); } }); - -const SELECTOR_TARGET = { kind: 'selector', selector: 'label=Continue' }; - -test('a command advertises readinessTimeoutMs exactly when its descriptor declares the budget', () => { - for (const metadata of listCommandMetadata()) { - const properties = metadata.inputSchema.properties ?? {}; - expect('readinessTimeoutMs' in properties, metadata.name).toBe( - commandAcceptsReadinessBudget(metadata.name), - ); - } -}); - -test('press reads readinessTimeoutMs', () => { - const input = findCommandMetadata('press').readInput({ - target: SELECTOR_TARGET, - readinessTimeoutMs: 2_000, - }) as { readinessTimeoutMs?: number }; - expect(input.readinessTimeoutMs).toBe(2_000); -}); - -test('fill refuses readinessTimeoutMs with an input error, and reads without it', () => { - const fill = findCommandMetadata('fill'); - expect(() => fill.readInput({ target: SELECTOR_TARGET, text: 'hi' })).not.toThrow(); - let refusal: unknown; - try { - fill.readInput({ target: SELECTOR_TARGET, text: 'hi', readinessTimeoutMs: 2_000 }); - } catch (error) { - refusal = error; - } - expect(refusal instanceof AppError && refusal.code).toBe('INVALID_ARGS'); -}); diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 2643579f69..0a2dcfedcd 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -21,7 +21,6 @@ import { SWIPE_REPETITION_MAX, } from '@agent-device/contracts/scroll-gesture'; import { FIND_LOCATORS } from '@agent-device/selectors'; -import { commandAcceptsReadinessBudget } from '@agent-device/command-registry/registry'; import { booleanField, elementTargetField, @@ -80,37 +79,12 @@ const interactionCommandDescriptions = { type InteractionCommandName = keyof typeof interactionCommandDescriptions; -/** - * The input field a command's `targetReadiness: 'budgeted'` descriptor trait entitles it to. Only - * those commands declare `readinessTimeoutMs`; the common input reader refuses the key for every - * command whose fields do not (`common-input-fields.ts`). Fails closed at module load for a command - * without the trait, so a field map cannot advertise a budget its runtime never polls under. - */ -function targetReadinessFields(command: InteractionCommandName) { - if (!commandAcceptsReadinessBudget(command)) { - throw new Error(`${command} does not declare targetReadiness: 'budgeted'`); - } - return { - readinessTimeoutMs: operatorField( - integerField( - "Operator-only: how long the command may poll for a target that does not exist yet, in milliseconds. Capped at the promotedTarget row's maxTimeoutMs; omitted takes the one-attempt resolution path.", - { min: 1 }, - ), - { - operatorPath: - 'Pass readinessTimeoutMs directly as CLI/Node.js command input; it is not exposed to model-facing tools.', - }, - ), - }; -} - const clickFields = { target: requiredField(interactionTargetField()), button: enumField(CLICK_BUTTONS, 'Pointer button for platforms that support mouse buttons.'), ...selectorSnapshotFields(), ...repeatedFields(), ...postActionObservationFields('click'), - ...targetReadinessFields('click'), }; const pressFields = { @@ -118,7 +92,6 @@ const pressFields = { ...selectorSnapshotFields(), ...repeatedFields(), ...postActionObservationFields('press'), - ...targetReadinessFields('press'), }; const fillFields = { @@ -150,7 +123,6 @@ const longPressFields = { durationMs: integerField('Long press duration in milliseconds.', { min: 0 }), ...selectorSnapshotFields(), ...postActionObservationFields('longpress'), - ...targetReadinessFields('longpress'), }; const hoverFields = { diff --git a/src/commands/interaction/runtime/__tests__/selector-read-policy.test.ts b/src/commands/interaction/runtime/__tests__/selector-read-policy.test.ts index 8dd908d398..445a283cdf 100644 --- a/src/commands/interaction/runtime/__tests__/selector-read-policy.test.ts +++ b/src/commands/interaction/runtime/__tests__/selector-read-policy.test.ts @@ -147,7 +147,7 @@ test('a read with no match at all still reports selector_not_found', async () => assert.equal(details.candidates, undefined); // And no dispatch disclosure: the shared not-found builder threads // `dispatched` as a parameter because the acting route proves 'no' and - // this route proves nothing about reaching the device (selector-readiness + // this route proves nothing about reaching the device (acting selector resolution // pins its side). assert.equal(details.dispatched, undefined); }); diff --git a/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/src/commands/interaction/runtime/__tests__/test-utils/index.ts index 6dc7dc7c8e..df2a15f664 100644 --- a/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -357,7 +357,7 @@ export function createInteractionDevice( > & { platform?: AgentDeviceBackend['platform']; sessionMetadata?: Record; - /** An advancing clock, for interactions that poll (the promotedTarget readiness loop). */ + /** An advancing clock for wait and post-action observation loops. */ clock?: { now: () => number; sleep: (ms: number) => Promise }; } = {}, ) { diff --git a/src/commands/interaction/runtime/gestures.ts b/src/commands/interaction/runtime/gestures.ts index 0f468f669f..c5d51bfc38 100644 --- a/src/commands/interaction/runtime/gestures.ts +++ b/src/commands/interaction/runtime/gestures.ts @@ -95,11 +95,6 @@ export type FocusCommandResult = ResolvedInteractionTarget & BackendResultEnvelo export type LongPressCommandOptions = CommandContext & { target: InteractionTarget; durationMs?: number; - /** - * Polls for the target to exist and become actionable, capped at the promotedTarget row's - * maxTimeoutMs. Absent takes one attempt. - */ - readinessTimeoutMs?: number; /** ADR 0012 step 4: replay-only post-resolution guard; see resolution.ts. */ expectedResolvedTarget?: ExpectedResolvedTarget; } & SettlePostActionObservationOptions; @@ -147,7 +142,6 @@ export const longPressCommand: RuntimeCommand< pipeline: SELECTOR_PIPELINE_POLICIES.promotedTarget, captureEvidenceBaseline: observation.needsPreActionBaseline, expectedResolvedTarget: options.expectedResolvedTarget, - readinessTimeoutMs: options.readinessTimeoutMs, }); if (!runtime.backend.longPress) { throw new AppError('UNSUPPORTED_OPERATION', 'longPress is not supported by this backend'); diff --git a/src/commands/interaction/runtime/interaction-resolution-request.ts b/src/commands/interaction/runtime/interaction-resolution-request.ts index 1a2507c776..0eca2e1931 100644 --- a/src/commands/interaction/runtime/interaction-resolution-request.ts +++ b/src/commands/interaction/runtime/interaction-resolution-request.ts @@ -45,12 +45,6 @@ export type ResolveInteractionTargetParams = { * the element they resolved. */ pipeline: ActingPipelinePolicy; - /** - * How long a `promotedTarget` row may poll for a target that does not exist yet (never model- or - * CLI-writable); `resolveSelectorInteractionTarget` caps it at the row's `maxTimeoutMs`. Anything - * other than a positive integer, and every `resolvedTarget` row, takes one attempt. - */ - readinessTimeoutMs?: number; /** * `--verify` (#1047): also capture the pre-action node set for a `point` target * so `changedFromBefore` evidence has a baseline. Ref/selector targets already diff --git a/src/commands/interaction/runtime/interaction-snapshot-capture.ts b/src/commands/interaction/runtime/interaction-snapshot-capture.ts index a5a68747d4..f9a3650ae3 100644 --- a/src/commands/interaction/runtime/interaction-snapshot-capture.ts +++ b/src/commands/interaction/runtime/interaction-snapshot-capture.ts @@ -5,7 +5,7 @@ import { now, toBackendContext } from '../../runtime-common.ts'; /** * A leaf shared by every interaction runtime module that needs a fresh (or session-cached) tree: - * `resolution.ts`'s ref/selector resolution, `selector-readiness.ts`'s readiness poll, + * `resolution.ts`'s ref/selector resolution, `selector-target-resolution.ts`'s acting selector lookup, * `gestures.ts`'s viewport resolution, and `post-action-observation.ts`'s `--verify` baseline. * Deliberately self-contained — it imports no sibling of this directory, so those siblings can * import it without ever risking a cycle back. diff --git a/src/commands/interaction/runtime/interactions.ts b/src/commands/interaction/runtime/interactions.ts index 7b53b2aa04..1266494ad7 100644 --- a/src/commands/interaction/runtime/interactions.ts +++ b/src/commands/interaction/runtime/interactions.ts @@ -42,11 +42,6 @@ export type PressCommandOptions = CommandContext & RepeatedInput & { target: InteractionTarget; button?: ClickButton; - /** - * Polls for the target to exist and become actionable, capped at the promotedTarget row's - * maxTimeoutMs. Absent takes one attempt. - */ - readinessTimeoutMs?: number; /** ADR 0012 step 4: replay-only post-resolution guard; see resolution.ts. */ expectedResolvedTarget?: ExpectedResolvedTarget; /** #1654: a mutating `find`'s already-resolved node; see resolution.ts. */ @@ -160,7 +155,6 @@ async function tapCommand( captureEvidenceBaseline: observation.needsPreActionBaseline, expectedResolvedTarget: options.expectedResolvedTarget, preresolvedTarget: options.preresolvedTarget, - readinessTimeoutMs: options.readinessTimeoutMs, }; const resolved = await resolveInteractionTarget(runtime, options, resolutionRequest); if (!runtime.backend.tap) { diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index a2255c6569..c7e0155d91 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -1,17 +1,10 @@ import { AppError } from '@agent-device/kernel/errors'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; -import type { SelectorResolution } from '@agent-device/selectors'; -import { readinessScheduleFor } from '@agent-device/selectors/selector-pipeline-policy'; import { attemptSelectorResolution, - pollForSelectorReadiness, - type SelectorReadinessWait, selectorInteractionFailure, -} from './selector-readiness.ts'; -import { - captureInteractionSnapshot, - type InteractionSnapshot, -} from './interaction-snapshot-capture.ts'; +} from './selector-target-resolution.ts'; +import { captureInteractionSnapshot } from './interaction-snapshot-capture.ts'; import { containsPoint } from '@agent-device/kernel/rect'; import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; import { surfaceScopedNodes } from './post-action-surface.ts'; @@ -137,34 +130,20 @@ async function resolveSelectorInteractionTarget( params: ResolveInteractionTargetParams, ): Promise { const selectorExpression = target.selector; - let capture: InteractionSnapshot; - let resolved: SelectorResolution | null; - let readiness: SelectorReadinessWait | undefined; - const readinessSchedule = readinessScheduleFor(params.pipeline.poll, params.readinessTimeoutMs); - if (!readinessSchedule) { - const attempt = await attemptSelectorResolution(runtime, options, selectorExpression, params); - capture = attempt.capture; - resolved = attempt.resolved; - if (!resolved || !resolved.node.rect) { - throw await selectorInteractionFailure({ - runtime, - nodes: capture.snapshot.nodes, - selectorExpression, - action: params.action, - resolved, - }); - } - } else { - const ready = await pollForSelectorReadiness( + const { capture, resolved } = await attemptSelectorResolution( + runtime, + options, + selectorExpression, + params, + ); + if (!resolved || !resolved.node.rect) { + throw await selectorInteractionFailure({ runtime, - options, + snapshot: capture.snapshot, selectorExpression, - params, - readinessSchedule, - ); - capture = ready.capture; - resolved = ready.resolved; - readiness = ready.readiness; + action: params.action, + resolved, + }); } // #1542: see the ref-target twin in ref-target-resolution.ts. const selected = resolved; @@ -191,7 +170,6 @@ async function resolveSelectorInteractionTarget( kind: 'selector', point, target: { kind: 'selector', selector: resolved.selector }, - ...(readiness ? { readiness } : {}), ...describeResolvedInteractionNode( runtime, visibleNode, diff --git a/src/commands/interaction/runtime/selector-readiness.test.ts b/src/commands/interaction/runtime/selector-readiness.test.ts deleted file mode 100644 index 5869560aa2..0000000000 --- a/src/commands/interaction/runtime/selector-readiness.test.ts +++ /dev/null @@ -1,143 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { AppError } from '@agent-device/kernel/errors'; -import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; -import { selector } from './selector-read-utils.ts'; -import { createFakeClock, createInteractionDevice } from './__tests__/test-utils/index.ts'; - -// promotedTarget's readiness poll runs only when the caller supplies `readinessTimeoutMs`, capped -// at the row's `maxTimeoutMs` (2_000). These pin the gating/capping decision at the runtime layer; the end-to-end poll mechanics (interactive/fresh -// capture sequencing, covered-target diagnosis, ridden-out capture errors) are unit-tested at the -// daemon level in test/integration/provider-scenarios/press-target-readiness.test.ts. - -const CONTINUE_BUTTON = { - index: 0, - depth: 0, - type: 'Button', - label: 'Continue', - rect: { x: 10, y: 20, width: 100, height: 40 }, - hittable: true, -}; - -test('runtime press without readinessTimeoutMs takes the one-attempt path and reports no readiness evidence', async () => { - let captures = 0; - const device = createInteractionDevice(makeSnapshotState([]), { - captureSnapshot: async () => { - captures += 1; - return { snapshot: makeSnapshotState([]) }; - }, - }); - - await assert.rejects( - () => device.interactions.press(selector('label=Continue'), { session: 'default' }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.details?.readiness, undefined); - // The shared not-found builder owes this route its dispatch disclosure: - // the acting row proves the request never reached the device. The read - // rows prove nothing and omit the field (see selector-read-policy). - assert.equal(error.details?.dispatched, 'no'); - return true; - }, - ); - // The interactive-capture-then-full-capture-fallback pair, not the readiness loop's repeated - // polling. - assert.equal(captures, 2); -}); - -test('runtime press with readinessTimeoutMs polls until the target appears', async () => { - let captures = 0; - const device = createInteractionDevice(makeSnapshotState([]), { - clock: createFakeClock(), - captureSnapshot: async () => { - captures += 1; - // Every capture before the fourth misses; the fourth (and every one after) has the button. - return { snapshot: makeSnapshotState(captures >= 4 ? [CONTINUE_BUTTON] : []) }; - }, - tap: async () => ({ ok: true }), - }); - - const result = await device.interactions.press(selector('label=Continue'), { - session: 'default', - readinessTimeoutMs: 2_000, - }); - - assert.equal(result.kind, 'selector'); - assert.equal(result.node?.label, 'Continue'); - assert.equal(result.kind === 'selector' ? result.readiness?.polls : undefined, 2); - assert.ok( - captures >= 4, - `expected at least 4 captures before the target appeared, got ${captures}`, - ); -}); - -test('runtime press that resolves on the first capture reports no readiness despite a budget', async () => { - const device = createInteractionDevice(makeSnapshotState([CONTINUE_BUTTON]), { - clock: createFakeClock(), - tap: async () => ({ ok: true }), - }); - - const result = await device.interactions.press(selector('label=Continue'), { - session: 'default', - readinessTimeoutMs: 2_000, - }); - - assert.equal(result.kind, 'selector'); - assert.equal('readiness' in result, false); -}); - -test('runtime press caps a readinessTimeoutMs larger than the row maxTimeoutMs at 2_000ms', async () => { - const device = createInteractionDevice(makeSnapshotState([]), { - clock: createFakeClock(), - captureSnapshot: async () => ({ snapshot: makeSnapshotState([]) }), - }); - - await assert.rejects( - () => - device.interactions.press(selector('label=Continue'), { - session: 'default', - // Far past the promotedTarget row's own maxTimeoutMs (2_000): the row's cap governs, not - // this caller-supplied value. - readinessTimeoutMs: 999_000, - }), - (error: unknown) => { - assert.ok(error instanceof AppError); - const readiness = error.details?.readiness as { waitedMs: number; end: string } | undefined; - assert.ok(readiness, 'expected readiness evidence on an exhausted poll'); - assert.equal(readiness.end, 'expired'); - assert.ok( - readiness.waitedMs >= 2_000 && readiness.waitedMs <= 2_200, - `expected waitedMs capped near 2_000ms, got ${readiness.waitedMs}`, - ); - return true; - }, - ); -}); - -test('runtime press whose every capture is unreadable fails with the unreadable-content error, not a selector miss', async () => { - let captures = 0; - const device = createInteractionDevice(makeSnapshotState([]), { - clock: createFakeClock(), - captureSnapshot: async () => { - captures += 1; - throw new AppError('COMMAND_FAILED', 'Android snapshot has no readable app content', { - androidSnapshotHelperFailureReason: 'system-window-only', - }); - }, - }); - - await assert.rejects( - () => - device.interactions.press(selector('label=Continue'), { - session: 'default', - readinessTimeoutMs: 2_000, - }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.details?.androidSnapshotHelperFailureReason, 'system-window-only'); - assert.notEqual(error.details?.reason, 'selector_not_found'); - return true; - }, - ); - assert.ok(captures >= 2, `expected the unreadable captures to be ridden out, got ${captures}`); -}); diff --git a/src/commands/interaction/runtime/selector-readiness.ts b/src/commands/interaction/runtime/selector-readiness.ts deleted file mode 100644 index 1380c41bff..0000000000 --- a/src/commands/interaction/runtime/selector-readiness.ts +++ /dev/null @@ -1,305 +0,0 @@ -import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; -import type { SnapshotState } from '@agent-device/kernel/snapshot'; -import { inheritPostGestureOutcome } from '@agent-device/kernel/snapshot'; -import type { SelectorResolution } from '@agent-device/selectors'; -import { resolveSelectorPipeline } from '@agent-device/selectors/selector-pipeline'; -import { - SELECTOR_PIPELINE_POLICIES, - type ReadinessSchedule, -} from '@agent-device/selectors/selector-pipeline-policy'; -import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-error'; -import { observeUntil } from '@agent-device/capture-kit/observe-until'; -import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; -import { isUnreadableCaptureContentError } from '@agent-device/contracts/android-snapshot-quality'; -import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; -import { - captureInteractionSnapshot, - type InteractionSnapshot, -} from './interaction-snapshot-capture.ts'; -import { buildCoveredInteractionError } from './target-visibility-stages.ts'; -import { resolveActionSelector } from './selector-action-resolution.ts'; -import { selectorNotFoundFailure } from './selector-read-shared.ts'; -import type { - InteractionAction, - ResolveInteractionTargetParams, -} from './interaction-resolution-request.ts'; - -/** - * `promotedTarget`'s readiness poll: the one-attempt capture-and-resolve, the covered-target - * diagnosis it shares with the exhausted-budget failure, and the `observeUntil`-driven loop itself. - * `resolution.ts#resolveSelectorInteractionTarget` is this module's one caller, deciding whether a - * call polls at all and under what capped budget. - */ - -/** One poll's outcome: the capture it read the tree from, and what it resolved (if anything). */ -export type SelectorResolutionAttempt = { - capture: InteractionSnapshot; - resolved: SelectorResolution | null; -}; - -/** The narrowed attempt a readiness poll accepts: a resolution with a usable rect. */ -export type ResolvedSelectorAttempt = { - capture: InteractionSnapshot; - resolved: SelectorResolution; - /** The wait that preceded the hit; absent when the first capture resolved. */ - readiness?: SelectorReadinessWait; -}; - -/** What a successful readiness wait reports: present only when more than one capture was needed. */ -export type SelectorReadinessWait = Pick; - -/** The readiness poll's evidence, attached to a target-not-found failure only (never to a refusal). */ -export type SelectorReadinessDetails = { - polls: number; - waitedMs: number; - end: 'expired' | 'stalled' | 'sparse'; -}; - -/** - * One capture-and-resolve attempt: interactive capture, resolve; on a miss that requires - * interactivity, fall back to a full (non-interactive) capture and resolve again. Identical body - * whether run once (`promotedTarget.poll`'s one-attempt twin, `resolvedTarget`) or repeated under - * `observeUntil` for a call that polls — this function draws no distinction, and does not decide - * whether the miss is a refusal (ambiguity, occlusion, off-screen) or a plain no-match: those stay - * the caller's decisions, made from `resolved`/thrown errors exactly as before. - */ -export async function attemptSelectorResolution( - runtime: AgentDeviceRuntime, - options: CommandContext, - selectorExpression: string, - params: ResolveInteractionTargetParams, -): Promise { - let capture = await captureInteractionSnapshot(runtime, options, params.requireInteractive); - let resolved = resolveActionSelector( - capture.snapshot.nodes, - selectorExpression, - runtime.backend.platform, - params.pipeline, - ); - if ((!resolved || !resolved.node.rect) && params.requireInteractive) { - const interactive = capture.snapshot; - capture = await captureInteractionSnapshot(runtime, options, false); - inheritPostGestureOutcome(interactive, capture.snapshot); - resolved = resolveActionSelector( - capture.snapshot.nodes, - selectorExpression, - runtime.backend.platform, - params.pipeline, - ); - } - return { capture, resolved }; -} - -/** - * No usable acting target. Before reporting "did not match", re-probe the same - * tree through the diagnosis row: a selector that DOES match but landed on a - * covered node is a different failure with a different recovery, and the - * acting row — rect-required, candidates rejected — cannot tell the caller - * that. Both probes name a policy row, so the two contracts stay visible side - * by side instead of as two sets of engine knobs. - * - * The diagnosis row keeps covered nodes as candidates precisely so its occlusion stage can report - * them: "matched but covered" is a different failure than "did not match", produced by re-probing - * the same tree under `coveredDiagnosis` rather than by the acting row's own (candidate-excluding) - * resolution. Shared by the single-attempt failure path and the readiness poll: a covered target is - * a refusal, not an absence, and must end a poll loop rather than spend its budget (a `promotedTarget` - * poll cannot otherwise tell "excluded because covered" apart from "does not exist yet"). - */ -async function detectCoveredSelectorTarget(params: { - runtime: AgentDeviceRuntime; - nodes: SnapshotState['nodes']; - selectorExpression: string; - action: InteractionAction; -}): Promise { - const { runtime, nodes, selectorExpression, action } = params; - const covered = await resolveSelectorPipeline( - SELECTOR_PIPELINE_POLICIES.coveredDiagnosis, - nodes, - selectorExpression, - { platform: runtime.backend.platform }, - ); - if (covered.kind !== 'occluded') return undefined; - return buildCoveredInteractionError({ - label: `Selector ${covered.selector}`, - node: covered.node, - action, - selector: covered.selector, - }); -} - -/** - * Shared by the one-attempt path (`resolution.ts#resolveSelectorInteractionTarget`) and this - * module's own exhausted-budget path: the same "selector not found" shape either way, covered - * targets disclosed through `detectCoveredSelectorTarget` first. - */ -export async function selectorInteractionFailure(params: { - runtime: AgentDeviceRuntime; - nodes: SnapshotState['nodes']; - selectorExpression: string; - action: InteractionAction; - resolved: SelectorResolution | null; -}): Promise { - const { runtime, nodes, selectorExpression, action, resolved } = params; - const covered = await detectCoveredSelectorTarget({ runtime, nodes, selectorExpression, action }); - if (covered) return covered; - return selectorNotFoundFailure(selectorExpression, { - diagnostics: resolved?.diagnostics ?? [], - unique: true, - dispatched: 'no', - }); -} - -/** - * One readiness poll: a capture-and-resolve attempt, the previous poll's post-gesture outcome - * carried forward, and the covered-target probe on a miss. A covered candidate is excluded by - * promotedTarget's own occlusion stage before it reaches `resolved`, so it looks identical to - * "not found yet"; detecting it here ends the loop on this poll instead of spending the budget on a - * target that will never stop being covered. - */ -async function pollSelectorReadinessOnce( - runtime: AgentDeviceRuntime, - options: CommandContext, - selectorExpression: string, - params: ResolveInteractionTargetParams, - previousPoll: InteractionSnapshot | undefined, -): Promise { - const attempt = await attemptSelectorResolution(runtime, options, selectorExpression, params); - if (previousPoll) inheritPostGestureOutcome(previousPoll.snapshot, attempt.capture.snapshot); - if (attempt.resolved?.node.rect) return attempt; - const quality = attempt.capture.snapshot.snapshotQuality; - if (isSparseSnapshotQualityVerdict(quality)) { - throw discloseDispatch( - new AppError( - 'COMMAND_FAILED', - `Selector ${selectorExpression} was not found in a sparse capture; the tree cannot prove it absent`, - { - reason: INTERACTION_ERROR_REASONS.captureSparse, - snapshotQuality: quality, - hint: 'Re-run after the screen settles, or capture a snapshot to inspect the tree.', - }, - ), - 'no', - ); - } - const covered = await detectCoveredSelectorTarget({ - runtime, - nodes: attempt.capture.snapshot.nodes, - selectorExpression, - action: params.action, - }); - if (covered) throw covered; - return attempt; -} - -/** - * The budget is spent or a capture stalled at the deadline. When no poll ever observed the tree, the - * last ridden-out capture error is the cause and is raised as such; otherwise the ordinary - * selector failure carries the readiness evidence. - */ -async function readinessExhaustedFailure( - runtime: AgentDeviceRuntime, - selectorExpression: string, - params: ResolveInteractionTargetParams, - observed: Extract< - Awaited>>, - { kind: 'expired' | 'stalled' } - >, -): Promise { - if (observed.last === undefined && observed.lastError !== undefined) return observed.lastError; - const readiness: SelectorReadinessDetails = { - polls: observed.polls.length, - waitedMs: observed.waitedMs, - end: observed.kind, - }; - const failure = await selectorInteractionFailure({ - runtime, - nodes: observed.last?.capture.snapshot.nodes ?? [], - selectorExpression, - action: params.action, - resolved: observed.last?.resolved ?? null, - }); - failure.details = { ...failure.details, readiness }; - return failure; -} - -/** - * `promotedTarget`'s readiness budget: the target may not exist yet, so a plain no-match (no - * resolution, or a resolution with no usable rect) keeps polling under `schedule` instead of - * refusing on the first capture. Every other outcome stays terminal exactly as a single attempt - * would produce it — an ambiguity throw from `resolveActionSelector`, or a capture error the loop - * does not ride out, ends the loop on this same poll, and is rethrown unchanged. Occlusion, - * off-screen, non-hittable, and keyboard refusals are not judged here: they run once, after this - * loop returns a rect-bearing resolution, as `runInteractionPipelineStages` does. - * - * The first poll is unbounded, so a caller whose first capture already matches pays the - * one-or-two-capture cost of a single attempt. - * - * ADR 0011 registry anchor: interaction-guarantees.ts cites this as the runtime-selector - * `targetReadiness` `via` symbol. - */ -export async function pollForSelectorReadiness( - runtime: AgentDeviceRuntime, - options: CommandContext, - selectorExpression: string, - params: ResolveInteractionTargetParams, - schedule: ReadinessSchedule, -): Promise { - const signal = options.signal ?? runtime.signal; - let previousPoll: InteractionSnapshot | undefined; - const observed = await observeUntil({ - capture: async (pollSignal) => { - const attempt = await pollSelectorReadinessOnce( - runtime, - { ...options, signal: pollSignal }, - selectorExpression, - params, - previousPoll, - ); - previousPoll = attempt.capture; - return attempt; - }, - verdict: (latest) => - latest.resolved && latest.resolved.node.rect - ? { kind: 'done', result: { capture: latest.capture, resolved: latest.resolved } } - : { kind: 'continue' }, - schedule: { - ...schedule, - // The poll signal reaches the platform as CaptureSnapshotInput.signal, which the snapshot - // binding joins (captureSnapshotSignal): the same per-capture cancellation `wait` relies on. - captureDeadline: 'cancel', - }, - rideOut: isUnreadableCaptureContentError, - ...(signal ? { signal } : {}), - ...(runtime.clock ? { clock: runtime.clock } : {}), - phase: 'interaction_target_readiness', - }); - if (observed.kind === 'done') { - return observed.polls.length > 1 - ? { - ...observed.result, - readiness: { polls: observed.polls.length, waitedMs: observed.waitedMs }, - } - : observed.result; - } - if (observed.kind === 'failed') throw withSparseReadiness(observed.error, observed); - throw await readinessExhaustedFailure(runtime, selectorExpression, params, observed); -} - -function withSparseReadiness( - error: unknown, - observed: { polls: readonly unknown[]; waitedMs: number }, -): unknown { - if ( - !(error instanceof AppError) || - error.details?.reason !== INTERACTION_ERROR_REASONS.captureSparse - ) { - return error; - } - const readiness: SelectorReadinessDetails = { - polls: observed.polls.length, - waitedMs: observed.waitedMs, - end: 'sparse', - }; - error.details = { ...error.details, readiness }; - return error; -} diff --git a/src/commands/interaction/runtime/selector-target-resolution.test.ts b/src/commands/interaction/runtime/selector-target-resolution.test.ts new file mode 100644 index 0000000000..d567048559 --- /dev/null +++ b/src/commands/interaction/runtime/selector-target-resolution.test.ts @@ -0,0 +1,55 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { selector } from './selector-read-utils.ts'; +import { createInteractionDevice } from './__tests__/test-utils/index.ts'; + +test('a selector press refuses a miss after one interactive and full capture attempt', async () => { + let captures = 0; + let taps = 0; + const device = createInteractionDevice(makeSnapshotState([]), { + captureSnapshot: async () => { + captures += 1; + return { snapshot: makeSnapshotState([]) }; + }, + tap: async () => { + taps += 1; + }, + }); + + await assert.rejects( + () => device.interactions.press(selector('label=Continue'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'selector_not_found'); + assert.equal(error.details?.dispatched, 'no'); + return true; + }, + ); + assert.equal(captures, 2); + assert.equal(taps, 0); +}); + +test('an unreadable selector capture fails with its content-quality reason without retrying', async () => { + let captures = 0; + const device = createInteractionDevice(makeSnapshotState([]), { + captureSnapshot: async () => { + captures += 1; + throw new AppError('COMMAND_FAILED', 'Android snapshot has no readable app content', { + androidSnapshotHelperFailureReason: 'system-window-only', + }); + }, + }); + + await assert.rejects( + () => device.interactions.press(selector('label=Continue'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.androidSnapshotHelperFailureReason, 'system-window-only'); + assert.notEqual(error.details?.reason, 'selector_not_found'); + return true; + }, + ); + assert.equal(captures, 1); +}); diff --git a/src/commands/interaction/runtime/selector-target-resolution.ts b/src/commands/interaction/runtime/selector-target-resolution.ts new file mode 100644 index 0000000000..8f5c058d3e --- /dev/null +++ b/src/commands/interaction/runtime/selector-target-resolution.ts @@ -0,0 +1,106 @@ +import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; +import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import { inheritPostGestureOutcome } from '@agent-device/kernel/snapshot'; +import type { SelectorResolution } from '@agent-device/selectors'; +import { resolveSelectorPipeline } from '@agent-device/selectors/selector-pipeline'; +import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; +import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-error'; +import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; +import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; +import { + captureInteractionSnapshot, + type InteractionSnapshot, +} from './interaction-snapshot-capture.ts'; +import { buildCoveredInteractionError } from './target-visibility-stages.ts'; +import { resolveActionSelector } from './selector-action-resolution.ts'; +import { selectorNotFoundFailure } from './selector-read-shared.ts'; +import type { + InteractionAction, + ResolveInteractionTargetParams, +} from './interaction-resolution-request.ts'; + +export type SelectorResolutionAttempt = { + capture: InteractionSnapshot; + resolved: SelectorResolution | null; +}; + +export async function attemptSelectorResolution( + runtime: AgentDeviceRuntime, + options: CommandContext, + selectorExpression: string, + params: ResolveInteractionTargetParams, +): Promise { + let capture = await captureInteractionSnapshot(runtime, options, params.requireInteractive); + let resolved = resolveActionSelector( + capture.snapshot.nodes, + selectorExpression, + runtime.backend.platform, + params.pipeline, + ); + if ((!resolved || !resolved.node.rect) && params.requireInteractive) { + const interactive = capture.snapshot; + capture = await captureInteractionSnapshot(runtime, options, false); + inheritPostGestureOutcome(interactive, capture.snapshot); + resolved = resolveActionSelector( + capture.snapshot.nodes, + selectorExpression, + runtime.backend.platform, + params.pipeline, + ); + } + return { capture, resolved }; +} + +async function detectCoveredSelectorTarget(params: { + runtime: AgentDeviceRuntime; + nodes: SnapshotState['nodes']; + selectorExpression: string; + action: InteractionAction; +}): Promise { + const { runtime, nodes, selectorExpression, action } = params; + const covered = await resolveSelectorPipeline( + SELECTOR_PIPELINE_POLICIES.coveredDiagnosis, + nodes, + selectorExpression, + { platform: runtime.backend.platform }, + ); + if (covered.kind !== 'occluded') return undefined; + return buildCoveredInteractionError({ + label: `Selector ${covered.selector}`, + node: covered.node, + action, + selector: covered.selector, + }); +} + +export async function selectorInteractionFailure(params: { + runtime: AgentDeviceRuntime; + snapshot: SnapshotState; + selectorExpression: string; + action: InteractionAction; + resolved: SelectorResolution | null; +}): Promise { + const { runtime, snapshot, selectorExpression, action, resolved } = params; + if (isSparseSnapshotQualityVerdict(snapshot.snapshotQuality)) { + return discloseDispatch( + new AppError( + 'COMMAND_FAILED', + `Selector ${selectorExpression} was not found in a sparse capture; the tree cannot prove it absent`, + { + reason: INTERACTION_ERROR_REASONS.captureSparse, + snapshotQuality: snapshot.snapshotQuality, + hint: 'Re-run after the screen settles, or capture a snapshot to inspect the tree.', + }, + ), + 'no', + ); + } + const nodes = snapshot.nodes; + const covered = await detectCoveredSelectorTarget({ runtime, nodes, selectorExpression, action }); + if (covered) return covered; + return selectorNotFoundFailure(selectorExpression, { + diagnostics: resolved?.diagnostics ?? [], + unique: true, + dispatched: 'no', + }); +} diff --git a/src/commands/interaction/runtime/target-visibility-stages.ts b/src/commands/interaction/runtime/target-visibility-stages.ts index b107fc2c65..b621ef50a8 100644 --- a/src/commands/interaction/runtime/target-visibility-stages.ts +++ b/src/commands/interaction/runtime/target-visibility-stages.ts @@ -25,7 +25,7 @@ import type { InteractionAction } from './interaction-resolution-request.ts'; /** * The one construction site for "covered by another visible element" refusals, shared by the - * node-stage runner below (ref, selector, native-ref preflight) and `selector-readiness.ts`'s + * node-stage runner below (ref, selector, native-ref preflight) and `selector-target-resolution.ts`'s * covered-target diagnosis probe. `find ... focus` builds the same detail shape through * {@link coveredInteractionDetails}, so both producers report one shape. * diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index a7a11e50cd..f9adaa26b3 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -166,6 +166,7 @@ Script paths are the caller's: Script line grammar: A .ad line is [positional...] [flag...]. Whitespace splits tokens; a token quoted with " or ' is one argument, and single quotes keep a double quote literal, exactly as at the shell (press 'id="far"' is one selector). Values in double quotes are JSON strings (escape \\\\, \\", \\t, \\n). Values in single quotes are literal, as at the shell: a backslash keeps its own character and the only escape is \\' for an apostrophe. A script carries only the flags declared for that command and marked recorded, so the script form of a step matches the CLI form: scroll down --until 'id="x"', wait 'label="Sign in"' --raw. CLI-only spellings and per-request options are not part of a step: --settle, --verify, scroll --pixels/--duration-ms, and the device-selection flags (--platform, --serial, --device) are the common ones a script does not carry. Reaching an off-screen target is viewport-independent in a script the same way it is at the CLI: write scroll down --until , not a fixed scroll amount that passes on one screen size and fails on another. + press/click/longpress resolve their target once. Before interacting with a loading screen, add an explicit landmark wait: wait 'label="Continue"' 5000, then press 'label="Continue"'. Reusable open-to-destination scripts: Arm recording on the first open, perform the full journey, verify the destination with a selector-targeted wait, then publish without closing: diff --git a/src/mcp/__tests__/command-tools-operator-inputs.test.ts b/src/mcp/__tests__/command-tools-operator-inputs.test.ts index 303b64a524..3ab9fd9f2e 100644 --- a/src/mcp/__tests__/command-tools-operator-inputs.test.ts +++ b/src/mcp/__tests__/command-tools-operator-inputs.test.ts @@ -86,33 +86,6 @@ test('MCP refuses every explicit operator-owned argument with guidance', async ( assert.deepEqual(calls, [], 'a refused operator input must never reach the command route'); }); -test('MCP neither advertises nor admits the operator-only readiness budget', async () => { - for (const tool of listCommandTools()) { - const properties = tool.inputSchema.properties ?? {}; - assert.equal('readinessTimeoutMs' in properties, false, `${tool.name} advertises it`); - } - const calls: unknown[] = []; - const executor = createCommandToolExecutor({ - createClient: () => ({}) as AgentDeviceClient, - runCommand: async (_client, name, input) => { - calls.push({ name, input }); - return {}; - }, - }); - - const result = await executor.execute('press', { - target: { kind: 'selector', selector: 'label=Continue' }, - readinessTimeoutMs: 2_000, - }); - - assert.equal(result.isError, true); - assert.match( - result.content[0]?.text ?? '', - /readinessTimeoutMs is not accepted as a tool argument/, - ); - assert.deepEqual(calls, [], 'a refused operator input must never reach the command route'); -}); - test('MCP refuses an explicit daemonAuthToken argument with env guidance', async () => { const calls: unknown[] = []; const executor = createCommandToolExecutor({ diff --git a/test/integration/interaction-contract/runtime-selector.contract.test.ts b/test/integration/interaction-contract/runtime-selector.contract.test.ts index 26574ee8a4..f708be01c3 100644 --- a/test/integration/interaction-contract/runtime-selector.contract.test.ts +++ b/test/integration/interaction-contract/runtime-selector.contract.test.ts @@ -207,35 +207,6 @@ test(scenario('nonHittable'), async () => { assert.match(result.hint ?? '', /hittable: false/); }); -test(scenario('targetReadiness'), async () => { - const taps: Point[] = []; - let captures = 0; - const device = createContractDevice(viewportOnlySnapshot(), { - captureSnapshot: async () => { - captures += 1; - return { - // The first poll's interactive capture and its full-capture fallback both miss; the button - // appears only from the second poll on, so a loop that never polled again leaves the tap - // unsent. - snapshot: captures >= 3 ? continueButtonSnapshot() : viewportOnlySnapshot(), - }; - }, - tap: async (_context, point) => { - taps.push(point); - }, - }); - - const result = await device.interactions.press(selector('label=Continue'), { - session: 'default', - // Never CLI- or model-writable, so the scenario supplies it directly. - readinessTimeoutMs: 2_000, - }); - - assert.equal(result.kind, 'selector'); - assert.ok(captures >= 3, `expected at least 2 polls (>=3 captures), got ${captures}`); - assert.deepEqual(taps, [{ x: 60, y: 40 }]); -}); - // #3471 targetStability: the session's last action animated the UI, so the rect this press resolved // may be a position the target has already left. The gate re-observes past the window: two agreeing // observations are the resting position it taps, and two disagreeing ones are a refusal. diff --git a/test/integration/interaction-contract/runtime-selector.coverage.ts b/test/integration/interaction-contract/runtime-selector.coverage.ts index 6119511b2c..9e962ea218 100644 --- a/test/integration/interaction-contract/runtime-selector.coverage.ts +++ b/test/integration/interaction-contract/runtime-selector.coverage.ts @@ -31,8 +31,6 @@ export const RUNTIME_SELECTOR_COVERAGE = definePathCoverage('runtime-selector', 'runtime-selector resolutionDisclosure: a unique match discloses the unique runtime shape', 'runtime-selector resolutionDisclosure: an equivalent wrapper chain discloses matchCount, winnerDiagnostic, and structural equivalence', ], - targetReadiness: - 'runtime-selector targetReadiness: a target missing on the first capture resolves once a later poll observes it', targetStability: [ 'runtime-selector targetStability: a settled slide-in is tapped where it came to rest', 'runtime-selector targetStability: a target still moving at the end of the window is refused with target_still_moving', diff --git a/test/integration/provider-scenarios/press-selector-resolution.test.ts b/test/integration/provider-scenarios/press-selector-resolution.test.ts new file mode 100644 index 0000000000..bf151f5e54 --- /dev/null +++ b/test/integration/provider-scenarios/press-selector-resolution.test.ts @@ -0,0 +1,167 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { assertRpcError, assertRpcOk } from './assertions.ts'; +import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; +import { + createAppleRunnerProviderFromTranscript, + createRecordingAppleToolProvider, + simctlDeviceLifecycleHandler, +} from './providers.ts'; +import { createProviderTranscript, type ProviderScenarioProviderEntry } from './transcript.ts'; + +const APP = 'com.example.app'; +const DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id; + +const APPLICATION_ONLY_NODES = [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, +]; + +const CONTINUE_BUTTON_NODES = [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + hittable: true, + rect: { x: 100, y: 300, width: 200, height: 44 }, + }, +]; + +function snapshotEntry(nodes: readonly unknown[]): ProviderScenarioProviderEntry { + return { + command: 'ios.runner.snapshot', + deviceId: DEVICE_ID, + platform: 'apple', + result: { nodes, truncated: false }, + }; +} + +function tapEntry(x: number, y: number): ProviderScenarioProviderEntry { + return { + command: 'ios.runner.tap', + deviceId: DEVICE_ID, + platform: 'apple', + result: { x, y }, + }; +} + +async function withPressSelectorDaemon( + entries: readonly ProviderScenarioProviderEntry[], + run: ( + daemon: Awaited>, + transcript: ReturnType, + ) => Promise, +): Promise { + // The scripted open is not read back, so the first capture after it settles over a quiet pair of + // reads. A snapshot consumes that pair here, keeping the scenario's own captures one-for-one. + const runnerTranscript = createProviderTranscript([ + snapshotEntry(APPLICATION_ONLY_NODES), + snapshotEntry(APPLICATION_ONLY_NODES), + ...entries, + ]); + const appleRunnerProvider = createAppleRunnerProviderFromTranscript( + runnerTranscript, + 'ios.runner', + ); + const appleTool = createRecordingAppleToolProvider({ + simctl: simctlDeviceLifecycleHandler('com.apple.CoreSimulator.SimRuntime.iOS-18-0', [ + { name: PROVIDER_SCENARIO_IOS_SIMULATOR.name, udid: DEVICE_ID }, + ]), + }); + + await withProviderScenarioResource( + async () => + await createProviderScenarioHarness({ + appleRunnerProvider: () => appleRunnerProvider, + appleToolProvider: () => appleTool.provider, + deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + }), + async (daemon) => { + const open = await daemon.callCommand('open', [APP], { platform: 'ios', udid: DEVICE_ID }); + assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); + await run(daemon, runnerTranscript); + }, + ); +} + +test('press resolving on the first capture costs exactly one snapshot call', async () => { + await withPressSelectorDaemon( + [snapshotEntry(CONTINUE_BUTTON_NODES), tapEntry(200, 322)], + async (daemon, transcript) => { + const callsBeforePress = transcript.calls.length; + const press = await daemon.callCommand('press', ['label=Continue']); + assertRpcOk(press); + assert.deepEqual( + transcript.calls.slice(callsBeforePress).map((call) => call.command), + ['ios.runner.snapshot', 'ios.runner.tap'], + ); + transcript.assertComplete(); + }, + ); +}); + +test('press refuses a selector miss after one interactive and full capture attempt', async () => { + await withPressSelectorDaemon( + [snapshotEntry(APPLICATION_ONLY_NODES), snapshotEntry(APPLICATION_ONLY_NODES)], + async (daemon, transcript) => { + const callsBeforePress = transcript.calls.length; + const press = await daemon.callCommand('press', ['label=Continue']); + const error = assertRpcError(press, 'COMMAND_FAILED', /Selector did not match/); + const details = error.details as { reason: unknown }; + assert.equal(details.reason, 'selector_not_found'); + + const commands = transcript.calls.slice(callsBeforePress).map((call) => call.command); + assert.deepEqual(commands, ['ios.runner.snapshot', 'ios.runner.snapshot']); + transcript.assertComplete(); + }, + ); +}); + +test('press refuses a sparse capture with capture_sparse and never taps', async () => { + await withPressSelectorDaemon( + [ + { + command: 'ios.runner.snapshot', + deviceId: DEVICE_ID, + platform: 'apple', + repeat: true, + result: { + nodes: APPLICATION_ONLY_NODES, + truncated: false, + snapshotQuality: { + state: 'sparse', + backend: 'tree', + reasonCode: 'sparse-tree', + }, + }, + }, + ], + async (daemon, transcript) => { + const callsBeforePress = transcript.calls.length; + const press = await daemon.callCommand('press', ['label=Continue']); + const error = assertRpcError(press, 'COMMAND_FAILED', /sparse capture/); + const details = error.details as { + reason: unknown; + snapshotQuality: { state: string }; + }; + assert.equal(details.reason, 'capture_sparse'); + assert.equal((error.details as { dispatched?: unknown }).dispatched, 'no'); + assert.equal(details.snapshotQuality.state, 'sparse'); + const commands = transcript.calls.slice(callsBeforePress).map((call) => call.command); + assert.ok(!commands.includes('ios.runner.tap'), 'no tap on a sparse capture'); + }, + ); +}); diff --git a/test/integration/provider-scenarios/press-target-readiness.test.ts b/test/integration/provider-scenarios/press-target-readiness.test.ts deleted file mode 100644 index 78fbde08ff..0000000000 --- a/test/integration/provider-scenarios/press-target-readiness.test.ts +++ /dev/null @@ -1,284 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { assertRpcError, assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; -import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; -import { - createAppleRunnerProviderFromTranscript, - createRecordingAppleToolProvider, - simctlDeviceLifecycleHandler, -} from './providers.ts'; -import { createProviderTranscript, type ProviderScenarioProviderEntry } from './transcript.ts'; - -// promotedTarget readiness (press/click/longpress poll for the target to exist and become -// actionable before refusing): end-to-end proof through the real daemon stack, not the plain -// runtime harness. The press route captures through the interaction backend, which keeps no -// selector capture cache, so every poll reaches the runner transcript below. - -const APP = 'com.example.app'; -const DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id; - -const APPLICATION_ONLY_NODES = [ - { - index: 0, - type: 'Application', - label: 'Example', - rect: { x: 0, y: 0, width: 400, height: 800 }, - }, -]; - -const CONTINUE_BUTTON_NODES = [ - { - index: 0, - type: 'Application', - label: 'Example', - rect: { x: 0, y: 0, width: 400, height: 800 }, - }, - { - index: 1, - parentIndex: 0, - type: 'Button', - label: 'Continue', - hittable: true, - rect: { x: 100, y: 300, width: 200, height: 44 }, - }, -]; - -function snapshotEntry(nodes: readonly unknown[]): ProviderScenarioProviderEntry { - return { - command: 'ios.runner.snapshot', - deviceId: DEVICE_ID, - platform: 'apple', - result: { nodes, truncated: false }, - }; -} - -function repeatSnapshotEntry(nodes: readonly unknown[]): ProviderScenarioProviderEntry { - return { ...snapshotEntry(nodes), repeat: true }; -} - -function tapEntry(x: number, y: number): ProviderScenarioProviderEntry { - return { - command: 'ios.runner.tap', - deviceId: DEVICE_ID, - platform: 'apple', - result: { x, y }, - }; -} - -async function withPressReadinessDaemon( - entries: readonly ProviderScenarioProviderEntry[], - run: ( - daemon: Awaited>, - transcript: ReturnType, - ) => Promise, -): Promise { - // The scripted open is not read back, so the first capture after it settles over a quiet pair of - // reads. A snapshot consumes that pair here, keeping the scenario's own captures one-for-one. - const runnerTranscript = createProviderTranscript([ - snapshotEntry(APPLICATION_ONLY_NODES), - snapshotEntry(APPLICATION_ONLY_NODES), - ...entries, - ]); - const appleRunnerProvider = createAppleRunnerProviderFromTranscript( - runnerTranscript, - 'ios.runner', - ); - const appleTool = createRecordingAppleToolProvider({ - simctl: simctlDeviceLifecycleHandler('com.apple.CoreSimulator.SimRuntime.iOS-18-0', [ - { name: PROVIDER_SCENARIO_IOS_SIMULATOR.name, udid: DEVICE_ID }, - ]), - }); - - await withProviderScenarioResource( - async () => - await createProviderScenarioHarness({ - appleRunnerProvider: () => appleRunnerProvider, - appleToolProvider: () => appleTool.provider, - deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], - }), - async (daemon) => { - const open = await daemon.callCommand('open', [APP], { platform: 'ios', udid: DEVICE_ID }); - assertRpcOk(open); - assertRpcOk(await daemon.callCommand('snapshot')); - await run(daemon, runnerTranscript); - }, - ); -} - -test('press waits for a selector missing on the first two captures, then taps once it appears', async () => { - await withPressReadinessDaemon( - [ - // Poll 1: interactive capture, then the interactive->full fallback — both miss because the - // button is not in the tree yet. - snapshotEntry(APPLICATION_ONLY_NODES), - snapshotEntry(APPLICATION_ONLY_NODES), - // Poll 2 onward: the button has appeared. - repeatSnapshotEntry(CONTINUE_BUTTON_NODES), - tapEntry(200, 322), - ], - async (daemon, transcript) => { - // Never CLI- or model-writable, so the scenario supplies it directly as a request flag. - const press = await daemon.callCommand('press', ['label=Continue'], { - readinessTimeoutMs: 2_000, - }); - const data = assertRpcOk(press); - assert.equal(data.x, 200); - assert.equal(data.y, 322); - - const commands = transcript.calls.map((call) => call.command); - const tapIndex = commands.indexOf('ios.runner.tap'); - assert.ok(tapIndex >= 0, 'expected the runner to receive a tap'); - assert.equal( - commands.filter((command) => command === 'ios.runner.tap').length, - 1, - 'expected exactly one tap dispatch', - ); - // Every snapshot call precedes the single tap: the loop never taps before a poll resolves. - assert.ok( - commands.slice(0, tapIndex).every((command) => command === 'ios.runner.snapshot'), - `expected only snapshot calls before the tap, got ${JSON.stringify(commands)}`, - ); - assert.ok( - tapIndex >= 3, - `expected at least 3 snapshot calls before the tap (interactive+fallback misses, then a resolved poll), got ${tapIndex}`, - ); - - transcript.assertComplete(); - }, - ); -}); - -test('press whose target appears on the third poll reports readiness with polls 3 on the success response', async () => { - await withPressReadinessDaemon( - [ - snapshotEntry(APPLICATION_ONLY_NODES), - snapshotEntry(APPLICATION_ONLY_NODES), - snapshotEntry(APPLICATION_ONLY_NODES), - snapshotEntry(APPLICATION_ONLY_NODES), - repeatSnapshotEntry(CONTINUE_BUTTON_NODES), - tapEntry(200, 322), - ], - async (daemon) => { - const press = await daemon.callCommand('press', ['label=Continue'], { - readinessTimeoutMs: 2_000, - }); - const data = assertRpcOk(press); - const readiness = data.readiness as { polls: number; waitedMs: number } | undefined; - assert.equal(readiness?.polls, 3); - assert.equal(typeof readiness?.waitedMs, 'number'); - }, - ); -}); - -test('press whose first capture hits carries no readiness field even with a readiness budget', async () => { - await withPressReadinessDaemon( - [snapshotEntry(CONTINUE_BUTTON_NODES), tapEntry(200, 322)], - async (daemon) => { - const press = await daemon.callCommand('press', ['label=Continue'], { - readinessTimeoutMs: 2_000, - }); - const data = assertRpcOk(press); - assert.equal('readiness' in data, false); - }, - ); -}); - -test('press fails with the standard no-match error, carrying readiness evidence, when the target never appears', async () => { - await withPressReadinessDaemon([repeatSnapshotEntry(APPLICATION_ONLY_NODES)], async (daemon) => { - // No fake clock is wired into this daemon-composed runtime path (AgentDeviceRuntime.clock is - // never set by daemon composition), so this genuinely spends the ~2s promotedTarget readiness - // budget in wall-clock time before failing. - const press = await daemon.callCommand('press', ['label=Continue'], { - readinessTimeoutMs: 2_000, - }); - const error = assertRpcError(press, 'COMMAND_FAILED', /Selector did not match/); - const details = error.details as { - reason: unknown; - readiness: { polls: number; waitedMs: number; end: string }; - }; - assert.equal(details.reason, 'selector_not_found'); - assert.ok(details.readiness.polls >= 2, `expected >=2 polls, got ${details.readiness.polls}`); - assert.ok( - details.readiness.waitedMs >= 2000, - `expected >=2000ms waited, got ${details.readiness.waitedMs}`, - ); - assert.equal(details.readiness.end, 'expired'); - }); -}, 15_000); - -test('press resolving on the first capture costs exactly one snapshot call (zero readiness cost on the success path)', async () => { - await withPressReadinessDaemon( - [snapshotEntry(CONTINUE_BUTTON_NODES), tapEntry(200, 322)], - async (daemon) => { - const press = await daemon.callCommand('press', ['label=Continue']); - assertRpcOk(press); - }, - ); -}); - -// (d) Agent misses are usually a wrong selector, so fast feedback beats absorbing a render race. -// Without an explicit readinessTimeoutMs (never CLI- or model-writable), a miss takes the -// one-attempt path: exactly one capture-and-resolve attempt (the interactive-then-full-capture -// fallback, not the readiness loop's repeated polling), and no readiness evidence to attach. -test('press without a readinessTimeoutMs flag fails on the first capture attempt, with no readiness poll', async () => { - await withPressReadinessDaemon( - // Interactive capture, then the interactive->full fallback — both miss, exactly like poll 1 of - // the readiness loop above, because this IS that same one attempt, just never repeated. - [snapshotEntry(APPLICATION_ONLY_NODES), snapshotEntry(APPLICATION_ONLY_NODES)], - async (daemon, transcript) => { - const callsBeforePress = transcript.calls.length; - const press = await daemon.callCommand('press', ['label=Continue']); - const error = assertRpcError(press, 'COMMAND_FAILED', /Selector did not match/); - const details = error.details as { reason: unknown; readiness: unknown }; - assert.equal(details.reason, 'selector_not_found'); - assert.equal(details.readiness, undefined); - - const commands = transcript.calls.slice(callsBeforePress).map((call) => call.command); - assert.deepEqual(commands, ['ios.runner.snapshot', 'ios.runner.snapshot']); - transcript.assertComplete(); - }, - ); -}); - -test('press ends the wait at once on a sparse capture with capture_sparse, and never taps', async () => { - await withPressReadinessDaemon( - [ - { - command: 'ios.runner.snapshot', - deviceId: DEVICE_ID, - platform: 'apple', - repeat: true, - result: { - nodes: APPLICATION_ONLY_NODES, - truncated: false, - snapshotQuality: { - state: 'sparse', - backend: 'tree', - reasonCode: 'sparse-tree', - }, - }, - }, - ], - async (daemon, transcript) => { - const callsBeforePress = transcript.calls.length; - const startedAt = Date.now(); - const press = await daemon.callCommand('press', ['label=Continue'], { - readinessTimeoutMs: 2_000, - }); - const error = assertRpcError(press, 'COMMAND_FAILED', /sparse capture/); - const details = error.details as { - reason: unknown; - snapshotQuality: { state: string }; - readiness: { polls: number; end: string }; - }; - assert.equal(details.reason, 'capture_sparse'); - assert.equal((error.details as { dispatched?: unknown }).dispatched, 'no'); - assert.equal(details.snapshotQuality.state, 'sparse'); - assert.equal(details.readiness.end, 'sparse'); - assert.ok(Date.now() - startedAt < 1_500, 'a sparse capture must not burn the budget'); - const commands = transcript.calls.slice(callsBeforePress).map((call) => call.command); - assert.ok(!commands.includes('ios.runner.tap'), 'no tap on a sparse capture'); - }, - ); -}); diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index b5dc409165..208659ce8e 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -207,16 +207,14 @@ await client.command.fold({ `fold` accepts either `pose` or `keyframes`. Keyframes use linear interpolation at roughly 60 updates per second; repeat an angle to hold it. Timestamps must start at zero and increase strictly, with 2–64 frames and a final timestamp no greater than 60,000ms. Angles must be finite and between 0° and 180°. The final timestamp bounds motion, excluding helper preparation and final hinge verification. A custom final angle is verified within 0.5°; interior angles must also settle. Cancellation stops the motion at its current angle. Re-snapshot afterwards, including after interrupted motion. Android emulators accept `pose` only. -`press`, `click`, and `longpress` take `readinessTimeoutMs`. With it, the command waits up to that many milliseconds for a target that is not on screen yet, then performs the requested interaction. Without it, the command looks once and fails at once, which is the right choice for an agent that most often misses because the selector is wrong. Use it in scripted flows, where a step can run before the screen finishes rendering: +`press`, `click`, and `longpress` resolve their target once. When a screen is still loading, wait for its landmark before interacting: ```ts -await client.interactions.press({ - selector: 'label="Continue"', - readinessTimeoutMs: 2_000, -}); +await client.command.wait({ text: 'Continue', timeoutMs: 5_000 }); +await client.interactions.press({ selector: 'label="Continue"' }); ``` -The wait is capped at 2 seconds and covers only a target that has not appeared. When the target is still missing after the wait, the error carries `error.details.readiness` with `waitedMs`, `polls`, and `end` (`expired` or `stalled`). A capture that shows an empty accessibility tree ends the wait at once with `capture_sparse` and `readiness.end: sparse`. When the command had to wait and then succeeded, the result carries `data.readiness` with `polls` and `waitedMs`. A command that found its target on the first look has no `readiness` field. A covered, off-screen, or ambiguous target fails at once, and a screen that stays unreadable for the whole wait fails with its own error; neither carries `readiness`. `readinessTimeoutMs` is not an MCP tool argument and has no CLI flag. +If the wait times out, check the selector and app state with a fresh snapshot. A covered, off-screen, or ambiguous target still fails the interaction. Vega OS client support is VVD-only and covers device discovery, app open/close, `back`, `home`, and `tvRemote`. Physical Fire TV, capture, selector, install, logging, and performance methods report unsupported for Vega targets. diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index fe19916642..8c1bc3f991 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -91,18 +91,15 @@ agent-device replay ~/.agent-device/sessions/e2e-2026-02-09T12-00-00-000Z.ad --s Earlier `close` actions still run. `test` does not accept the flag because each suite attempt cleans up its own session. -- `press`, `click`, and `longpress` steps wait up to 2 seconds for their target to appear before - they fail, so a step recorded against a screen that was still loading passes once the target - shows up. The wait covers only a target that is not on screen yet: a target that is covered, - off-screen, or matched by more than one element fails at once, as it does live. The replay output - does not show the wait for a step that passed; run with `--debug` to see it in the diagnostics. -- When the target never appears, replay stops with `REPLAY_DIVERGENCE`, and - `error.details.readiness` says how long the step waited and how many times it looked (`waitedMs`, - `polls`, `end`). For a step recorded with a target annotation (the `# agent-device:target-v1` +- `press`, `click`, and `longpress` steps resolve their target once. If a screen needs time to + load, add an explicit wait before the interaction, such as `wait 'label="Continue"' 5000`. + Covered, off-screen, and ambiguous targets fail as they do in live commands. +- When the target is missing, replay stops with `REPLAY_DIVERGENCE`. + For a step recorded with a target annotation (the `# agent-device:target-v1` line above it), `error.details.divergence.kind` is `selector-miss` and the step is never sent. For a step without an annotation, `error.details.reason` is `selector_not_found`, as for a live - command. If the app shows an empty accessibility tree during that wait, `error.details.reason` - is `capture_sparse` instead; take a snapshot to see where the app is. + command. If the app shows an empty accessibility tree, `error.details.reason` is + `capture_sparse` instead; take a snapshot to see where the app is. - A step whose read names one element (`is` with a predicate other than `exists`/`absent`, or `get attrs`) that reaches dispatch with an ambiguous selector fails with `AMBIGUOUS_MATCH` as the divergence cause — the same code the live command reports, so an ambiguous recorded @@ -396,8 +393,8 @@ Passing `--plan-digest` that no longer matches the current script — because yo which repeats until the element is actually on screen. - A `press` or `click` step fails because its target was not found, but the element is on the screenshot: - - A `selector-miss` divergence, or `error.details.readiness.end: expired`, means the element was - not in the accessibility tree for the whole 2-second wait: the selector is wrong for this - build, or the element is not exposed to accessibility. `readiness.end: sparse` means the app - showed an empty tree: the screen was mid-transition or the app had left. Add a `wait` step for - a landmark on the new screen before the press. + - A `selector-miss` divergence means the target was absent from the captured accessibility + tree. Check the selector for this build and whether the app exposes the element to + accessibility. A `capture_sparse` cause means the capture had no usable tree; take a fresh + snapshot to check the app state. Add a `wait` step for a landmark on a loading screen before + the press. From 6eedc86dee8e504d7d1ce67334a36ca588b8cdb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 08:18:11 +0200 Subject: [PATCH 2/7] fix(interaction): keep one-attempt selector failures as they were The sparse-capture refusal existed only inside the readiness poll. Moving it into the shared failure path widened it to every live selector action, so restore the one-attempt behaviour and drop the capture_sparse reason that no longer has a producer. Co-Authored-By: Claude Opus 5.5 --- packages/selectors/src/interaction-error.ts | 6 ---- .../interaction/runtime/resolution.ts | 2 +- .../runtime/selector-target-resolution.ts | 23 ++---------- .../press-selector-resolution.test.ts | 36 ------------------- website/docs/docs/replay-e2e.md | 6 ++-- 5 files changed, 6 insertions(+), 67 deletions(-) diff --git a/packages/selectors/src/interaction-error.ts b/packages/selectors/src/interaction-error.ts index ba09c40e56..d1124c6036 100644 --- a/packages/selectors/src/interaction-error.ts +++ b/packages/selectors/src/interaction-error.ts @@ -21,12 +21,6 @@ export const INTERACTION_ERROR_REASONS = { refUnlabeled: 'ref_unlabeled', /** The target names a node with no usable centre to touch: a missing, non-finite, or negative rect. */ targetBoundsInvalid: 'target_bounds_invalid', - /** - * A capture's snapshot-quality verdict is sparse and the selector did not resolve in it: - * the tree is untrustworthy, so absence proves nothing and the interaction is refused. - * `details.snapshotQuality` carries the verdict. - */ - captureSparse: 'capture_sparse', /** The target resolved, but another visible element covers it. */ targetCovered: 'target_covered', /** diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index c7e0155d91..57ecd729ff 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -139,7 +139,7 @@ async function resolveSelectorInteractionTarget( if (!resolved || !resolved.node.rect) { throw await selectorInteractionFailure({ runtime, - snapshot: capture.snapshot, + nodes: capture.snapshot.nodes, selectorExpression, action: params.action, resolved, diff --git a/src/commands/interaction/runtime/selector-target-resolution.ts b/src/commands/interaction/runtime/selector-target-resolution.ts index 8f5c058d3e..951dd0c8ca 100644 --- a/src/commands/interaction/runtime/selector-target-resolution.ts +++ b/src/commands/interaction/runtime/selector-target-resolution.ts @@ -1,11 +1,9 @@ -import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; +import type { AppError } from '@agent-device/kernel/errors'; import type { SnapshotState } from '@agent-device/kernel/snapshot'; import { inheritPostGestureOutcome } from '@agent-device/kernel/snapshot'; import type { SelectorResolution } from '@agent-device/selectors'; import { resolveSelectorPipeline } from '@agent-device/selectors/selector-pipeline'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; -import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-error'; -import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; import { captureInteractionSnapshot, @@ -75,27 +73,12 @@ async function detectCoveredSelectorTarget(params: { export async function selectorInteractionFailure(params: { runtime: AgentDeviceRuntime; - snapshot: SnapshotState; + nodes: SnapshotState['nodes']; selectorExpression: string; action: InteractionAction; resolved: SelectorResolution | null; }): Promise { - const { runtime, snapshot, selectorExpression, action, resolved } = params; - if (isSparseSnapshotQualityVerdict(snapshot.snapshotQuality)) { - return discloseDispatch( - new AppError( - 'COMMAND_FAILED', - `Selector ${selectorExpression} was not found in a sparse capture; the tree cannot prove it absent`, - { - reason: INTERACTION_ERROR_REASONS.captureSparse, - snapshotQuality: snapshot.snapshotQuality, - hint: 'Re-run after the screen settles, or capture a snapshot to inspect the tree.', - }, - ), - 'no', - ); - } - const nodes = snapshot.nodes; + const { runtime, nodes, selectorExpression, action, resolved } = params; const covered = await detectCoveredSelectorTarget({ runtime, nodes, selectorExpression, action }); if (covered) return covered; return selectorNotFoundFailure(selectorExpression, { diff --git a/test/integration/provider-scenarios/press-selector-resolution.test.ts b/test/integration/provider-scenarios/press-selector-resolution.test.ts index bf151f5e54..3f2f659d5e 100644 --- a/test/integration/provider-scenarios/press-selector-resolution.test.ts +++ b/test/integration/provider-scenarios/press-selector-resolution.test.ts @@ -129,39 +129,3 @@ test('press refuses a selector miss after one interactive and full capture attem }, ); }); - -test('press refuses a sparse capture with capture_sparse and never taps', async () => { - await withPressSelectorDaemon( - [ - { - command: 'ios.runner.snapshot', - deviceId: DEVICE_ID, - platform: 'apple', - repeat: true, - result: { - nodes: APPLICATION_ONLY_NODES, - truncated: false, - snapshotQuality: { - state: 'sparse', - backend: 'tree', - reasonCode: 'sparse-tree', - }, - }, - }, - ], - async (daemon, transcript) => { - const callsBeforePress = transcript.calls.length; - const press = await daemon.callCommand('press', ['label=Continue']); - const error = assertRpcError(press, 'COMMAND_FAILED', /sparse capture/); - const details = error.details as { - reason: unknown; - snapshotQuality: { state: string }; - }; - assert.equal(details.reason, 'capture_sparse'); - assert.equal((error.details as { dispatched?: unknown }).dispatched, 'no'); - assert.equal(details.snapshotQuality.state, 'sparse'); - const commands = transcript.calls.slice(callsBeforePress).map((call) => call.command); - assert.ok(!commands.includes('ios.runner.tap'), 'no tap on a sparse capture'); - }, - ); -}); diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 8c1bc3f991..2bcadc7bbf 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -98,8 +98,7 @@ agent-device replay ~/.agent-device/sessions/e2e-2026-02-09T12-00-00-000Z.ad --s For a step recorded with a target annotation (the `# agent-device:target-v1` line above it), `error.details.divergence.kind` is `selector-miss` and the step is never sent. For a step without an annotation, `error.details.reason` is `selector_not_found`, as for a live - command. If the app shows an empty accessibility tree, `error.details.reason` is - `capture_sparse` instead; take a snapshot to see where the app is. + command. - A step whose read names one element (`is` with a predicate other than `exists`/`absent`, or `get attrs`) that reaches dispatch with an ambiguous selector fails with `AMBIGUOUS_MATCH` as the divergence cause — the same code the live command reports, so an ambiguous recorded @@ -395,6 +394,5 @@ Passing `--plan-digest` that no longer matches the current script — because yo screenshot: - A `selector-miss` divergence means the target was absent from the captured accessibility tree. Check the selector for this build and whether the app exposes the element to - accessibility. A `capture_sparse` cause means the capture had no usable tree; take a fresh - snapshot to check the app state. Add a `wait` step for a landmark on a loading screen before + accessibility. Add a `wait` step for a landmark on a loading screen before the press. From e8a3d26123c8299af5540b9da894c79df56c4876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 08:18:11 +0200 Subject: [PATCH 3/7] refactor(capture-kit): drop observeUntil options only the readiness wait used budgetFrom, captureDeadline, rideOut, signal and phase have no remaining production caller after the readiness wait's removal. Co-Authored-By: Claude Opus 5.5 --- .../capture-kit/src/observe-until.test.ts | 163 +------------ packages/capture-kit/src/observe-until.ts | 220 +++--------------- .../capture-kit/src/post-gesture-stability.ts | 2 - .../daemon/src/execution/scroll-movement.ts | 1 - 4 files changed, 43 insertions(+), 343 deletions(-) diff --git a/packages/capture-kit/src/observe-until.test.ts b/packages/capture-kit/src/observe-until.test.ts index 3a1426e622..0e7020952a 100644 --- a/packages/capture-kit/src/observe-until.test.ts +++ b/packages/capture-kit/src/observe-until.test.ts @@ -1,7 +1,6 @@ import assert from 'node:assert/strict'; import { describe, test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; -import { isUnreadableCaptureContentError } from '@agent-device/contracts/android-snapshot-quality'; import { observeUntil, type ObservationClock } from './observe-until.ts'; /** A clock that advances only when the loop sleeps or a capture declares its own cost. */ @@ -21,16 +20,7 @@ function fakeClock(): ObservationClock & { advance(ms: number): void; slept: num }; } -/** No per-capture deadline: the engine default. */ -const UNBOUNDED_SCHEDULE = { intervalMs: 200, budgetMs: 1_000 } as const; -const SCHEDULE = { ...UNBOUNDED_SCHEDULE, captureDeadline: 'cancel' } as const; - -/** The Android helper's content verdict: the capture ran but held no readable app content. */ -function unreadableContent(): AppError { - return new AppError('COMMAND_FAILED', 'no readable content', { - androidSnapshotHelperFailureReason: 'system-window-only', - }); -} +const SCHEDULE = { intervalMs: 200, budgetMs: 1_000 } as const; describe('observeUntil', () => { test('answers done on the poll whose verdict accepts, with the timeline', async () => { @@ -72,28 +62,7 @@ describe('observeUntil', () => { assert.equal(observed.waitedMs, 1_000); }); - test('rides out only the errors the caller classifies, and keeps polling', async () => { - const clock = fakeClock(); - let captures = 0; - const observed = await observeUntil({ - capture: async () => { - captures += 1; - if (captures === 1) throw unreadableContent(); - return captures; - }, - verdict: (latest) => ({ kind: 'done', result: latest }), - schedule: SCHEDULE, - rideOut: isUnreadableCaptureContentError, - clock, - }); - assert.equal(observed.kind, 'done'); - assert.deepEqual( - observed.polls.map((poll) => poll.outcome), - ['rode-out', 'observed'], - ); - }); - - test('fails on the first error the caller does not ride out', async () => { + test('fails on the first capture error', async () => { const clock = fakeClock(); const failure = new AppError('COMMAND_FAILED', 'wedged'); const observed = await observeUntil({ @@ -102,7 +71,6 @@ describe('observeUntil', () => { }, verdict: () => ({ kind: 'continue' }), schedule: SCHEDULE, - rideOut: isUnreadableCaptureContentError, clock, }); assert.equal(observed.kind, 'failed'); @@ -113,47 +81,6 @@ describe('observeUntil', () => { ); }); - test('reports the last ridden-out error when every poll was ridden out', async () => { - const clock = fakeClock(); - const failures: AppError[] = []; - const observed = await observeUntil({ - capture: async () => { - const failure = unreadableContent(); - failures.push(failure); - throw failure; - }, - verdict: (latest) => ({ kind: 'done', result: latest }), - schedule: SCHEDULE, - rideOut: isUnreadableCaptureContentError, - clock, - }); - assert.equal(observed.kind, 'expired'); - assert.equal(observed.kind === 'expired' && observed.last, undefined); - assert.equal(observed.kind === 'expired' && observed.lastError, failures.at(-1)); - assert.ok(observed.polls.every((poll) => poll.outcome === 'rode-out')); - }); - - test('cancels and joins a capture still in flight at the deadline', async () => { - let aborted = false; - const observed = await observeUntil({ - capture: (signal) => - new Promise((resolve) => { - signal.addEventListener('abort', () => { - aborted = true; - setTimeout(() => resolve(1), 5); - }); - }), - verdict: () => ({ kind: 'done', result: true }), - schedule: { intervalMs: 5, budgetMs: 20, captureDeadline: 'cancel' }, - }); - assert.equal(observed.kind, 'stalled'); - assert.equal(aborted, true); - assert.deepEqual( - observed.polls.map((poll) => poll.outcome), - ['stalled'], - ); - }); - test('always completes minPolls even past the budget, so a quiet pair can form', async () => { const clock = fakeClock(); let captures = 0; @@ -167,7 +94,7 @@ describe('observeUntil', () => { previous !== undefined ? { kind: 'done', result: [previous, latest] } : { kind: 'continue' }, - schedule: { ...UNBOUNDED_SCHEDULE, minPolls: 2 }, + schedule: { ...SCHEDULE, minPolls: 2 }, clock, }); assert.equal(observed.kind, 'done'); @@ -210,59 +137,7 @@ describe('observeUntil', () => { }); }); -describe('observeUntil cancellation', () => { - test('a signal aborted during the sleep ends the loop canceled before another capture', async () => { - const clock = fakeClock(); - const controller = new AbortController(); - let captures = 0; - await assert.rejects( - observeUntil({ - capture: async () => { - captures += 1; - return captures; - }, - verdict: () => ({ kind: 'continue' }), - schedule: UNBOUNDED_SCHEDULE, - signal: controller.signal, - clock: { - now: clock.now, - sleep: async (ms) => { - await clock.sleep(ms); - controller.abort(); - }, - }, - }), - (error) => error instanceof AppError && error.details?.reason === 'request_canceled', - ); - assert.equal(captures, 1); - }); -}); - -describe('observeUntil budgetFrom first-capture', () => { - test('never bounds the first capture and spends the budget only on retries', async () => { - const clock = fakeClock(); - let captures = 0; - const observed = await observeUntil({ - capture: async () => { - captures += 1; - if (captures === 1) clock.advance(5_000); - return captures; - }, - verdict: (latest) => (latest === 3 ? { kind: 'done', result: latest } : { kind: 'continue' }), - schedule: { - intervalMs: 200, - budgetMs: 1_000, - budgetFrom: 'first-capture', - captureDeadline: 'cancel', - }, - clock, - }); - assert.equal(observed.kind, 'done'); - assert.equal(observed.polls.length, 3); - assert.equal(observed.polls[0]?.durationMs, 5_000); - assert.equal(observed.waitedMs, 5_400); - }); - +describe('observeUntil initial value', () => { test('sleeps the interval between an initial observation and the first poll', async () => { const clock = fakeClock(); const observed = await observeUntil({ @@ -271,7 +146,7 @@ describe('observeUntil budgetFrom first-capture', () => { previous === undefined ? { kind: 'continue' } : { kind: 'done', result: [previous, latest] }, - schedule: { intervalMs: 200, budgetMs: 1_000, minPolls: 2, captureDeadline: 'cancel' }, + schedule: { ...SCHEDULE, minPolls: 2 }, initial: 1, clock, }); @@ -281,8 +156,8 @@ describe('observeUntil budgetFrom first-capture', () => { }); }); -describe('observeUntil captureDeadline', () => { - /** A second capture that returns only after the remaining budget is spent, ignoring its signal. */ +describe('observeUntil late capture', () => { + /** A second capture that returns only after the remaining budget is spent. */ function lateSecondCapture(clock: ReturnType) { let captures = 0; return async () => { @@ -292,12 +167,12 @@ describe('observeUntil captureDeadline', () => { }; } - test('by default judges a capture that finishes past the budget and can end done', async () => { + test('judges a capture that finishes past the budget and can end done', async () => { const clock = fakeClock(); const observed = await observeUntil({ capture: lateSecondCapture(clock), verdict: (latest) => (latest === 2 ? { kind: 'done', result: latest } : { kind: 'continue' }), - schedule: UNBOUNDED_SCHEDULE, + schedule: SCHEDULE, clock, }); assert.equal(observed.kind, 'done'); @@ -307,32 +182,16 @@ describe('observeUntil captureDeadline', () => { ); }); - test("by default ends expired, never stalled, when the late capture's verdict continues", async () => { + test("ends expired when the late capture's verdict continues", async () => { const clock = fakeClock(); const observed = await observeUntil({ capture: lateSecondCapture(clock), verdict: () => ({ kind: 'continue' }), - schedule: UNBOUNDED_SCHEDULE, + schedule: SCHEDULE, clock, }); assert.equal(observed.kind, 'expired'); assert.equal(observed.kind === 'expired' && observed.last, 2); assert.equal(observed.polls.length, 2); }); - - test("'cancel' ends stalled on a capture that finishes past its deadline", async () => { - const clock = fakeClock(); - const observed = await observeUntil({ - capture: lateSecondCapture(clock), - verdict: (latest) => (latest === 2 ? { kind: 'done', result: latest } : { kind: 'continue' }), - schedule: SCHEDULE, - clock, - }); - assert.equal(observed.kind, 'stalled'); - assert.equal(observed.kind === 'stalled' && observed.last, 1); - assert.deepEqual( - observed.polls.map((poll) => poll.outcome), - ['observed', 'stalled'], - ); - }); }); diff --git a/packages/capture-kit/src/observe-until.ts b/packages/capture-kit/src/observe-until.ts index 89d7bdfbc9..55d8245cb9 100644 --- a/packages/capture-kit/src/observe-until.ts +++ b/packages/capture-kit/src/observe-until.ts @@ -1,52 +1,20 @@ -import { createRequestCanceledError } from '@agent-device/kernel/errors'; -import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; - /** * The one observation loop: capture until a verdict accepts, on a schedule, under a budget. Owns - * cadence, the deadline, abort-and-join of an in-flight capture under `captureDeadline: 'cancel'`, - * which capture errors are ridden out, and the per-poll timeline. Owns nothing about WHAT is - * observed: the verdict is the caller's, and so is the equality rule behind it (digest, signature, - * pixel). + * cadence, the deadline, and the per-poll timeline. Owns nothing about WHAT is observed: the verdict + * is the caller's, and so is the equality rule behind it (digest, signature, pixel). */ -/** - * How an observation loop ended. - * - * - `done` — the verdict accepted a capture. - * - `expired` — the budget ran out while the verdict still said continue. - * - `stalled` — a capture was still in flight at its deadline; it was cancelled and joined, so no - * late result can mutate state after the loop returned. - * - `failed` — a capture error the loop does not ride out. - */ -export type ObservationEnd = 'done' | 'expired' | 'stalled' | 'failed'; - export type ObservationSchedule = Readonly<{ /** Delay between polls, clamped to the remaining budget. */ intervalMs: number; /** - * Wall-clock budget from the first capture. It bounds when a poll may start; under - * `captureDeadline: 'cancel'` a capture that starts inside it may overrun it by at most one - * interval. The sample after the last sleep is always taken, so a verdict that reads elapsed time - * against a cap sees a poll at or past it. + * Wall-clock budget from the first capture. It bounds when a poll may start; a capture that + * finishes past it is still judged. The sample after the last sleep is always taken, so a verdict + * that reads elapsed time against a cap sees a poll at or past it. */ budgetMs: number; /** Observations (an `initial` counts) the loop always completes before the budget may end it. */ minPolls?: number; - /** - * `'start'` (default) bounds every capture, the first included. `'first-capture'` leaves the first - * capture unbounded and spends the budget only on retries, so a caller whose first attempt is a - * one-shot pays nothing on its success path. - */ - budgetFrom?: 'start' | 'first-capture'; - /** - * Whether a capture is bounded by its own deadline. `'cancel'` arms each capture with the - * remaining budget (at least one interval) as an abort signal, and a capture that ends at or past - * that deadline ends the loop `stalled`: declare it only when the capture honors the signal. - * `'none'` hands the capture no deadline; a capture that finishes past the budget is still judged, - * so the loop ends `done` on an accepting verdict and `expired` otherwise. Default `'none'`, so a - * capture that ignores the signal cannot end the loop `stalled` by omission. - */ - captureDeadline?: 'cancel' | 'none'; }>; export type ObservationClock = Readonly<{ @@ -59,13 +27,10 @@ export type ObservationVerdict = /** Keep polling; `budgetMs` raises the budget measured from the first capture (never lowers it). */ | Readonly<{ kind: 'continue'; budgetMs?: number }>; -/** `failed` is a capture error the loop does not ride out; it always ends the loop. */ -export type ObservationPollOutcome = 'observed' | 'rode-out' | 'stalled' | 'failed'; - export type ObservationPoll = Readonly<{ startedMs: number; durationMs: number; - outcome: ObservationPollOutcome; + outcome: 'observed' | 'failed'; }>; export type ObservationEvidence = Readonly<{ @@ -73,30 +38,25 @@ export type ObservationEvidence = Readonly<{ waitedMs: number; }>; +/** + * How an observation loop ended: `done` when the verdict accepted a capture, `expired` when the + * budget ran out while the verdict still said continue, `failed` when a capture threw. + */ type ObservedEnd = | Readonly<{ kind: 'done'; result: R; value: T }> - | Readonly<{ kind: 'expired'; last: T | undefined; lastError: unknown }> - | Readonly<{ kind: 'stalled'; last: T | undefined; lastError: unknown; error: unknown }> + | Readonly<{ kind: 'expired'; last: T | undefined }> | Readonly<{ kind: 'failed'; last: T | undefined; error: unknown }>; export type Observed = ObservationEvidence & ObservedEnd; export type ObserveUntilParams = Readonly<{ - capture: (signal: AbortSignal) => Promise; + capture: () => Promise; /** Judges the latest capture; `previous` is the last observed value, for quiet-pair predicates. */ verdict: (latest: T, previous: T | undefined, polls: number) => ObservationVerdict; schedule: ObservationSchedule; - /** - * True keeps polling past this capture error. Default: no error is ridden out. The last - * ridden-out error is reported as `lastError` when the loop expires or stalls. - */ - rideOut?: (error: unknown) => boolean; - signal?: AbortSignal; clock?: ObservationClock; /** A capture the caller already holds; judged first, without a poll. */ initial?: T; - /** Diagnostic phase for the end-of-loop line; nothing is logged without it. */ - phase?: string; }>; const REAL_CLOCK: ObservationClock = { @@ -106,13 +66,11 @@ const REAL_CLOCK: ObservationClock = { type LoopState = { readonly startedMs: number; - budgetStartedMs: number; budgetMs: number; observations: number; readonly polls: ObservationPoll[]; previous: T | undefined; last: T | undefined; - lastError: unknown; }; type Loop = Readonly<{ @@ -125,19 +83,16 @@ export async function observeUntil( params: ObserveUntilParams, ): Promise> { const clock = params.clock ?? REAL_CLOCK; - const startedMs = clock.now(); const loop: Loop = { params, clock, state: { - startedMs, - budgetStartedMs: startedMs, + startedMs: clock.now(), budgetMs: params.schedule.budgetMs, observations: 0, polls: [], previous: undefined, last: undefined, - lastError: undefined, }, }; if (params.initial !== undefined) { @@ -146,17 +101,15 @@ export async function observeUntil( if (done) return done; } while (true) { - if (params.signal?.aborted) throw createRequestCanceledError(); const expired = await waitBeforePoll(loop); if (expired) return expired; - if (params.signal?.aborted) throw createRequestCanceledError(); const ended = await pollOnce(loop); if (ended) return ended; } } function remainingMs({ state, clock }: Loop): number { - return state.budgetStartedMs + state.budgetMs - clock.now(); + return state.startedMs + state.budgetMs - clock.now(); } function mustPoll({ params, state }: Loop): boolean { @@ -168,8 +121,7 @@ async function waitBeforePoll(loop: Loop): Promise | const { state, clock, params } = loop; if (state.observations === 0) return undefined; const forced = mustPoll(loop); - if (!forced && remainingMs(loop) <= 0) - return finish(loop, { kind: 'expired', last: state.last, lastError: state.lastError }); + if (!forced && remainingMs(loop) <= 0) return finish(loop, { kind: 'expired', last: state.last }); const { intervalMs } = params.schedule; await clock.sleep(forced ? intervalMs : Math.max(0, Math.min(intervalMs, remainingMs(loop)))); return undefined; @@ -177,46 +129,24 @@ async function waitBeforePoll(loop: Loop): Promise | async function pollOnce(loop: Loop): Promise | undefined> { const { params, clock, state } = loop; - const unbounded = params.schedule.budgetFrom === 'first-capture' && state.polls.length === 0; const pollStartedMs = clock.now(); - const bounded = !unbounded && params.schedule.captureDeadline === 'cancel'; - const poll = await captureWithin( - bounded ? Math.max(remainingMs(loop), params.schedule.intervalMs) : undefined, - params.signal, - params.capture, - clock, - ); - state.observations += 1; - if (unbounded) state.budgetStartedMs = clock.now(); - const outcome = pollOutcome(loop, poll); - state.polls.push({ - startedMs: pollStartedMs - state.startedMs, - durationMs: clock.now() - pollStartedMs, - outcome, - }); - if (poll.kind === 'observed') return judge(loop, poll.value); - if (outcome === 'rode-out') { - state.lastError = poll.error; - return undefined; - } - if (poll.kind === 'stalled') { - return finish(loop, { - kind: 'stalled', - last: state.last, - lastError: state.lastError, - error: poll.error, + const record = (outcome: ObservationPoll['outcome']) => { + state.observations += 1; + state.polls.push({ + startedMs: pollStartedMs - state.startedMs, + durationMs: clock.now() - pollStartedMs, + outcome, }); + }; + let value: T; + try { + value = await params.capture(); + } catch (error) { + record('failed'); + return finish(loop, { kind: 'failed', last: state.last, error }); } - return finish(loop, { kind: 'failed', last: state.last, error: poll.error }); -} - -function pollOutcome( - loop: Loop, - poll: CaptureWithinOutcome, -): ObservationPollOutcome { - if (poll.kind === 'stalled') return 'stalled'; - if (poll.kind === 'failed') return loop.params.rideOut?.(poll.error) ? 'rode-out' : 'failed'; - return 'observed'; + record('observed'); + return judge(loop, value); } function judge(loop: Loop, value: T): Observed | undefined { @@ -230,92 +160,6 @@ function judge(loop: Loop, value: T): Observed | undefined { } function finish(loop: Loop, end: ObservedEnd): Observed { - const { params, state, clock } = loop; - const observed: Observed = { - ...end, - polls: state.polls, - waitedMs: clock.now() - state.startedMs, - }; - if (params.phase) { - emitDiagnostic({ - level: 'debug', - phase: params.phase, - data: { - end: observed.kind satisfies ObservationEnd, - polls: state.polls.length, - waitedMs: observed.waitedMs, - }, - }); - } - return observed; -} - -type CaptureWithinOutcome = - | Readonly<{ kind: 'observed'; value: T }> - | Readonly<{ kind: 'failed'; error: unknown }> - | Readonly<{ kind: 'stalled'; error: unknown }>; - -/** - * Runs one capture under the remaining budget by cancellation, then waits for it to quiesce. - * Deliberately not race-and-abandon: a late capture could otherwise mutate session state or keep a - * platform helper after the loop has returned. - */ -async function captureWithin( - remainingMs: number | undefined, - parent: AbortSignal | undefined, - capture: (signal: AbortSignal) => Promise, - clock: ObservationClock, -): Promise> { - const deadline = armDeadline(remainingMs, clock); - const signal = parent ? AbortSignal.any([parent, deadline.signal]) : deadline.signal; - try { - const value = await capture(signal); - return endOfCapture(parent, deadline.expired(), undefined) ?? { kind: 'observed', value }; - } catch (error) { - return endOfCapture(parent, deadline.expired(), error) ?? { kind: 'failed', error }; - } finally { - deadline.dispose(); - } -} - -/** A capture that ended after the deadline stalled; one ended by the caller's signal was canceled. */ -function endOfCapture( - parent: AbortSignal | undefined, - deadlineExpired: boolean, - error: unknown, -): CaptureWithinOutcome | undefined { - if (parent?.aborted) throw createRequestCanceledError(); - return deadlineExpired ? { kind: 'stalled', error } : undefined; -} - -/** The deadline has passed once its timer fired or the loop's clock reached it, whichever is first. */ -function armDeadline( - remainingMs: number | undefined, - clock: ObservationClock, -): { - signal: AbortSignal; - expired: () => boolean; - dispose: () => void; -} { - const controller = new AbortController(); - const deadlineAtMs = remainingMs === undefined ? undefined : clock.now() + remainingMs; - let fired = false; - const timer = - remainingMs === undefined - ? undefined - : setTimeout( - () => { - fired = true; - controller.abort(new DOMException('Observation deadline exceeded', 'TimeoutError')); - }, - Math.max(0, remainingMs), - ); - timer?.unref(); - return { - signal: controller.signal, - expired: () => fired || (deadlineAtMs !== undefined && clock.now() >= deadlineAtMs), - dispose: () => { - if (timer !== undefined) clearTimeout(timer); - }, - }; + const { state, clock } = loop; + return { ...end, polls: state.polls, waitedMs: clock.now() - state.startedMs }; } diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index 9fb7bd3644..838209d326 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -18,8 +18,6 @@ import { observeUntil, type ObservationClock, type ObservationSchedule } from '. * Cadence and budget for the quiet-window loop: poll every 200ms, allow 1.5s * before an unsettled surface times out, and always complete two observations * (an `initial` counts) so a quiet pair can form even under a tight budget. - * No per-capture deadline: `hooks.capture` takes no signal, so a late capture - * is judged when it returns. */ const POST_GESTURE_STABILITY_SCHEDULE: ObservationSchedule = { intervalMs: 200, diff --git a/packages/daemon/src/execution/scroll-movement.ts b/packages/daemon/src/execution/scroll-movement.ts index f66ae5b209..250c25fb07 100644 --- a/packages/daemon/src/execution/scroll-movement.ts +++ b/packages/daemon/src/execution/scroll-movement.ts @@ -67,7 +67,6 @@ import type { SessionState } from '../core/session-state.ts'; /** * How long a scroll keeps asking whether an untouched surface is really untouched (#1542's window). - * No per-capture deadline: the scroll capture takes no signal, so a late capture is judged. */ const SCROLL_MOVEMENT_SCHEDULE: ObservationSchedule = { intervalMs: 200, From af02fbf5b07719c6d4df5a2dd899ad96f8d16365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 08:26:32 +0200 Subject: [PATCH 4/7] docs(interaction): describe the removal without overstating single resolution Co-Authored-By: Claude Opus 5.5 --- packages/selectors/src/selector-pipeline-policy.ts | 2 +- .../interaction/runtime/__tests__/test-utils/index.ts | 2 +- .../interaction/runtime/interaction-snapshot-capture.ts | 3 ++- .../provider-scenarios/press-selector-resolution.test.ts | 3 ++- website/docs/docs/client-api.md | 2 +- website/docs/docs/replay-e2e.md | 5 +++-- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/selectors/src/selector-pipeline-policy.ts b/packages/selectors/src/selector-pipeline-policy.ts index 1cbcdb4efb..75554a5f02 100644 --- a/packages/selectors/src/selector-pipeline-policy.ts +++ b/packages/selectors/src/selector-pipeline-policy.ts @@ -107,7 +107,7 @@ export type SelectorPipelinePolicy = SelectorListPolicy & { const WAIT_POLL_BUDGET: WaitPollBudget = { defaultTimeoutMs: 10_000, intervalMs: 300 }; export const SELECTOR_PIPELINE_POLICIES = { - /** `click`/`press`/`longpress`: the tap lands on the actionable owner of the match. */ + /** `click`/`press`/`longpress`: the action dispatches to the actionable owner of the match. */ promotedTarget: { resolution: SELECTOR_RESOLUTION_POLICIES.act, occlusion: 'exclude-and-refuse', diff --git a/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/src/commands/interaction/runtime/__tests__/test-utils/index.ts index df2a15f664..df199692a1 100644 --- a/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -357,7 +357,7 @@ export function createInteractionDevice( > & { platform?: AgentDeviceBackend['platform']; sessionMetadata?: Record; - /** An advancing clock for wait and post-action observation loops. */ + /** An advancing clock for wait, post-action observation, and moving-target recheck loops. */ clock?: { now: () => number; sleep: (ms: number) => Promise }; } = {}, ) { diff --git a/src/commands/interaction/runtime/interaction-snapshot-capture.ts b/src/commands/interaction/runtime/interaction-snapshot-capture.ts index f9a3650ae3..920c5abd2b 100644 --- a/src/commands/interaction/runtime/interaction-snapshot-capture.ts +++ b/src/commands/interaction/runtime/interaction-snapshot-capture.ts @@ -6,7 +6,8 @@ import { now, toBackendContext } from '../../runtime-common.ts'; /** * A leaf shared by every interaction runtime module that needs a fresh (or session-cached) tree: * `resolution.ts`'s ref/selector resolution, `selector-target-resolution.ts`'s acting selector lookup, - * `gestures.ts`'s viewport resolution, and `post-action-observation.ts`'s `--verify` baseline. + * `moving-target-gate.ts`'s re-observation, `gestures.ts`'s viewport resolution, and + * `post-action-observation.ts`'s `--verify` baseline. * Deliberately self-contained — it imports no sibling of this directory, so those siblings can * import it without ever risking a cycle back. */ diff --git a/test/integration/provider-scenarios/press-selector-resolution.test.ts b/test/integration/provider-scenarios/press-selector-resolution.test.ts index 3f2f659d5e..8928c630da 100644 --- a/test/integration/provider-scenarios/press-selector-resolution.test.ts +++ b/test/integration/provider-scenarios/press-selector-resolution.test.ts @@ -120,8 +120,9 @@ test('press refuses a selector miss after one interactive and full capture attem const callsBeforePress = transcript.calls.length; const press = await daemon.callCommand('press', ['label=Continue']); const error = assertRpcError(press, 'COMMAND_FAILED', /Selector did not match/); - const details = error.details as { reason: unknown }; + const details = error.details as { reason: unknown; dispatched?: unknown }; assert.equal(details.reason, 'selector_not_found'); + assert.equal(details.dispatched, 'no'); const commands = transcript.calls.slice(callsBeforePress).map((call) => call.command); assert.deepEqual(commands, ['ios.runner.snapshot', 'ios.runner.snapshot']); diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 208659ce8e..7216b91197 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -207,7 +207,7 @@ await client.command.fold({ `fold` accepts either `pose` or `keyframes`. Keyframes use linear interpolation at roughly 60 updates per second; repeat an angle to hold it. Timestamps must start at zero and increase strictly, with 2–64 frames and a final timestamp no greater than 60,000ms. Angles must be finite and between 0° and 180°. The final timestamp bounds motion, excluding helper preparation and final hinge verification. A custom final angle is verified within 0.5°; interior angles must also settle. Cancellation stops the motion at its current angle. Re-snapshot afterwards, including after interrupted motion. Android emulators accept `pose` only. -`press`, `click`, and `longpress` resolve their target once. When a screen is still loading, wait for its landmark before interacting: +`press`, `click`, and `longpress` do not wait for a missing target to appear. When a screen is still loading, wait for its landmark before interacting: ```ts await client.command.wait({ text: 'Continue', timeoutMs: 5_000 }); diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 2bcadc7bbf..497177f3dd 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -91,8 +91,9 @@ agent-device replay ~/.agent-device/sessions/e2e-2026-02-09T12-00-00-000Z.ad --s Earlier `close` actions still run. `test` does not accept the flag because each suite attempt cleans up its own session. -- `press`, `click`, and `longpress` steps resolve their target once. If a screen needs time to - load, add an explicit wait before the interaction, such as `wait 'label="Continue"' 5000`. +- `press`, `click`, and `longpress` steps do not wait for a missing target to appear. If a screen + needs time to load, add an explicit wait before the interaction, such as + `wait 'label="Continue"' 5000`. Covered, off-screen, and ambiguous targets fail as they do in live commands. - When the target is missing, replay stops with `REPLAY_DIVERGENCE`. For a step recorded with a target annotation (the `# agent-device:target-v1` From 40a5cb0b1b15536f2bc270b3710ee96ed5d3df6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 12:53:12 +0200 Subject: [PATCH 5/7] fix(interaction): refuse the retired readinessTimeoutMs with migration guidance Removing the option let readFieldInput and the tap writers drop it silently, so a released caller's press became a one-attempt press with no signal. Declare it retired at the common input row and refuse it in the click, press and longpress writers that the Node client reaches directly. Co-Authored-By: Claude Opus 5.5 --- ...lient-interactions-input-filtering.test.ts | 49 ++++++++++--------- src/commands/__tests__/input-audience.test.ts | 16 +++++- src/commands/common-input-fields.ts | 20 +++++++- src/commands/interaction/interactions.ts | 28 ++++++----- .../command-tools-operator-inputs.test.ts | 21 ++++++++ 5 files changed, 96 insertions(+), 38 deletions(-) diff --git a/src/__tests__/client-interactions-input-filtering.test.ts b/src/__tests__/client-interactions-input-filtering.test.ts index 415bbbdc0d..9d2a20951d 100644 --- a/src/__tests__/client-interactions-input-filtering.test.ts +++ b/src/__tests__/client-interactions-input-filtering.test.ts @@ -3,35 +3,36 @@ import { test } from 'vitest'; import { createAgentDeviceClient } from '../agent-device-client.ts'; import { createTransport } from './client-transport-fixture.ts'; -test('tap commands filter undeclared input without losing post-action observation', async () => { +const RETIRED = /readinessTimeoutMs was removed; add an explicit wait for the target/; + +test('tap commands refuse the retired readinessTimeoutMs before sending anything', async () => { const setup = createTransport(async () => ({ ok: true, data: {} })); const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); - await client.interactions.press({ - selector: 'label=Foo', - settle: true, + const calls = [ // @ts-expect-error readinessTimeoutMs is not a client option. - readinessTimeoutMs: 2_000, - }); - await client.interactions.click({ - selector: 'label=Foo', - settle: true, + () => client.interactions.press({ selector: 'label=Foo', readinessTimeoutMs: 2_000 }), // @ts-expect-error readinessTimeoutMs is not a client option. - readinessTimeoutMs: 1_500, - }); - await client.interactions.longPress({ - selector: 'label=Foo', - settle: true, + () => client.interactions.click({ selector: 'label=Foo', readinessTimeoutMs: 1_500 }), // @ts-expect-error readinessTimeoutMs is not a client option. - readinessTimeoutMs: 900, - }); - - assert.deepEqual( - setup.calls.map((call) => call.command), - ['press', 'click', 'longpress'], - ); - for (const call of setup.calls) { - assert.equal(Object.hasOwn(call.flags ?? {}, 'readinessTimeoutMs'), false); - assert.equal(call.flags?.settle, true); + () => client.interactions.longPress({ selector: 'label=Foo', readinessTimeoutMs: 900 }), + ]; + for (const call of calls) { + await assert.rejects(call(), (error: { code?: string; message: string }) => { + assert.equal(error.code, 'INVALID_ARGS'); + assert.match(error.message, RETIRED); + return true; + }); } + assert.deepEqual(setup.calls, []); +}); + +test('tap commands still carry post-action observation flags', async () => { + const setup = createTransport(async () => ({ ok: true, data: {} })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await client.interactions.press({ selector: 'label=Foo', settle: true }); + + assert.equal(setup.calls[0]?.command, 'press'); + assert.equal(setup.calls[0]?.flags?.settle, true); }); diff --git a/src/commands/__tests__/input-audience.test.ts b/src/commands/__tests__/input-audience.test.ts index eb4eb67562..ed74f1f610 100644 --- a/src/commands/__tests__/input-audience.test.ts +++ b/src/commands/__tests__/input-audience.test.ts @@ -49,14 +49,26 @@ test('a key neither env nor config resolves states its own operator path', () => // in `mcp/__tests__/command-tools-operator-inputs.test.ts`.) test('an operator-owned common key stays in the CLI and Node input schema', () => { const advertised = commonProperties(); - const operatorKeys = Object.keys(COMMON_INPUT_AUDIENCE); + const operatorKeys = Object.keys(COMMON_INPUT_AUDIENCE).filter( + (key) => COMMON_INPUT_AUDIENCE[key]?.kind === 'operator', + ); assert.ok(operatorKeys.length > 0, 'the table declares operator-owned common keys'); for (const key of operatorKeys) { - assert.equal(COMMON_INPUT_AUDIENCE[key]?.kind, 'operator'); assert.ok(key in advertised, `${key} must stay readable by the CLI and Node client`); } }); +test('a retired common key is advertised on no input schema', () => { + const advertised = commonProperties(); + const retiredKeys = Object.keys(COMMON_INPUT_AUDIENCE).filter( + (key) => COMMON_INPUT_AUDIENCE[key]?.kind === 'retired', + ); + assert.deepEqual(retiredKeys, ['readinessTimeoutMs']); + for (const key of retiredKeys) { + assert.equal(key in advertised, false, `${key} is retired`); + } +}); + // Schema, readers, and projection are three derivations of one table. A row // that loses its reader or its projection entry drops the key silently at the // seam nothing else covers, which is the failure `--no-record` hit twice diff --git a/src/commands/common-input-fields.ts b/src/commands/common-input-fields.ts index b172c1109c..66eaf0d12d 100644 --- a/src/commands/common-input-fields.ts +++ b/src/commands/common-input-fields.ts @@ -86,6 +86,17 @@ type CommonInputFieldSpec = { */ const ENV_OR_OPERATOR_CONFIG = operatorAudience({ operatorConfig: true }); +const RETIRED_READINESS_TIMEOUT_MESSAGE = + 'readinessTimeoutMs was removed; add an explicit wait for the target before the interaction.'; + +/** Refuses the released `readinessTimeoutMs` key, which the target-readiness wait owned. */ +export function assertNoRetiredReadinessTimeout(record: Record): undefined { + if (Object.hasOwn(record, 'readinessTimeoutMs')) { + throw new AppError('INVALID_ARGS', RETIRED_READINESS_TIMEOUT_MESSAGE); + } + return undefined; +} + const COMMON_INPUT_FIELDS = { session: { schema: { type: 'string', description: 'Agent-device session name.' }, @@ -215,6 +226,10 @@ const COMMON_INPUT_FIELDS = { flagKey: 'noRecord', flagIn: ['input', 'selection'], }, + readinessTimeoutMs: { + audience: { kind: 'retired', message: RETIRED_READINESS_TIMEOUT_MESSAGE }, + read: assertNoRetiredReadinessTimeout, + }, daemonBaseUrl: { schema: { type: 'string', description: 'Remote daemon base URL.' }, read: (record) => optionalString(record, 'daemonBaseUrl'), @@ -249,7 +264,10 @@ const COMMON_INPUT_FIELDS = { schema: { type: 'boolean', description: 'Enable debug diagnostics.' }, read: (record) => optionalBoolean(record, 'debug'), }, -} as const satisfies Record; +} as const satisfies Record< + keyof CommonCommandInput | 'target' | 'readinessTimeoutMs', + CommonInputFieldSpec +>; const COMMON_INPUT_ROWS: ReadonlyArray = Object.entries(COMMON_INPUT_FIELDS); diff --git a/src/commands/interaction/interactions.ts b/src/commands/interaction/interactions.ts index 79154b99d1..a30b947483 100644 --- a/src/commands/interaction/interactions.ts +++ b/src/commands/interaction/interactions.ts @@ -33,6 +33,7 @@ import { targetInputFromClientTarget, } from '../cli-grammar/common.ts'; import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; +import { assertNoRetiredReadinessTimeout } from '../common-input-fields.ts'; import type { ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; export const interactionCliReaders = { @@ -120,17 +121,22 @@ export const interactionCliReaders = { } satisfies Record; export const interactionDaemonWriters = { - click: (input) => - request(PUBLIC_COMMANDS.click, interactionTargetPositionals(input as InteractionTarget), { - ...input, - clickButton: input.button, - }), - press: direct(PUBLIC_COMMANDS.press, (input) => - interactionTargetPositionals(input as InteractionTarget), - ), - longpress: direct(PUBLIC_COMMANDS.longPress, (input) => - longPressPositionals(input as LongPressOptions), - ), + click: (input) => { + assertNoRetiredReadinessTimeout(input); + return request( + PUBLIC_COMMANDS.click, + interactionTargetPositionals(input as InteractionTarget), + { ...input, clickButton: input.button }, + ); + }, + press: direct(PUBLIC_COMMANDS.press, (input) => { + assertNoRetiredReadinessTimeout(input); + return interactionTargetPositionals(input as InteractionTarget); + }), + longpress: direct(PUBLIC_COMMANDS.longPress, (input) => { + assertNoRetiredReadinessTimeout(input); + return longPressPositionals(input as LongPressOptions); + }), hover: direct(PUBLIC_COMMANDS.hover, (input) => interactionTargetPositionals(input as InteractionTarget), ), diff --git a/src/mcp/__tests__/command-tools-operator-inputs.test.ts b/src/mcp/__tests__/command-tools-operator-inputs.test.ts index 3ab9fd9f2e..3912304ea6 100644 --- a/src/mcp/__tests__/command-tools-operator-inputs.test.ts +++ b/src/mcp/__tests__/command-tools-operator-inputs.test.ts @@ -86,6 +86,27 @@ test('MCP refuses every explicit operator-owned argument with guidance', async ( assert.deepEqual(calls, [], 'a refused operator input must never reach the command route'); }); +test('MCP neither advertises nor admits the retired readiness budget', async () => { + for (const tool of listCommandTools()) { + const properties = tool.inputSchema.properties ?? {}; + assert.equal('readinessTimeoutMs' in properties, false, `${tool.name} advertises it`); + } + // The real command route (no runCommand injection), like the retired maxSize regressions: + // retired keys pass admission so the command's own reader answers with migration guidance. + const executor = createCommandToolExecutor({ + createClient: () => ({}) as AgentDeviceClient, + }); + + for (const [command, input] of [ + ['press', { target: { kind: 'selector', selector: 'label=Continue' } }], + ['fill', { target: { kind: 'selector', selector: 'label=Name' }, text: 'x' }], + ] as const) { + const result = await executor.execute(command, { ...input, readinessTimeoutMs: 2_000 }); + assert.equal(result.isError, true, command); + assert.match(result.content[0]?.text ?? '', /readinessTimeoutMs was removed/, command); + } +}); + test('MCP refuses an explicit daemonAuthToken argument with env guidance', async () => { const calls: unknown[] = []; const executor = createCommandToolExecutor({ From e419d18ccf7d4b9bd6f0dbf14e88c1ecd755049a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 15:34:03 +0200 Subject: [PATCH 6/7] fix: drop unused import after main merge --- src/commands/interaction/runtime/resolution.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index 9494448a8c..57ecd729ff 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -25,7 +25,6 @@ import { buildSelectorResolutionDisclosure, describeResolvedInteractionNode, } from './resolution-disclosure.ts'; -import type { ActionSelectorResolution } from './selector-action-resolution.ts'; import { assertVisibleSelectorTarget, runInteractionPipelineStages, From 1f26100dab5f45a02fc13dda2375624f0e2afea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 18:36:44 +0200 Subject: [PATCH 7/7] refactor(commands): refuse retired common keys once at the Node projection seam The Node client reaches daemon writers without readFieldInput, so per-writer guards covered only click, press and longpress. Refuse every retired common key in prepareDaemonCommandRequest instead, and share one refusal between retiredField and the common row. Co-Authored-By: Claude Opus 5.5 --- ...lient-interactions-input-filtering.test.ts | 16 ++++++++++- src/commands/command-input.ts | 14 +++++----- src/commands/command-projection.ts | 2 ++ src/commands/common-input-fields.ts | 28 ++++++++++++------- src/commands/input-audience.ts | 11 ++++++++ src/commands/interaction/interactions.ts | 28 ++++++++----------- 6 files changed, 64 insertions(+), 35 deletions(-) diff --git a/src/__tests__/client-interactions-input-filtering.test.ts b/src/__tests__/client-interactions-input-filtering.test.ts index 9d2a20951d..3ec44b2d78 100644 --- a/src/__tests__/client-interactions-input-filtering.test.ts +++ b/src/__tests__/client-interactions-input-filtering.test.ts @@ -5,7 +5,7 @@ import { createTransport } from './client-transport-fixture.ts'; const RETIRED = /readinessTimeoutMs was removed; add an explicit wait for the target/; -test('tap commands refuse the retired readinessTimeoutMs before sending anything', async () => { +test('client commands refuse the retired readinessTimeoutMs before sending anything', async () => { const setup = createTransport(async () => ({ ok: true, data: {} })); const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); @@ -16,6 +16,20 @@ test('tap commands refuse the retired readinessTimeoutMs before sending anything () => client.interactions.click({ selector: 'label=Foo', readinessTimeoutMs: 1_500 }), // @ts-expect-error readinessTimeoutMs is not a client option. () => client.interactions.longPress({ selector: 'label=Foo', readinessTimeoutMs: 900 }), + () => + client.interactions.fill({ + selector: 'label=Name', + text: 'x', + // @ts-expect-error readinessTimeoutMs is not a client option. + readinessTimeoutMs: 900, + }), + () => + client.interactions.swipe({ + from: { x: 1, y: 1 }, + to: { x: 2, y: 2 }, + // @ts-expect-error readinessTimeoutMs is not a client option. + readinessTimeoutMs: 900, + }), ]; for (const call of calls) { await assert.rejects(call(), (error: { code?: string; message: string }) => { diff --git a/src/commands/command-input.ts b/src/commands/command-input.ts index 1fa4b59ebb..fbce60a059 100644 --- a/src/commands/command-input.ts +++ b/src/commands/command-input.ts @@ -20,7 +20,12 @@ import { readCommonInput, type CommonCommandInput, } from './common-input-fields.ts'; -import type { InputAudience, InputAudienceMap, OperatorInputSource } from './input-audience.ts'; +import { + refuseRetiredInput, + type InputAudience, + type InputAudienceMap, + type OperatorInputSource, +} from './input-audience.ts'; import { compactRecord, optionalAnyString, @@ -211,12 +216,7 @@ export function retiredField(message: string): CommandField { schema: { type: 'null' }, required: false, audience: { kind: 'retired', message }, - read: (record, key) => { - if (Object.hasOwn(record, key)) { - throw new AppError('INVALID_ARGS', message); - } - return undefined; - }, + read: (record, key) => refuseRetiredInput(record, key, message), }; } diff --git a/src/commands/command-projection.ts b/src/commands/command-projection.ts index 4ccabc5355..1e9aba520e 100644 --- a/src/commands/command-projection.ts +++ b/src/commands/command-projection.ts @@ -1,6 +1,7 @@ import { createBatchDaemonWriter } from './batch/index.ts'; import type { AnyDaemonWriter, CommandInput, DaemonCommandRequest } from './cli-grammar/types.ts'; import { findCommandMetadata } from './command-metadata.ts'; +import { refuseRetiredCommonInput } from './common-input-fields.ts'; import { readMetadataCommandFlags } from './command-flags.ts'; import { listCommandFamilyDaemonWriters } from './family/registry.ts'; import { AppError } from '@agent-device/kernel/errors'; @@ -50,6 +51,7 @@ export async function prepareDaemonCommandRequest( if (!writer) { throw new Error(`Missing daemon writer for command: ${command}`); } + refuseRetiredCommonInput(input as Record); const metadata = findCommandMetadata(command); return await prepareRequestWithMetadataFlags(writer, metadata, input); } diff --git a/src/commands/common-input-fields.ts b/src/commands/common-input-fields.ts index 66eaf0d12d..55169762e5 100644 --- a/src/commands/common-input-fields.ts +++ b/src/commands/common-input-fields.ts @@ -11,7 +11,12 @@ import { } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import type { JsonSchema } from './command-contract.ts'; -import { operatorAudience, type InputAudience, type InputAudienceMap } from './input-audience.ts'; +import { + operatorAudience, + refuseRetiredInput, + type InputAudience, + type InputAudienceMap, +} from './input-audience.ts'; import { compactRecord, optionalBoolean, optionalEnum, optionalString } from './input-readers.ts'; export type CommonCommandInput = Pick< @@ -89,14 +94,6 @@ const ENV_OR_OPERATOR_CONFIG = operatorAudience({ operatorConfig: true }); const RETIRED_READINESS_TIMEOUT_MESSAGE = 'readinessTimeoutMs was removed; add an explicit wait for the target before the interaction.'; -/** Refuses the released `readinessTimeoutMs` key, which the target-readiness wait owned. */ -export function assertNoRetiredReadinessTimeout(record: Record): undefined { - if (Object.hasOwn(record, 'readinessTimeoutMs')) { - throw new AppError('INVALID_ARGS', RETIRED_READINESS_TIMEOUT_MESSAGE); - } - return undefined; -} - const COMMON_INPUT_FIELDS = { session: { schema: { type: 'string', description: 'Agent-device session name.' }, @@ -228,7 +225,8 @@ const COMMON_INPUT_FIELDS = { }, readinessTimeoutMs: { audience: { kind: 'retired', message: RETIRED_READINESS_TIMEOUT_MESSAGE }, - read: assertNoRetiredReadinessTimeout, + read: (record) => + refuseRetiredInput(record, 'readinessTimeoutMs', RETIRED_READINESS_TIMEOUT_MESSAGE), }, daemonBaseUrl: { schema: { type: 'string', description: 'Remote daemon base URL.' }, @@ -277,6 +275,16 @@ export const COMMON_INPUT_AUDIENCE: InputAudienceMap = Object.fromEntries( COMMON_INPUT_ROWS.flatMap(([key, field]) => (field.audience ? [[key, field.audience]] : [])), ); +/** + * Refuses every retired common key. `readFieldInput` reaches the same rows; this covers the Node + * client, which hands its options to the daemon writers without reading them through fields. + */ +export function refuseRetiredCommonInput(record: Record): void { + for (const [key, audience] of Object.entries(COMMON_INPUT_AUDIENCE)) { + if (audience.kind === 'retired') refuseRetiredInput(record, key, audience.message); + } +} + export function commonProperties(): Record { return Object.fromEntries( COMMON_INPUT_ROWS.flatMap(([key, field]) => (field.schema ? [[key, field.schema]] : [])), diff --git a/src/commands/input-audience.ts b/src/commands/input-audience.ts index 3d7a73e61d..5aa21d91f1 100644 --- a/src/commands/input-audience.ts +++ b/src/commands/input-audience.ts @@ -1,3 +1,4 @@ +import { AppError } from '@agent-device/kernel/errors'; import { buildPrimaryEnvVarName } from '@agent-device/kernel/source-value'; /** @@ -43,6 +44,16 @@ export type InputAudience = /** Non-model audiences by input key. Keys absent from the map are model-writable. */ export type InputAudienceMap = Readonly>; +/** A retired key present in the input is refused with its migration message. */ +export function refuseRetiredInput( + record: Record, + key: string, + message: string, +): undefined { + if (Object.hasOwn(record, key)) throw new AppError('INVALID_ARGS', message); + return undefined; +} + export function operatorAudience(source: OperatorInputSource): InputAudience { return { kind: 'operator', source }; } diff --git a/src/commands/interaction/interactions.ts b/src/commands/interaction/interactions.ts index a30b947483..79154b99d1 100644 --- a/src/commands/interaction/interactions.ts +++ b/src/commands/interaction/interactions.ts @@ -33,7 +33,6 @@ import { targetInputFromClientTarget, } from '../cli-grammar/common.ts'; import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; -import { assertNoRetiredReadinessTimeout } from '../common-input-fields.ts'; import type { ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; export const interactionCliReaders = { @@ -121,22 +120,17 @@ export const interactionCliReaders = { } satisfies Record; export const interactionDaemonWriters = { - click: (input) => { - assertNoRetiredReadinessTimeout(input); - return request( - PUBLIC_COMMANDS.click, - interactionTargetPositionals(input as InteractionTarget), - { ...input, clickButton: input.button }, - ); - }, - press: direct(PUBLIC_COMMANDS.press, (input) => { - assertNoRetiredReadinessTimeout(input); - return interactionTargetPositionals(input as InteractionTarget); - }), - longpress: direct(PUBLIC_COMMANDS.longPress, (input) => { - assertNoRetiredReadinessTimeout(input); - return longPressPositionals(input as LongPressOptions); - }), + click: (input) => + request(PUBLIC_COMMANDS.click, interactionTargetPositionals(input as InteractionTarget), { + ...input, + clickButton: input.button, + }), + press: direct(PUBLIC_COMMANDS.press, (input) => + interactionTargetPositionals(input as InteractionTarget), + ), + longpress: direct(PUBLIC_COMMANDS.longPress, (input) => + longPressPositionals(input as LongPressOptions), + ), hover: direct(PUBLIC_COMMANDS.hover, (input) => interactionTargetPositionals(input as InteractionTarget), ),