From 0b82302f4ca64cbdd935c5ff8321d6d620828d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 09:25:17 +0200 Subject: [PATCH 1/2] fix(interaction): disclose an unsettled post-gesture surface on interactions A selector click, press, or fill right after a scroll whose surface never settled missed on that moving tree and answered a plain selector_not_found. The request's capture proof now records the unsettledGesture fact beside the foreground repair, first fact wins, and the interaction route discloses both on every response it builds. Closes #2853 --- CHANGELOG.md | 11 +-- ...pture-disclosure-target-activation.test.ts | 10 +-- .../selector-capture-runtime.test.ts | 16 ++-- src/daemon/capture-disclosure.ts | 75 ++++++++++++------- ...=> interaction-capture-disclosure.test.ts} | 70 ++++++++++++++++- .../internal/find-target-capture.ts | 6 +- src/daemon/interaction/internal/find.ts | 12 +-- .../internal/interaction-runtime.ts | 4 +- .../interaction/internal/interaction.ts | 12 ++- src/daemon/interaction/internal/types.ts | 4 +- src/daemon/selector-capture-runtime.ts | 8 +- src/daemon/selector-runtime-backend.ts | 8 +- src/daemon/selector-runtime.ts | 6 +- src/daemon/snapshot-runtime.ts | 10 +-- src/daemon/wait-current-surface.test.ts | 8 +- src/daemon/wait-current-surface.ts | 8 +- src/daemon/wait-runtime.ts | 6 +- website/docs/docs/commands.md | 2 +- 18 files changed, 182 insertions(+), 94 deletions(-) rename src/daemon/interaction/internal/__tests__/{interaction-target-activation-disclosure.test.ts => interaction-capture-disclosure.test.ts} (59%) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4ee6cdf56..3f522377e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,12 @@ - Fixed (mobile): a read taken right after a `scroll`, `swipe`, or `gesture swipe` no longer reports a definite miss when the surface never settled. When post-gesture stabilization ran out of budget on a surface still moving, `is visible` answered a plain `selector_not_found` and `is absent` - passed. That capture now carries `unsettledGesture`: `is`, `get`, `find`, and `wait` report it (in - `error.details` or `data`) with an appended warning, `snapshot` appends the warning, `is absent` - refuses with `observation: "unsettled"`, `wait absent` keeps polling, and the next read captures - afresh. Click, press, and fill by selector do not disclose it yet. A failed read now also carries - `targetActivation` in `error.details`, the same place as `unsettledGesture`. + passed. That capture now carries `unsettledGesture`: `is`, `get`, `find`, `wait`, and every + interaction that captured it (`click`, `press`, `fill`, and the other touch and gesture commands) + report it (in `error.details` or `data`) with an appended warning, `snapshot` appends the warning, + `is absent` refuses with `observation: "unsettled"`, `wait absent` keeps polling, and the next read + captures afresh. A failed read now also carries `targetActivation` in `error.details`, the same + place as `unsettledGesture`. - Fixed (ios): `open` on a local Simulator now waits for the launched app's discovery before it decides whether the app is observable. On a loaded host `simctl spawn launchctl list` outlasts one 1.5 s discovery wait slice, and the launch observation read that slice as an unobservable app, so diff --git a/src/daemon/__tests__/capture-disclosure-target-activation.test.ts b/src/daemon/__tests__/capture-disclosure-target-activation.test.ts index b381114465..4ded8d7d85 100644 --- a/src/daemon/__tests__/capture-disclosure-target-activation.test.ts +++ b/src/daemon/__tests__/capture-disclosure-target-activation.test.ts @@ -77,7 +77,7 @@ test('a failure does not borrow a repair the stored snapshot happens to carry', const response = withCaptureDisclosures({ response: failed, consumedTree: { targetActivation: FACT }, - activationProof: {}, + captureProof: {}, }); assert.equal(response, failed); @@ -95,7 +95,7 @@ test('surface and foreground disclosures ride one response together', () => { iosSystemSurfaceBundleId: 'com.apple.SafariViewService', targetActivation: FACT, }, - activationProof: { state: { targetActivation: FACT } }, + captureProof: { targetActivation: FACT }, }); const data = dataOf(response); assert.match(String(data.warning), /system web sign-in sheet/); @@ -117,17 +117,17 @@ test('a repair that passes through two wrappers is named once in the failure hin details: { hint: 'Use snapshot to see the current tree.' }, }, }; - const proof = { state: { targetActivation: FACT } }; + const proof = { targetActivation: FACT }; const once = withCaptureDisclosures({ response: missed, consumedTree: { targetActivation: FACT }, - activationProof: proof, + captureProof: proof, }); const twice = withCaptureDisclosures({ response: once, consumedTree: { targetActivation: FACT }, - activationProof: proof, + captureProof: proof, }); assert.equal(twice.ok, false); diff --git a/src/daemon/__tests__/selector-capture-runtime.test.ts b/src/daemon/__tests__/selector-capture-runtime.test.ts index 64cf3655e8..9384dc7553 100644 --- a/src/daemon/__tests__/selector-capture-runtime.test.ts +++ b/src/daemon/__tests__/selector-capture-runtime.test.ts @@ -9,7 +9,7 @@ import { type SnapshotState, } from '@agent-device/kernel/snapshot'; import type { DaemonResponse } from '../daemon-request.ts'; -import { type RequestActivationProof, withCaptureDisclosures } from '../capture-disclosure.ts'; +import { type RequestCaptureProof, withCaptureDisclosures } from '../capture-disclosure.ts'; import { makeIosSession } from '../../__tests__/test-utils/session-factories.ts'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; import { createSelectorCaptureRuntime } from '../selector-capture-runtime.ts'; @@ -223,14 +223,14 @@ function proofRuntime(params: { : {}), } as never); const consumedSnapshot: { state?: SnapshotState } = {}; - const activationProof: RequestActivationProof = {}; + const captureProof: RequestCaptureProof = {}; const runtime = createSelectorCaptureRuntime({ device: session.device, session, sessionStore, sessionName: params.sessionName, consumedSnapshot, - activationProof, + captureProof, capture: boundCapture, req: { token: 't', @@ -240,7 +240,7 @@ function proofRuntime(params: { flags: {}, }, }); - return { runtime, consumedSnapshot, activationProof }; + return { runtime, consumedSnapshot, captureProof }; } /** @@ -264,12 +264,12 @@ test('a session-snapshot cache hit consumes a repaired tree without earning the expect(boundCapture).not.toHaveBeenCalled(); expect(holders.consumedSnapshot.state?.targetActivation).toEqual(REPAIR); - expect(holders.activationProof.state).toBeUndefined(); + expect(holders.captureProof.targetActivation).toBeUndefined(); const response = withCaptureDisclosures({ response: { ok: true, data: { nodes: [] } } as DaemonResponse, consumedTree: holders.consumedSnapshot.state, - activationProof: holders.activationProof, + captureProof: holders.captureProof, }); expect(response.ok).toBe(true); if (response.ok) { @@ -287,7 +287,7 @@ test('a capture the request took itself earns the repair proof', async () => { await holders.runtime.capture({ flags: {}, cache: { useSessionSnapshot: true } }); expect(boundCapture).toHaveBeenCalledTimes(1); - expect(holders.activationProof.state?.targetActivation).toEqual(REPAIR); + expect(holders.captureProof.targetActivation).toEqual(REPAIR); }); /** @@ -310,7 +310,7 @@ test('a later fact-less capture does not erase an earlier repair proof', async ( await holders.runtime.capture({ flags: {}, cache: { forceFresh: true } }); expect(boundCapture).toHaveBeenCalledTimes(2); - expect(holders.activationProof.state?.targetActivation).toEqual(REPAIR); + expect(holders.captureProof.targetActivation).toEqual(REPAIR); }); function makeCaptureRuntime(sessionName: string) { diff --git a/src/daemon/capture-disclosure.ts b/src/daemon/capture-disclosure.ts index 1e594f1613..e3ec742979 100644 --- a/src/daemon/capture-disclosure.ts +++ b/src/daemon/capture-disclosure.ts @@ -12,8 +12,9 @@ export type CaptureProvenance = Pick< >; /** - * The foreground repair THIS request paid for, filled by the capture path only when the request - * actually captured a tree (#2682). + * The whole-tree facts THIS request's own captures observed, filled by the capture path only when the + * request actually captured a tree: the foreground repair it paid for (#2682) and the gesture whose + * surface it read before that surface settled. * * Separate from the consumed tree on purpose. A selector read may answer from a cached or stored * tree — that tree still describes the surface the response is about, which is what #2438 discloses @@ -21,23 +22,21 @@ export type CaptureProvenance = Pick< * consumed tree would tell a command "you found the session app out of foreground" when it never * looked, which is a fabricated observation rather than a disclosure. */ -export type RequestActivationProof = { - state?: CaptureProvenance; -}; +export type RequestCaptureProof = Pick; /** - * Note the repair a capture paid for, and hand that capture back. First fact wins: a later capture in - * the same request that reports no repair — a sparse recovery's fresh tree, a poll's fact-less read — - * cannot erase the capture that did (#2682). One rule, because three capture paths owe it and a - * hand-copied condition drifts from the other two the moment one of them learns something. + * Note the facts a capture observed, and hand that capture back. First fact wins: a later capture in + * the same request that reports neither — a sparse recovery's fresh tree, a poll's fact-less read, an + * interaction's full-tree retry — cannot erase the capture that did. Only the first capture after a + * gesture is compared for settling, so a later one proves nothing about the surface having stopped. */ -export function recordActivationProof( - proof: RequestActivationProof | undefined, +export function recordCaptureProof( + proof: RequestCaptureProof | undefined, snapshot: T, ): T { - if (proof !== undefined && proof.state === undefined && snapshot.targetActivation !== undefined) { - proof.state = snapshot; - } + if (proof === undefined) return snapshot; + if (snapshot.targetActivation) proof.targetActivation ??= snapshot.targetActivation; + if (snapshot.unsettledGesture) proof.unsettledGesture ??= snapshot.unsettledGesture; return snapshot; } @@ -94,15 +93,16 @@ export function withTargetActivationDisclosure( /** * Every capture-provenance disclosure a response owes, from the two different things a capture can * prove: what the answered tree describes (#2438 — cache tiers included, because the surface is - * still on screen) and what this request's own capture found (#2682 — cache hits excluded, because - * a request that captured nothing repaired nothing). + * still on screen; an unsettled gesture — a poll that later answered from a settled tree owes none) + * and what this request's own capture found (#2682 — cache hits excluded, because a request that + * captured nothing repaired nothing). */ export function withCaptureDisclosures(params: { response: DaemonResponse; consumedTree: CaptureProvenance | undefined; - activationProof?: RequestActivationProof; + captureProof?: RequestCaptureProof; }): DaemonResponse { - const { response, consumedTree, activationProof } = params; + const { response, consumedTree, captureProof } = params; return withTargetActivationDisclosure( withTreeFactDisclosure( withSystemSurfaceDisclosure(response, consumedTree), @@ -110,14 +110,34 @@ export function withCaptureDisclosures(params: { consumedTree?.unsettledGesture, formatGestureUnsettledWarning, ), - activationProof?.state, + captureProof, + ); +} + +/** + * Every fact an interaction's own captures observed. The gesture aims at the trees this request + * captured, so an unsettled read anywhere in it is disclosed on success and failure alike. + */ +export function withRequestCaptureDisclosures( + response: DaemonResponse, + captureProof: RequestCaptureProof, +): DaemonResponse { + return withTargetActivationDisclosure( + withTreeFactDisclosure( + response, + 'unsettledGesture', + captureProof.unsettledGesture, + formatGestureUnsettledWarning, + ), + captureProof, ); } /** * Which success-side field a disclosure enters. #2438's surface sentence shipped on the singular * `warning`; the repair sentence ships on the `warnings` array beside the typed fact that travels - * with it. Failure has one carrier for both: `error.details.hint`. + * with it. Failure has one carrier for both: the hint request finalization keeps, which is + * `error.hint` when the route set one and `error.details.hint` otherwise. */ type DisclosureCarrier = 'warning' | 'warnings'; @@ -135,13 +155,16 @@ function appendDisclosure( ): DaemonResponse { if (carriesDisclosure(response, disclosure)) return response; if (!response.ok) { - const details = response.error.details ?? {}; + const { hint, details = {} } = response.error; return { ...response, - error: { - ...response.error, - details: { ...details, hint: appended(details.hint, disclosure) }, - }, + error: + typeof hint === 'string' + ? { ...response.error, hint: appended(hint, disclosure) } + : { + ...response.error, + details: { ...details, hint: appended(details.hint, disclosure) }, + }, }; } if (carrier === 'warnings') { @@ -166,7 +189,7 @@ function appendDisclosure( function carriesDisclosure(response: DaemonResponse, disclosure: string): boolean { const carriers: unknown[] = response.ok ? [response.data?.warning, ...responseWarnings(response.data?.warnings)] - : [response.error.details?.hint]; + : [response.error.hint, response.error.details?.hint]; return carriers.some((carrier) => typeof carrier === 'string' && carrier.includes(disclosure)); } diff --git a/src/daemon/interaction/internal/__tests__/interaction-target-activation-disclosure.test.ts b/src/daemon/interaction/internal/__tests__/interaction-capture-disclosure.test.ts similarity index 59% rename from src/daemon/interaction/internal/__tests__/interaction-target-activation-disclosure.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-capture-disclosure.test.ts index d915ae7be9..1c6a70e56c 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-target-activation-disclosure.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-capture-disclosure.test.ts @@ -1,4 +1,4 @@ -import { test, expect } from 'vitest'; +import { test, expect, vi } from 'vitest'; import { attachRefs, type IosTargetActivation, @@ -9,6 +9,14 @@ import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory import { handleInteractionCommands } from '../../index.ts'; import { getRuntimeBindings } from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, makeSession } from './interaction-touch-fixtures.ts'; +import { legacyDispatchCapture } from '../../../__tests__/legacy-snapshot-capture-fixture.ts'; +import { markDeferredInteractionOutcome } from '../../../deferred-interaction-outcome.ts'; +import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; + +vi.mock('../../../snapshot-interactor-capture.ts', async () => { + const fixture = await import('../../../__tests__/legacy-snapshot-capture-fixture.ts'); + return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; +}); const FACT: IosTargetActivation = { reason: 'stale_target', @@ -124,3 +132,63 @@ test('a press that consumes no capture is not disclosed against an older tree', expect(response.data?.targetActivation).toBeUndefined(); } }); + +/** + * A press by selector after a scroll whose list never stops: the post-gesture capture misses on a + * surface still moving, and the full-tree retry right after it proves nothing about the surface + * having stopped. The miss must not read as a definite absence. + */ +test('a press that misses on a surface still moving after a scroll reports the unsettled fact', async () => { + const sessionStore = makeSessionStore(); + const session = makeSession('default'); + markDeferredInteractionOutcome({ session, command: 'scroll', positionals: ['down'], flags: {} }); + sessionStore.set('default', session); + let call = 0; + legacyDispatchCapture.mockImplementation(async () => { + call += 1; + return { + backend: 'xctest', + nodes: [ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + depth: 1, + type: 'Cell', + label: 'Wi-Fi', + rect: { x: 16, y: 600 - call * 37, width: 370, height: 52 }, + hittable: true, + }, + ], + }; + }); + const realSetTimeout = globalThis.setTimeout; + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); + let done = false; + const pending = handleInteractionCommands({ + req: { + token: 't', + session: 'default', + command: 'press', + positionals: ['label="General"'], + flags: {}, + }, + sessionName: 'default', + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }).finally(() => (done = true)); + // The route awaits real I/O between polls, so the faked clock advances while the test yields. + while (!done) { + await vi.advanceTimersByTimeAsync(50); + await new Promise((resolve) => realSetTimeout(resolve, 1)); + } + const response = await pending; + vi.useRealTimers(); + + const gesture = { action: 'scroll', positionals: ['down'] }; + expect(response?.ok === false && response.error).toMatchObject({ + hint: expect.stringContaining(formatGestureUnsettledWarning(gesture)), + details: { reason: 'selector_not_found', unsettledGesture: gesture }, + }); +}); diff --git a/src/daemon/interaction/internal/find-target-capture.ts b/src/daemon/interaction/internal/find-target-capture.ts index f1698f340d..e0b906f539 100644 --- a/src/daemon/interaction/internal/find-target-capture.ts +++ b/src/daemon/interaction/internal/find-target-capture.ts @@ -1,7 +1,7 @@ import type { FindLocator } from '@agent-device/selectors'; import type { BoundSelectorCapture } from '../../selector-capture-binding.ts'; import type { SnapshotQualityVerdict, SnapshotState } from '@agent-device/kernel/snapshot'; -import type { CaptureProvenance, RequestActivationProof } from '../../capture-disclosure.ts'; +import type { CaptureProvenance, RequestCaptureProof } from '../../capture-disclosure.ts'; import { createSelectorCaptureRuntime } from '../../selector-capture-runtime.ts'; import { SessionStore } from '../../session-store.ts'; import type { DaemonRequest, DaemonResponse } from '../../daemon-request.ts'; @@ -32,7 +32,7 @@ export function createFindTargetCapture( * Filled by whichever capture this find actually took, including a re-capture that replaced a * sparse first tree — find's response is owed the repair its own first capture paid for. */ - activationProof: RequestActivationProof; + captureProof: RequestCaptureProof; }>, ): () => Promise { const { device, session, req, logPath, locator, query, sessionStore, sessionName } = params; @@ -44,7 +44,7 @@ export function createFindTargetCapture( req, logPath, capture: params.capture, - activationProof: params.activationProof, + captureProof: params.captureProof, }); return async () => { // Interaction targets need the full interactive tree so duplicate labels can diff --git a/src/daemon/interaction/internal/find.ts b/src/daemon/interaction/internal/find.ts index 552e4b68fd..8a24349bc5 100644 --- a/src/daemon/interaction/internal/find.ts +++ b/src/daemon/interaction/internal/find.ts @@ -18,7 +18,7 @@ import type { SessionState } from '../../session-state.ts'; import { SessionStore } from '../../session-store.ts'; import { contextFromFlags } from '../../context.ts'; import { readCommandMessage, successText } from '@agent-device/kernel/success-text'; -import type { RequestActivationProof } from '../../capture-disclosure.ts'; +import type { RequestCaptureProof } from '../../capture-disclosure.ts'; import { withCaptureDisclosures } from '../../capture-disclosure.ts'; import { recordSessionAction } from '../../session-action-recorder.ts'; import { stripInternalInteractionFlags } from '../../interaction-outcome-policy.ts'; @@ -129,7 +129,7 @@ export async function handleFindCommands(params: FindRouteInput): Promise { - const activationProof: RequestActivationProof = {}; - const routed = { ...params, refSnapshotFlagGuardResponse, activationProof }; + const captureProof: RequestCaptureProof = {}; + const routed = { ...params, refSnapshotFlagGuardResponse, captureProof }; const response = await dispatchInteractionCommand(routed); - // The interaction's own capture is what the gesture was aimed at, so a foreground repair inside - // it belongs on this response even though the interaction routes never read the stored snapshot. - return response ? withTargetActivationDisclosure(response, activationProof.state) : response; + return response ? withRequestCaptureDisclosures(response, captureProof) : response; } type RoutedInteractionInput = InteractionRouteInput & { diff --git a/src/daemon/interaction/internal/types.ts b/src/daemon/interaction/internal/types.ts index 8a63ae5524..9236285c0d 100644 --- a/src/daemon/interaction/internal/types.ts +++ b/src/daemon/interaction/internal/types.ts @@ -1,7 +1,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; import type { Rect, SnapshotPreferredBackend, SnapshotState } from '@agent-device/kernel/snapshot'; -import type { RequestActivationProof } from '../../capture-disclosure.ts'; +import type { RequestCaptureProof } from '../../capture-disclosure.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { CommandSessionStore } from '../../../runtime-contract.ts'; import type { DeferredInteractionOutcomeMark } from '../../deferred-interaction-outcome.ts'; @@ -35,7 +35,7 @@ export type InteractionRouteInput = { * The foreground repair this request's own capture reported, when it captured at all. A coordinate * press consumes no capture and must not be disclosed against an earlier request's tree (#2682). */ - activationProof?: RequestActivationProof; + captureProof?: RequestCaptureProof; }; export type FindRouteInput = { diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index 1b24f5510a..cae50dcc56 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -9,8 +9,8 @@ import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapsh import type { DaemonRequest } from './daemon-request.ts'; import type { SessionState } from './session-state.ts'; import { SessionStore } from './session-store.ts'; -import { recordActivationProof } from './capture-disclosure.ts'; -import type { RequestActivationProof } from './capture-disclosure.ts'; +import { recordCaptureProof } from './capture-disclosure.ts'; +import type { RequestCaptureProof } from './capture-disclosure.ts'; import { captureSnapshot } from './snapshot-capture.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; import { getActiveAndroidSnapshotFreshness } from './session-snapshot-freshness.ts'; @@ -35,7 +35,7 @@ export type SelectorCaptureRuntimeParams = { * Filled ONLY by a capture this request actually took, never by a cache tier — the foreground * repair a route may disclose has to be one the route paid for (#2682). */ - activationProof?: RequestActivationProof; + captureProof?: RequestCaptureProof; /** * The request-bound capture from `resolveBoundSelectorCapture`: every cache tier, recovery * re-capture, and poll below reaches the platform through it. Required since find (R35) — @@ -221,7 +221,7 @@ async function runCapture( // Recorded here rather than at the caller that consumes the result: a sparse recovery re-capture // DISCARDS this tree and returns a fresh one, and the repair this capture paid for belongs to the // request, not to whichever tree survives (#2682). - return recordActivationProof(params.activationProof, capture.snapshot); + return recordCaptureProof(params.captureProof, capture.snapshot); } function readReusableLastSnapshot(params: { diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index a256ba0ec1..326e8bce5a 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -27,7 +27,7 @@ import type { AndroidObservationAdapter } from '@agent-device/contracts/android- import type { PlatformResourceCleanup } from './platform-resource-cleanup.ts'; import { getRequestSignal } from '@agent-device/host-kit/request'; import { snapshotOptionsToFlags } from '../backend-snapshot-options.ts'; -import type { RequestActivationProof } from './capture-disclosure.ts'; +import type { RequestCaptureProof } from './capture-disclosure.ts'; import { checkIsArgs } from '@agent-device/selectors'; import { noActiveSessionError } from '@agent-device/kernel/contracts'; @@ -41,7 +41,7 @@ export type SelectorRuntimeParams = { // sessionless routes disclose from here because no session record stores the capture. consumedSnapshot?: { state?: SnapshotState }; /** The repair this request's own capture reported, when it captured at all (#2682). */ - activationProof?: RequestActivationProof; + captureProof?: RequestCaptureProof; signal?: AbortSignal; inspectFacts?: InspectDeviceRuntimeFacts; bindDevice?: BindDeviceRuntime; @@ -93,7 +93,7 @@ async function resolveSelectorRuntimeDevice( requireSession: boolean, ): Promise { params.consumedSnapshot ??= {}; - params.activationProof ??= {}; + params.captureProof ??= {}; const session = params.sessionStore.get(params.sessionName); if (!session && requireSession) return { ok: false, response: noActiveSessionError() }; const device = session?.device ?? (await resolveTargetDevice(params.req.flags ?? {})); @@ -171,7 +171,7 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice sessionName, req, consumedSnapshot: params.consumedSnapshot, - activationProof: params.activationProof, + captureProof: params.captureProof, logPath, capture: boundOperations.capture, }); diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index c49279293a..13ea73daa6 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -104,7 +104,7 @@ export async function dispatchFindReadOnlyViaRuntime( return withCaptureDisclosures({ response, consumedTree: consumedSessionSnapshot(params), - activationProof: params.activationProof, + captureProof: params.captureProof, }); } @@ -182,7 +182,7 @@ export async function dispatchGetViaRuntime( return withCaptureDisclosures({ response, consumedTree: consumedSessionSnapshot(params), - activationProof: params.activationProof, + captureProof: params.captureProof, }); } @@ -242,7 +242,7 @@ export async function dispatchIsViaRuntime( return withCaptureDisclosures({ response: await maybeAndroidForegroundBlockerResponse(params, response, `is ${predicate}`), consumedTree: consumedSessionSnapshot(params), - activationProof: params.activationProof, + captureProof: params.captureProof, }); } diff --git a/src/daemon/snapshot-runtime.ts b/src/daemon/snapshot-runtime.ts index 96c938473f..b7cf97cf30 100644 --- a/src/daemon/snapshot-runtime.ts +++ b/src/daemon/snapshot-runtime.ts @@ -4,7 +4,7 @@ import { SNAPSHOT_COMMAND_OPTION_KEYS, snapshotOptionsFromFlags, } from '@agent-device/kernel/snapshot'; -import type { RequestActivationProof } from './capture-disclosure.ts'; +import type { RequestCaptureProof } from './capture-disclosure.ts'; import { withTargetActivationDisclosure } from './capture-disclosure.ts'; import { dispatchSnapshotRuntimeCommand } from './snapshot-command-runtime.ts'; import { captureSparseFallbackScreenshot } from './sparse-fallback-screenshot.ts'; @@ -15,7 +15,7 @@ import type { SessionState } from './session-state.ts'; export async function dispatchSnapshotViaRuntime( params: SnapshotRuntimeRouteParams, ): Promise { - const activationProof: RequestActivationProof = {}; + const captureProof: RequestCaptureProof = {}; const response = await dispatchSnapshotRuntimeCommand({ ...params, command: 'snapshot', @@ -33,9 +33,7 @@ export async function dispatchSnapshotViaRuntime( }); // This request's own capture, read here rather than off the stored snapshot: a snapshot that // failed before capturing must not inherit the previous command's repair (#2682). - if (result.targetActivation && !activationProof.state) { - activationProof.state = { targetActivation: result.targetActivation }; - } + if (result.targetActivation) captureProof.targetActivation ??= result.targetActivation; const refsGeneration = publishedSnapshotGeneration( request, params.sessionStore.get(resolvedSessionName), @@ -83,7 +81,7 @@ export async function dispatchSnapshotViaRuntime( // own annotations, so re-deriving it from the stored snapshot would copy a sentence the response // already carries — and on a snapshot that failed before capturing, would credit it with a surface // it never observed (#2682). - return withTargetActivationDisclosure(response, activationProof.state); + return withTargetActivationDisclosure(response, captureProof); } function publishedSnapshotGeneration( diff --git a/src/daemon/wait-current-surface.test.ts b/src/daemon/wait-current-surface.test.ts index f11a6aa40b..3b1865a865 100644 --- a/src/daemon/wait-current-surface.test.ts +++ b/src/daemon/wait-current-surface.test.ts @@ -1,7 +1,7 @@ import { expect, test, vi } from 'vitest'; import { ANDROID_EMULATOR } from '../__tests__/test-utils/device-fixtures.ts'; -import type { RequestActivationProof } from './capture-disclosure.ts'; +import type { RequestCaptureProof } from './capture-disclosure.ts'; import { maybeWaitTimeoutSurfaceResponse } from './wait-current-surface.ts'; import type { BoundSelectorCapture } from './selector-capture-binding.ts'; @@ -113,10 +113,10 @@ test('the decoration capture records the foreground repair it paid for', async ( targetActivation: repair, nodes: [{ index: 0, depth: 0, type: 'Button', label: 'Receipt' }], })) as unknown as BoundSelectorCapture; - const activationProof: RequestActivationProof = {}; + const captureProof: RequestCaptureProof = {}; await maybeWaitTimeoutSurfaceResponse( - { req, session: undefined, device: ANDROID_EMULATOR, capture, activationProof }, + { req, session: undefined, device: ANDROID_EMULATOR, capture, captureProof }, { ok: false as const, error: { @@ -127,5 +127,5 @@ test('the decoration capture records the foreground repair it paid for', async ( }, ); - expect(activationProof.state).toMatchObject({ targetActivation: repair }); + expect(captureProof.targetActivation).toEqual(repair); }); diff --git a/src/daemon/wait-current-surface.ts b/src/daemon/wait-current-surface.ts index 5238b2fb37..db7de91bcd 100644 --- a/src/daemon/wait-current-surface.ts +++ b/src/daemon/wait-current-surface.ts @@ -1,7 +1,7 @@ import { WAIT_REASONS } from '@agent-device/contracts/wait'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; -import { recordActivationProof } from './capture-disclosure.ts'; -import type { RequestActivationProof } from './capture-disclosure.ts'; +import { recordCaptureProof } from './capture-disclosure.ts'; +import type { RequestCaptureProof } from './capture-disclosure.ts'; import type { DaemonRequest, DaemonResponse } from './daemon-request.ts'; import type { SessionState } from './session-state.ts'; import { captureSnapshot } from './snapshot-capture.ts'; @@ -25,7 +25,7 @@ type WaitCurrentSurfaceParams = { * A timed-out wait still consumed that capture to describe its surface, so the repair it paid for * belongs to the response this module decorates. */ - activationProof?: RequestActivationProof; + captureProof?: RequestCaptureProof; }; type CurrentSurfaceDetails = { @@ -89,7 +89,7 @@ async function inspectCurrentSurface( }), ), }); - recordActivationProof(params.activationProof, capture.snapshot); + recordCaptureProof(params.captureProof, capture.snapshot); const orderedNodes = [...capture.snapshot.nodes].sort(compareSurfacePriority); const labels = topSurfaceTexts(orderedNodes, 6, { includeIdentifiers: true }); if (labels.length === 0) return null; diff --git a/src/daemon/wait-runtime.ts b/src/daemon/wait-runtime.ts index 025061dabd..766f7df922 100644 --- a/src/daemon/wait-runtime.ts +++ b/src/daemon/wait-runtime.ts @@ -54,7 +54,7 @@ export async function dispatchWaitViaRuntime(params: DispatchWaitParams): Promis // Wait builds its runtime directly (no createBoundSelectorRuntime), so the consumed-snapshot slot // must be initialized here too or sessionless waits have nowhere to report the capture from. params.consumedSnapshot ??= {}; - params.activationProof ??= {}; + params.captureProof ??= {}; // A pure sleep consumes no capture, so it never earns the system-surface disclosure below. if (parsed.kind === 'sleep') { return await executeWaitRequest( @@ -86,7 +86,7 @@ export async function dispatchWaitViaRuntime(params: DispatchWaitParams): Promis params.platformResourceCleanup, ), consumedTree: consumedSessionSnapshot(params), - activationProof: params.activationProof, + captureProof: params.captureProof, }); } @@ -187,7 +187,7 @@ async function executeWaitRequest( session, device, capture: waitOperations.capture, - activationProof: params.activationProof, + captureProof: params.captureProof, }, response, ) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 516cc31350..97bbe343ca 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -558,7 +558,7 @@ agent-device is text 'id="greeting"' "Welcome back" - `is visible` checks whether the resolved element is present in the current visible snapshot viewport. A node without its own rect still passes when a visible ancestor within the viewport provides the on-screen geometry. - `is exists` only checks whether the selector matches in the current snapshot. - `is absent` passes only when the selector has zero matches in one readable, complete, settled, unscoped, full-depth accessibility capture. It does not mean hidden; `--scope` and `--depth` are rejected, and sparse, unreadable, truncated, or unsettled captures fail closed. -- A read that answers from the first capture after a `scroll`, `swipe`, or `gesture swipe` waits for two consecutive captures to agree. When the surface is still changing when that budget runs out, `is`, `get`, `find`, and `wait` carry `unsettledGesture` (`{ "action", "positionals" }`) in `error.details` or `data` and append a warning, and `snapshot` appends the warning. A miss on that capture is not proof of absence: read again. `is absent` refuses it with `observation: "unsettled"`, and `wait absent` keeps polling. +- A read that answers from the first capture after a `scroll`, `swipe`, or `gesture swipe` waits for two consecutive captures to agree. When the surface is still changing when that budget runs out, `is`, `get`, `find`, `wait`, and an interaction that captured it (such as `click`, `press`, or `fill`) carry `unsettledGesture` (`{ "action", "positionals" }`) in `error.details` or `data` and append a warning, and `snapshot` appends the warning. A miss on that capture is not proof of absence: read again. `is absent` refuses it with `observation: "unsettled"`, and `wait absent` keeps polling. - `wait text` is a text-presence wait, not a hittability assertion. - Strict `wait absent` is not exported to Maestro's lenient `notVisible` condition; Maestro export reports it as unsupported unless an exact zero-candidate primitive becomes available. - `is text ` compares the resolved element text against the expected value. From e7dc1f8318aeab482e8f91a07d491ca8391ddd62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 09:41:07 +0200 Subject: [PATCH 2/2] refactor(capture): carry the gesture no-effect fact on the snapshot tree A proven no-effect gesture reached the agent only through capture annotations, which selector reads and interactions drop, so is, get, and find never showed it. The fact now rides SnapshotState.gestureNoEffect beside unsettledGesture and every route derives its disclosure from that stamp. withGestureNoEffectWarning and its annotation channel are gone. Closes #2856 --- CHANGELOG.md | 4 ++ .../capture-kit/src/post-gesture-stability.ts | 4 +- packages/kernel/src/snapshot.ts | 2 + src/commands/capture/runtime/snapshot.test.ts | 21 +++++- src/commands/capture/runtime/snapshot.ts | 8 ++- .../deferred-interaction-outcome.test.ts | 5 +- src/daemon/__tests__/is-runtime.test.ts | 38 +++++++++- .../post-gesture-no-effect-claim.test.ts | 17 +++-- src/daemon/capture-disclosure.ts | 72 +++++++++++-------- src/daemon/deferred-interaction-outcome.ts | 6 +- src/daemon/gesture-no-effect.ts | 22 ------ website/docs/docs/commands.md | 1 + 12 files changed, 135 insertions(+), 65 deletions(-) delete mode 100644 src/daemon/gesture-no-effect.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f522377e7..751aa2fd55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- Fixed (mobile): the warning for a `scroll`, `swipe`, or `gesture swipe` that had no visible effect + now reaches `is`, `get`, `find`, `wait`, and interactions, not only `snapshot`. The capture it + was proven on carries `gestureNoEffect` (`{ action, positionals }`), and those commands report it + in `data` or `error.details` with the appended warning. - Fixed (mobile): a read taken right after a `scroll`, `swipe`, or `gesture swipe` no longer reports a definite miss when the surface never settled. When post-gesture stabilization ran out of budget on a surface still moving, `is visible` answered a plain `selector_not_found` and `is absent` diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index 438a674bbb..47d8a104d0 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -83,9 +83,9 @@ function describePostGestureAction(gesture: PostGestureAction): string { * moved a stuck list when synthesized scrolls did not (#1600, element-18: * raw `swipe` worked where scroll/fling/pan all silently no-opped). */ -export function formatGestureNoEffectWarning(action: string, positionals: string[]): string { +export function formatGestureNoEffectWarning(gesture: PostGestureAction): string { return ( - `${describePostGestureAction({ action, positionals })} produced no visible change: the tree still matches its pre-gesture state. ` + + `${describePostGestureAction(gesture)} produced no visible change: the tree still matches its pre-gesture state. ` + 'Either the container is already at its edge, or it ignores synthesized scrolls — ' + 'a raw drag moves such lists: swipe x1 y1 x2 y2 (start inside the list).' ); diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 33eeafed20..2a3c471e3b 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -502,6 +502,8 @@ export type SnapshotState = { targetActivation?: IosTargetActivation; /** The gesture whose surface was still changing when stabilization gave up on this capture. */ unsettledGesture?: PostGestureAction; + /** The gesture this capture proved had no visible effect (#1600). */ + gestureNoEffect?: PostGestureAction; } & SnapshotStateProvenance; /** The gesture a post-gesture outcome fact names: the command and its positionals. */ diff --git a/src/commands/capture/runtime/snapshot.test.ts b/src/commands/capture/runtime/snapshot.test.ts index d24f27a490..0de5e19ec8 100644 --- a/src/commands/capture/runtime/snapshot.test.ts +++ b/src/commands/capture/runtime/snapshot.test.ts @@ -13,7 +13,10 @@ import { type CommandSessionStore, } from '../../../runtime.ts'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; -import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; +import { + formatGestureNoEffectWarning, + formatGestureUnsettledWarning, +} from '@agent-device/capture-kit/post-gesture-stability'; test('runtime snapshot captures nodes and updates the session baseline', async () => { let stored: Parameters[0] | undefined; @@ -803,3 +806,19 @@ test('runtime snapshot warns when its tree was read on a surface still moving af assert.deepEqual(result.warnings, [formatGestureUnsettledWarning(gesture)]); }); + +test('runtime snapshot warns when its tree proved the gesture before it had no effect', async () => { + const gesture = { action: 'scroll', positionals: ['down'] }; + const device = createSnapshotOnlyDevice({ + snapshot: { + ...makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], { + backend: 'xctest', + }), + gestureNoEffect: gesture, + }, + }); + + const result = await device.capture.snapshot({ session: 'default' }); + + assert.deepEqual(result.warnings, [formatGestureNoEffectWarning(gesture)]); +}); diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index 21f058fe53..5b67ae0085 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -26,7 +26,10 @@ import { renderSnapshotQualityWarnings, truncatedCaptureWarning, } from '@agent-device/capture-kit/quality-warnings'; -import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; +import { + formatGestureNoEffectWarning, + formatGestureUnsettledWarning, +} from '@agent-device/capture-kit/post-gesture-stability'; import { buildSnapshotVisibility } from '@agent-device/capture-kit/snapshot-visibility'; import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '@agent-device/contracts/android-system-surface-disclosure'; import { formatReactNativeOverlayWarning } from '../../react-native/overlay.ts'; @@ -264,6 +267,9 @@ function buildSnapshotWarnings(params: { if (params.snapshot.unsettledGesture) { warnings.push(formatGestureUnsettledWarning(params.snapshot.unsettledGesture)); } + if (params.snapshot.gestureNoEffect) { + warnings.push(formatGestureNoEffectWarning(params.snapshot.gestureNoEffect)); + } warnings.push(...buildEmptyAndroidInteractiveWarnings(params)); if (!params.annotations.quality) { // Legacy runners without a structured verdict keep the old daemon-side heuristics. diff --git a/src/daemon/__tests__/deferred-interaction-outcome.test.ts b/src/daemon/__tests__/deferred-interaction-outcome.test.ts index 1dbc8374c7..ee91bbdf9f 100644 --- a/src/daemon/__tests__/deferred-interaction-outcome.test.ts +++ b/src/daemon/__tests__/deferred-interaction-outcome.test.ts @@ -280,7 +280,7 @@ test('a pending stabilization resolves through the quiet-window loop and clears assert.equal(result?.warnings, undefined); }); -test('a proven no-effect gesture surfaces its warning on the resolved capture (iOS accept-stale)', async () => { +test('a proven no-effect gesture is stamped on the resolved capture tree (iOS accept-stale)', async () => { vi.useFakeTimers(); const session = makeSession('ios'); session.snapshot = pickupSnapshot(); @@ -299,8 +299,7 @@ test('a proven no-effect gesture surfaces its warning on the resolved capture (i } const result = await pendingResult; - assert.equal(result?.warnings?.length, 1); - assert.match(result?.warnings?.[0] ?? '', /produced no visible change/); + assert.deepEqual(result?.snapshot.gestureNoEffect, { action: 'scroll', positionals: ['down'] }); assert.equal(isPostGestureStabilizationPending(session), false); }); diff --git a/src/daemon/__tests__/is-runtime.test.ts b/src/daemon/__tests__/is-runtime.test.ts index 722948d9ab..3608b4baaf 100644 --- a/src/daemon/__tests__/is-runtime.test.ts +++ b/src/daemon/__tests__/is-runtime.test.ts @@ -13,7 +13,10 @@ import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fix import type { DaemonRequest } from '../daemon-request.ts'; import { selectorCaptureFixture } from './selector-capture-fixture.ts'; import { markDeferredInteractionOutcome } from '../deferred-interaction-outcome.ts'; -import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; +import { + formatGestureNoEffectWarning, + formatGestureUnsettledWarning, +} from '@agent-device/capture-kit/post-gesture-stability'; const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ mockRunAppleRunnerCommand: vi.fn() })); @@ -468,3 +471,36 @@ test('a miss on a surface that never settled carries the unsettled fact, and the expect(fixture.captures.length).toBe(captures + 1); expect(reread?.ok === false && reread.error.details?.unsettledGesture).toBeUndefined(); }); + +test('a read after a scroll that moved nothing carries the no-effect fact', async () => { + vi.useFakeTimers(); + const row = { + index: 0, + type: 'Cell', + identifier: 'row', + rect: { x: 0, y: 200, width: 390, height: 60 }, + }; + const fixture = selectorCaptureFixture({ + snapshot: () => ({ nodes: [row], backend: 'xctest', producer: 'apple-runner' }), + }); + const sessionStore = makeSessionStore(); + const session = makeIosAppSession('is-no-effect', { snapshot: makeSnapshotState([row]) }); + markDeferredInteractionOutcome({ session, command: 'scroll', positionals: ['down'], flags: {} }); + sessionStore.set('is-no-effect', session); + const pending = dispatchIsViaRuntime({ + req: isRequest('is-no-effect', ['visible', 'id=row']), + sessionName: 'is-no-effect', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + // A tree that still matches its pre-gesture baseline is distrusted up to the 3.5s cap. + await vi.advanceTimersByTimeAsync(3_700); + const gesture = { action: 'scroll', positionals: ['down'] }; + + const response = await pending; + expect(response?.ok && response.data).toMatchObject({ + gestureNoEffect: gesture, + warnings: [formatGestureNoEffectWarning(gesture)], + }); +}); diff --git a/src/daemon/__tests__/post-gesture-no-effect-claim.test.ts b/src/daemon/__tests__/post-gesture-no-effect-claim.test.ts index bfe0ddca12..bf6121a578 100644 --- a/src/daemon/__tests__/post-gesture-no-effect-claim.test.ts +++ b/src/daemon/__tests__/post-gesture-no-effect-claim.test.ts @@ -362,19 +362,28 @@ test('summarizeDiscriminatingSurfaceDivergence counts one-sided keys and moved r test('formatGestureNoEffectWarning names the gesture and the raw-drag escape hatch', () => { // Positionals echo verbatim: the warning names the gesture the agent issued, // and `scroll down 1` is what they issued. - const scrollWarning = formatGestureNoEffectWarning('scroll', ['down', '1']); + const scrollWarning = formatGestureNoEffectWarning({ + action: 'scroll', + positionals: ['down', '1'], + }); assert.match(scrollWarning, /scroll down 1 produced no visible change/); assert.match(scrollWarning, /swipe x1 y1 x2 y2/); assert.match(scrollWarning, /already at its edge/); - const gestureWarning = formatGestureNoEffectWarning('gesture', ['swipe', 'left']); + const gestureWarning = formatGestureNoEffectWarning({ + action: 'gesture', + positionals: ['swipe', 'left'], + }); assert.match(gestureWarning, /gesture swipe left produced no visible change/); - const bareWarning = formatGestureNoEffectWarning('swipe', []); + const bareWarning = formatGestureNoEffectWarning({ action: 'swipe', positionals: [] }); assert.match(bareWarning, /swipe produced no visible change/); // The regression the deleted heuristic caused: every positional of a swipe is // a coordinate, so "drop anything numeric-looking" left a contentless "swipe". - const swipeWarning = formatGestureNoEffectWarning('swipe', ['10', '20', '30', '40']); + const swipeWarning = formatGestureNoEffectWarning({ + action: 'swipe', + positionals: ['10', '20', '30', '40'], + }); assert.match(swipeWarning, /^swipe 10 20 30 40 produced no visible change/); }); diff --git a/src/daemon/capture-disclosure.ts b/src/daemon/capture-disclosure.ts index e3ec742979..2c92d7e89a 100644 --- a/src/daemon/capture-disclosure.ts +++ b/src/daemon/capture-disclosure.ts @@ -2,19 +2,26 @@ import type { SnapshotState } from '@agent-device/kernel/snapshot'; import { systemSurfaceDisclosure } from '@agent-device/contracts/android-system-surface-disclosure'; import { iosSystemSurfaceDisclosure } from '@agent-device/contracts/ios-system-surface'; import { iosTargetActivationDisclosure } from '@agent-device/contracts/ios-target-activation'; -import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; +import { + formatGestureNoEffectWarning, + formatGestureUnsettledWarning, +} from '@agent-device/capture-kit/post-gesture-stability'; import type { DaemonResponse } from './daemon-request.ts'; -/** The capture provenance a response must be disclosed against (#2438, #2682, unsettled gestures). */ +/** The capture provenance a response must be disclosed against (#2438, #2682, gesture outcomes). */ export type CaptureProvenance = Pick< SnapshotState, - 'systemSurfaceOnly' | 'iosSystemSurfaceBundleId' | 'targetActivation' | 'unsettledGesture' ->; + 'systemSurfaceOnly' | 'iosSystemSurfaceBundleId' +> & + TreeFacts; + +/** Facts about a whole captured tree, each disclosed as a typed field plus its sentence. */ +type TreeFacts = Pick; /** * The whole-tree facts THIS request's own captures observed, filled by the capture path only when the - * request actually captured a tree: the foreground repair it paid for (#2682) and the gesture whose - * surface it read before that surface settled. + * request actually captured a tree: the foreground repair it paid for (#2682) and the outcome of the + * gesture it read after: a surface that never settled, or a proven no-effect gesture (#1600). * * Separate from the consumed tree on purpose. A selector read may answer from a cached or stored * tree — that tree still describes the surface the response is about, which is what #2438 discloses @@ -22,13 +29,13 @@ export type CaptureProvenance = Pick< * consumed tree would tell a command "you found the session app out of foreground" when it never * looked, which is a fabricated observation rather than a disclosure. */ -export type RequestCaptureProof = Pick; +export type RequestCaptureProof = TreeFacts; /** * Note the facts a capture observed, and hand that capture back. First fact wins: a later capture in * the same request that reports neither — a sparse recovery's fresh tree, a poll's fact-less read, an * interaction's full-tree retry — cannot erase the capture that did. Only the first capture after a - * gesture is compared for settling, so a later one proves nothing about the surface having stopped. + * gesture is compared against it, so a later one proves nothing about that gesture. */ export function recordCaptureProof( proof: RequestCaptureProof | undefined, @@ -37,6 +44,7 @@ export function recordCaptureProof( if (proof === undefined) return snapshot; if (snapshot.targetActivation) proof.targetActivation ??= snapshot.targetActivation; if (snapshot.unsettledGesture) proof.unsettledGesture ??= snapshot.unsettledGesture; + if (snapshot.gestureNoEffect) proof.gestureNoEffect ??= snapshot.gestureNoEffect; return snapshot; } @@ -58,15 +66,15 @@ export function withSystemSurfaceDisclosure( } /** - * Disclose a fact about the whole answered tree (#2682 foreground repair, an unsettled gesture). Its - * sentence is APPENDED, never replacing an earlier warning, and the typed fact lands on either - * outcome (`data` or `error.details`) even when the sentence was already carried. + * Disclose a fact about the whole answered tree. Its sentence is APPENDED, never replacing an earlier + * warning, and the typed fact lands on either outcome (`data` or `error.details`) even when the + * sentence was already carried. */ -function withTreeFactDisclosure( +function withTreeFactDisclosure( response: DaemonResponse, key: K, - fact: CaptureProvenance[K], - sentence: (fact: NonNullable) => string, + fact: TreeFacts[K], + sentence: (fact: NonNullable) => string, ): DaemonResponse { if (!fact) return response; const disclosed = appendDisclosure(response, sentence(fact), 'warnings'); @@ -90,10 +98,28 @@ export function withTargetActivationDisclosure( ); } +/** The outcome of the gesture a post-gesture capture read: it never settled, or it moved nothing. */ +function withGestureOutcomeDisclosure( + response: DaemonResponse, + facts: TreeFacts | undefined, +): DaemonResponse { + return withTreeFactDisclosure( + withTreeFactDisclosure( + response, + 'unsettledGesture', + facts?.unsettledGesture, + formatGestureUnsettledWarning, + ), + 'gestureNoEffect', + facts?.gestureNoEffect, + formatGestureNoEffectWarning, + ); +} + /** * Every capture-provenance disclosure a response owes, from the two different things a capture can * prove: what the answered tree describes (#2438 — cache tiers included, because the surface is - * still on screen; an unsettled gesture — a poll that later answered from a settled tree owes none) + * still on screen; a gesture outcome — a poll that later answered from another tree owes none) * and what this request's own capture found (#2682 — cache hits excluded, because a request that * captured nothing repaired nothing). */ @@ -104,31 +130,21 @@ export function withCaptureDisclosures(params: { }): DaemonResponse { const { response, consumedTree, captureProof } = params; return withTargetActivationDisclosure( - withTreeFactDisclosure( - withSystemSurfaceDisclosure(response, consumedTree), - 'unsettledGesture', - consumedTree?.unsettledGesture, - formatGestureUnsettledWarning, - ), + withGestureOutcomeDisclosure(withSystemSurfaceDisclosure(response, consumedTree), consumedTree), captureProof, ); } /** * Every fact an interaction's own captures observed. The gesture aims at the trees this request - * captured, so an unsettled read anywhere in it is disclosed on success and failure alike. + * captured, so a gesture outcome read anywhere in it is disclosed on success and failure alike. */ export function withRequestCaptureDisclosures( response: DaemonResponse, captureProof: RequestCaptureProof, ): DaemonResponse { return withTargetActivationDisclosure( - withTreeFactDisclosure( - response, - 'unsettledGesture', - captureProof.unsettledGesture, - formatGestureUnsettledWarning, - ), + withGestureOutcomeDisclosure(response, captureProof), captureProof, ); } diff --git a/src/daemon/deferred-interaction-outcome.ts b/src/daemon/deferred-interaction-outcome.ts index 23208ad04a..5c61bbf449 100644 --- a/src/daemon/deferred-interaction-outcome.ts +++ b/src/daemon/deferred-interaction-outcome.ts @@ -13,7 +13,6 @@ import { isNavigationSensitiveAction, type SnapshotFreshnessMode, } from '@agent-device/capture-kit/snapshot-freshness'; -import { withGestureNoEffectWarning } from './gesture-no-effect.ts'; import { areInteractionSurfaceSignaturesStable, buildInteractionSurfaceSignature, @@ -354,13 +353,14 @@ export async function capturePostGestureStabilizedResult(params: { return outcome; } -/** The stabilized attempt as a capture result: the tree carries an unsettled outcome as its own fact. */ +/** The stabilized attempt as a capture result: the tree carries the gesture's outcome as its own fact. */ function resolvedPostGestureCapture( stabilized: PostGestureStabilityOutcome, ): DeferredOutcomeCaptureResult { const { snapshot, annotations } = stabilized.value; if (stabilized.unsettledGesture) snapshot.unsettledGesture = stabilized.unsettledGesture; - return { snapshot, ...withGestureNoEffectWarning(annotations, stabilized.gestureNoEffect) }; + if (stabilized.gestureNoEffect) snapshot.gestureNoEffect = stabilized.gestureNoEffect; + return { snapshot, ...annotations }; } function isPostGestureStabilizingAction( diff --git a/src/daemon/gesture-no-effect.ts b/src/daemon/gesture-no-effect.ts deleted file mode 100644 index 38ce4d9cb3..0000000000 --- a/src/daemon/gesture-no-effect.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { SnapshotCaptureAnnotations } from '@agent-device/contracts/capture'; -import { formatGestureNoEffectWarning } from '@agent-device/capture-kit/post-gesture-stability'; - -/** - * #1600: a proven no-effect gesture must reach the agent inside the very - * response it reads next, not only the diagnostics stream. Warnings ride the - * existing annotations channel so every renderer that already prints capture - * warnings picks this up with no new plumbing. - */ -export function withGestureNoEffectWarning( - annotations: SnapshotCaptureAnnotations, - gestureNoEffect: { action: string; positionals: string[] } | undefined, -): SnapshotCaptureAnnotations { - if (!gestureNoEffect) return annotations; - return { - ...annotations, - warnings: [ - ...(annotations.warnings ?? []), - formatGestureNoEffectWarning(gestureNoEffect.action, gestureNoEffect.positionals), - ], - }; -} diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 97bbe343ca..9a8bac5f53 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -559,6 +559,7 @@ agent-device is text 'id="greeting"' "Welcome back" - `is exists` only checks whether the selector matches in the current snapshot. - `is absent` passes only when the selector has zero matches in one readable, complete, settled, unscoped, full-depth accessibility capture. It does not mean hidden; `--scope` and `--depth` are rejected, and sparse, unreadable, truncated, or unsettled captures fail closed. - A read that answers from the first capture after a `scroll`, `swipe`, or `gesture swipe` waits for two consecutive captures to agree. When the surface is still changing when that budget runs out, `is`, `get`, `find`, `wait`, and an interaction that captured it (such as `click`, `press`, or `fill`) carry `unsettledGesture` (`{ "action", "positionals" }`) in `error.details` or `data` and append a warning, and `snapshot` appends the warning. A miss on that capture is not proof of absence: read again. `is absent` refuses it with `observation: "unsettled"`, and `wait absent` keeps polling. +- When that first capture still matches the tree from before the gesture, the gesture is reported as having no visible effect: the same readers carry `gestureNoEffect` (`{ "action", "positionals" }`) with an appended warning, and `snapshot` appends the warning. The container may be at its edge or may ignore synthesized scrolls; a raw `swipe x1 y1 x2 y2` inside the list moves such lists. - `wait text` is a text-presence wait, not a hittability assertion. - Strict `wait absent` is not exported to Maestro's lenient `notVisible` condition; Maestro export reports it as unsupported unless an exact zero-candidate primitive becomes available. - `is text ` compares the resolved element text against the expected value.