diff --git a/CHANGELOG.md b/CHANGELOG.md index 45a21de44e..b94914b0e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ (`apple_toolchain_probe_unavailable`) now fails once, not three times with a fresh startup budget each time. A `retriable` failure from an external Apple runner provider is also no longer resent. The error keeps `retriable: true`, so the caller's next request still tries again. (#2862) +- Fixed (mobile): a `get`, `is`, `find`, or `wait` selector read no longer answers from a tree that + a later command outdated. Selector reads reuse the session's stored tree for 750 ms after its + capture. Two cases outdated that tree without replacing it: a `wait text` satisfied by the + owner's native text reading after an earlier miss capture, and a mutation that captured nothing + (for example a coordinate `press`). The next read then reused the pre-change tree and could + report `Selector did not match` for an element that was on screen. A side effect or a native read + now retires the stored tree, and the next read captures again. - Fixed (mobile): a read taken right after a `scroll`, `swipe`, or `gesture swipe` no longer reports a definite miss when the surface never settled. When post-gesture stabilization ran out of budget on a surface still moving, `is visible` answered a plain `selector_not_found` and `is absent` diff --git a/docs/agents/selector-capture.md b/docs/agents/selector-capture.md index 44c3b89ccf..6a6411f457 100644 --- a/docs/agents/selector-capture.md +++ b/docs/agents/selector-capture.md @@ -9,7 +9,9 @@ These are cross-route behavior requirements; their rationale and owning decision resolution; ambiguity and other runner failures remain failures. - Regular selector reads are capture-backed. `@ref` resolves against its authorized ref frame, while `get`, `is`, `find`, and `wait` selectors capture through the backend. Polling bypasses the - snapshot cache, as do active freshness recovery and stabilization. + snapshot cache, as do active freshness recovery and stabilization. The cache serves a stored tree + only while it is the newest observation: a side-effect seam or a native read such as `wait + text`'s owner text reading retires it. - Sparse capture verdicts are observable failures and never replace the session snapshot. Only a user-facing snapshot may publish a fallback screenshot; internal polling must not create one artifact per attempt. diff --git a/src/daemon/__tests__/wait-runtime.test.ts b/src/daemon/__tests__/wait-runtime.test.ts index 4177872305..fe84f4fa1c 100644 --- a/src/daemon/__tests__/wait-runtime.test.ts +++ b/src/daemon/__tests__/wait-runtime.test.ts @@ -26,6 +26,7 @@ import { unavailableDeploymentSnapshotAndShutdownOperationFacts } from '../../__ import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; import { handleSnapshotCommands } from '../handlers/snapshot.ts'; import { resolveBoundSelectorCapture } from '../selector-capture-binding.ts'; +import { dispatchGetViaRuntime } from '../selector-runtime.ts'; import type { DaemonRequest } from '../daemon-request.ts'; const webDevice = { @@ -389,6 +390,53 @@ test('a text wait is satisfied by the owner native reading when the tree never c expect(harness.captureSnapshot).toHaveBeenCalled(); }); +test('a read after a natively satisfied text wait captures instead of reusing the older tree', async () => { + // Poll 1: the native reading misses and the capture still shows the previous screen, which it + // publishes to the session. The app then navigates, and poll 2's native reading sees the + // destination, so the stored tree is older than the observation that satisfied the wait. + let nativeReads = 0; + const harness = waitRuntimeHarness({ + findText: available, + findTextAnswers: () => { + nativeReads += 1; + return nativeReads > 1; + }, + nodesPerPoll: [ + [{ index: 0, depth: 0, type: 'StaticText', label: 'Home' }], + [ + { index: 0, depth: 0, type: 'StaticText', label: 'Automation lab' }, + { index: 1, depth: 0, type: 'StaticText', label: 'cold.start' }, + ], + ], + }); + const { + response: waited, + session, + sessionStore, + } = await runWait(['text', 'Automation lab', '2000'], harness); + expect(waited).toMatchObject({ ok: true, data: { text: 'Automation lab' } }); + expect(harness.captureSnapshot).toHaveBeenCalledOnce(); + + const read = await dispatchGetViaRuntime({ + req: { + command: 'get', + positionals: ['text', 'label="cold.start"'], + token: 't', + session: session.name, + flags: {}, + meta: { requestId: 'wait-runtime-get' }, + } as unknown as DaemonRequest, + sessionName: session.name, + logPath: '/tmp/daemon.log', + sessionStore, + inspectFacts: harness.inspectFacts, + bindDevice: harness.bindDevice, + }); + + expect(read).toMatchObject({ ok: true, data: { text: 'cold.start' } }); + expect(harness.captureSnapshot).toHaveBeenCalledTimes(2); +}); + test('a satisfied native reading short-circuits the poll without capturing', async () => { const harness = waitRuntimeHarness({ findText: available, diff --git a/src/daemon/interaction/internal/__tests__/interaction-touch-press.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-press.test.ts index d12d43c8eb..be2046aeda 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-touch-press.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-press.test.ts @@ -1,5 +1,6 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; +import { makeIosSession } from '../../../../__tests__/test-utils/session-factories.ts'; import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; import { handleInteractionCommands } from '../../index.ts'; import { @@ -391,3 +392,55 @@ test('#1654: the shared guards still run on the pre-resolved node', async () => } expect(readPressPoint(mockTapPoint)).toBeUndefined(); }); + +test('a read right after a press captures the post-tap screen instead of reusing the pre-tap tree', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'read-after-press'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + const screen = (step: string) => ({ + backend: 'xctest' as const, + producer: 'apple-runner' as const, + nodes: [ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 393, height: 852 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Next', + rect: { x: 24, y: 120, width: 120, height: 44 }, + enabled: true, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'StaticText', + label: step, + identifier: 'step', + rect: { x: 24, y: 220, width: 320, height: 24 }, + }, + ], + }); + mockCaptureSnapshotForSession + .mockResolvedValueOnce(screen('Step 1')) + .mockResolvedValue(screen('Step 2')); + const run = async (command: string, positionals: string[]) => + await handleInteractionCommands({ + req: { token: 't', session: sessionName, command, positionals, flags: {} }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(await run('is', ['visible', 'label=Next'])).toMatchObject({ ok: true }); + expect(await run('press', ['84', '142'])).toMatchObject({ ok: true }); + expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); + + const read = await run('get', ['text', 'id="step"']); + + expect(read).toMatchObject({ ok: true, data: { text: 'Step 2' } }); + expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(2); +}); diff --git a/src/daemon/ref-frame.ts b/src/daemon/ref-frame.ts index bdb5e2e341..b51c332084 100644 --- a/src/daemon/ref-frame.ts +++ b/src/daemon/ref-frame.ts @@ -10,6 +10,7 @@ import { import type { SessionState } from './session-state.ts'; const runtimeRevisions = new WeakMap(); +const outdatedObservations = new WeakSet(); /** * ADR 0014 session ref-frame lifetime — the authorization model for mutation @@ -89,10 +90,25 @@ export function refFrameTree(session: SessionState): SnapshotState | undefined { */ export function expireRefFrame(session: SessionState): void { advanceSessionRuntimeRevision(session); + markSessionSnapshotOutdated(session); session.refFrame = expiredRefFrame(refFrame(session)); session.snapshotScopeSource = undefined; } +/** + * Record that the device was observed or changed after the session's stored tree was + * captured: a side-effect seam above, or a native read that produces no tree (such as `wait + * text`'s owner text reading). An outdated tree stays the session's latest stored observation, + * but a selector read never reuses it in place of a capture. + */ +export function markSessionSnapshotOutdated(session: SessionState): void { + if (session.snapshot) outdatedObservations.add(session.snapshot); +} + +export function isOutdatedObservation(snapshot: SnapshotState): boolean { + return outdatedObservations.has(snapshot); +} + /** * Monotonic, daemon-private revision for side-effect lineage. Unlike the * client-visible snapshot/ref generations, this advances for every possible diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index 1b24f5510a..bfbe7849c5 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -15,6 +15,7 @@ import { captureSnapshot } from './snapshot-capture.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; import { getActiveAndroidSnapshotFreshness } from './session-snapshot-freshness.ts'; import { isPostGestureStabilizationPending } from './deferred-interaction-outcome.ts'; +import { isOutdatedObservation } from './ref-frame.ts'; import type { BoundSelectorCapture } from './selector-capture-binding.ts'; import { buildRuntimeCaptureInput } from './snapshot-runtime-capture-input.ts'; import { isLegacySparseIosInteractiveSnapshot } from '@agent-device/selectors/absence-observation'; @@ -258,7 +259,7 @@ function reusableSessionSnapshot(params: { }): SnapshotState | undefined { const { session, timestamp, request } = params; const snapshot = session?.snapshot; - if (!snapshot) return undefined; + if (!snapshot || isOutdatedObservation(snapshot)) return undefined; if (!canUseSessionSnapshotCache(session, request)) return undefined; if (!isFreshSelectorSnapshot(snapshot, timestamp)) return undefined; if (snapshot.presentationKey !== presentationKeyFor(request)) return undefined; diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index a256ba0ec1..2348d31c3f 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -12,6 +12,7 @@ import { createDaemonRuntimeSessionStore } from './runtime-session.ts'; import { contextFromFlags, type BoundContextFromFlags } from './context.ts'; import { readTextForNode } from './selector-text-runtime.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; +import { markSessionSnapshotOutdated } from './ref-frame.ts'; import { SessionStore } from './session-store.ts'; import type { DaemonRequest, DaemonResponse } from './daemon-request.ts'; import type { SessionState } from './session-state.ts'; @@ -220,16 +221,16 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice // reports `found: false` and the poll consults the canonical tree. ...(boundFindText ? { - findText: async (context: BackendCommandContext, text: string) => ({ - found: ( - await boundFindText({ - text, - options: { appBundleId: session?.appBundleId, surface: session?.surface }, - execution: runnerExecution, - ...(context.signal ? { signal: context.signal } : {}), - }) - ).found, - }), + findText: async (context: BackendCommandContext, text: string) => { + const { found } = await boundFindText({ + text, + options: { appBundleId: session?.appBundleId, surface: session?.surface }, + execution: runnerExecution, + ...(context.signal ? { signal: context.signal } : {}), + }); + if (session) markSessionSnapshotOutdated(session); + return { found }; + }, } : {}), };