From a9b3878bab71c7f0f6e5750e8315f9bfb7754e91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 09:51:39 +0200 Subject: [PATCH] fix(selectors): retire the stored tree after a side effect or a native read Selector reads reuse the session's stored tree for 750 ms after its capture. Two events outdated that tree without replacing it: a wait text satisfied by the owner's native text reading after an earlier miss capture, and a mutation that captured nothing. The next get or is then reused the pre-change tree and reported Selector did not match. Every side-effect seam (expireRefFrame) and every native text reading now marks the session's current tree outdated, and the session cache tier refuses an outdated tree. The mark is keyed by the tree, so it survives the session record copies that reset the per-session runtime revision. Closes #2866 --- CHANGELOG.md | 7 +++ docs/agents/selector-capture.md | 4 +- src/daemon/__tests__/wait-runtime.test.ts | 48 +++++++++++++++++ .../__tests__/interaction-touch-press.test.ts | 53 +++++++++++++++++++ src/daemon/ref-frame.ts | 16 ++++++ src/daemon/selector-capture-runtime.ts | 3 +- src/daemon/selector-runtime-backend.ts | 21 ++++---- 7 files changed, 140 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45a21de44e..b94914b0e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ (`apple_toolchain_probe_unavailable`) now fails once, not three times with a fresh startup budget each time. A `retriable` failure from an external Apple runner provider is also no longer resent. The error keeps `retriable: true`, so the caller's next request still tries again. (#2862) +- Fixed (mobile): a `get`, `is`, `find`, or `wait` selector read no longer answers from a tree that + a later command outdated. Selector reads reuse the session's stored tree for 750 ms after its + capture. Two cases outdated that tree without replacing it: a `wait text` satisfied by the + owner's native text reading after an earlier miss capture, and a mutation that captured nothing + (for example a coordinate `press`). The next read then reused the pre-change tree and could + report `Selector did not match` for an element that was on screen. A side effect or a native read + now retires the stored tree, and the next read captures again. - Fixed (mobile): a read taken right after a `scroll`, `swipe`, or `gesture swipe` no longer reports a definite miss when the surface never settled. When post-gesture stabilization ran out of budget on a surface still moving, `is visible` answered a plain `selector_not_found` and `is absent` diff --git a/docs/agents/selector-capture.md b/docs/agents/selector-capture.md index 44c3b89ccf..6a6411f457 100644 --- a/docs/agents/selector-capture.md +++ b/docs/agents/selector-capture.md @@ -9,7 +9,9 @@ These are cross-route behavior requirements; their rationale and owning decision resolution; ambiguity and other runner failures remain failures. - Regular selector reads are capture-backed. `@ref` resolves against its authorized ref frame, while `get`, `is`, `find`, and `wait` selectors capture through the backend. Polling bypasses the - snapshot cache, as do active freshness recovery and stabilization. + snapshot cache, as do active freshness recovery and stabilization. The cache serves a stored tree + only while it is the newest observation: a side-effect seam or a native read such as `wait + text`'s owner text reading retires it. - Sparse capture verdicts are observable failures and never replace the session snapshot. Only a user-facing snapshot may publish a fallback screenshot; internal polling must not create one artifact per attempt. diff --git a/src/daemon/__tests__/wait-runtime.test.ts b/src/daemon/__tests__/wait-runtime.test.ts index 4177872305..fe84f4fa1c 100644 --- a/src/daemon/__tests__/wait-runtime.test.ts +++ b/src/daemon/__tests__/wait-runtime.test.ts @@ -26,6 +26,7 @@ import { unavailableDeploymentSnapshotAndShutdownOperationFacts } from '../../__ import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; import { handleSnapshotCommands } from '../handlers/snapshot.ts'; import { resolveBoundSelectorCapture } from '../selector-capture-binding.ts'; +import { dispatchGetViaRuntime } from '../selector-runtime.ts'; import type { DaemonRequest } from '../daemon-request.ts'; const webDevice = { @@ -389,6 +390,53 @@ test('a text wait is satisfied by the owner native reading when the tree never c expect(harness.captureSnapshot).toHaveBeenCalled(); }); +test('a read after a natively satisfied text wait captures instead of reusing the older tree', async () => { + // Poll 1: the native reading misses and the capture still shows the previous screen, which it + // publishes to the session. The app then navigates, and poll 2's native reading sees the + // destination, so the stored tree is older than the observation that satisfied the wait. + let nativeReads = 0; + const harness = waitRuntimeHarness({ + findText: available, + findTextAnswers: () => { + nativeReads += 1; + return nativeReads > 1; + }, + nodesPerPoll: [ + [{ index: 0, depth: 0, type: 'StaticText', label: 'Home' }], + [ + { index: 0, depth: 0, type: 'StaticText', label: 'Automation lab' }, + { index: 1, depth: 0, type: 'StaticText', label: 'cold.start' }, + ], + ], + }); + const { + response: waited, + session, + sessionStore, + } = await runWait(['text', 'Automation lab', '2000'], harness); + expect(waited).toMatchObject({ ok: true, data: { text: 'Automation lab' } }); + expect(harness.captureSnapshot).toHaveBeenCalledOnce(); + + const read = await dispatchGetViaRuntime({ + req: { + command: 'get', + positionals: ['text', 'label="cold.start"'], + token: 't', + session: session.name, + flags: {}, + meta: { requestId: 'wait-runtime-get' }, + } as unknown as DaemonRequest, + sessionName: session.name, + logPath: '/tmp/daemon.log', + sessionStore, + inspectFacts: harness.inspectFacts, + bindDevice: harness.bindDevice, + }); + + expect(read).toMatchObject({ ok: true, data: { text: 'cold.start' } }); + expect(harness.captureSnapshot).toHaveBeenCalledTimes(2); +}); + test('a satisfied native reading short-circuits the poll without capturing', async () => { const harness = waitRuntimeHarness({ findText: available, diff --git a/src/daemon/interaction/internal/__tests__/interaction-touch-press.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-press.test.ts index d12d43c8eb..be2046aeda 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-touch-press.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-press.test.ts @@ -1,5 +1,6 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; +import { makeIosSession } from '../../../../__tests__/test-utils/session-factories.ts'; import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; import { handleInteractionCommands } from '../../index.ts'; import { @@ -391,3 +392,55 @@ test('#1654: the shared guards still run on the pre-resolved node', async () => } expect(readPressPoint(mockTapPoint)).toBeUndefined(); }); + +test('a read right after a press captures the post-tap screen instead of reusing the pre-tap tree', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'read-after-press'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + const screen = (step: string) => ({ + backend: 'xctest' as const, + producer: 'apple-runner' as const, + nodes: [ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 393, height: 852 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Next', + rect: { x: 24, y: 120, width: 120, height: 44 }, + enabled: true, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'StaticText', + label: step, + identifier: 'step', + rect: { x: 24, y: 220, width: 320, height: 24 }, + }, + ], + }); + mockCaptureSnapshotForSession + .mockResolvedValueOnce(screen('Step 1')) + .mockResolvedValue(screen('Step 2')); + const run = async (command: string, positionals: string[]) => + await handleInteractionCommands({ + req: { token: 't', session: sessionName, command, positionals, flags: {} }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(await run('is', ['visible', 'label=Next'])).toMatchObject({ ok: true }); + expect(await run('press', ['84', '142'])).toMatchObject({ ok: true }); + expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1); + + const read = await run('get', ['text', 'id="step"']); + + expect(read).toMatchObject({ ok: true, data: { text: 'Step 2' } }); + expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(2); +}); diff --git a/src/daemon/ref-frame.ts b/src/daemon/ref-frame.ts index bdb5e2e341..b51c332084 100644 --- a/src/daemon/ref-frame.ts +++ b/src/daemon/ref-frame.ts @@ -10,6 +10,7 @@ import { import type { SessionState } from './session-state.ts'; const runtimeRevisions = new WeakMap(); +const outdatedObservations = new WeakSet(); /** * ADR 0014 session ref-frame lifetime — the authorization model for mutation @@ -89,10 +90,25 @@ export function refFrameTree(session: SessionState): SnapshotState | undefined { */ export function expireRefFrame(session: SessionState): void { advanceSessionRuntimeRevision(session); + markSessionSnapshotOutdated(session); session.refFrame = expiredRefFrame(refFrame(session)); session.snapshotScopeSource = undefined; } +/** + * Record that the device was observed or changed after the session's stored tree was + * captured: a side-effect seam above, or a native read that produces no tree (such as `wait + * text`'s owner text reading). An outdated tree stays the session's latest stored observation, + * but a selector read never reuses it in place of a capture. + */ +export function markSessionSnapshotOutdated(session: SessionState): void { + if (session.snapshot) outdatedObservations.add(session.snapshot); +} + +export function isOutdatedObservation(snapshot: SnapshotState): boolean { + return outdatedObservations.has(snapshot); +} + /** * Monotonic, daemon-private revision for side-effect lineage. Unlike the * client-visible snapshot/ref generations, this advances for every possible diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index 1b24f5510a..bfbe7849c5 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -15,6 +15,7 @@ import { captureSnapshot } from './snapshot-capture.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; import { getActiveAndroidSnapshotFreshness } from './session-snapshot-freshness.ts'; import { isPostGestureStabilizationPending } from './deferred-interaction-outcome.ts'; +import { isOutdatedObservation } from './ref-frame.ts'; import type { BoundSelectorCapture } from './selector-capture-binding.ts'; import { buildRuntimeCaptureInput } from './snapshot-runtime-capture-input.ts'; import { isLegacySparseIosInteractiveSnapshot } from '@agent-device/selectors/absence-observation'; @@ -258,7 +259,7 @@ function reusableSessionSnapshot(params: { }): SnapshotState | undefined { const { session, timestamp, request } = params; const snapshot = session?.snapshot; - if (!snapshot) return undefined; + if (!snapshot || isOutdatedObservation(snapshot)) return undefined; if (!canUseSessionSnapshotCache(session, request)) return undefined; if (!isFreshSelectorSnapshot(snapshot, timestamp)) return undefined; if (snapshot.presentationKey !== presentationKeyFor(request)) return undefined; diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index a256ba0ec1..2348d31c3f 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -12,6 +12,7 @@ import { createDaemonRuntimeSessionStore } from './runtime-session.ts'; import { contextFromFlags, type BoundContextFromFlags } from './context.ts'; import { readTextForNode } from './selector-text-runtime.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; +import { markSessionSnapshotOutdated } from './ref-frame.ts'; import { SessionStore } from './session-store.ts'; import type { DaemonRequest, DaemonResponse } from './daemon-request.ts'; import type { SessionState } from './session-state.ts'; @@ -220,16 +221,16 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice // reports `found: false` and the poll consults the canonical tree. ...(boundFindText ? { - findText: async (context: BackendCommandContext, text: string) => ({ - found: ( - await boundFindText({ - text, - options: { appBundleId: session?.appBundleId, surface: session?.surface }, - execution: runnerExecution, - ...(context.signal ? { signal: context.signal } : {}), - }) - ).found, - }), + findText: async (context: BackendCommandContext, text: string) => { + const { found } = await boundFindText({ + text, + options: { appBundleId: session?.appBundleId, surface: session?.surface }, + execution: runnerExecution, + ...(context.signal ? { signal: context.signal } : {}), + }); + if (session) markSessionSnapshotOutdated(session); + return { found }; + }, } : {}), };