Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@
(`apple_toolchain_probe_unavailable`) now fails once, not three times with a fresh startup budget
each time. A `retriable` failure from an external Apple runner provider is also no longer resent.
The error keeps `retriable: true`, so the caller's next request still tries again. (#2862)
- Fixed (mobile): a `get`, `is`, `find`, or `wait` selector read no longer answers from a tree that
a later command outdated. Selector reads reuse the session's stored tree for 750 ms after its
capture. Two cases outdated that tree without replacing it: a `wait text` satisfied by the
owner's native text reading after an earlier miss capture, and a mutation that captured nothing
(for example a coordinate `press`). The next read then reused the pre-change tree and could
report `Selector did not match` for an element that was on screen. A side effect or a native read
now retires the stored tree, and the next read captures again.
- Fixed (mobile): a read taken right after a `scroll`, `swipe`, or `gesture swipe` no longer reports
a definite miss when the surface never settled. When post-gesture stabilization ran out of budget
on a surface still moving, `is visible` answered a plain `selector_not_found` and `is absent`
Expand Down
4 changes: 3 additions & 1 deletion docs/agents/selector-capture.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ These are cross-route behavior requirements; their rationale and owning decision
resolution; ambiguity and other runner failures remain failures.
- Regular selector reads are capture-backed. `@ref` resolves against its authorized ref frame,
while `get`, `is`, `find`, and `wait` selectors capture through the backend. Polling bypasses the
snapshot cache, as do active freshness recovery and stabilization.
snapshot cache, as do active freshness recovery and stabilization. The cache serves a stored tree
only while it is the newest observation: a side-effect seam or a native read such as `wait
text`'s owner text reading retires it.
- Sparse capture verdicts are observable failures and never replace the session snapshot. Only a
user-facing snapshot may publish a fallback screenshot; internal polling must not create one
artifact per attempt.
Expand Down
48 changes: 48 additions & 0 deletions src/daemon/__tests__/wait-runtime.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import { unavailableDeploymentSnapshotAndShutdownOperationFacts } from '../../__
import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts';
import { handleSnapshotCommands } from '../handlers/snapshot.ts';
import { resolveBoundSelectorCapture } from '../selector-capture-binding.ts';
import { dispatchGetViaRuntime } from '../selector-runtime.ts';
import type { DaemonRequest } from '../daemon-request.ts';

const webDevice = {
Expand Down Expand Up @@ -389,6 +390,53 @@ test('a text wait is satisfied by the owner native reading when the tree never c
expect(harness.captureSnapshot).toHaveBeenCalled();
});

test('a read after a natively satisfied text wait captures instead of reusing the older tree', async () => {
// Poll 1: the native reading misses and the capture still shows the previous screen, which it
// publishes to the session. The app then navigates, and poll 2's native reading sees the
// destination, so the stored tree is older than the observation that satisfied the wait.
let nativeReads = 0;
const harness = waitRuntimeHarness({
findText: available,
findTextAnswers: () => {
nativeReads += 1;
return nativeReads > 1;
},
nodesPerPoll: [
[{ index: 0, depth: 0, type: 'StaticText', label: 'Home' }],
[
{ index: 0, depth: 0, type: 'StaticText', label: 'Automation lab' },
{ index: 1, depth: 0, type: 'StaticText', label: 'cold.start' },
],
],
});
const {
response: waited,
session,
sessionStore,
} = await runWait(['text', 'Automation lab', '2000'], harness);
expect(waited).toMatchObject({ ok: true, data: { text: 'Automation lab' } });
expect(harness.captureSnapshot).toHaveBeenCalledOnce();

const read = await dispatchGetViaRuntime({
req: {
command: 'get',
positionals: ['text', 'label="cold.start"'],
token: 't',
session: session.name,
flags: {},
meta: { requestId: 'wait-runtime-get' },
} as unknown as DaemonRequest,
sessionName: session.name,
logPath: '/tmp/daemon.log',
sessionStore,
inspectFacts: harness.inspectFacts,
bindDevice: harness.bindDevice,
});

expect(read).toMatchObject({ ok: true, data: { text: 'cold.start' } });
expect(harness.captureSnapshot).toHaveBeenCalledTimes(2);
});

test('a satisfied native reading short-circuits the poll without capturing', async () => {
const harness = waitRuntimeHarness({
findText: available,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { test, expect, vi, beforeEach } from 'vitest';
import { attachRefs } from '@agent-device/kernel/snapshot';
import { makeIosSession } from '../../../../__tests__/test-utils/session-factories.ts';
import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts';
import { handleInteractionCommands } from '../../index.ts';
import {
Expand Down Expand Up @@ -391,3 +392,55 @@ test('#1654: the shared guards still run on the pre-resolved node', async () =>
}
expect(readPressPoint(mockTapPoint)).toBeUndefined();
});

test('a read right after a press captures the post-tap screen instead of reusing the pre-tap tree', async () => {
const sessionStore = makeSessionStore();
const sessionName = 'read-after-press';
sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' }));
const screen = (step: string) => ({
backend: 'xctest' as const,
producer: 'apple-runner' as const,
nodes: [
{ index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 393, height: 852 } },
{
index: 1,
depth: 1,
parentIndex: 0,
type: 'Button',
label: 'Next',
rect: { x: 24, y: 120, width: 120, height: 44 },
enabled: true,
hittable: true,
},
{
index: 2,
depth: 1,
parentIndex: 0,
type: 'StaticText',
label: step,
identifier: 'step',
rect: { x: 24, y: 220, width: 320, height: 24 },
},
],
});
mockCaptureSnapshotForSession
.mockResolvedValueOnce(screen('Step 1'))
.mockResolvedValue(screen('Step 2'));
const run = async (command: string, positionals: string[]) =>
await handleInteractionCommands({
req: { token: 't', session: sessionName, command, positionals, flags: {} },
sessionName,
sessionStore,
contextFromFlags,
...getRuntimeBindings(),
});

expect(await run('is', ['visible', 'label=Next'])).toMatchObject({ ok: true });
expect(await run('press', ['84', '142'])).toMatchObject({ ok: true });
expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(1);

const read = await run('get', ['text', 'id="step"']);

expect(read).toMatchObject({ ok: true, data: { text: 'Step 2' } });
expect(mockCaptureSnapshotForSession).toHaveBeenCalledTimes(2);
});
16 changes: 16 additions & 0 deletions src/daemon/ref-frame.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
import type { SessionState } from './session-state.ts';

const runtimeRevisions = new WeakMap<SessionState, number>();
const outdatedObservations = new WeakSet<SnapshotState>();

/**
* ADR 0014 session ref-frame lifetime — the authorization model for mutation
Expand Down Expand Up @@ -89,10 +90,25 @@ export function refFrameTree(session: SessionState): SnapshotState | undefined {
*/
export function expireRefFrame(session: SessionState): void {
advanceSessionRuntimeRevision(session);
markSessionSnapshotOutdated(session);
session.refFrame = expiredRefFrame(refFrame(session));
session.snapshotScopeSource = undefined;
}

/**
* Record that the device was observed or changed after the session's stored tree was
* captured: a side-effect seam above, or a native read that produces no tree (such as `wait
* text`'s owner text reading). An outdated tree stays the session's latest stored observation,
* but a selector read never reuses it in place of a capture.
*/
export function markSessionSnapshotOutdated(session: SessionState): void {
if (session.snapshot) outdatedObservations.add(session.snapshot);
}

export function isOutdatedObservation(snapshot: SnapshotState): boolean {
return outdatedObservations.has(snapshot);
}

/**
* Monotonic, daemon-private revision for side-effect lineage. Unlike the
* client-visible snapshot/ref generations, this advances for every possible
Expand Down
3 changes: 2 additions & 1 deletion src/daemon/selector-capture-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { captureSnapshot } from './snapshot-capture.ts';
import { setSessionSnapshot } from './session-snapshot.ts';
import { getActiveAndroidSnapshotFreshness } from './session-snapshot-freshness.ts';
import { isPostGestureStabilizationPending } from './deferred-interaction-outcome.ts';
import { isOutdatedObservation } from './ref-frame.ts';
import type { BoundSelectorCapture } from './selector-capture-binding.ts';
import { buildRuntimeCaptureInput } from './snapshot-runtime-capture-input.ts';
import { isLegacySparseIosInteractiveSnapshot } from '@agent-device/selectors/absence-observation';
Expand Down Expand Up @@ -258,7 +259,7 @@ function reusableSessionSnapshot(params: {
}): SnapshotState | undefined {
const { session, timestamp, request } = params;
const snapshot = session?.snapshot;
if (!snapshot) return undefined;
if (!snapshot || isOutdatedObservation(snapshot)) return undefined;
if (!canUseSessionSnapshotCache(session, request)) return undefined;
if (!isFreshSelectorSnapshot(snapshot, timestamp)) return undefined;
if (snapshot.presentationKey !== presentationKeyFor(request)) return undefined;
Expand Down
21 changes: 11 additions & 10 deletions src/daemon/selector-runtime-backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { createDaemonRuntimeSessionStore } from './runtime-session.ts';
import { contextFromFlags, type BoundContextFromFlags } from './context.ts';
import { readTextForNode } from './selector-text-runtime.ts';
import { setSessionSnapshot } from './session-snapshot.ts';
import { markSessionSnapshotOutdated } from './ref-frame.ts';
import { SessionStore } from './session-store.ts';
import type { DaemonRequest, DaemonResponse } from './daemon-request.ts';
import type { SessionState } from './session-state.ts';
Expand Down Expand Up @@ -220,16 +221,16 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice
// reports `found: false` and the poll consults the canonical tree.
...(boundFindText
? {
findText: async (context: BackendCommandContext, text: string) => ({
found: (
await boundFindText({
text,
options: { appBundleId: session?.appBundleId, surface: session?.surface },
execution: runnerExecution,
...(context.signal ? { signal: context.signal } : {}),
})
).found,
}),
findText: async (context: BackendCommandContext, text: string) => {
const { found } = await boundFindText({
text,
options: { appBundleId: session?.appBundleId, surface: session?.surface },
execution: runnerExecution,
...(context.signal ? { signal: context.signal } : {}),
});
if (session) markSessionSnapshotOutdated(session);
return { found };
},
}
: {}),
};
Expand Down
Loading