From 4cde58eb71ad8943dde4822ca7a86618a70dd64a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 16:41:00 +0200 Subject: [PATCH 1/2] fix(settle): observe a settled screen through the capture the request admitted --settle on back/scroll and on a coordinate-targeted press/fill polled the screen through the legacy interactor while every command the caller ran next (snapshot, diff, a selector press --settle, wait stable) read it through the request-bound runtime. On an iOS simulator those are two projections of one screen: the settled diff, the refs issued from it, and the settled: true verdict described a tree the next command could never resolve against, and a screen nobody had changed reported five additions and six removals. Both routes now admit the same capture-only plan the selector family polls under and the loop polls that one binding. An owner whose facts refuse the plan keeps observing through the interactor its own commands answer with, disclosed in the request's diagnostics. --- .../0019-request-bound-platform-runtime.md | 8 + src/daemon/__tests__/generic-settle.test.ts | 143 ++++++++-- .../__tests__/ios-comparison-key-fixture.ts | 32 ++- .../settle-observation-capture.test.ts | 137 +++++++++ src/daemon/generic-settle.ts | 20 +- src/daemon/interaction/index.ts | 12 +- .../interaction-ios-tap-outcome-fixtures.ts | 2 +- .../interaction-ios-tap-outcome.test.ts | 76 +++++ .../interaction-touch-prepare.test.ts | 261 ++++++++++++++++++ .../internal/interaction-ios-tap-outcome.ts | 20 +- .../internal/interaction-runtime.ts | 55 +++- .../internal/interaction-touch-fill.ts | 23 +- .../internal/interaction-touch-prepare.ts | 41 ++- .../internal/interaction-touch-press.ts | 1 + .../internal/interaction-touch-runtime.ts | 8 + src/daemon/interaction/internal/types.ts | 20 ++ src/daemon/request-generic-dispatch.ts | 8 + src/daemon/request-router.ts | 4 + src/daemon/session-runtime-admission.ts | 3 +- src/daemon/settle-observation-capture.ts | 100 +++++++ src/daemon/snapshot-runtime-binding.ts | 23 ++ 21 files changed, 918 insertions(+), 79 deletions(-) create mode 100644 src/daemon/__tests__/settle-observation-capture.test.ts create mode 100644 src/daemon/interaction/internal/__tests__/interaction-touch-prepare.test.ts create mode 100644 src/daemon/settle-observation-capture.ts diff --git a/docs/adr/0019-request-bound-platform-runtime.md b/docs/adr/0019-request-bound-platform-runtime.md index 49fbf69e06..45bd0f0d83 100644 --- a/docs/adr/0019-request-bound-platform-runtime.md +++ b/docs/adr/0019-request-bound-platform-runtime.md @@ -783,6 +783,14 @@ set to read facts — is retired for new units: they may not introduce admission an already-migrated handler onto facts inspection changes production admission behavior; it is its own independently reviewed unit with parity evidence, never a side effect of a structural change. +One second bind per request is permitted, and is not the retired idiom: a post-action observation +(`--settle`) whose action use carries no capture admits the `capture-only` plan for its own reads +(#3510). It is not an admission-only bind — it never reads facts through a binding, admits through +the facts probe, and binds only on admission — and where the action's use already required +`captureSnapshot` (`scroll top|bottom`, `scroll --until`) it projects the request's cached device +binding rather than acquiring a second one. It stays capture-only: a second bind that carries a +mutating operation is the execution use the plan should have selected. + Use declarations share one neutral `defineUse`; per-domain currying wrappers add a module per domain for no information and are not added. Required-only declarations are the default. Declaring a preferred operation requires a recorded measurement of the fast path's benefit in the unit diff --git a/src/daemon/__tests__/generic-settle.test.ts b/src/daemon/__tests__/generic-settle.test.ts index 3cc0f4ac1f..c745d57cdf 100644 --- a/src/daemon/__tests__/generic-settle.test.ts +++ b/src/daemon/__tests__/generic-settle.test.ts @@ -16,6 +16,7 @@ import type { SessionStore } from '../session-store.ts'; import type { DaemonRequest, DaemonResponse } from '../daemon-request.ts'; import type { SessionRef, SessionState } from '../session-state.ts'; import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; +import { selectorCaptureFixture } from './selector-capture-fixture.ts'; import { testCreateCommandSurface } from './command-surface-fixture.ts'; // #1638 `--settle` on the GENERIC daemon route (scroll/back): the settled diff, @@ -90,14 +91,21 @@ async function emulateCaptureSnapshotForSession( ref: SessionRef, flags: CommandFlags | undefined, sessionStore: SessionStore, - options: { interactiveOnly: boolean }, + options: { + interactiveOnly: boolean; + boundCapture?: Parameters[4]['boundCapture']; + }, ) { captureObservations.push({ postGestureStabilizationPending: sessionStore.requireCurrent(ref).postGestureStabilization !== undefined, }); const effectiveFlags = { ...(flags ?? {}), snapshotInteractiveOnly: options.interactiveOnly }; - const snapshotData = (await mockDispatch('snapshot')) as Parameters[0]; + // Same rule the real seam applies: a request-bound capture is the tree this request reads, and + // only a request that holds none dispatches for one (#3510). + const snapshotData = options.boundCapture + ? await options.boundCapture({}) + : ((await mockDispatch('snapshot')) as Parameters[0]); const snapshot = buildSnapshotState(snapshotData ?? {}, effectiveFlags); setSessionSnapshot(sessionStore.requireCurrent(ref), snapshot); return snapshot; @@ -138,8 +146,9 @@ function seedSession( sessionName: string, sessionStore: SessionStore, baseline: GenericSettleTree = { nodes: BEFORE_NODES }, + overrides?: Partial, ): SessionState { - const session = makeIosSession(sessionName); + const session = makeIosSession(sessionName, overrides); setSessionSnapshot(session, buildSnapshotState(snapshotPayload(baseline), {})); activateCompleteRefFrame(session); sessionStore.publish(sessionName, session); @@ -172,13 +181,28 @@ function mockSurfaceDispatch(tree: GenericSettleTree, result: Record[0], 'inspectFacts' | 'bindDevice'> + >; }): Promise { const req: DaemonRequest = { token: 't', @@ -197,6 +221,8 @@ async function dispatchGeneric(params: { sessionStore: params.sessionStore, contextFromFlags, executePlatformCommand: platformExecution, + ...legacyCaptureRuntimeBindings(), + ...params.runtimeBindings, }); } @@ -222,7 +248,7 @@ beforeEach(() => { test('scroll --settle answers with the settled diff against the stored pre-action tree', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-scroll'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); // Every settle capture sees the post-scroll tree; the diff baseline is the // tree already stored on the session, not a capture taken here. mockCommandDispatch([AFTER_NODES]); @@ -230,7 +256,6 @@ test('scroll --settle answers with the settled diff against the stored pre-actio const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'scroll', positionals: ['down'], flags: { ...SETTLE_FLAGS }, @@ -268,7 +293,7 @@ test('scroll --settle answers with the settled diff against the stored pre-actio test('back --settle answers with the settled diff alongside the command result', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-back'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); mockDispatch.mockImplementation(async (command) => { if (command === 'snapshot') return { nodes: AFTER_NODES, backend: 'xctest', producer: 'apple-runner' }; @@ -278,7 +303,6 @@ test('back --settle answers with the settled diff alongside the command result', const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'back', flags: { ...SETTLE_FLAGS }, }); @@ -382,7 +406,7 @@ function expectNoPublishedRefFrame(sessionStore: SessionStore, sessionName: stri test('scroll --settle attaches no diff across an app-to-sheet surface change and discloses it', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-scroll-to-sheet'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); // The stored pre-action tree is app content; every settle capture reads the sheet now presented // over that still-foreground app. mockSurfaceDispatch({ nodes: WEB_SIGN_IN_SHEET_NODES, systemSurface: WEB_SIGN_IN_SHEET }); @@ -390,7 +414,6 @@ test('scroll --settle attaches no diff across an app-to-sheet surface change and const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'scroll', positionals: ['down'], flags: { ...SETTLE_FLAGS }, @@ -412,7 +435,7 @@ test('scroll --settle attaches no diff across a sheet-to-app surface change and const sessionName = 'generic-settle-scroll-from-sheet'; // The baseline is the sheet this scroll acts on; the settled tree is the app content that // returns once the sheet completes and dismisses itself. - const session = seedSession(sessionName, sessionStore, { + seedSession(sessionName, sessionStore, { nodes: WEB_SIGN_IN_SHEET_NODES, systemSurface: WEB_SIGN_IN_SHEET, }); @@ -421,7 +444,6 @@ test('scroll --settle attaches no diff across a sheet-to-app surface change and const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'scroll', positionals: ['down'], flags: { ...SETTLE_FLAGS }, @@ -444,7 +466,7 @@ test('scroll --settle attaches no diff across a sheet-to-app surface change and test('back --settle attaches no diff across an app-to-sheet surface change and discloses it', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-back-to-sheet'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); mockSurfaceDispatch( { nodes: WEB_SIGN_IN_SHEET_NODES, systemSurface: WEB_SIGN_IN_SHEET }, { @@ -457,7 +479,6 @@ test('back --settle attaches no diff across an app-to-sheet surface change and d const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'back', flags: { ...SETTLE_FLAGS }, }); @@ -478,7 +499,7 @@ test('back --settle attaches no diff across an app-to-sheet surface change and d test('back --settle attaches no diff across a sheet-to-app surface change and discloses it', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-back-from-sheet'; - const session = seedSession(sessionName, sessionStore, { + seedSession(sessionName, sessionStore, { nodes: WEB_SIGN_IN_SHEET_NODES, systemSurface: WEB_SIGN_IN_SHEET, }); @@ -494,7 +515,6 @@ test('back --settle attaches no diff across a sheet-to-app surface change and di const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'back', flags: { ...SETTLE_FLAGS }, }); @@ -513,13 +533,12 @@ test('back --settle attaches no diff across a sheet-to-app surface change and di test('the settle observation runs after the post-gesture stabilization marker', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-order'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); mockCommandDispatch([AFTER_NODES]); await dispatchGeneric({ sessionName, sessionStore, - session, command: 'scroll', positionals: ['down'], flags: { ...SETTLE_FLAGS }, @@ -532,22 +551,93 @@ test('the settle observation runs after the post-gesture stabilization marker', expect(captureObservations[0]?.postGestureStabilizationPending).toBe(true); }); +test('scroll --settle observes through the request-bound capture, once admitted', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-bound-capture'; + // A tracked app: the plan this request must admit is the active-app one. The fixture refuses the + // without-active-app plan, so an observation that resolved the wrong cell would degrade to the + // legacy dispatch below and fail these assertions rather than pass by accident. + seedSession( + sessionName, + sessionStore, + { nodes: BEFORE_NODES }, + { appBundleId: 'com.example.app' }, + ); + mockCommandDispatch([AFTER_NODES]); + const runtimeBindings = selectorCaptureFixture({ + withoutActiveApp: { available: false, reason: 'owner-capability-missing' }, + snapshot: () => ({ + nodes: AFTER_NODES, + backend: 'xctest', + producer: 'simulator-ax-bridge', + }), + }); + + const response = await dispatchGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + runtimeBindings, + }); + + const settle = expectOkData(response).settle as SettlePayload; + expect(settle.settled).toBe(true); + // One admit-and-bind for the whole observation, and every capture taken through it: the settled + // tree and the tree the caller's next `snapshot` reads are the same producer's output (#3510). + expect(runtimeBindings.binds.length).toBe(1); + expect(runtimeBindings.captures.length).toBe(settle.captures); + expect(mockDispatch).not.toHaveBeenCalledWith('snapshot'); + expect((sessionStore.get(sessionName) as SessionState).snapshot?.producer).toBe( + 'simulator-ax-bridge', + ); +}); + +test('scroll --settle still observes when the owner cannot capture', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-unbound-capture'; + seedSession(sessionName, sessionStore); + mockCommandDispatch([AFTER_NODES]); + const runtimeBindings = selectorCaptureFixture({ + capture: { available: false, reason: 'unsupported-platform-leaf' }, + }); + + const response = await dispatchGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + runtimeBindings, + }); + + // The owner's own snapshot path is its only snapshot path: the observation stays complete and + // nothing pretends a second capture implementation produced it. + expect(expectOkData(response).settle).toMatchObject({ settled: true }); + expect(runtimeBindings.binds).toEqual([]); + expect(mockDispatch).toHaveBeenCalledWith('snapshot'); +}); + test('scroll without --settle takes no observation captures and issues no refs', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-off'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); mockCommandDispatch([AFTER_NODES]); + const runtimeBindings = selectorCaptureFixture(); const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'scroll', positionals: ['down'], + runtimeBindings, }); expect(expectOkData(response).settle).toBeUndefined(); expect(captureObservations).toEqual([]); + // Settling is what asks for an observation capture, so a plain scroll never pays for one. + expect(runtimeBindings.binds).toEqual([]); // ADR 0014: the leaf side-effect seam expired the frame and nothing // re-published it. expect(refFrameState(sessionStore.get(sessionName) as SessionState)).toBe('expired'); @@ -577,6 +667,7 @@ test('an ended generic lifetime is refused before dispatch or settle constructio logPath: '', contextFromFlags, executePlatformCommand: platformExecution, + ...legacyCaptureRuntimeBindings(), }), ).rejects.toMatchObject({ details: { reason: 'session_lifetime_ended' } }); expect(mockDispatch).not.toHaveBeenCalled(); @@ -586,7 +677,7 @@ test('an ended generic lifetime is refused before dispatch or settle constructio test('a generic command without the observation trait ignores a stray settle flag', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-traitless'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); mockDispatch.mockResolvedValue({ action: 'home', message: 'Home' }); // A hand-built daemon request can put settle flags on any command; only the @@ -595,7 +686,6 @@ test('a generic command without the observation trait ignores a stray settle fla const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'home', flags: { settle: true }, }); @@ -607,13 +697,12 @@ test('a generic command without the observation trait ignores a stray settle fla test('an orphaned --settle-quiet is rejected before the command dispatches', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-orphan'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); mockDispatch.mockRejectedValue(new Error('dispatch must not run for an orphaned settle flag')); const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'scroll', positionals: ['down'], flags: { settleQuietMs: 25 }, @@ -628,7 +717,7 @@ test('an orphaned --settle-quiet is rejected before the command dispatches', asy test('scroll stays successful when its lifetime retires during optional settle observation', async () => { const store = makeSessionStore(); const name = 'generic-settle-retired'; - const session = seedSession(name, store); + seedSession(name, store); const ref = store.lookup(name)!; mockCommandDispatch([AFTER_NODES]); mockCaptureSnapshotForSession.mockImplementationOnce( @@ -649,7 +738,6 @@ test('scroll stays successful when its lifetime retires during optional settle o const response = await dispatchGeneric({ sessionName: name, sessionStore: store, - session, command: 'scroll', positionals: ['down'], flags: { ...SETTLE_FLAGS }, @@ -663,7 +751,7 @@ for (const change of ['rebuild', 'retire'] as const) { test(`held generic settle capture respects its lifetime after ${change}`, async () => { const store = makeSessionStore(); const name = `generic-held-settle-${change}`; - const session = seedSession(name, store); + seedSession(name, store); const ref = store.lookup(name)!; let entered!: () => void; let release!: () => void; @@ -682,7 +770,6 @@ for (const change of ['rebuild', 'retire'] as const) { const pending = dispatchGeneric({ sessionName: name, sessionStore: store, - session, command: 'scroll', positionals: ['down'], flags: { ...SETTLE_FLAGS }, diff --git a/src/daemon/__tests__/ios-comparison-key-fixture.ts b/src/daemon/__tests__/ios-comparison-key-fixture.ts index b581749a42..95a39f7d74 100644 --- a/src/daemon/__tests__/ios-comparison-key-fixture.ts +++ b/src/daemon/__tests__/ios-comparison-key-fixture.ts @@ -3,29 +3,41 @@ import { createIosSnapshotRequest, iosSnapshotComparisonIdentityKey, } from '@agent-device/capture-kit/ios-snapshot-planning'; +import type { IosSnapshotComparisonIdentity } from '@agent-device/contracts/ios-snapshot'; import { IOS_SYSTEM_SURFACE_HOSTS } from '@agent-device/contracts/ios-system-surface'; /** - * The comparison keys the iOS snapshot route stamps on XCTest captures, built through the production - * key function: an app capture is lineaged to the app, a capture of an in-place system surface (a web - * sign-in sheet) to its HOST, because that is the surface it describes (#2438). Regressions that rely - * on a cross-surface pair being incomparable build both keys here, so they fail if the construction - * ever stops distinguishing them. + * The comparison identity the iOS snapshot route stamps on a capture it served from the XCTest + * runner, built through the production key function: an app capture is lineaged to the app, a + * capture of an in-place system surface (a web sign-in sheet) to its HOST, because that is the + * surface it describes (#2438). Regressions that rely on a cross-surface pair being incomparable + * build both here, so they fail if the construction ever stops distinguishing them. A test that + * needs both halves of one capture — a stored baseline keyed from it, and a capture result carrying + * it — takes the identity rather than rewriting the shape (#3510). */ +export function appCaptureComparisonIdentity( + deviceId: string, + appBundleId: string, +): IosSnapshotComparisonIdentity { + return runnerComparisonIdentity(`${deviceId}:${appBundleId}`); +} + export function appCaptureComparisonKey(deviceId: string, appBundleId: string): string { - return runnerComparisonKey(`${deviceId}:${appBundleId}`); + return iosSnapshotComparisonIdentityKey(appCaptureComparisonIdentity(deviceId, appBundleId)); } export function systemSurfaceCaptureComparisonKey(deviceId: string): string { - return runnerComparisonKey(`${deviceId}:${IOS_SYSTEM_SURFACE_HOSTS[0]!.bundleId}`); + return iosSnapshotComparisonIdentityKey( + runnerComparisonIdentity(`${deviceId}:${IOS_SYSTEM_SURFACE_HOSTS[0]!.bundleId}`), + ); } -function runnerComparisonKey(targetId: string): string { - return iosSnapshotComparisonIdentityKey({ +function runnerComparisonIdentity(targetId: string): IosSnapshotComparisonIdentity { + return { producer: 'apple-runner', intent: 'full', lineage: { targetId }, presentationKey: buildIosSnapshotPresentationKey(createIosSnapshotRequest()), residue: [{ kind: 'fallback-source', producer: 'apple-runner' }], - }); + }; } diff --git a/src/daemon/__tests__/settle-observation-capture.test.ts b/src/daemon/__tests__/settle-observation-capture.test.ts new file mode 100644 index 0000000000..7f74cc6def --- /dev/null +++ b/src/daemon/__tests__/settle-observation-capture.test.ts @@ -0,0 +1,137 @@ +import { beforeEach, expect, test, vi } from 'vitest'; +import { IOS_SIMULATOR } from '@agent-device/test-kit/device-fixtures'; +import { makeIosSession } from './support/session-factories.ts'; +import { daemonPlatformServicesFixture } from './platform-services-fixture.ts'; +import { selectorCaptureFixture } from './selector-capture-fixture.ts'; +import { resolveSettleObservationCapture } from '../settle-observation-capture.ts'; +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; + +vi.mock('@agent-device/host-kit/diagnostics', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, emitDiagnostic: vi.fn(actual.emitDiagnostic) }; +}); + +// #3510: the capture a `--settle` observation polls on a route whose action runtime use carries +// none. The guarantee it owes is producer identity with the command the caller runs next; the one it +// must not break is that settling never fails the action it follows. These pin which plan gets +// admitted, how often, and that every way an observation capture goes away degrades to no capture +// while disclosing its typed reason. + +const unavailablePhase = ['settle_observation_capture_unavailable']; + +/** Level included: a degrade the caller cannot see must not be info-only, which success drops. */ +function disclosures(): { level: string | undefined; data: Record }[] { + return vi + .mocked(emitDiagnostic) + .mock.calls.flatMap(([event]) => + event.phase === unavailablePhase[0] && event.data + ? [{ level: event.level, data: event.data }] + : [], + ); +} + +beforeEach(() => { + vi.mocked(emitDiagnostic).mockClear(); +}); + +test('an available plan inspects once and binds once for the whole observation', async () => { + const fixture = selectorCaptureFixture(); + + const capture = await resolveSettleObservationCapture({ + command: 'back', + session: makeIosSession('settle', { appBundleId: 'com.example.app' }), + inspectFacts: fixture.inspectFacts, + platformServices: daemonPlatformServicesFixture(), + bindDevice: fixture.bindDevice, + }); + if (!capture) throw new Error('expected an admitted capture'); + + expect(fixture.inspections).toEqual([IOS_SIMULATOR]); + expect(fixture.binds).toEqual([IOS_SIMULATOR]); + + await capture({}); + await capture({}); + expect(fixture.binds).toEqual([IOS_SIMULATOR]); + expect(fixture.captures).toHaveLength(2); +}); + +// One rule over the two plan cells: an owner that refuses the observation's plan keeps the loop on +// its own snapshot path, and the refusal is disclosed with the typed fact rather than passing as a +// silently unchanged observation. The first row is the Apple no-tracked-app cell, where the owner's +// own `snapshot` refuses for the same declared reason. +test.each([ + { + plan: 'without-active-app', + refusal: { withoutActiveApp: { available: false, reason: 'owner-capability-missing' } }, + session: () => makeIosSession('no-app'), + operation: 'captureSnapshotWithoutActiveApp', + reason: 'owner-capability-missing', + }, + { + plan: 'active-app', + refusal: { capture: { available: false, reason: 'unsupported-platform-leaf' } }, + session: () => makeIosSession('settle', { appBundleId: 'com.example.app' }), + operation: 'captureSnapshot', + reason: 'unsupported-platform-leaf', + }, +] as const)( + 'a $plan plan the owner refuses binds nothing and discloses its reason', + async ({ refusal, session, operation, reason }) => { + const fixture = selectorCaptureFixture(refusal); + + const capture = await resolveSettleObservationCapture({ + command: 'back', + session: session(), + inspectFacts: fixture.inspectFacts, + platformServices: daemonPlatformServicesFixture(), + bindDevice: fixture.bindDevice, + }); + + expect(capture).toBeUndefined(); + expect(fixture.binds).toEqual([]); + expect(disclosures()).toEqual([ + { level: 'warn', data: { command: 'back', outcome: 'refused', operation, reason } }, + ]); + }, +); + +// The two ways the owner's own machinery can fail the observation after a valid plan. Both are +// best-effort by contract: the command the loop follows must still run, and each is disclosed at +// `warn` — an info-only record is dropped on the success path, and these cells answer `ok: true`. +test.each([ + // Apple's inspection reads the host's screen-recording state and re-throws on a cancelled probe. + ['admission-probe-failed', Object.assign(new Error('probe cancelled'), { name: 'AbortError' })], + ['bind-failed', Object.assign(new Error('gateway closed'), { name: 'AppError' })], +] as const)( + 'a %s costs the action nothing and discloses its own failure', + async (outcome, failure) => { + const fixture = selectorCaptureFixture(); + const failing = + outcome === 'admission-probe-failed' + ? { + inspectFacts: async () => { + throw failure; + }, + bindDevice: fixture.bindDevice, + } + : { + inspectFacts: fixture.inspectFacts, + bindDevice: async () => { + throw failure; + }, + }; + + const capture = await resolveSettleObservationCapture({ + command: 'back', + session: makeIosSession('settle', { appBundleId: 'com.example.app' }), + platformServices: daemonPlatformServicesFixture(), + ...failing, + }); + + expect(capture).toBeUndefined(); + expect(fixture.binds).toEqual([]); + expect(disclosures()).toEqual([ + { level: 'warn', data: { command: 'back', outcome, errorName: failure.name } }, + ]); + }, +); diff --git a/src/daemon/generic-settle.ts b/src/daemon/generic-settle.ts index 91fc2e97ff..01559d9598 100644 --- a/src/daemon/generic-settle.ts +++ b/src/daemon/generic-settle.ts @@ -13,7 +13,9 @@ import { settleFlagGuardResponse, } from './interaction/index.ts'; import type { BoundContextFromFlags } from './context.ts'; +import type { RuntimeAdmissionBindings } from './request-runtime-binding.ts'; import type { DaemonPlatformServices } from './platform-services.ts'; +import { resolveSettleObservationCapture } from './settle-observation-capture.ts'; import { issueSettleRefs } from './session-snapshot.ts'; import type { SessionStore } from './session-store.ts'; import type { DaemonRequest, DaemonResponse } from './daemon-request.ts'; @@ -66,7 +68,7 @@ type GenericSettleContext = { sessionStore: SessionStore; contextFromFlags: BoundContextFromFlags; platformServices: DaemonPlatformServices; -}; +} & RuntimeAdmissionBindings; /** * Rejects an orphaned `--settle-quiet` up front (the shared flag grammar), and @@ -120,10 +122,25 @@ async function observeSettled( return refsGeneration === undefined ? observation : { ...observation, refsGeneration }; } +/** + * Resolves the observation's capture before the runtime is constructed, so the loop reads the screen + * through the same producer the caller's next command reads it through (#3510). `back` and + * `scroll ` are the cells that need it: their action use is the movement alone. + * `scroll top|bottom` and `scroll --until` already require `captureSnapshot` for their between-pass + * verification; there this is the request's second admit-and-bind for one operation, served from the + * gateway's binding cache rather than a second device acquisition. + */ async function createGenericSettleRuntime( context: GenericSettleContext, ): Promise> | undefined> { try { + const boundCapture = await resolveSettleObservationCapture({ + command: context.req.command, + session: context.session, + inspectFacts: context.inspectFacts, + platformServices: context.platformServices, + bindDevice: context.bindDevice, + }); return await createInteractionRuntime({ createCommandSurface: context.createCommandSurface, req: context.req, @@ -134,6 +151,7 @@ async function createGenericSettleRuntime( contextFromFlags: context.contextFromFlags, platformServices: context.platformServices, captureSnapshotForSession, + settleObservationCapture: boundCapture, }); } catch { return undefined; diff --git a/src/daemon/interaction/index.ts b/src/daemon/interaction/index.ts index 4f66f480f8..612abde8c5 100644 --- a/src/daemon/interaction/index.ts +++ b/src/daemon/interaction/index.ts @@ -10,6 +10,7 @@ import type { CaptureSnapshotForSession, FindRouteInput, InteractionRouteInput, + SettleObservationCaptureInput, } from './internal/types.ts'; export type { FindRouteInput, InteractionRouteInput } from './internal/types.ts'; @@ -67,11 +68,12 @@ export const captureSnapshotForSession: CaptureSnapshotForSession = async ( }; export async function createInteractionRuntime( - params: InteractionRouteInput & { - pairedGestureViewport?: Rect; - touchExecutor?: import('../touch-runtime.ts').BoundTouchExecutor; - gestures?: import('../gesture-runtime.ts').BoundGestureExecutor; - }, + params: SettleObservationCaptureInput & + InteractionRouteInput & { + pairedGestureViewport?: Rect; + touchExecutor?: import('../touch-runtime.ts').BoundTouchExecutor; + gestures?: import('../gesture-runtime.ts').BoundGestureExecutor; + }, ) { return await createInteractionRuntimeForRoute({ ...bindInteractionSession(params), diff --git a/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures.ts b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures.ts index 4574dc36c7..d95baf646c 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures.ts @@ -58,7 +58,7 @@ export function snapshotPayload( backend: 'tree' | 'queries' | 'private-ax' = 'tree', ) { return { - backend: 'xctest', + backend: 'xctest' as const, producer: 'apple-runner' as const, nodes, quality: { state: 'healthy' as const, backend }, diff --git a/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts index cf7557a2e0..f58f23989a 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts @@ -6,6 +6,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; import { buildSnapshotPresentationKey } from '@agent-device/kernel/snapshot'; +import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; import { handleInteractionCommands } from '../../index.ts'; import { makeIosSession, @@ -22,6 +23,7 @@ import { snapshotPayload, } from './interaction-ios-tap-outcome-fixtures.ts'; import { + appCaptureComparisonIdentity, appCaptureComparisonKey, systemSurfaceCaptureComparisonKey, } from '../../../__tests__/ios-comparison-key-fixture.ts'; @@ -294,6 +296,80 @@ test('a changed capture from a different iOS backend keeps the tap failure', asy expect(sessionStore.get(sessionName)?.actions).toHaveLength(0); }); +/** + * #3510: `--settle` on a coordinate press binds the request's own capture, and the pre-action + * baseline the settle plan takes goes through it. The corroboration probe is an observation too, so + * it answers through the same capture AND under the same backend pin: an unkeyed baseline next to a + * keyed capture, or a keyed baseline next to an unkeyed one, is an identity mismatch that would + * report a tap that landed as a recorded failure. + * + * A private-AX baseline is the unkeyed half of that pair: a pinned capture leaves the planned route, + * and an app capture off that route carries no identity (#2438). Dropping the pin whenever a bound + * capture exists would answer it from the bridge and refuse the rescue the pin exists to earn. + */ +test.each([ + { + name: 'a request-bound bridge capture corroborates its own keyed baseline', + baseline: () => + buildSnapshotState( + { ...snapshotPayload(profileNodes), comparisonIdentity: identity() }, + { snapshotInteractiveOnly: false }, + ), + after: () => ({ ...snapshotPayload(imageViewerNodes), comparisonIdentity: identity() }), + expectedPreferredBackend: undefined, + }, + { + name: 'a private-AX baseline pins the backend of the request-bound capture', + baseline: () => + buildSnapshotState(snapshotPayload(profileNodes, 'private-ax'), { + snapshotInteractiveOnly: false, + }), + // A pinned capture leaves the planned route, so the runner answers and stamps no identity. + after: () => snapshotPayload(imageViewerNodes, 'private-ax'), + expectedPreferredBackend: 'private-ax', + }, +] as const)( + 'the corroboration probe answers $name', + async ({ baseline, after, expectedPreferredBackend }) => { + const sessionName = 'ios-corroboration-bound-capture'; + const sessionStore = makeSessionStore(); + sessionStore.publish( + sessionName, + makeIosSession(sessionName, { appBundleId: 'com.example.app', snapshot: baseline() }), + ); + const askedWith: Record[] = []; + const boundCapture = vi.fn(async () => after()); + + const corroboration = await corroborateIosTapFailure({ + error: new AppError('XCTEST_RECORDED_FAILURE', 'tap failed'), + command: 'click', + requestId: undefined, + flags: {}, + ref: sessionStore.lookup(sessionName)!, + sessionStore, + contextFromFlags, + // Mirrors the real seam: one capture ask, answered by the bound capture under the pin. + captureSnapshotForSession: async (_ref, _flags, _store, _context, options) => { + askedWith.push(options as unknown as Record); + return buildSnapshotState(await options.boundCapture!([] as never), { + snapshotInteractiveOnly: options.interactiveOnly, + }); + }, + boundCapture: boundCapture as unknown as Parameters< + typeof corroborateIosTapFailure + >[0]['boundCapture'], + }); + + expect(boundCapture).toHaveBeenCalledTimes(1); + expect(askedWith[0]?.preferredBackend).toBe(expectedPreferredBackend); + expect(corroboration?.warning).toMatch(/post-action accessibility capture changed/); + }, +); + +function identity() { + return appCaptureComparisonIdentity(IOS_SIMULATOR.id, 'com.example.app'); +} + test('a producer or generation switch cannot corroborate a failed tap', async () => { const sessionName = 'ios-comparison-identity-mismatch'; const sessionStore = makeSessionStore(); diff --git a/src/daemon/interaction/internal/__tests__/interaction-touch-prepare.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-prepare.test.ts new file mode 100644 index 0000000000..1689563fd6 --- /dev/null +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-prepare.test.ts @@ -0,0 +1,261 @@ +import { beforeEach, expect, test, vi } from 'vitest'; +import type { CommandFlags } from '@agent-device/contracts/command'; +import { + type DeviceBinding, + type RuntimeFacts, + localRuntimeOwner, + narrowDeviceBinding, +} from '@agent-device/contracts/platform-runtime'; +import type { PlatformRuntimeOperations } from '@agent-device/contracts/platform-runtime-operations'; +import { snapshotRuntimeOperationFacts } from '@agent-device/contracts/snapshot-runtime'; +import { + type FillPointInput, + type TapPointInput, + touchRuntimeOperationFacts, +} from '@agent-device/contracts/touch-runtime'; +import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { createUnavailableRuntimeFactsForTest } from '@agent-device/test-kit/runtime-operation-facts'; +import { makeIosAppSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import type { + BindDeviceRuntime, + InspectDeviceRuntimeFacts, +} from '../../../request-runtime-binding.ts'; +import { daemonPlatformServicesFixture } from '../../../__tests__/platform-services-fixture.ts'; +import { testCreateCommandSurface } from '../../../__tests__/command-surface-fixture.ts'; +import { handleInteractionCommands } from '../../index.ts'; +import type { SessionStore } from '../../../session-store.ts'; + +// #3510: the capture an interaction's post-action observation polls. A coordinate-targeted press or +// fill acts through a use that declares no capture, so before this its settle loop read the screen +// through the session's legacy snapshot path while the command the caller ran next read the +// request-bound runtime: on an iOS simulator, two projections of one screen. + +const available = Object.freeze({ available: true } as const); +const refused = Object.freeze({ available: false, reason: 'unsupported-platform-leaf' } as const); + +const TREE: RawSnapshotNode[] = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + identifier: 'save', + label: 'Save', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, +]; + +/** The tree only the request-bound capture can produce, so producer identity is decidable. */ +const BRIDGE_PRODUCED = Object.freeze({ + nodes: TREE, + backend: 'xctest' as const, + producer: 'simulator-ax-bridge' as const, +}); + +const mockInteractorCapture = vi.hoisted(() => vi.fn()); +vi.mock('../../../snapshot-interactor-capture.ts', () => ({ + captureSnapshotWithInteractor: mockInteractorCapture, +})); + +const contextFromFlags = (_flags: CommandFlags | undefined) => ({}); + +/** An owner that captures, taps points, fills points, and does nothing else. */ +function settleFixture() { + const binds: Array> = []; + let captures = 0; + const facts = (device: DeviceInfo): RuntimeFacts => { + const base = createUnavailableRuntimeFactsForTest(device, localRuntimeOwner('apple')); + return Object.freeze({ + device: base.device, + operations: { + ...base.operations, + ...snapshotRuntimeOperationFacts({ + capture: available, + customActions: refused, + withoutActiveApp: available, + }), + ...touchRuntimeOperationFacts({ + unsupported: refused, + tap: available, + longPress: refused, + fill: available, + }), + }, + }); + }; + const capture = async () => { + captures += 1; + return BRIDGE_PRODUCED; + }; + const inspectFacts: InspectDeviceRuntimeFacts = async (device) => facts(device); + const bindDevice: BindDeviceRuntime = async (device, use) => { + binds.push({ device, required: [...use.required] }); + return await narrowDeviceBinding( + Object.freeze({ + device, + owner: localRuntimeOwner('apple'), + facts: facts(device), + operations: Object.freeze({ + captureSnapshot: capture, + captureSnapshotWithoutActiveApp: capture, + tapPoint: async (input: TapPointInput) => ({ x: input.point.x, y: input.point.y }), + fillPoint: async (input: FillPointInput) => ({ x: input.point.x, y: input.point.y }), + }), + [Symbol.asyncDispose]: async () => undefined, + }) as unknown as DeviceBinding, + use, + ); + }; + return { + binds, + inspectFacts, + bindDevice, + captureCount: () => captures, + }; +} + +const SETTLE_FLAGS = { settle: true, settleQuietMs: 25, timeoutMs: 2_000 } satisfies CommandFlags; + +function publishSession(sessionStore: SessionStore, sessionName: string) { + const session = makeIosAppSession(sessionName); + session.snapshot = { + nodes: attachRefs(TREE), + createdAt: Date.now(), + backend: 'xctest', + producer: 'apple-runner', + }; + sessionStore.publish(sessionName, session); + return session; +} + +async function runCommand( + sessionStore: SessionStore, + sessionName: string, + command: string, + positionals: string[], + bindings: ReturnType, + flags: CommandFlags = {}, + inspectFacts: InspectDeviceRuntimeFacts = bindings.inspectFacts, +) { + return await handleInteractionCommands({ + createCommandSurface: testCreateCommandSurface, + req: { token: 't', session: sessionName, command, positionals, flags: { ...flags } }, + sessionName, + sessionStore, + contextFromFlags, + platformServices: daemonPlatformServicesFixture(), + inspectFacts, + bindDevice: bindings.bindDevice, + }); +} + +function settled(response: unknown) { + return (response as { data: { settle?: { settled: boolean } } }).data.settle; +} + +beforeEach(() => { + mockInteractorCapture.mockReset(); +}); + +// The observation's bind is a second, capture-only use beside the action's exact one — taken once +// for the request, not once per capture. +test.each([ + ['press', ['10', '20'], ['tapPoint']], + ['fill', ['10', '20', 'kit'], ['fillPoint']], +] as const)( + 'a coordinate %s with --settle observes through the capture the request admitted', + async (command, positionals, actionUse) => { + const sessionStore = makeSessionStore(); + const sessionName = `point-${command}-settle`; + const session = publishSession(sessionStore, sessionName); + const bindings = settleFixture(); + + const response = await runCommand( + sessionStore, + sessionName, + command, + [...positionals], + bindings, + SETTLE_FLAGS, + ); + + expect(response?.ok).toBe(true); + expect(settled(response)).toMatchObject({ settled: true }); + expect(bindings.binds.map((bind) => bind.required)).toEqual([actionUse, ['captureSnapshot']]); + // The tree the observation published is the bound capture's, and the legacy path was never asked + // for a tree: one producer for the whole request, which is what lets the refs the settle response + // issued resolve against the tree the caller's next `snapshot` reads. + expect(session.snapshot?.producer).toBe('simulator-ax-bridge'); + expect(bindings.captureCount()).toBeGreaterThan(1); + expect(mockInteractorCapture).not.toHaveBeenCalled(); + }, +); + +test('a coordinate press without --settle binds no observation capture', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'point-press-no-settle'; + publishSession(sessionStore, sessionName); + const bindings = settleFixture(); + + const response = await runCommand(sessionStore, sessionName, 'press', ['10', '20'], bindings); + + expect(response?.ok).toBe(true); + expect(bindings.binds.map((bind) => bind.required)).toEqual([['tapPoint']]); +}); + +test('an observation capture the owner cannot admit still lets the coordinate press act', async () => { + // The action's own facts inspection answers first, then the observation's admission is asked. An + // owner that fails the second ask must not cost the first: settling is best-effort by contract, so + // the tap still lands and the observation degrades to the owner's own snapshot path (#3510). + const sessionStore = makeSessionStore(); + const sessionName = 'point-press-probe-failure'; + publishSession(sessionStore, sessionName); + const bindings = settleFixture(); + mockInteractorCapture.mockResolvedValue(BRIDGE_PRODUCED); + let inspections = 0; + + const response = await runCommand( + sessionStore, + sessionName, + 'press', + ['10', '20'], + bindings, + SETTLE_FLAGS, + async (device) => { + inspections += 1; + // Stands in for the Apple facts probe re-throwing a cancelled host read on the second ask. + if (inspections > 1) + throw Object.assign(new Error('probe cancelled'), { name: 'AbortError' }); + return await bindings.inspectFacts(device); + }, + ); + + expect(response?.ok).toBe(true); + expect(settled(response)).toMatchObject({ settled: true }); + expect(mockInteractorCapture).toHaveBeenCalled(); +}); + +test('a tree-resolved press reuses the capture its action binding already carries', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'selector-press-settle'; + publishSession(sessionStore, sessionName); + const bindings = settleFixture(); + + const response = await runCommand( + sessionStore, + sessionName, + 'press', + ['label="Save"'], + bindings, + SETTLE_FLAGS, + ); + + expect(response?.ok).toBe(true); + // No extra bind: this route's use already requires the capture, which is why a selector press + // never had the split a coordinate press had. + expect(bindings.binds.map((bind) => bind.required)).toEqual([['captureSnapshot', 'tapPoint']]); + expect(sessionStore.get(sessionName)?.snapshot?.producer).toBe('simulator-ax-bridge'); +}); diff --git a/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts b/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts index 5fd0566d49..daa210dd46 100644 --- a/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts +++ b/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts @@ -12,6 +12,7 @@ import { isLocalIosRunnerSession } from '../../direct-ios-selector.ts'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { SessionStore } from '../../session-store.ts'; import type { SessionRef, SessionState } from '../../session-state.ts'; +import type { BoundSnapshotCapture } from '../../snapshot-runtime-binding.ts'; import type { BoundContextFromFlags, CaptureSnapshotForSession } from './types.ts'; const XCTEST_RECORDED_FAILURE = 'XCTEST_RECORDED_FAILURE'; @@ -39,6 +40,12 @@ export type IosTapCorroborationParams = { sessionStore: SessionStore; contextFromFlags: BoundContextFromFlags; captureSnapshotForSession: CaptureSnapshotForSession; + /** + * The capture this request's own pre-action baseline was taken with, when it holds one. The probe + * is an observation too and answers through it: a keyed baseline next to an unkeyed legacy capture + * is an identity mismatch, which would report a tap that landed as a recorded failure (#3510). + */ + boundCapture?: BoundSnapshotCapture; }; export type IosTapCorroboration = { @@ -131,10 +138,15 @@ async function captureCorroborationSnapshot( params.contextFromFlags, { interactiveOnly: presentation?.interactiveOnly ?? true, - // Evidence comparison is only valid same-backend, and the recorded-failure - // screens are exactly where the capture plan flips between XCTest and - // private-AX (the penalty boundary) — pin the probe to the baseline's - // backend instead of failing closed on the mismatch. + // Two independent facts about the one capture, both forwarded: which implementation answers + // (the request's own binding when it holds one) and which backend that capture must present. + ...(params.boundCapture ? { boundCapture: params.boundCapture } : {}), + // Evidence comparison is only valid same-backend, and the recorded-failure screens are + // exactly where the capture plan flips between XCTest and private-AX (the penalty + // boundary) — pin the probe to the baseline's backend instead of failing closed on the + // mismatch. The pin is an option OF the capture, so a bound capture takes it too: dropping + // it there would answer a private-AX baseline with a bridge capture, whose comparison + // identity no unkeyed runner baseline can match. ...(preferredBackend ? { preferredBackend } : {}), signal: getRequestSignal(params.requestId), }, diff --git a/src/daemon/interaction/internal/interaction-runtime.ts b/src/daemon/interaction/internal/interaction-runtime.ts index c50fb21d41..bee3cd7c63 100644 --- a/src/daemon/interaction/internal/interaction-runtime.ts +++ b/src/daemon/interaction/internal/interaction-runtime.ts @@ -20,6 +20,7 @@ import type { CaptureSnapshotForSession, InteractionRouteInput, InteractionRuntimeInput, + SettleObservationCaptureInput, } from './types.ts'; import { finalizeTouchInteraction as finalizeInteraction } from './interaction-common.ts'; import { markDeferredInteractionOutcome } from '../../deferred-interaction-outcome.ts'; @@ -30,23 +31,25 @@ import type { DaemonResponse } from '../../daemon-request.ts'; import type { SessionRef } from '../../session-state.ts'; import type { BoundTouchExecutor } from '../../touch-runtime.ts'; import type { BoundGestureExecutor } from '../../gesture-runtime.ts'; +import type { BoundSnapshotCapture } from '../../snapshot-runtime-binding.ts'; import { NO_ACTIVE_SESSION_MESSAGE } from '@agent-device/kernel/contracts'; export async function createInteractionRuntimeForRoute( - params: InteractionRouteInput & { - captureSnapshotForSession: CaptureSnapshotForSession; - pairedGestureViewport?: Rect; - touchExecutor?: BoundTouchExecutor; - gestures?: BoundGestureExecutor; - /** - * Filled with the LAST capture this request consumed. ADR 0014 ambiguity - * issuance is only valid against a tree the session stored under the - * generation it freezes, and a sparse-quality capture deliberately stores - * nothing — the refusal seam needs the same consumed-capture slot the - * selector routes carry (`consumedSnapshot`), owned per dispatch. - */ - consumedCapture?: { state?: SnapshotState }; - }, + params: SettleObservationCaptureInput & + InteractionRouteInput & { + captureSnapshotForSession: CaptureSnapshotForSession; + pairedGestureViewport?: Rect; + touchExecutor?: BoundTouchExecutor; + gestures?: BoundGestureExecutor; + /** + * Filled with the LAST capture this request consumed. ADR 0014 ambiguity + * issuance is only valid against a tree the session stored under the + * generation it freezes, and a sparse-quality capture deliberately stores + * nothing — the refusal seam needs the same consumed-capture slot the + * selector routes carry (`consumedSnapshot`), owned per dispatch. + */ + consumedCapture?: { state?: SnapshotState }; + }, ) { const ref = bindInteractionSession(params).sessionRef; if (!ref) throw new KernelAppError('SESSION_NOT_FOUND', NO_ACTIVE_SESSION_MESSAGE); @@ -90,6 +93,7 @@ export async function createInteractionRuntimeForRoute( pairedGestureViewport: params.pairedGestureViewport, touchExecutor: params.touchExecutor, gestures: params.gestures, + boundCapture: resolveBoundCapture(params), }); } @@ -145,6 +149,27 @@ export function finalizeTouchInteraction(params: FinalizeTouchInteractionInput): }); } +/** + * The capture implementation every observation on this request reads the screen through: the action + * binding's own capture when its runtime use carries one, otherwise the capture the request admitted + * for its observations (`settle-observation-capture.ts`). Resolved once, at construction, so a route + * cannot answer one capture from the request-bound runtime and the next from the legacy path. Which + * backend a bound capture presents stays per-capture: the settle loop pins it once it sees a + * `private-ax` verdict. + */ +function resolveBoundCapture( + params: SettleObservationCaptureInput & { + touchExecutor?: BoundTouchExecutor; + gestures?: BoundGestureExecutor; + }, +): BoundSnapshotCapture | undefined { + return ( + params.touchExecutor?.captureSnapshot ?? + params.gestures?.captureSnapshot ?? + params.settleObservationCapture + ); +} + function createInteractionAgentDevice(params: InteractionRuntimeInput) { return params.createCommandSurface({ backend: createInteractionBackend(params), @@ -166,7 +191,7 @@ function createInteractionBackend(params: InteractionRuntimeInput): AgentDeviceB preferredBackend: options?.preferredBackend, includeRects: options?.includeRects === true, signal: context.signal, - boundCapture: params.touchExecutor?.captureSnapshot ?? params.gestures?.captureSnapshot, + boundCapture: params.boundCapture, }), }), ...gestureBackendMembers(params, gestureContext), diff --git a/src/daemon/interaction/internal/interaction-touch-fill.ts b/src/daemon/interaction/internal/interaction-touch-fill.ts index 5d6523125a..0142edac5d 100644 --- a/src/daemon/interaction/internal/interaction-touch-fill.ts +++ b/src/daemon/interaction/internal/interaction-touch-fill.ts @@ -14,6 +14,7 @@ import type { CaptureSnapshotForSession, InteractionRouteInput, RefSnapshotFlagGuardResponse, + SettleObservationCaptureInput, } from './types.ts'; import { refreshAndroidRefSnapshotIfFreshnessActive } from './interaction-touch-android-freshness.ts'; import { unsupportedMacOsDesktopSurfaceInteraction } from './interaction-touch-policy.ts'; @@ -47,18 +48,21 @@ type AdmittedFill = { parsedTarget: Extract, { ok: true }>; touchExecutor: BoundTouchExecutor; staleRefsWarning: string | undefined; + settleObservationCapture: SettleObservationCaptureInput['settleObservationCapture']; }; export async function dispatchFillViaRuntime(params: FillParams): Promise { params = bindInteractionSession(params); const admission = await admitFill(params); if ('response' in admission) return admission.response; - const { parsedTarget, touchExecutor, staleRefsWarning } = admission.admitted; + const { parsedTarget, touchExecutor, staleRefsWarning, settleObservationCapture } = + admission.admitted; const { req, sessionName } = params; const replayTargetGuard = req.internal?.replayTargetGuard; return await dispatchRuntimeInteraction(params, { touchExecutor, + settleObservationCapture, refContext: parsedTarget.target.kind === 'ref' && req.internal?.findResolvedTarget === undefined ? { @@ -115,13 +119,8 @@ async function readFillAdmission( const target = readFillTarget(params); if ('response' in target) return target; const { session, parsedTarget } = target; - const prepared = await prepareTouchDispatch( - params, - session, - 'fill', - parsedTarget.target.kind !== 'point', - ); - if (!prepared.ok) return { response: prepared.response }; + // Pure input refusals first: a rejected fill must not pay a facts inspection or a bind, matching + // the order the press route keeps by validating inside its own admission. assertRecordedFillParameterization({ flags: req.flags, replayPlanStep: req.internal?.replayPlanStep === true, @@ -129,6 +128,13 @@ async function readFillAdmission( }); const invalidSettleFlags = settleFlagGuardResponse('fill', req.flags); if (invalidSettleFlags) return { response: invalidSettleFlags }; + const prepared = await prepareTouchDispatch( + params, + session, + 'fill', + parsedTarget.target.kind !== 'point', + ); + if (!prepared.ok) return { response: prepared.response }; const refPreamble = await prepareFillRefTarget( params, session, @@ -142,6 +148,7 @@ async function readFillAdmission( parsedTarget, touchExecutor: prepared.touchExecutor, staleRefsWarning: refPreamble.staleRefsWarning, + settleObservationCapture: prepared.settleObservationCapture, }, }; } diff --git a/src/daemon/interaction/internal/interaction-touch-prepare.ts b/src/daemon/interaction/internal/interaction-touch-prepare.ts index 34bafeab8f..6cc9ab7ba5 100644 --- a/src/daemon/interaction/internal/interaction-touch-prepare.ts +++ b/src/daemon/interaction/internal/interaction-touch-prepare.ts @@ -5,15 +5,30 @@ import { type BoundTouchExecutor, type TouchRuntimeCommand, } from '../../touch-runtime.ts'; +import { readSettleRequest } from './interaction-flags.ts'; +import type { BoundSnapshotCapture } from '../../snapshot-runtime-binding.ts'; +import { resolveSettleObservationCapture } from '../../settle-observation-capture.ts'; import type { InteractionRouteInput } from './types.ts'; import type { DaemonFailureResponse } from '@agent-device/kernel/contracts'; import { refusedBeforeDispatch } from '../../request-dispatch-disclosure.ts'; export type PreparedTouchDispatch = | Readonly<{ ok: false; response: DaemonFailureResponse }> - | Readonly<{ ok: true; touchExecutor: BoundTouchExecutor }>; + | Readonly<{ + ok: true; + touchExecutor: BoundTouchExecutor; + /** + * The observation capture this request still owes: only a route whose action runtime use + * carries no capture and owes a settle observation can produce one, so every other touch + * dispatch pays no admission and no bind. + */ + settleObservationCapture: BoundSnapshotCapture | undefined; + }>; -/** Exact-owner admission, one bind, and command-context projection shared by every touch route. */ +/** + * Exact-owner admission, the command-context projection, and — for a coordinate route that owes a + * settle observation — the request's own capture-only bind, shared by every touch route. + */ export async function prepareTouchDispatch( params: InteractionRouteInput, session: SessionState, @@ -29,11 +44,25 @@ export async function prepareTouchDispatch( bindDevice: params.bindDevice, }); if (!bound.ok) return { ok: false, response: refusedBeforeDispatch(bound.response) }; + const touchExecutor = createBoundTouchExecutor( + bound.runtime, + params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), + ); + const owesObservationCapture = + touchExecutor.captureSnapshot === undefined && + readSettleRequest(params.req.flags) !== undefined; + const settleObservationCapture = owesObservationCapture + ? await resolveSettleObservationCapture({ + command: params.req.command, + session, + inspectFacts: params.inspectFacts, + platformServices: params.platformServices, + bindDevice: params.bindDevice, + }) + : undefined; return { ok: true, - touchExecutor: createBoundTouchExecutor( - bound.runtime, - params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), - ), + touchExecutor, + settleObservationCapture, }; } diff --git a/src/daemon/interaction/internal/interaction-touch-press.ts b/src/daemon/interaction/internal/interaction-touch-press.ts index 6c6f26588d..1bd6944889 100644 --- a/src/daemon/interaction/internal/interaction-touch-press.ts +++ b/src/daemon/interaction/internal/interaction-touch-press.ts @@ -84,6 +84,7 @@ export async function dispatchTargetedTouchViaRuntime( return await dispatchRuntimeInteraction(params, { ...buildTargetedRuntimeOptions(params, command, boundAdmitted), touchExecutor, + settleObservationCapture: prepared.settleObservationCapture, }); } diff --git a/src/daemon/interaction/internal/interaction-touch-runtime.ts b/src/daemon/interaction/internal/interaction-touch-runtime.ts index 36ec2a12e5..feb9f5cc34 100644 --- a/src/daemon/interaction/internal/interaction-touch-runtime.ts +++ b/src/daemon/interaction/internal/interaction-touch-runtime.ts @@ -32,6 +32,7 @@ import { type InteractionResponsePayloads, } from './interaction-touch-response.ts'; import type { BoundTouchExecutor } from '../../touch-runtime.ts'; +import type { BoundSnapshotCapture } from '../../snapshot-runtime-binding.ts'; import { noActiveSessionError } from '@agent-device/kernel/contracts'; /** @@ -48,6 +49,8 @@ export async function dispatchRuntimeInteraction< }, options: { touchExecutor: BoundTouchExecutor; + /** The request's own observation capture when its action use carries none (#3510). */ + settleObservationCapture?: BoundSnapshotCapture; androidFreshnessBaseline?: SessionState['snapshot']; /** * Present when the action targets a `@ref`: if Android dialog recovery @@ -75,6 +78,7 @@ export async function dispatchRuntimeInteraction< const runtime = await createInteractionRuntimeForRoute({ ...params, touchExecutor: options.touchExecutor, + settleObservationCapture: options.settleObservationCapture, consumedCapture, }); const actionStartedAt = Date.now(); @@ -134,6 +138,7 @@ export async function dispatchRuntimeInteraction< const corroboratedResponse = await buildRuntimeIosCorroboratedResponse({ error: appError, handlerParams: params, + observationCapture: options.settleObservationCapture, session, target: options.iosTapCorroboration?.target, extra: options.iosTapCorroboration?.extra, @@ -150,6 +155,8 @@ async function buildRuntimeIosCorroboratedResponse(params: { handlerParams: InteractionRouteInput & { captureSnapshotForSession: CaptureSnapshotForSession; }; + /** The capture this request's own pre-action baseline was taken with, when it holds one (#3510). */ + observationCapture: BoundSnapshotCapture | undefined; session: SessionState; target: InteractionTarget | undefined; extra: Record | undefined; @@ -168,6 +175,7 @@ async function buildRuntimeIosCorroboratedResponse(params: { sessionStore: params.handlerParams.sessionStore, contextFromFlags: params.handlerParams.contextFromFlags, captureSnapshotForSession: params.handlerParams.captureSnapshotForSession, + boundCapture: params.observationCapture, }); if (!corroboration) return undefined; diff --git a/src/daemon/interaction/internal/types.ts b/src/daemon/interaction/internal/types.ts index 57346e6aed..f9a0ade0f3 100644 --- a/src/daemon/interaction/internal/types.ts +++ b/src/daemon/interaction/internal/types.ts @@ -44,6 +44,20 @@ export type InteractionRouteInput = { captureProof?: RequestCaptureProof; }; +/** + * The observation capture a request admitted for itself. Only the two constructors that build an + * observation runtime carry it — never the route input every interaction command shares — so a + * capture reaches an observation only from the module that admitted it on the owner's facts (#3510). + */ +export type SettleObservationCaptureInput = { + /** + * For a route whose action runtime use carries no capture: a coordinate-targeted touch + * (`tapPoint`/`fillPoint` declare none) or a generic-route action. Without it the post-action + * observation reads the screen through a different producer than the caller's next command does. + */ + settleObservationCapture?: BoundSnapshotCapture; +}; + export type FindRouteInput = { createCommandSurface: CreateDaemonCommandSurface; req: DaemonRequest; @@ -105,6 +119,12 @@ export type InteractionRuntimeInput = { pairedGestureViewport?: Rect; touchExecutor?: BoundTouchExecutor; gestures?: BoundGestureExecutor; + /** + * The capture every observation on this request reads the screen through: the action binding's + * own capture when its runtime use carries one, otherwise the capture the request admitted for + * its observations. Resolved once at construction so no route can mix producers (#3510). + */ + boundCapture?: BoundSnapshotCapture; }; export type InteractionGestureVisualization = ( diff --git a/src/daemon/request-generic-dispatch.ts b/src/daemon/request-generic-dispatch.ts index 04b6099b83..4c76de00e2 100644 --- a/src/daemon/request-generic-dispatch.ts +++ b/src/daemon/request-generic-dispatch.ts @@ -26,6 +26,7 @@ import { buildActionEventResult } from '@agent-device/session-journal/session-ev import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; import type { CreateDaemonCommandSurface } from '@agent-device/contracts/daemon-command-surface'; import type { GenericPlatformExecution, RecordedGenericRequest } from './generic-leaf-execution.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from './request-runtime-binding.ts'; export async function dispatchGenericCommand(params: { req: DaemonRequest; @@ -43,6 +44,9 @@ export async function dispatchGenericCommand(params: { recordedRequest?: RecordedGenericRequest; androidObservation?: AndroidObservationAdapter; platformServices: DaemonPlatformServices; + /** The request-scoped runtime seams `--settle` admits its own observation capture through (#3510). */ + inspectFacts: InspectDeviceRuntimeFacts; + bindDevice: BindDeviceRuntime; }): Promise { const { req, ref: sessionRef, logPath, sessionStore, contextFromFlags } = params; const session = sessionStore.requireCurrent(sessionRef); @@ -68,6 +72,8 @@ export async function dispatchGenericCommand(params: { command: platformCommand, flags: req.flags, createCommandSurface: params.createCommandSurface, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, }); if ('response' in settlePlan) return settlePlan.response; const preflightReadiness = await ensureNoAndroidBlockingDialogReady( @@ -224,6 +230,8 @@ async function planGenericSettleObservation(params: { platformServices: DaemonPlatformServices; command: string; flags: CommandFlags | undefined; + inspectFacts: InspectDeviceRuntimeFacts; + bindDevice: BindDeviceRuntime; }): Promise< { response: DaemonResponse } | { observe?: () => Promise } > { diff --git a/src/daemon/request-router.ts b/src/daemon/request-router.ts index 9df1bab385..c9a0467cf1 100644 --- a/src/daemon/request-router.ts +++ b/src/daemon/request-router.ts @@ -647,6 +647,10 @@ async function dispatchGenericForLockedScope(params: { executePlatformCommand: runtimeExecution.execute, androidObservation, createCommandSurface, + // The settle observation binds its own capture on this route, so it needs the same + // request-scoped seams the action's admission above already used (#3510). + inspectFacts: lockedScope.inspectFacts, + bindDevice: lockedScope.bindDevice, ...(runtimeExecution.recorded ? { recordedRequest: runtimeExecution.recorded } : {}), }); return dispatchResponse; diff --git a/src/daemon/session-runtime-admission.ts b/src/daemon/session-runtime-admission.ts index 340ab88272..20e0aa48e1 100644 --- a/src/daemon/session-runtime-admission.ts +++ b/src/daemon/session-runtime-admission.ts @@ -3,6 +3,7 @@ import type { PlatformRuntimeOperations } from '@agent-device/contracts/platform import type { RuntimeOperationFact, RuntimeOperationKey, + RuntimeOperationUnavailability, } from '@agent-device/contracts/platform-runtime'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -124,7 +125,7 @@ export function unwrapAdmittedRuntimePlan( export type RefusedRuntimePlan = Readonly<{ admitted: false; operation: Plan['use']['required'][number]; - fact: RuntimeOperationFact; + fact: RuntimeOperationUnavailability; }>; /** diff --git a/src/daemon/settle-observation-capture.ts b/src/daemon/settle-observation-capture.ts new file mode 100644 index 0000000000..ad149ef413 --- /dev/null +++ b/src/daemon/settle-observation-capture.ts @@ -0,0 +1,100 @@ +import type { BoundSnapshotCapture } from './snapshot-runtime-binding.ts'; +import { bindAdmittedSnapshotCapture } from './snapshot-runtime-binding.ts'; +import { admitRuntimePlan } from './session-runtime-admission.ts'; +import { + resolveSelectorCaptureRuntimePlan, + type SelectorCaptureRuntimePlan, +} from '@agent-device/contracts/platform-runtime-operations'; +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import type { RuntimeOperationUnavailability } from '@agent-device/contracts/platform-runtime'; +import type { RuntimeAdmissionBindings } from './request-runtime-binding.ts'; +import type { SessionState } from './session-state.ts'; + +/** + * The capture a post-action observation (`--settle`) polls, for a request whose own action binding + * carries none: a coordinate-targeted `press`/`fill` (whose point use declares none) or a + * generic-route action (#3510). Those routes polled the legacy interactor while the commands the + * caller runs next read the request-bound runtime, and on an iOS simulator those are two projections + * of one tree. + * + * Admitting the `capture-only` plan the selector family polls under closes the split; where both + * paths already reached the same tree, the admitted operation resolves to the implementation used + * before. Settling is best-effort by contract, so nothing here may fail or cancel the action it + * follows: every unavailability degrades to `undefined`, leaving the observation on the owner's own + * snapshot path, and discloses its typed reason. Admitting through the side-effect-free facts probe + * and binding only on admission keeps an optional observation from building a command-facing refusal + * response, whose Apple branch live-probes the booted simulator set for a hint nothing shows. + */ +export async function resolveSettleObservationCapture( + params: Readonly<{ command: string; session: SessionState }> & RuntimeAdmissionBindings, +): Promise { + const plan = resolveSelectorCaptureRuntimePlan({ + hasActiveApp: params.session.appBundleId !== undefined, + intent: 'capture-only', + }); + const admission = await admitObservationCapturePlan(params, plan); + if (admission === undefined) return undefined; + if (!admission.admitted) { + return unavailable({ + command: params.command, + outcome: 'refused', + operation: admission.operation, + reason: admission.fact.reason, + }); + } + try { + return await bindAdmittedSnapshotCapture({ + admission, + bindDevice: params.bindDevice, + platformServices: params.platformServices, + }); + } catch (error) { + return unavailable({ command: params.command, outcome: 'bind-failed', error }); + } +} + +/** `undefined` means the owner's facts probe failed: the caller degrades like any other refusal. */ +async function admitObservationCapturePlan( + params: Readonly<{ command: string; session: SessionState }> & RuntimeAdmissionBindings, + plan: SelectorCaptureRuntimePlan, +) { + try { + return await admitRuntimePlan({ + device: params.session.device, + plan, + inspectFacts: params.inspectFacts, + }); + } catch (error) { + return unavailable({ command: params.command, outcome: 'admission-probe-failed', error }); + } +} + +/** Why the observation has to go without a capture, as one typed fact the caller can read. */ +type ObservationCaptureUnavailable = Readonly<{ command: string }> & + ( + | Readonly<{ + outcome: 'refused'; + operation: string; + reason: RuntimeOperationUnavailability['reason']; + }> + | Readonly<{ outcome: 'admission-probe-failed' | 'bind-failed'; error: unknown }> + ); + +function unavailable(detail: ObservationCaptureUnavailable): undefined { + emitDiagnostic({ + // `warn`, not the default `info`: a degrade that answers `ok: true` is dropped at info level on + // the success path, which is the passing-unnoticed this discloses against. + level: 'warn', + phase: 'settle_observation_capture_unavailable', + // The error's name travels, never its message: request diagnostics keep user input out. + data: + detail.outcome === 'refused' + ? { ...detail } + : { + command: detail.command, + outcome: detail.outcome, + errorName: (detail.error as Error | undefined)?.name, + }, + }); + return undefined; +} diff --git a/src/daemon/snapshot-runtime-binding.ts b/src/daemon/snapshot-runtime-binding.ts index 518288fe83..7b2e9ec5b1 100644 --- a/src/daemon/snapshot-runtime-binding.ts +++ b/src/daemon/snapshot-runtime-binding.ts @@ -200,6 +200,29 @@ export async function resolveBoundSnapshotCaptureRuntime( }); } +/** + * Binds a plan that the caller already admitted, and keeps only its capture operation. For a caller + * that owns its refusal path — an optional capture whose refusal degrades an observation instead of + * failing a command — this is the half of `admitAndBindSnapshotCapture` that carries no + * command-facing refusal wording, and so spends none of the host probes that wording builds. A + * caller with a session, or one that owes a readiness check, uses the full form instead. + */ +export async function bindAdmittedSnapshotCapture( + params: Readonly<{ + admission: AdmittedRuntimePlan; + bindDevice?: BindDeviceRuntime; + platformServices: DaemonPlatformServices; + }>, +): Promise { + const bound = await bindSnapshotCaptureRuntime( + params.bindDevice, + params.admission, + undefined, + params.platformServices, + ); + return async (input: CaptureSnapshotInput) => await bound.captureSnapshot(input); +} + /** * Binds only an admitted plan, on the device it was admitted for: the token is minted by * `admitRuntimePlan` alone and unwrapped by exact identity, so nothing that was not admitted — From 8fe9c58ada7e5930c87ca2d03c013cbf86184b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 16:41:00 +0200 Subject: [PATCH 2/2] chore(gates): claim settle-observation-capture.ts in the daemon-core layer --- scripts/layering/daemon-layer-manifest.json | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/layering/daemon-layer-manifest.json b/scripts/layering/daemon-layer-manifest.json index c216ac3d39..b230cfca3f 100644 --- a/scripts/layering/daemon-layer-manifest.json +++ b/scripts/layering/daemon-layer-manifest.json @@ -55,6 +55,7 @@ "src/daemon/session-state.ts", "src/daemon/session-store.ts", "src/daemon/session-tenant-scope.ts", + "src/daemon/settle-observation-capture.ts", "src/daemon/snapshot-capture.ts", "src/daemon/snapshot-interactor-capture.ts", "src/daemon/snapshot-runtime-binding.ts",