diff --git a/docs/adr/0019-request-bound-platform-runtime.md b/docs/adr/0019-request-bound-platform-runtime.md index 49fbf69e06..45bd0f0d83 100644 --- a/docs/adr/0019-request-bound-platform-runtime.md +++ b/docs/adr/0019-request-bound-platform-runtime.md @@ -783,6 +783,14 @@ set to read facts — is retired for new units: they may not introduce admission an already-migrated handler onto facts inspection changes production admission behavior; it is its own independently reviewed unit with parity evidence, never a side effect of a structural change. +One second bind per request is permitted, and is not the retired idiom: a post-action observation +(`--settle`) whose action use carries no capture admits the `capture-only` plan for its own reads +(#3510). It is not an admission-only bind — it never reads facts through a binding, admits through +the facts probe, and binds only on admission — and where the action's use already required +`captureSnapshot` (`scroll top|bottom`, `scroll --until`) it projects the request's cached device +binding rather than acquiring a second one. It stays capture-only: a second bind that carries a +mutating operation is the execution use the plan should have selected. + Use declarations share one neutral `defineUse`; per-domain currying wrappers add a module per domain for no information and are not added. Required-only declarations are the default. Declaring a preferred operation requires a recorded measurement of the fast path's benefit in the unit diff --git a/scripts/layering/daemon-layer-manifest.json b/scripts/layering/daemon-layer-manifest.json index c216ac3d39..b230cfca3f 100644 --- a/scripts/layering/daemon-layer-manifest.json +++ b/scripts/layering/daemon-layer-manifest.json @@ -55,6 +55,7 @@ "src/daemon/session-state.ts", "src/daemon/session-store.ts", "src/daemon/session-tenant-scope.ts", + "src/daemon/settle-observation-capture.ts", "src/daemon/snapshot-capture.ts", "src/daemon/snapshot-interactor-capture.ts", "src/daemon/snapshot-runtime-binding.ts", diff --git a/src/daemon/__tests__/generic-settle.test.ts b/src/daemon/__tests__/generic-settle.test.ts index 3cc0f4ac1f..c745d57cdf 100644 --- a/src/daemon/__tests__/generic-settle.test.ts +++ b/src/daemon/__tests__/generic-settle.test.ts @@ -16,6 +16,7 @@ import type { SessionStore } from '../session-store.ts'; import type { DaemonRequest, DaemonResponse } from '../daemon-request.ts'; import type { SessionRef, SessionState } from '../session-state.ts'; import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; +import { selectorCaptureFixture } from './selector-capture-fixture.ts'; import { testCreateCommandSurface } from './command-surface-fixture.ts'; // #1638 `--settle` on the GENERIC daemon route (scroll/back): the settled diff, @@ -90,14 +91,21 @@ async function emulateCaptureSnapshotForSession( ref: SessionRef, flags: CommandFlags | undefined, sessionStore: SessionStore, - options: { interactiveOnly: boolean }, + options: { + interactiveOnly: boolean; + boundCapture?: Parameters[4]['boundCapture']; + }, ) { captureObservations.push({ postGestureStabilizationPending: sessionStore.requireCurrent(ref).postGestureStabilization !== undefined, }); const effectiveFlags = { ...(flags ?? {}), snapshotInteractiveOnly: options.interactiveOnly }; - const snapshotData = (await mockDispatch('snapshot')) as Parameters[0]; + // Same rule the real seam applies: a request-bound capture is the tree this request reads, and + // only a request that holds none dispatches for one (#3510). + const snapshotData = options.boundCapture + ? await options.boundCapture({}) + : ((await mockDispatch('snapshot')) as Parameters[0]); const snapshot = buildSnapshotState(snapshotData ?? {}, effectiveFlags); setSessionSnapshot(sessionStore.requireCurrent(ref), snapshot); return snapshot; @@ -138,8 +146,9 @@ function seedSession( sessionName: string, sessionStore: SessionStore, baseline: GenericSettleTree = { nodes: BEFORE_NODES }, + overrides?: Partial, ): SessionState { - const session = makeIosSession(sessionName); + const session = makeIosSession(sessionName, overrides); setSessionSnapshot(session, buildSnapshotState(snapshotPayload(baseline), {})); activateCompleteRefFrame(session); sessionStore.publish(sessionName, session); @@ -172,13 +181,28 @@ function mockSurfaceDispatch(tree: GenericSettleTree, result: Record[0], 'inspectFacts' | 'bindDevice'> + >; }): Promise { const req: DaemonRequest = { token: 't', @@ -197,6 +221,8 @@ async function dispatchGeneric(params: { sessionStore: params.sessionStore, contextFromFlags, executePlatformCommand: platformExecution, + ...legacyCaptureRuntimeBindings(), + ...params.runtimeBindings, }); } @@ -222,7 +248,7 @@ beforeEach(() => { test('scroll --settle answers with the settled diff against the stored pre-action tree', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-scroll'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); // Every settle capture sees the post-scroll tree; the diff baseline is the // tree already stored on the session, not a capture taken here. mockCommandDispatch([AFTER_NODES]); @@ -230,7 +256,6 @@ test('scroll --settle answers with the settled diff against the stored pre-actio const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'scroll', positionals: ['down'], flags: { ...SETTLE_FLAGS }, @@ -268,7 +293,7 @@ test('scroll --settle answers with the settled diff against the stored pre-actio test('back --settle answers with the settled diff alongside the command result', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-back'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); mockDispatch.mockImplementation(async (command) => { if (command === 'snapshot') return { nodes: AFTER_NODES, backend: 'xctest', producer: 'apple-runner' }; @@ -278,7 +303,6 @@ test('back --settle answers with the settled diff alongside the command result', const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'back', flags: { ...SETTLE_FLAGS }, }); @@ -382,7 +406,7 @@ function expectNoPublishedRefFrame(sessionStore: SessionStore, sessionName: stri test('scroll --settle attaches no diff across an app-to-sheet surface change and discloses it', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-scroll-to-sheet'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); // The stored pre-action tree is app content; every settle capture reads the sheet now presented // over that still-foreground app. mockSurfaceDispatch({ nodes: WEB_SIGN_IN_SHEET_NODES, systemSurface: WEB_SIGN_IN_SHEET }); @@ -390,7 +414,6 @@ test('scroll --settle attaches no diff across an app-to-sheet surface change and const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'scroll', positionals: ['down'], flags: { ...SETTLE_FLAGS }, @@ -412,7 +435,7 @@ test('scroll --settle attaches no diff across a sheet-to-app surface change and const sessionName = 'generic-settle-scroll-from-sheet'; // The baseline is the sheet this scroll acts on; the settled tree is the app content that // returns once the sheet completes and dismisses itself. - const session = seedSession(sessionName, sessionStore, { + seedSession(sessionName, sessionStore, { nodes: WEB_SIGN_IN_SHEET_NODES, systemSurface: WEB_SIGN_IN_SHEET, }); @@ -421,7 +444,6 @@ test('scroll --settle attaches no diff across a sheet-to-app surface change and const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'scroll', positionals: ['down'], flags: { ...SETTLE_FLAGS }, @@ -444,7 +466,7 @@ test('scroll --settle attaches no diff across a sheet-to-app surface change and test('back --settle attaches no diff across an app-to-sheet surface change and discloses it', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-back-to-sheet'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); mockSurfaceDispatch( { nodes: WEB_SIGN_IN_SHEET_NODES, systemSurface: WEB_SIGN_IN_SHEET }, { @@ -457,7 +479,6 @@ test('back --settle attaches no diff across an app-to-sheet surface change and d const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'back', flags: { ...SETTLE_FLAGS }, }); @@ -478,7 +499,7 @@ test('back --settle attaches no diff across an app-to-sheet surface change and d test('back --settle attaches no diff across a sheet-to-app surface change and discloses it', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-back-from-sheet'; - const session = seedSession(sessionName, sessionStore, { + seedSession(sessionName, sessionStore, { nodes: WEB_SIGN_IN_SHEET_NODES, systemSurface: WEB_SIGN_IN_SHEET, }); @@ -494,7 +515,6 @@ test('back --settle attaches no diff across a sheet-to-app surface change and di const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'back', flags: { ...SETTLE_FLAGS }, }); @@ -513,13 +533,12 @@ test('back --settle attaches no diff across a sheet-to-app surface change and di test('the settle observation runs after the post-gesture stabilization marker', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-order'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); mockCommandDispatch([AFTER_NODES]); await dispatchGeneric({ sessionName, sessionStore, - session, command: 'scroll', positionals: ['down'], flags: { ...SETTLE_FLAGS }, @@ -532,22 +551,93 @@ test('the settle observation runs after the post-gesture stabilization marker', expect(captureObservations[0]?.postGestureStabilizationPending).toBe(true); }); +test('scroll --settle observes through the request-bound capture, once admitted', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-bound-capture'; + // A tracked app: the plan this request must admit is the active-app one. The fixture refuses the + // without-active-app plan, so an observation that resolved the wrong cell would degrade to the + // legacy dispatch below and fail these assertions rather than pass by accident. + seedSession( + sessionName, + sessionStore, + { nodes: BEFORE_NODES }, + { appBundleId: 'com.example.app' }, + ); + mockCommandDispatch([AFTER_NODES]); + const runtimeBindings = selectorCaptureFixture({ + withoutActiveApp: { available: false, reason: 'owner-capability-missing' }, + snapshot: () => ({ + nodes: AFTER_NODES, + backend: 'xctest', + producer: 'simulator-ax-bridge', + }), + }); + + const response = await dispatchGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + runtimeBindings, + }); + + const settle = expectOkData(response).settle as SettlePayload; + expect(settle.settled).toBe(true); + // One admit-and-bind for the whole observation, and every capture taken through it: the settled + // tree and the tree the caller's next `snapshot` reads are the same producer's output (#3510). + expect(runtimeBindings.binds.length).toBe(1); + expect(runtimeBindings.captures.length).toBe(settle.captures); + expect(mockDispatch).not.toHaveBeenCalledWith('snapshot'); + expect((sessionStore.get(sessionName) as SessionState).snapshot?.producer).toBe( + 'simulator-ax-bridge', + ); +}); + +test('scroll --settle still observes when the owner cannot capture', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-unbound-capture'; + seedSession(sessionName, sessionStore); + mockCommandDispatch([AFTER_NODES]); + const runtimeBindings = selectorCaptureFixture({ + capture: { available: false, reason: 'unsupported-platform-leaf' }, + }); + + const response = await dispatchGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + runtimeBindings, + }); + + // The owner's own snapshot path is its only snapshot path: the observation stays complete and + // nothing pretends a second capture implementation produced it. + expect(expectOkData(response).settle).toMatchObject({ settled: true }); + expect(runtimeBindings.binds).toEqual([]); + expect(mockDispatch).toHaveBeenCalledWith('snapshot'); +}); + test('scroll without --settle takes no observation captures and issues no refs', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-off'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); mockCommandDispatch([AFTER_NODES]); + const runtimeBindings = selectorCaptureFixture(); const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'scroll', positionals: ['down'], + runtimeBindings, }); expect(expectOkData(response).settle).toBeUndefined(); expect(captureObservations).toEqual([]); + // Settling is what asks for an observation capture, so a plain scroll never pays for one. + expect(runtimeBindings.binds).toEqual([]); // ADR 0014: the leaf side-effect seam expired the frame and nothing // re-published it. expect(refFrameState(sessionStore.get(sessionName) as SessionState)).toBe('expired'); @@ -577,6 +667,7 @@ test('an ended generic lifetime is refused before dispatch or settle constructio logPath: '', contextFromFlags, executePlatformCommand: platformExecution, + ...legacyCaptureRuntimeBindings(), }), ).rejects.toMatchObject({ details: { reason: 'session_lifetime_ended' } }); expect(mockDispatch).not.toHaveBeenCalled(); @@ -586,7 +677,7 @@ test('an ended generic lifetime is refused before dispatch or settle constructio test('a generic command without the observation trait ignores a stray settle flag', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-traitless'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); mockDispatch.mockResolvedValue({ action: 'home', message: 'Home' }); // A hand-built daemon request can put settle flags on any command; only the @@ -595,7 +686,6 @@ test('a generic command without the observation trait ignores a stray settle fla const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'home', flags: { settle: true }, }); @@ -607,13 +697,12 @@ test('a generic command without the observation trait ignores a stray settle fla test('an orphaned --settle-quiet is rejected before the command dispatches', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-orphan'; - const session = seedSession(sessionName, sessionStore); + seedSession(sessionName, sessionStore); mockDispatch.mockRejectedValue(new Error('dispatch must not run for an orphaned settle flag')); const response = await dispatchGeneric({ sessionName, sessionStore, - session, command: 'scroll', positionals: ['down'], flags: { settleQuietMs: 25 }, @@ -628,7 +717,7 @@ test('an orphaned --settle-quiet is rejected before the command dispatches', asy test('scroll stays successful when its lifetime retires during optional settle observation', async () => { const store = makeSessionStore(); const name = 'generic-settle-retired'; - const session = seedSession(name, store); + seedSession(name, store); const ref = store.lookup(name)!; mockCommandDispatch([AFTER_NODES]); mockCaptureSnapshotForSession.mockImplementationOnce( @@ -649,7 +738,6 @@ test('scroll stays successful when its lifetime retires during optional settle o const response = await dispatchGeneric({ sessionName: name, sessionStore: store, - session, command: 'scroll', positionals: ['down'], flags: { ...SETTLE_FLAGS }, @@ -663,7 +751,7 @@ for (const change of ['rebuild', 'retire'] as const) { test(`held generic settle capture respects its lifetime after ${change}`, async () => { const store = makeSessionStore(); const name = `generic-held-settle-${change}`; - const session = seedSession(name, store); + seedSession(name, store); const ref = store.lookup(name)!; let entered!: () => void; let release!: () => void; @@ -682,7 +770,6 @@ for (const change of ['rebuild', 'retire'] as const) { const pending = dispatchGeneric({ sessionName: name, sessionStore: store, - session, command: 'scroll', positionals: ['down'], flags: { ...SETTLE_FLAGS }, diff --git a/src/daemon/__tests__/ios-comparison-key-fixture.ts b/src/daemon/__tests__/ios-comparison-key-fixture.ts index b581749a42..95a39f7d74 100644 --- a/src/daemon/__tests__/ios-comparison-key-fixture.ts +++ b/src/daemon/__tests__/ios-comparison-key-fixture.ts @@ -3,29 +3,41 @@ import { createIosSnapshotRequest, iosSnapshotComparisonIdentityKey, } from '@agent-device/capture-kit/ios-snapshot-planning'; +import type { IosSnapshotComparisonIdentity } from '@agent-device/contracts/ios-snapshot'; import { IOS_SYSTEM_SURFACE_HOSTS } from '@agent-device/contracts/ios-system-surface'; /** - * The comparison keys the iOS snapshot route stamps on XCTest captures, built through the production - * key function: an app capture is lineaged to the app, a capture of an in-place system surface (a web - * sign-in sheet) to its HOST, because that is the surface it describes (#2438). Regressions that rely - * on a cross-surface pair being incomparable build both keys here, so they fail if the construction - * ever stops distinguishing them. + * The comparison identity the iOS snapshot route stamps on a capture it served from the XCTest + * runner, built through the production key function: an app capture is lineaged to the app, a + * capture of an in-place system surface (a web sign-in sheet) to its HOST, because that is the + * surface it describes (#2438). Regressions that rely on a cross-surface pair being incomparable + * build both here, so they fail if the construction ever stops distinguishing them. A test that + * needs both halves of one capture — a stored baseline keyed from it, and a capture result carrying + * it — takes the identity rather than rewriting the shape (#3510). */ +export function appCaptureComparisonIdentity( + deviceId: string, + appBundleId: string, +): IosSnapshotComparisonIdentity { + return runnerComparisonIdentity(`${deviceId}:${appBundleId}`); +} + export function appCaptureComparisonKey(deviceId: string, appBundleId: string): string { - return runnerComparisonKey(`${deviceId}:${appBundleId}`); + return iosSnapshotComparisonIdentityKey(appCaptureComparisonIdentity(deviceId, appBundleId)); } export function systemSurfaceCaptureComparisonKey(deviceId: string): string { - return runnerComparisonKey(`${deviceId}:${IOS_SYSTEM_SURFACE_HOSTS[0]!.bundleId}`); + return iosSnapshotComparisonIdentityKey( + runnerComparisonIdentity(`${deviceId}:${IOS_SYSTEM_SURFACE_HOSTS[0]!.bundleId}`), + ); } -function runnerComparisonKey(targetId: string): string { - return iosSnapshotComparisonIdentityKey({ +function runnerComparisonIdentity(targetId: string): IosSnapshotComparisonIdentity { + return { producer: 'apple-runner', intent: 'full', lineage: { targetId }, presentationKey: buildIosSnapshotPresentationKey(createIosSnapshotRequest()), residue: [{ kind: 'fallback-source', producer: 'apple-runner' }], - }); + }; } diff --git a/src/daemon/__tests__/settle-observation-capture.test.ts b/src/daemon/__tests__/settle-observation-capture.test.ts new file mode 100644 index 0000000000..7f74cc6def --- /dev/null +++ b/src/daemon/__tests__/settle-observation-capture.test.ts @@ -0,0 +1,137 @@ +import { beforeEach, expect, test, vi } from 'vitest'; +import { IOS_SIMULATOR } from '@agent-device/test-kit/device-fixtures'; +import { makeIosSession } from './support/session-factories.ts'; +import { daemonPlatformServicesFixture } from './platform-services-fixture.ts'; +import { selectorCaptureFixture } from './selector-capture-fixture.ts'; +import { resolveSettleObservationCapture } from '../settle-observation-capture.ts'; +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; + +vi.mock('@agent-device/host-kit/diagnostics', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, emitDiagnostic: vi.fn(actual.emitDiagnostic) }; +}); + +// #3510: the capture a `--settle` observation polls on a route whose action runtime use carries +// none. The guarantee it owes is producer identity with the command the caller runs next; the one it +// must not break is that settling never fails the action it follows. These pin which plan gets +// admitted, how often, and that every way an observation capture goes away degrades to no capture +// while disclosing its typed reason. + +const unavailablePhase = ['settle_observation_capture_unavailable']; + +/** Level included: a degrade the caller cannot see must not be info-only, which success drops. */ +function disclosures(): { level: string | undefined; data: Record }[] { + return vi + .mocked(emitDiagnostic) + .mock.calls.flatMap(([event]) => + event.phase === unavailablePhase[0] && event.data + ? [{ level: event.level, data: event.data }] + : [], + ); +} + +beforeEach(() => { + vi.mocked(emitDiagnostic).mockClear(); +}); + +test('an available plan inspects once and binds once for the whole observation', async () => { + const fixture = selectorCaptureFixture(); + + const capture = await resolveSettleObservationCapture({ + command: 'back', + session: makeIosSession('settle', { appBundleId: 'com.example.app' }), + inspectFacts: fixture.inspectFacts, + platformServices: daemonPlatformServicesFixture(), + bindDevice: fixture.bindDevice, + }); + if (!capture) throw new Error('expected an admitted capture'); + + expect(fixture.inspections).toEqual([IOS_SIMULATOR]); + expect(fixture.binds).toEqual([IOS_SIMULATOR]); + + await capture({}); + await capture({}); + expect(fixture.binds).toEqual([IOS_SIMULATOR]); + expect(fixture.captures).toHaveLength(2); +}); + +// One rule over the two plan cells: an owner that refuses the observation's plan keeps the loop on +// its own snapshot path, and the refusal is disclosed with the typed fact rather than passing as a +// silently unchanged observation. The first row is the Apple no-tracked-app cell, where the owner's +// own `snapshot` refuses for the same declared reason. +test.each([ + { + plan: 'without-active-app', + refusal: { withoutActiveApp: { available: false, reason: 'owner-capability-missing' } }, + session: () => makeIosSession('no-app'), + operation: 'captureSnapshotWithoutActiveApp', + reason: 'owner-capability-missing', + }, + { + plan: 'active-app', + refusal: { capture: { available: false, reason: 'unsupported-platform-leaf' } }, + session: () => makeIosSession('settle', { appBundleId: 'com.example.app' }), + operation: 'captureSnapshot', + reason: 'unsupported-platform-leaf', + }, +] as const)( + 'a $plan plan the owner refuses binds nothing and discloses its reason', + async ({ refusal, session, operation, reason }) => { + const fixture = selectorCaptureFixture(refusal); + + const capture = await resolveSettleObservationCapture({ + command: 'back', + session: session(), + inspectFacts: fixture.inspectFacts, + platformServices: daemonPlatformServicesFixture(), + bindDevice: fixture.bindDevice, + }); + + expect(capture).toBeUndefined(); + expect(fixture.binds).toEqual([]); + expect(disclosures()).toEqual([ + { level: 'warn', data: { command: 'back', outcome: 'refused', operation, reason } }, + ]); + }, +); + +// The two ways the owner's own machinery can fail the observation after a valid plan. Both are +// best-effort by contract: the command the loop follows must still run, and each is disclosed at +// `warn` — an info-only record is dropped on the success path, and these cells answer `ok: true`. +test.each([ + // Apple's inspection reads the host's screen-recording state and re-throws on a cancelled probe. + ['admission-probe-failed', Object.assign(new Error('probe cancelled'), { name: 'AbortError' })], + ['bind-failed', Object.assign(new Error('gateway closed'), { name: 'AppError' })], +] as const)( + 'a %s costs the action nothing and discloses its own failure', + async (outcome, failure) => { + const fixture = selectorCaptureFixture(); + const failing = + outcome === 'admission-probe-failed' + ? { + inspectFacts: async () => { + throw failure; + }, + bindDevice: fixture.bindDevice, + } + : { + inspectFacts: fixture.inspectFacts, + bindDevice: async () => { + throw failure; + }, + }; + + const capture = await resolveSettleObservationCapture({ + command: 'back', + session: makeIosSession('settle', { appBundleId: 'com.example.app' }), + platformServices: daemonPlatformServicesFixture(), + ...failing, + }); + + expect(capture).toBeUndefined(); + expect(fixture.binds).toEqual([]); + expect(disclosures()).toEqual([ + { level: 'warn', data: { command: 'back', outcome, errorName: failure.name } }, + ]); + }, +); diff --git a/src/daemon/generic-settle.ts b/src/daemon/generic-settle.ts index 91fc2e97ff..01559d9598 100644 --- a/src/daemon/generic-settle.ts +++ b/src/daemon/generic-settle.ts @@ -13,7 +13,9 @@ import { settleFlagGuardResponse, } from './interaction/index.ts'; import type { BoundContextFromFlags } from './context.ts'; +import type { RuntimeAdmissionBindings } from './request-runtime-binding.ts'; import type { DaemonPlatformServices } from './platform-services.ts'; +import { resolveSettleObservationCapture } from './settle-observation-capture.ts'; import { issueSettleRefs } from './session-snapshot.ts'; import type { SessionStore } from './session-store.ts'; import type { DaemonRequest, DaemonResponse } from './daemon-request.ts'; @@ -66,7 +68,7 @@ type GenericSettleContext = { sessionStore: SessionStore; contextFromFlags: BoundContextFromFlags; platformServices: DaemonPlatformServices; -}; +} & RuntimeAdmissionBindings; /** * Rejects an orphaned `--settle-quiet` up front (the shared flag grammar), and @@ -120,10 +122,25 @@ async function observeSettled( return refsGeneration === undefined ? observation : { ...observation, refsGeneration }; } +/** + * Resolves the observation's capture before the runtime is constructed, so the loop reads the screen + * through the same producer the caller's next command reads it through (#3510). `back` and + * `scroll ` are the cells that need it: their action use is the movement alone. + * `scroll top|bottom` and `scroll --until` already require `captureSnapshot` for their between-pass + * verification; there this is the request's second admit-and-bind for one operation, served from the + * gateway's binding cache rather than a second device acquisition. + */ async function createGenericSettleRuntime( context: GenericSettleContext, ): Promise> | undefined> { try { + const boundCapture = await resolveSettleObservationCapture({ + command: context.req.command, + session: context.session, + inspectFacts: context.inspectFacts, + platformServices: context.platformServices, + bindDevice: context.bindDevice, + }); return await createInteractionRuntime({ createCommandSurface: context.createCommandSurface, req: context.req, @@ -134,6 +151,7 @@ async function createGenericSettleRuntime( contextFromFlags: context.contextFromFlags, platformServices: context.platformServices, captureSnapshotForSession, + settleObservationCapture: boundCapture, }); } catch { return undefined; diff --git a/src/daemon/interaction/index.ts b/src/daemon/interaction/index.ts index 4f66f480f8..612abde8c5 100644 --- a/src/daemon/interaction/index.ts +++ b/src/daemon/interaction/index.ts @@ -10,6 +10,7 @@ import type { CaptureSnapshotForSession, FindRouteInput, InteractionRouteInput, + SettleObservationCaptureInput, } from './internal/types.ts'; export type { FindRouteInput, InteractionRouteInput } from './internal/types.ts'; @@ -67,11 +68,12 @@ export const captureSnapshotForSession: CaptureSnapshotForSession = async ( }; export async function createInteractionRuntime( - params: InteractionRouteInput & { - pairedGestureViewport?: Rect; - touchExecutor?: import('../touch-runtime.ts').BoundTouchExecutor; - gestures?: import('../gesture-runtime.ts').BoundGestureExecutor; - }, + params: SettleObservationCaptureInput & + InteractionRouteInput & { + pairedGestureViewport?: Rect; + touchExecutor?: import('../touch-runtime.ts').BoundTouchExecutor; + gestures?: import('../gesture-runtime.ts').BoundGestureExecutor; + }, ) { return await createInteractionRuntimeForRoute({ ...bindInteractionSession(params), diff --git a/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures.ts b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures.ts index 4574dc36c7..d95baf646c 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures.ts @@ -58,7 +58,7 @@ export function snapshotPayload( backend: 'tree' | 'queries' | 'private-ax' = 'tree', ) { return { - backend: 'xctest', + backend: 'xctest' as const, producer: 'apple-runner' as const, nodes, quality: { state: 'healthy' as const, backend }, diff --git a/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts index cf7557a2e0..f58f23989a 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts @@ -6,6 +6,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; import { buildSnapshotPresentationKey } from '@agent-device/kernel/snapshot'; +import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; import { handleInteractionCommands } from '../../index.ts'; import { makeIosSession, @@ -22,6 +23,7 @@ import { snapshotPayload, } from './interaction-ios-tap-outcome-fixtures.ts'; import { + appCaptureComparisonIdentity, appCaptureComparisonKey, systemSurfaceCaptureComparisonKey, } from '../../../__tests__/ios-comparison-key-fixture.ts'; @@ -294,6 +296,80 @@ test('a changed capture from a different iOS backend keeps the tap failure', asy expect(sessionStore.get(sessionName)?.actions).toHaveLength(0); }); +/** + * #3510: `--settle` on a coordinate press binds the request's own capture, and the pre-action + * baseline the settle plan takes goes through it. The corroboration probe is an observation too, so + * it answers through the same capture AND under the same backend pin: an unkeyed baseline next to a + * keyed capture, or a keyed baseline next to an unkeyed one, is an identity mismatch that would + * report a tap that landed as a recorded failure. + * + * A private-AX baseline is the unkeyed half of that pair: a pinned capture leaves the planned route, + * and an app capture off that route carries no identity (#2438). Dropping the pin whenever a bound + * capture exists would answer it from the bridge and refuse the rescue the pin exists to earn. + */ +test.each([ + { + name: 'a request-bound bridge capture corroborates its own keyed baseline', + baseline: () => + buildSnapshotState( + { ...snapshotPayload(profileNodes), comparisonIdentity: identity() }, + { snapshotInteractiveOnly: false }, + ), + after: () => ({ ...snapshotPayload(imageViewerNodes), comparisonIdentity: identity() }), + expectedPreferredBackend: undefined, + }, + { + name: 'a private-AX baseline pins the backend of the request-bound capture', + baseline: () => + buildSnapshotState(snapshotPayload(profileNodes, 'private-ax'), { + snapshotInteractiveOnly: false, + }), + // A pinned capture leaves the planned route, so the runner answers and stamps no identity. + after: () => snapshotPayload(imageViewerNodes, 'private-ax'), + expectedPreferredBackend: 'private-ax', + }, +] as const)( + 'the corroboration probe answers $name', + async ({ baseline, after, expectedPreferredBackend }) => { + const sessionName = 'ios-corroboration-bound-capture'; + const sessionStore = makeSessionStore(); + sessionStore.publish( + sessionName, + makeIosSession(sessionName, { appBundleId: 'com.example.app', snapshot: baseline() }), + ); + const askedWith: Record[] = []; + const boundCapture = vi.fn(async () => after()); + + const corroboration = await corroborateIosTapFailure({ + error: new AppError('XCTEST_RECORDED_FAILURE', 'tap failed'), + command: 'click', + requestId: undefined, + flags: {}, + ref: sessionStore.lookup(sessionName)!, + sessionStore, + contextFromFlags, + // Mirrors the real seam: one capture ask, answered by the bound capture under the pin. + captureSnapshotForSession: async (_ref, _flags, _store, _context, options) => { + askedWith.push(options as unknown as Record); + return buildSnapshotState(await options.boundCapture!([] as never), { + snapshotInteractiveOnly: options.interactiveOnly, + }); + }, + boundCapture: boundCapture as unknown as Parameters< + typeof corroborateIosTapFailure + >[0]['boundCapture'], + }); + + expect(boundCapture).toHaveBeenCalledTimes(1); + expect(askedWith[0]?.preferredBackend).toBe(expectedPreferredBackend); + expect(corroboration?.warning).toMatch(/post-action accessibility capture changed/); + }, +); + +function identity() { + return appCaptureComparisonIdentity(IOS_SIMULATOR.id, 'com.example.app'); +} + test('a producer or generation switch cannot corroborate a failed tap', async () => { const sessionName = 'ios-comparison-identity-mismatch'; const sessionStore = makeSessionStore(); diff --git a/src/daemon/interaction/internal/__tests__/interaction-touch-prepare.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-prepare.test.ts new file mode 100644 index 0000000000..1689563fd6 --- /dev/null +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-prepare.test.ts @@ -0,0 +1,261 @@ +import { beforeEach, expect, test, vi } from 'vitest'; +import type { CommandFlags } from '@agent-device/contracts/command'; +import { + type DeviceBinding, + type RuntimeFacts, + localRuntimeOwner, + narrowDeviceBinding, +} from '@agent-device/contracts/platform-runtime'; +import type { PlatformRuntimeOperations } from '@agent-device/contracts/platform-runtime-operations'; +import { snapshotRuntimeOperationFacts } from '@agent-device/contracts/snapshot-runtime'; +import { + type FillPointInput, + type TapPointInput, + touchRuntimeOperationFacts, +} from '@agent-device/contracts/touch-runtime'; +import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { createUnavailableRuntimeFactsForTest } from '@agent-device/test-kit/runtime-operation-facts'; +import { makeIosAppSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import type { + BindDeviceRuntime, + InspectDeviceRuntimeFacts, +} from '../../../request-runtime-binding.ts'; +import { daemonPlatformServicesFixture } from '../../../__tests__/platform-services-fixture.ts'; +import { testCreateCommandSurface } from '../../../__tests__/command-surface-fixture.ts'; +import { handleInteractionCommands } from '../../index.ts'; +import type { SessionStore } from '../../../session-store.ts'; + +// #3510: the capture an interaction's post-action observation polls. A coordinate-targeted press or +// fill acts through a use that declares no capture, so before this its settle loop read the screen +// through the session's legacy snapshot path while the command the caller ran next read the +// request-bound runtime: on an iOS simulator, two projections of one screen. + +const available = Object.freeze({ available: true } as const); +const refused = Object.freeze({ available: false, reason: 'unsupported-platform-leaf' } as const); + +const TREE: RawSnapshotNode[] = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + identifier: 'save', + label: 'Save', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, +]; + +/** The tree only the request-bound capture can produce, so producer identity is decidable. */ +const BRIDGE_PRODUCED = Object.freeze({ + nodes: TREE, + backend: 'xctest' as const, + producer: 'simulator-ax-bridge' as const, +}); + +const mockInteractorCapture = vi.hoisted(() => vi.fn()); +vi.mock('../../../snapshot-interactor-capture.ts', () => ({ + captureSnapshotWithInteractor: mockInteractorCapture, +})); + +const contextFromFlags = (_flags: CommandFlags | undefined) => ({}); + +/** An owner that captures, taps points, fills points, and does nothing else. */ +function settleFixture() { + const binds: Array> = []; + let captures = 0; + const facts = (device: DeviceInfo): RuntimeFacts => { + const base = createUnavailableRuntimeFactsForTest(device, localRuntimeOwner('apple')); + return Object.freeze({ + device: base.device, + operations: { + ...base.operations, + ...snapshotRuntimeOperationFacts({ + capture: available, + customActions: refused, + withoutActiveApp: available, + }), + ...touchRuntimeOperationFacts({ + unsupported: refused, + tap: available, + longPress: refused, + fill: available, + }), + }, + }); + }; + const capture = async () => { + captures += 1; + return BRIDGE_PRODUCED; + }; + const inspectFacts: InspectDeviceRuntimeFacts = async (device) => facts(device); + const bindDevice: BindDeviceRuntime = async (device, use) => { + binds.push({ device, required: [...use.required] }); + return await narrowDeviceBinding( + Object.freeze({ + device, + owner: localRuntimeOwner('apple'), + facts: facts(device), + operations: Object.freeze({ + captureSnapshot: capture, + captureSnapshotWithoutActiveApp: capture, + tapPoint: async (input: TapPointInput) => ({ x: input.point.x, y: input.point.y }), + fillPoint: async (input: FillPointInput) => ({ x: input.point.x, y: input.point.y }), + }), + [Symbol.asyncDispose]: async () => undefined, + }) as unknown as DeviceBinding, + use, + ); + }; + return { + binds, + inspectFacts, + bindDevice, + captureCount: () => captures, + }; +} + +const SETTLE_FLAGS = { settle: true, settleQuietMs: 25, timeoutMs: 2_000 } satisfies CommandFlags; + +function publishSession(sessionStore: SessionStore, sessionName: string) { + const session = makeIosAppSession(sessionName); + session.snapshot = { + nodes: attachRefs(TREE), + createdAt: Date.now(), + backend: 'xctest', + producer: 'apple-runner', + }; + sessionStore.publish(sessionName, session); + return session; +} + +async function runCommand( + sessionStore: SessionStore, + sessionName: string, + command: string, + positionals: string[], + bindings: ReturnType, + flags: CommandFlags = {}, + inspectFacts: InspectDeviceRuntimeFacts = bindings.inspectFacts, +) { + return await handleInteractionCommands({ + createCommandSurface: testCreateCommandSurface, + req: { token: 't', session: sessionName, command, positionals, flags: { ...flags } }, + sessionName, + sessionStore, + contextFromFlags, + platformServices: daemonPlatformServicesFixture(), + inspectFacts, + bindDevice: bindings.bindDevice, + }); +} + +function settled(response: unknown) { + return (response as { data: { settle?: { settled: boolean } } }).data.settle; +} + +beforeEach(() => { + mockInteractorCapture.mockReset(); +}); + +// The observation's bind is a second, capture-only use beside the action's exact one — taken once +// for the request, not once per capture. +test.each([ + ['press', ['10', '20'], ['tapPoint']], + ['fill', ['10', '20', 'kit'], ['fillPoint']], +] as const)( + 'a coordinate %s with --settle observes through the capture the request admitted', + async (command, positionals, actionUse) => { + const sessionStore = makeSessionStore(); + const sessionName = `point-${command}-settle`; + const session = publishSession(sessionStore, sessionName); + const bindings = settleFixture(); + + const response = await runCommand( + sessionStore, + sessionName, + command, + [...positionals], + bindings, + SETTLE_FLAGS, + ); + + expect(response?.ok).toBe(true); + expect(settled(response)).toMatchObject({ settled: true }); + expect(bindings.binds.map((bind) => bind.required)).toEqual([actionUse, ['captureSnapshot']]); + // The tree the observation published is the bound capture's, and the legacy path was never asked + // for a tree: one producer for the whole request, which is what lets the refs the settle response + // issued resolve against the tree the caller's next `snapshot` reads. + expect(session.snapshot?.producer).toBe('simulator-ax-bridge'); + expect(bindings.captureCount()).toBeGreaterThan(1); + expect(mockInteractorCapture).not.toHaveBeenCalled(); + }, +); + +test('a coordinate press without --settle binds no observation capture', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'point-press-no-settle'; + publishSession(sessionStore, sessionName); + const bindings = settleFixture(); + + const response = await runCommand(sessionStore, sessionName, 'press', ['10', '20'], bindings); + + expect(response?.ok).toBe(true); + expect(bindings.binds.map((bind) => bind.required)).toEqual([['tapPoint']]); +}); + +test('an observation capture the owner cannot admit still lets the coordinate press act', async () => { + // The action's own facts inspection answers first, then the observation's admission is asked. An + // owner that fails the second ask must not cost the first: settling is best-effort by contract, so + // the tap still lands and the observation degrades to the owner's own snapshot path (#3510). + const sessionStore = makeSessionStore(); + const sessionName = 'point-press-probe-failure'; + publishSession(sessionStore, sessionName); + const bindings = settleFixture(); + mockInteractorCapture.mockResolvedValue(BRIDGE_PRODUCED); + let inspections = 0; + + const response = await runCommand( + sessionStore, + sessionName, + 'press', + ['10', '20'], + bindings, + SETTLE_FLAGS, + async (device) => { + inspections += 1; + // Stands in for the Apple facts probe re-throwing a cancelled host read on the second ask. + if (inspections > 1) + throw Object.assign(new Error('probe cancelled'), { name: 'AbortError' }); + return await bindings.inspectFacts(device); + }, + ); + + expect(response?.ok).toBe(true); + expect(settled(response)).toMatchObject({ settled: true }); + expect(mockInteractorCapture).toHaveBeenCalled(); +}); + +test('a tree-resolved press reuses the capture its action binding already carries', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'selector-press-settle'; + publishSession(sessionStore, sessionName); + const bindings = settleFixture(); + + const response = await runCommand( + sessionStore, + sessionName, + 'press', + ['label="Save"'], + bindings, + SETTLE_FLAGS, + ); + + expect(response?.ok).toBe(true); + // No extra bind: this route's use already requires the capture, which is why a selector press + // never had the split a coordinate press had. + expect(bindings.binds.map((bind) => bind.required)).toEqual([['captureSnapshot', 'tapPoint']]); + expect(sessionStore.get(sessionName)?.snapshot?.producer).toBe('simulator-ax-bridge'); +}); diff --git a/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts b/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts index 5fd0566d49..daa210dd46 100644 --- a/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts +++ b/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts @@ -12,6 +12,7 @@ import { isLocalIosRunnerSession } from '../../direct-ios-selector.ts'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { SessionStore } from '../../session-store.ts'; import type { SessionRef, SessionState } from '../../session-state.ts'; +import type { BoundSnapshotCapture } from '../../snapshot-runtime-binding.ts'; import type { BoundContextFromFlags, CaptureSnapshotForSession } from './types.ts'; const XCTEST_RECORDED_FAILURE = 'XCTEST_RECORDED_FAILURE'; @@ -39,6 +40,12 @@ export type IosTapCorroborationParams = { sessionStore: SessionStore; contextFromFlags: BoundContextFromFlags; captureSnapshotForSession: CaptureSnapshotForSession; + /** + * The capture this request's own pre-action baseline was taken with, when it holds one. The probe + * is an observation too and answers through it: a keyed baseline next to an unkeyed legacy capture + * is an identity mismatch, which would report a tap that landed as a recorded failure (#3510). + */ + boundCapture?: BoundSnapshotCapture; }; export type IosTapCorroboration = { @@ -131,10 +138,15 @@ async function captureCorroborationSnapshot( params.contextFromFlags, { interactiveOnly: presentation?.interactiveOnly ?? true, - // Evidence comparison is only valid same-backend, and the recorded-failure - // screens are exactly where the capture plan flips between XCTest and - // private-AX (the penalty boundary) — pin the probe to the baseline's - // backend instead of failing closed on the mismatch. + // Two independent facts about the one capture, both forwarded: which implementation answers + // (the request's own binding when it holds one) and which backend that capture must present. + ...(params.boundCapture ? { boundCapture: params.boundCapture } : {}), + // Evidence comparison is only valid same-backend, and the recorded-failure screens are + // exactly where the capture plan flips between XCTest and private-AX (the penalty + // boundary) — pin the probe to the baseline's backend instead of failing closed on the + // mismatch. The pin is an option OF the capture, so a bound capture takes it too: dropping + // it there would answer a private-AX baseline with a bridge capture, whose comparison + // identity no unkeyed runner baseline can match. ...(preferredBackend ? { preferredBackend } : {}), signal: getRequestSignal(params.requestId), }, diff --git a/src/daemon/interaction/internal/interaction-runtime.ts b/src/daemon/interaction/internal/interaction-runtime.ts index c50fb21d41..bee3cd7c63 100644 --- a/src/daemon/interaction/internal/interaction-runtime.ts +++ b/src/daemon/interaction/internal/interaction-runtime.ts @@ -20,6 +20,7 @@ import type { CaptureSnapshotForSession, InteractionRouteInput, InteractionRuntimeInput, + SettleObservationCaptureInput, } from './types.ts'; import { finalizeTouchInteraction as finalizeInteraction } from './interaction-common.ts'; import { markDeferredInteractionOutcome } from '../../deferred-interaction-outcome.ts'; @@ -30,23 +31,25 @@ import type { DaemonResponse } from '../../daemon-request.ts'; import type { SessionRef } from '../../session-state.ts'; import type { BoundTouchExecutor } from '../../touch-runtime.ts'; import type { BoundGestureExecutor } from '../../gesture-runtime.ts'; +import type { BoundSnapshotCapture } from '../../snapshot-runtime-binding.ts'; import { NO_ACTIVE_SESSION_MESSAGE } from '@agent-device/kernel/contracts'; export async function createInteractionRuntimeForRoute( - params: InteractionRouteInput & { - captureSnapshotForSession: CaptureSnapshotForSession; - pairedGestureViewport?: Rect; - touchExecutor?: BoundTouchExecutor; - gestures?: BoundGestureExecutor; - /** - * Filled with the LAST capture this request consumed. ADR 0014 ambiguity - * issuance is only valid against a tree the session stored under the - * generation it freezes, and a sparse-quality capture deliberately stores - * nothing — the refusal seam needs the same consumed-capture slot the - * selector routes carry (`consumedSnapshot`), owned per dispatch. - */ - consumedCapture?: { state?: SnapshotState }; - }, + params: SettleObservationCaptureInput & + InteractionRouteInput & { + captureSnapshotForSession: CaptureSnapshotForSession; + pairedGestureViewport?: Rect; + touchExecutor?: BoundTouchExecutor; + gestures?: BoundGestureExecutor; + /** + * Filled with the LAST capture this request consumed. ADR 0014 ambiguity + * issuance is only valid against a tree the session stored under the + * generation it freezes, and a sparse-quality capture deliberately stores + * nothing — the refusal seam needs the same consumed-capture slot the + * selector routes carry (`consumedSnapshot`), owned per dispatch. + */ + consumedCapture?: { state?: SnapshotState }; + }, ) { const ref = bindInteractionSession(params).sessionRef; if (!ref) throw new KernelAppError('SESSION_NOT_FOUND', NO_ACTIVE_SESSION_MESSAGE); @@ -90,6 +93,7 @@ export async function createInteractionRuntimeForRoute( pairedGestureViewport: params.pairedGestureViewport, touchExecutor: params.touchExecutor, gestures: params.gestures, + boundCapture: resolveBoundCapture(params), }); } @@ -145,6 +149,27 @@ export function finalizeTouchInteraction(params: FinalizeTouchInteractionInput): }); } +/** + * The capture implementation every observation on this request reads the screen through: the action + * binding's own capture when its runtime use carries one, otherwise the capture the request admitted + * for its observations (`settle-observation-capture.ts`). Resolved once, at construction, so a route + * cannot answer one capture from the request-bound runtime and the next from the legacy path. Which + * backend a bound capture presents stays per-capture: the settle loop pins it once it sees a + * `private-ax` verdict. + */ +function resolveBoundCapture( + params: SettleObservationCaptureInput & { + touchExecutor?: BoundTouchExecutor; + gestures?: BoundGestureExecutor; + }, +): BoundSnapshotCapture | undefined { + return ( + params.touchExecutor?.captureSnapshot ?? + params.gestures?.captureSnapshot ?? + params.settleObservationCapture + ); +} + function createInteractionAgentDevice(params: InteractionRuntimeInput) { return params.createCommandSurface({ backend: createInteractionBackend(params), @@ -166,7 +191,7 @@ function createInteractionBackend(params: InteractionRuntimeInput): AgentDeviceB preferredBackend: options?.preferredBackend, includeRects: options?.includeRects === true, signal: context.signal, - boundCapture: params.touchExecutor?.captureSnapshot ?? params.gestures?.captureSnapshot, + boundCapture: params.boundCapture, }), }), ...gestureBackendMembers(params, gestureContext), diff --git a/src/daemon/interaction/internal/interaction-touch-fill.ts b/src/daemon/interaction/internal/interaction-touch-fill.ts index 5d6523125a..0142edac5d 100644 --- a/src/daemon/interaction/internal/interaction-touch-fill.ts +++ b/src/daemon/interaction/internal/interaction-touch-fill.ts @@ -14,6 +14,7 @@ import type { CaptureSnapshotForSession, InteractionRouteInput, RefSnapshotFlagGuardResponse, + SettleObservationCaptureInput, } from './types.ts'; import { refreshAndroidRefSnapshotIfFreshnessActive } from './interaction-touch-android-freshness.ts'; import { unsupportedMacOsDesktopSurfaceInteraction } from './interaction-touch-policy.ts'; @@ -47,18 +48,21 @@ type AdmittedFill = { parsedTarget: Extract, { ok: true }>; touchExecutor: BoundTouchExecutor; staleRefsWarning: string | undefined; + settleObservationCapture: SettleObservationCaptureInput['settleObservationCapture']; }; export async function dispatchFillViaRuntime(params: FillParams): Promise { params = bindInteractionSession(params); const admission = await admitFill(params); if ('response' in admission) return admission.response; - const { parsedTarget, touchExecutor, staleRefsWarning } = admission.admitted; + const { parsedTarget, touchExecutor, staleRefsWarning, settleObservationCapture } = + admission.admitted; const { req, sessionName } = params; const replayTargetGuard = req.internal?.replayTargetGuard; return await dispatchRuntimeInteraction(params, { touchExecutor, + settleObservationCapture, refContext: parsedTarget.target.kind === 'ref' && req.internal?.findResolvedTarget === undefined ? { @@ -115,13 +119,8 @@ async function readFillAdmission( const target = readFillTarget(params); if ('response' in target) return target; const { session, parsedTarget } = target; - const prepared = await prepareTouchDispatch( - params, - session, - 'fill', - parsedTarget.target.kind !== 'point', - ); - if (!prepared.ok) return { response: prepared.response }; + // Pure input refusals first: a rejected fill must not pay a facts inspection or a bind, matching + // the order the press route keeps by validating inside its own admission. assertRecordedFillParameterization({ flags: req.flags, replayPlanStep: req.internal?.replayPlanStep === true, @@ -129,6 +128,13 @@ async function readFillAdmission( }); const invalidSettleFlags = settleFlagGuardResponse('fill', req.flags); if (invalidSettleFlags) return { response: invalidSettleFlags }; + const prepared = await prepareTouchDispatch( + params, + session, + 'fill', + parsedTarget.target.kind !== 'point', + ); + if (!prepared.ok) return { response: prepared.response }; const refPreamble = await prepareFillRefTarget( params, session, @@ -142,6 +148,7 @@ async function readFillAdmission( parsedTarget, touchExecutor: prepared.touchExecutor, staleRefsWarning: refPreamble.staleRefsWarning, + settleObservationCapture: prepared.settleObservationCapture, }, }; } diff --git a/src/daemon/interaction/internal/interaction-touch-prepare.ts b/src/daemon/interaction/internal/interaction-touch-prepare.ts index 34bafeab8f..6cc9ab7ba5 100644 --- a/src/daemon/interaction/internal/interaction-touch-prepare.ts +++ b/src/daemon/interaction/internal/interaction-touch-prepare.ts @@ -5,15 +5,30 @@ import { type BoundTouchExecutor, type TouchRuntimeCommand, } from '../../touch-runtime.ts'; +import { readSettleRequest } from './interaction-flags.ts'; +import type { BoundSnapshotCapture } from '../../snapshot-runtime-binding.ts'; +import { resolveSettleObservationCapture } from '../../settle-observation-capture.ts'; import type { InteractionRouteInput } from './types.ts'; import type { DaemonFailureResponse } from '@agent-device/kernel/contracts'; import { refusedBeforeDispatch } from '../../request-dispatch-disclosure.ts'; export type PreparedTouchDispatch = | Readonly<{ ok: false; response: DaemonFailureResponse }> - | Readonly<{ ok: true; touchExecutor: BoundTouchExecutor }>; + | Readonly<{ + ok: true; + touchExecutor: BoundTouchExecutor; + /** + * The observation capture this request still owes: only a route whose action runtime use + * carries no capture and owes a settle observation can produce one, so every other touch + * dispatch pays no admission and no bind. + */ + settleObservationCapture: BoundSnapshotCapture | undefined; + }>; -/** Exact-owner admission, one bind, and command-context projection shared by every touch route. */ +/** + * Exact-owner admission, the command-context projection, and — for a coordinate route that owes a + * settle observation — the request's own capture-only bind, shared by every touch route. + */ export async function prepareTouchDispatch( params: InteractionRouteInput, session: SessionState, @@ -29,11 +44,25 @@ export async function prepareTouchDispatch( bindDevice: params.bindDevice, }); if (!bound.ok) return { ok: false, response: refusedBeforeDispatch(bound.response) }; + const touchExecutor = createBoundTouchExecutor( + bound.runtime, + params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), + ); + const owesObservationCapture = + touchExecutor.captureSnapshot === undefined && + readSettleRequest(params.req.flags) !== undefined; + const settleObservationCapture = owesObservationCapture + ? await resolveSettleObservationCapture({ + command: params.req.command, + session, + inspectFacts: params.inspectFacts, + platformServices: params.platformServices, + bindDevice: params.bindDevice, + }) + : undefined; return { ok: true, - touchExecutor: createBoundTouchExecutor( - bound.runtime, - params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), - ), + touchExecutor, + settleObservationCapture, }; } diff --git a/src/daemon/interaction/internal/interaction-touch-press.ts b/src/daemon/interaction/internal/interaction-touch-press.ts index 6c6f26588d..1bd6944889 100644 --- a/src/daemon/interaction/internal/interaction-touch-press.ts +++ b/src/daemon/interaction/internal/interaction-touch-press.ts @@ -84,6 +84,7 @@ export async function dispatchTargetedTouchViaRuntime( return await dispatchRuntimeInteraction(params, { ...buildTargetedRuntimeOptions(params, command, boundAdmitted), touchExecutor, + settleObservationCapture: prepared.settleObservationCapture, }); } diff --git a/src/daemon/interaction/internal/interaction-touch-runtime.ts b/src/daemon/interaction/internal/interaction-touch-runtime.ts index 36ec2a12e5..feb9f5cc34 100644 --- a/src/daemon/interaction/internal/interaction-touch-runtime.ts +++ b/src/daemon/interaction/internal/interaction-touch-runtime.ts @@ -32,6 +32,7 @@ import { type InteractionResponsePayloads, } from './interaction-touch-response.ts'; import type { BoundTouchExecutor } from '../../touch-runtime.ts'; +import type { BoundSnapshotCapture } from '../../snapshot-runtime-binding.ts'; import { noActiveSessionError } from '@agent-device/kernel/contracts'; /** @@ -48,6 +49,8 @@ export async function dispatchRuntimeInteraction< }, options: { touchExecutor: BoundTouchExecutor; + /** The request's own observation capture when its action use carries none (#3510). */ + settleObservationCapture?: BoundSnapshotCapture; androidFreshnessBaseline?: SessionState['snapshot']; /** * Present when the action targets a `@ref`: if Android dialog recovery @@ -75,6 +78,7 @@ export async function dispatchRuntimeInteraction< const runtime = await createInteractionRuntimeForRoute({ ...params, touchExecutor: options.touchExecutor, + settleObservationCapture: options.settleObservationCapture, consumedCapture, }); const actionStartedAt = Date.now(); @@ -134,6 +138,7 @@ export async function dispatchRuntimeInteraction< const corroboratedResponse = await buildRuntimeIosCorroboratedResponse({ error: appError, handlerParams: params, + observationCapture: options.settleObservationCapture, session, target: options.iosTapCorroboration?.target, extra: options.iosTapCorroboration?.extra, @@ -150,6 +155,8 @@ async function buildRuntimeIosCorroboratedResponse(params: { handlerParams: InteractionRouteInput & { captureSnapshotForSession: CaptureSnapshotForSession; }; + /** The capture this request's own pre-action baseline was taken with, when it holds one (#3510). */ + observationCapture: BoundSnapshotCapture | undefined; session: SessionState; target: InteractionTarget | undefined; extra: Record | undefined; @@ -168,6 +175,7 @@ async function buildRuntimeIosCorroboratedResponse(params: { sessionStore: params.handlerParams.sessionStore, contextFromFlags: params.handlerParams.contextFromFlags, captureSnapshotForSession: params.handlerParams.captureSnapshotForSession, + boundCapture: params.observationCapture, }); if (!corroboration) return undefined; diff --git a/src/daemon/interaction/internal/types.ts b/src/daemon/interaction/internal/types.ts index 57346e6aed..f9a0ade0f3 100644 --- a/src/daemon/interaction/internal/types.ts +++ b/src/daemon/interaction/internal/types.ts @@ -44,6 +44,20 @@ export type InteractionRouteInput = { captureProof?: RequestCaptureProof; }; +/** + * The observation capture a request admitted for itself. Only the two constructors that build an + * observation runtime carry it — never the route input every interaction command shares — so a + * capture reaches an observation only from the module that admitted it on the owner's facts (#3510). + */ +export type SettleObservationCaptureInput = { + /** + * For a route whose action runtime use carries no capture: a coordinate-targeted touch + * (`tapPoint`/`fillPoint` declare none) or a generic-route action. Without it the post-action + * observation reads the screen through a different producer than the caller's next command does. + */ + settleObservationCapture?: BoundSnapshotCapture; +}; + export type FindRouteInput = { createCommandSurface: CreateDaemonCommandSurface; req: DaemonRequest; @@ -105,6 +119,12 @@ export type InteractionRuntimeInput = { pairedGestureViewport?: Rect; touchExecutor?: BoundTouchExecutor; gestures?: BoundGestureExecutor; + /** + * The capture every observation on this request reads the screen through: the action binding's + * own capture when its runtime use carries one, otherwise the capture the request admitted for + * its observations. Resolved once at construction so no route can mix producers (#3510). + */ + boundCapture?: BoundSnapshotCapture; }; export type InteractionGestureVisualization = ( diff --git a/src/daemon/request-generic-dispatch.ts b/src/daemon/request-generic-dispatch.ts index 04b6099b83..4c76de00e2 100644 --- a/src/daemon/request-generic-dispatch.ts +++ b/src/daemon/request-generic-dispatch.ts @@ -26,6 +26,7 @@ import { buildActionEventResult } from '@agent-device/session-journal/session-ev import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; import type { CreateDaemonCommandSurface } from '@agent-device/contracts/daemon-command-surface'; import type { GenericPlatformExecution, RecordedGenericRequest } from './generic-leaf-execution.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from './request-runtime-binding.ts'; export async function dispatchGenericCommand(params: { req: DaemonRequest; @@ -43,6 +44,9 @@ export async function dispatchGenericCommand(params: { recordedRequest?: RecordedGenericRequest; androidObservation?: AndroidObservationAdapter; platformServices: DaemonPlatformServices; + /** The request-scoped runtime seams `--settle` admits its own observation capture through (#3510). */ + inspectFacts: InspectDeviceRuntimeFacts; + bindDevice: BindDeviceRuntime; }): Promise { const { req, ref: sessionRef, logPath, sessionStore, contextFromFlags } = params; const session = sessionStore.requireCurrent(sessionRef); @@ -68,6 +72,8 @@ export async function dispatchGenericCommand(params: { command: platformCommand, flags: req.flags, createCommandSurface: params.createCommandSurface, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, }); if ('response' in settlePlan) return settlePlan.response; const preflightReadiness = await ensureNoAndroidBlockingDialogReady( @@ -224,6 +230,8 @@ async function planGenericSettleObservation(params: { platformServices: DaemonPlatformServices; command: string; flags: CommandFlags | undefined; + inspectFacts: InspectDeviceRuntimeFacts; + bindDevice: BindDeviceRuntime; }): Promise< { response: DaemonResponse } | { observe?: () => Promise } > { diff --git a/src/daemon/request-router.ts b/src/daemon/request-router.ts index 9df1bab385..c9a0467cf1 100644 --- a/src/daemon/request-router.ts +++ b/src/daemon/request-router.ts @@ -647,6 +647,10 @@ async function dispatchGenericForLockedScope(params: { executePlatformCommand: runtimeExecution.execute, androidObservation, createCommandSurface, + // The settle observation binds its own capture on this route, so it needs the same + // request-scoped seams the action's admission above already used (#3510). + inspectFacts: lockedScope.inspectFacts, + bindDevice: lockedScope.bindDevice, ...(runtimeExecution.recorded ? { recordedRequest: runtimeExecution.recorded } : {}), }); return dispatchResponse; diff --git a/src/daemon/session-runtime-admission.ts b/src/daemon/session-runtime-admission.ts index 340ab88272..20e0aa48e1 100644 --- a/src/daemon/session-runtime-admission.ts +++ b/src/daemon/session-runtime-admission.ts @@ -3,6 +3,7 @@ import type { PlatformRuntimeOperations } from '@agent-device/contracts/platform import type { RuntimeOperationFact, RuntimeOperationKey, + RuntimeOperationUnavailability, } from '@agent-device/contracts/platform-runtime'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -124,7 +125,7 @@ export function unwrapAdmittedRuntimePlan( export type RefusedRuntimePlan = Readonly<{ admitted: false; operation: Plan['use']['required'][number]; - fact: RuntimeOperationFact; + fact: RuntimeOperationUnavailability; }>; /** diff --git a/src/daemon/settle-observation-capture.ts b/src/daemon/settle-observation-capture.ts new file mode 100644 index 0000000000..ad149ef413 --- /dev/null +++ b/src/daemon/settle-observation-capture.ts @@ -0,0 +1,100 @@ +import type { BoundSnapshotCapture } from './snapshot-runtime-binding.ts'; +import { bindAdmittedSnapshotCapture } from './snapshot-runtime-binding.ts'; +import { admitRuntimePlan } from './session-runtime-admission.ts'; +import { + resolveSelectorCaptureRuntimePlan, + type SelectorCaptureRuntimePlan, +} from '@agent-device/contracts/platform-runtime-operations'; +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import type { RuntimeOperationUnavailability } from '@agent-device/contracts/platform-runtime'; +import type { RuntimeAdmissionBindings } from './request-runtime-binding.ts'; +import type { SessionState } from './session-state.ts'; + +/** + * The capture a post-action observation (`--settle`) polls, for a request whose own action binding + * carries none: a coordinate-targeted `press`/`fill` (whose point use declares none) or a + * generic-route action (#3510). Those routes polled the legacy interactor while the commands the + * caller runs next read the request-bound runtime, and on an iOS simulator those are two projections + * of one tree. + * + * Admitting the `capture-only` plan the selector family polls under closes the split; where both + * paths already reached the same tree, the admitted operation resolves to the implementation used + * before. Settling is best-effort by contract, so nothing here may fail or cancel the action it + * follows: every unavailability degrades to `undefined`, leaving the observation on the owner's own + * snapshot path, and discloses its typed reason. Admitting through the side-effect-free facts probe + * and binding only on admission keeps an optional observation from building a command-facing refusal + * response, whose Apple branch live-probes the booted simulator set for a hint nothing shows. + */ +export async function resolveSettleObservationCapture( + params: Readonly<{ command: string; session: SessionState }> & RuntimeAdmissionBindings, +): Promise { + const plan = resolveSelectorCaptureRuntimePlan({ + hasActiveApp: params.session.appBundleId !== undefined, + intent: 'capture-only', + }); + const admission = await admitObservationCapturePlan(params, plan); + if (admission === undefined) return undefined; + if (!admission.admitted) { + return unavailable({ + command: params.command, + outcome: 'refused', + operation: admission.operation, + reason: admission.fact.reason, + }); + } + try { + return await bindAdmittedSnapshotCapture({ + admission, + bindDevice: params.bindDevice, + platformServices: params.platformServices, + }); + } catch (error) { + return unavailable({ command: params.command, outcome: 'bind-failed', error }); + } +} + +/** `undefined` means the owner's facts probe failed: the caller degrades like any other refusal. */ +async function admitObservationCapturePlan( + params: Readonly<{ command: string; session: SessionState }> & RuntimeAdmissionBindings, + plan: SelectorCaptureRuntimePlan, +) { + try { + return await admitRuntimePlan({ + device: params.session.device, + plan, + inspectFacts: params.inspectFacts, + }); + } catch (error) { + return unavailable({ command: params.command, outcome: 'admission-probe-failed', error }); + } +} + +/** Why the observation has to go without a capture, as one typed fact the caller can read. */ +type ObservationCaptureUnavailable = Readonly<{ command: string }> & + ( + | Readonly<{ + outcome: 'refused'; + operation: string; + reason: RuntimeOperationUnavailability['reason']; + }> + | Readonly<{ outcome: 'admission-probe-failed' | 'bind-failed'; error: unknown }> + ); + +function unavailable(detail: ObservationCaptureUnavailable): undefined { + emitDiagnostic({ + // `warn`, not the default `info`: a degrade that answers `ok: true` is dropped at info level on + // the success path, which is the passing-unnoticed this discloses against. + level: 'warn', + phase: 'settle_observation_capture_unavailable', + // The error's name travels, never its message: request diagnostics keep user input out. + data: + detail.outcome === 'refused' + ? { ...detail } + : { + command: detail.command, + outcome: detail.outcome, + errorName: (detail.error as Error | undefined)?.name, + }, + }); + return undefined; +} diff --git a/src/daemon/snapshot-runtime-binding.ts b/src/daemon/snapshot-runtime-binding.ts index 518288fe83..7b2e9ec5b1 100644 --- a/src/daemon/snapshot-runtime-binding.ts +++ b/src/daemon/snapshot-runtime-binding.ts @@ -200,6 +200,29 @@ export async function resolveBoundSnapshotCaptureRuntime( }); } +/** + * Binds a plan that the caller already admitted, and keeps only its capture operation. For a caller + * that owns its refusal path — an optional capture whose refusal degrades an observation instead of + * failing a command — this is the half of `admitAndBindSnapshotCapture` that carries no + * command-facing refusal wording, and so spends none of the host probes that wording builds. A + * caller with a session, or one that owes a readiness check, uses the full form instead. + */ +export async function bindAdmittedSnapshotCapture( + params: Readonly<{ + admission: AdmittedRuntimePlan; + bindDevice?: BindDeviceRuntime; + platformServices: DaemonPlatformServices; + }>, +): Promise { + const bound = await bindSnapshotCaptureRuntime( + params.bindDevice, + params.admission, + undefined, + params.platformServices, + ); + return async (input: CaptureSnapshotInput) => await bound.captureSnapshot(input); +} + /** * Binds only an admitted plan, on the device it was admitted for: the token is minted by * `admitRuntimePlan` alone and unwrapped by exact identity, so nothing that was not admitted —