diff --git a/contracts/fixtures/dispatch-disclosure.json b/contracts/fixtures/dispatch-disclosure.json index 369a42d98f..a6a5e99444 100644 --- a/contracts/fixtures/dispatch-disclosure.json +++ b/contracts/fixtures/dispatch-disclosure.json @@ -17,6 +17,24 @@ "trigger": "fill admission (admitFill) refused the request before any backend call, e.g. a target with no text", "dispatched": "no" }, + { + "id": "daemon.refusal.gesture-drag-admission", + "producer": "daemon", + "trigger": "a drag target's @ref admission refused before the gesture runtime was bound, without an earlier readiness recovery", + "dispatched": "no" + }, + { + "id": "daemon.post-dispatch.gesture-drag-after-recovery", + "producer": "daemon", + "trigger": "Android readiness tapped Close app and relaunched the app, then drag ref admission refused before the gesture runtime was bound", + "dispatched": "unknown" + }, + { + "id": "daemon.post-dispatch.ref-after-android-recovery", + "producer": "daemon", + "trigger": "Android blocking-dialog readiness tapped Close app and relaunched the app, then re-admission aborted the requested ref press", + "dispatched": "unknown" + }, { "id": "daemon.refusal.selector-readiness-exhausted", "producer": "daemon", diff --git a/src/daemon/interaction/internal/__tests__/interaction-dispatch-disclosure.test.ts b/src/daemon/interaction/internal/__tests__/interaction-dispatch-disclosure.test.ts index 8e11181547..4c3a0187b2 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-dispatch-disclosure.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-dispatch-disclosure.test.ts @@ -25,11 +25,16 @@ import { } from '../../../request-dispatch-ledger.ts'; import type { BindDeviceRuntime } from '../../../request-runtime-binding.ts'; import { clearAndroidObservationFixture } from '../../../__tests__/android-observation-fixture.ts'; +import { activateCompleteRefFrame, expireRefFrame } from '../../../ref-frame.ts'; import { handleInteractionCommands } from '../../index.ts'; import type { InteractionRouteInput } from '../types.ts'; import { assertAndroidPressStayedInApp } from '../interaction-android-escape.ts'; import { gestureRuntimeBindingsFixture } from './gesture-runtime-bindings.fixtures.ts'; -import { contextFromFlags, makeSession } from './interaction-touch-fixtures.ts'; +import { + contextFromFlags, + makeSession, + makeStaleRefSession, +} from './interaction-touch-fixtures.ts'; // contracts/fixtures/dispatch-disclosure.json, daemon and post-action guard rows: the daemon rows // drive a real `press` through the daemon interaction handler, inside the router's dispatch seam, @@ -210,6 +215,110 @@ async function swipeRefusedOnSecondRepetition(): Promise { throw new AppError(response.error.code, response.error.message, response.error.details); } +/** A stale drag ref is rejected before binding any gesture operation. */ +async function gestureDragRefusedBeforeDispatch(): Promise { + const gestures = gestureRuntimeBindingsFixture(); + const sessionStore = makeSessionStore(); + const session = makeStaleRefSession('dispatch-disclosure-stale-drag'); + expireRefFrame(session); + sessionStore.publish(session.name, session); + const response = await routeInteraction({ + req: { + token: 't', + session: session.name, + command: 'gesture', + positionals: [], + flags: {}, + input: { + kind: 'drag', + source: '@e1', + destination: '@e2', + sourceHoldMs: 100, + moveMs: 100, + destinationHoldMs: 100, + }, + }, + sessionName: session.name, + sessionStore, + contextFromFlags, + inspectFacts: gestures.inspectFacts, + bindDevice: gestures.bindDevice, + }); + assert.ok(response && !response.ok, 'expected stale drag admission to fail'); + assert.equal(response.error.details?.reason, 'ref_frame_expired'); + assert.equal(gestures.bindDevice.mock.calls.length, 0, 'admission must precede runtime binding'); + throw new AppError(response.error.code, response.error.message, response.error.details); +} + +async function refRefusedAfterAndroidRecovery(command: 'press' | 'gesture'): Promise { + const gestures = gestureRuntimeBindingsFixture(); + const sessionStore = makeSessionStore(); + const session = makeAndroidSession('dispatch-disclosure-recovered-drag', { + appBundleId: 'com.example.app', + }); + session.snapshot = makeStaleRefSession('recovery-frame').snapshot; + activateCompleteRefFrame(session); + sessionStore.publish(session.name, session); + const tap = vi.fn(clearAndroidObservationFixture.tap); + const openApp = vi.fn(clearAndroidObservationFixture.openApp); + const androidObservation: AndroidObservationAdapter = { + ...clearAndroidObservationFixture, + readBlockingDialog: async () => ({ + status: 'dialog', + focus: { package: 'com.example.app', focusedWindow: 'Application Not Responding', raw: '' }, + }), + readSnapshotNodes: async () => [ + { + index: 0, + ref: 'e0', + type: 'Button', + label: 'Close app', + rect: { x: 10, y: 20, width: 100, height: 40 }, + }, + ], + tap, + openApp, + }; + const response = await routeInteraction({ + req: { + token: 't', + session: session.name, + command, + positionals: command === 'press' ? ['@e1'] : [], + flags: {}, + input: { + kind: 'drag', + source: '@e1', + destination: '@e2', + sourceHoldMs: 100, + moveMs: 100, + destinationHoldMs: 100, + }, + }, + sessionName: session.name, + sessionStore, + contextFromFlags, + ...(command === 'gesture' + ? { inspectFacts: gestures.inspectFacts, bindDevice: gestures.bindDevice } + : getRuntimeBindings()), + androidObservation, + }); + assert.ok(response && !response.ok, 'expected stale drag admission to fail after recovery'); + assert.equal(response.error.details?.reason, 'ref_frame_expired'); + assert.equal(tap.mock.calls.length, 1); + assert.equal(openApp.mock.calls.length, 1); + assert.equal(mockTapPoint.mock.calls.length, 0); + if (command === 'gesture') { + assert.equal(gestures.bindDevice.mock.calls.length, 0); + assert.equal( + response.error.details?.warning, + 'Recovered Android app ANR before gesture: closed and relaunched com.example.app.', + ); + assert.match(response.error.hint ?? '', /Capture a fresh interactive snapshot/); + } + throw new AppError(response.error.code, response.error.message, response.error.details); +} + /** An Android press whose tap returns and whose post-press foreground read is refused with `no`. */ async function pressThenForegroundReadRefused(): Promise { const session = makeAndroidSession('dispatch-disclosure-post-dispatch', { @@ -247,6 +356,10 @@ const DRIVERS: Record Promise> = { 'daemon.refusal.selector-readiness-exhausted': pressWhoseTargetNeverAppears, 'daemon.unclassified': () => pressAfterUnclassifiedTouchFailure(), 'daemon.series.swipe-later-repetition-refused': swipeRefusedOnSecondRepetition, + 'daemon.refusal.gesture-drag-admission': gestureDragRefusedBeforeDispatch, + 'daemon.post-dispatch.gesture-drag-after-recovery': () => + refRefusedAfterAndroidRecovery('gesture'), + 'daemon.post-dispatch.ref-after-android-recovery': () => refRefusedAfterAndroidRecovery('press'), 'daemon.post-dispatch.press-then-foreground-read-refused': pressThenForegroundReadRefused, 'daemon.read-only-command': () => press({ command: 'get', positionals: ['text', 'label="Missing"'] }), diff --git a/src/daemon/interaction/internal/interaction-gesture.ts b/src/daemon/interaction/internal/interaction-gesture.ts index 59eb620d8a..415a4983fe 100644 --- a/src/daemon/interaction/internal/interaction-gesture.ts +++ b/src/daemon/interaction/internal/interaction-gesture.ts @@ -18,7 +18,7 @@ import { SWIPE_REPETITION_MAX, SWIPE_SERIES_MAX_SCHEDULED_DURATION_MS, } from '@agent-device/contracts/scroll-gesture'; -import { AppError, normalizeError } from '@agent-device/kernel/errors'; +import { AppError, asAppError, normalizeError } from '@agent-device/kernel/errors'; import { REF_GRAMMAR_HINT, splitRefGenerationSuffix, @@ -27,9 +27,13 @@ import { import { resolveBoundGestureRuntime, type BoundGestureExecutor } from '../../gesture-runtime.ts'; import { isActiveProviderDevice } from '../../provider-device-admission.ts'; import { sleep } from '@agent-device/host-kit/retry'; -import { ensureAndroidBlockingSystemDialogReady } from '../../android-system-dialog.ts'; +import { + ensureAndroidBlockingSystemDialogReady, + type AndroidBlockingDialogReadinessResult, +} from '../../android-system-dialog.ts'; import { readRefMutationFrame } from '../../ref-frame.ts'; import type { DaemonResponse } from '../../daemon-request.ts'; +import { thrownBeforeDispatch } from '../../request-dispatch-disclosure.ts'; import type { SessionState } from '../../session-state.ts'; import { assertRefMutationAdmitted } from './interaction-ref-policy.ts'; import { @@ -73,17 +77,18 @@ export async function dispatchGestureViaRuntime( params: GestureHandlerParams, ): Promise { params = bindInteractionSession(params); - return await dispatchGestureInteraction(params, 'gesture', async (session) => - runGestureInteraction(params, session), + return await dispatchGestureInteraction(params, 'gesture', async (session, readiness) => + runGestureInteraction(params, session, readiness), ); } async function runGestureInteraction( params: GestureHandlerParams, session: SessionState, + readiness: AndroidBlockingDialogReadinessResult, ): Promise { const input = readGesturePayload(params.req.input); - const gesture = prepareGestureCommandInput(input, session); + const gesture = prepareGestureCommandInput(input, session, readiness); if (gesture.intent === 'pan' && params.req.internal?.gestureExecutionProfile) { gesture.executionProfile = params.req.internal.gestureExecutionProfile; } @@ -207,7 +212,10 @@ function createGestureRuntime(params: GestureHandlerParams, gestures: BoundGestu async function dispatchGestureInteraction( params: GestureHandlerParams, command: 'gesture' | 'swipe', - run: (session: SessionState) => Promise, + run: ( + session: SessionState, + readiness: AndroidBlockingDialogReadinessResult, + ) => Promise, ): Promise { const session = params.sessionStore.get(params.sessionName); if (!session) return noActiveSessionError(); @@ -222,7 +230,7 @@ async function dispatchGestureInteraction( phase: 'before-command', observation: params.androidObservation, }); - const outcome = await run(session); + const outcome = await run(session, readiness); if (isRefusal(outcome)) return outcome.refused; if (!providerDevice) { await ensureAndroidBlockingSystemDialogReady({ @@ -268,14 +276,26 @@ function resolveExecutionProfile( function prepareGestureCommandInput( input: GesturePayload, session: SessionState, + readiness: AndroidBlockingDialogReadinessResult, ): GestureCommandInput { const normalized = normalizeGestureCommandInput(input); if (normalized.intent !== 'drag') return normalized; - return { - ...normalized, - source: prepareDragTarget(normalized.source, session), - destination: prepareDragTarget(normalized.destination, session), - }; + try { + return { + ...normalized, + source: prepareDragTarget(normalized.source, session), + destination: prepareDragTarget(normalized.destination, session), + }; + } catch (error) { + if (readiness.status === 'recovered') { + const failure = asAppError(error); + const existingWarning = + typeof failure.details?.warning === 'string' ? failure.details.warning + ' ' : ''; + failure.details = { ...failure.details, warning: existingWarning + readiness.warning }; + throw failure; + } + throw thrownBeforeDispatch(error); + } } function prepareDragTarget(target: string, session: SessionState): string {