From 893753114bdf2331e56fb0568f7295988b43a306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 9 Oct 2026 21:08:04 +0200 Subject: [PATCH 1/3] fix(ios): settle the first capture after an open that saw the launch in flight (#3354) An iOS app open whose post-open observation came back `unobservable` returns on a fixed settle while the launch may still be laying out. A selector click resolved from that first capture tapped a point the target had already left. The published session now carries the existing post-gesture stabilization mark, so its next capture runs the same bounded quiet-window loop a scroll does. Fresh sessions are covered: the mark is placed after the session is published. Other verdicts, Android, and warm clicks are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../deferred-interaction-outcome.test.ts | 49 +++++++++++++++++ src/daemon/deferred-interaction-outcome.ts | 29 +++++++++- .../session-open-execution-runtime.test.ts | 54 +++++++++++++++++++ .../internal/session-open-execution.ts | 20 +++---- src/daemon/session-state.ts | 2 +- .../provider-scenarios/ios-world.ts | 25 +++++---- website/docs/docs/commands.md | 1 + 7 files changed, 155 insertions(+), 25 deletions(-) diff --git a/src/daemon/__tests__/deferred-interaction-outcome.test.ts b/src/daemon/__tests__/deferred-interaction-outcome.test.ts index 148e3ea915..0526a37eef 100644 --- a/src/daemon/__tests__/deferred-interaction-outcome.test.ts +++ b/src/daemon/__tests__/deferred-interaction-outcome.test.ts @@ -3,6 +3,7 @@ import { afterEach, test, vi } from 'vitest'; import { isPostGestureStabilizationPending, markDeferredInteractionOutcome, + markPostOpenStabilization, resolveDeferredInteractionOutcome, stripInternalInteractionFlags, type DeferredOutcomeSnapshotAttempt, @@ -97,6 +98,54 @@ test('freshness and stabilization eligibility read the action, not the outer com // --- capture-side resolution --- +test('an Apple open that could not observe its app marks stabilization without a baseline', () => { + const session = makeSession('ios'); + session.snapshot = pickupSnapshot(); + + markPostOpenStabilization(session, 'unobservable'); + + assert.equal(isPostGestureStabilizationPending(session), true); + assert.equal(session.postGestureStabilization?.action, 'open'); + assert.equal(session.postGestureStabilization?.baselineSignature, undefined); +}); + +test('only an open that saw the launch still in flight marks stabilization', () => { + const quiet = ( + ['observable', 'probe-failed', 'app-unidentified', 'not-eligible', undefined] as const + ).map((observation) => { + const session = makeSession('ios'); + markPostOpenStabilization(session, observation); + return isPostGestureStabilizationPending(session); + }); + const android = makeSession('android'); + markPostOpenStabilization(android, 'unobservable'); + + assert.deepEqual(quiet, [false, false, false, false, false]); + assert.equal(isPostGestureStabilizationPending(android), false); +}); + +test('the first capture after an unobserved open reads the list where it came to rest', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'unobservable'); + const { capture, calls } = scriptedCapture([ + pickupSnapshot(319), + pickupSnapshot(406), + pickupSnapshot(406), + ]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + for (let step = 0; step < 10; step += 1) { + await vi.advanceTimersByTimeAsync(200); + } + const result = await pendingResult; + + assert.equal(calls(), 3); + assert.equal(result?.snapshot.nodes[1]?.rect?.y, 406); + assert.equal(result?.snapshot.postGestureOutcome, undefined); + assert.equal(isPostGestureStabilizationPending(session), false); +}); + test('nothing deferred resolves to undefined without capturing', async () => { const session = makeSession('ios'); const { capture, calls } = scriptedCapture([pickupSnapshot()]); diff --git a/src/daemon/deferred-interaction-outcome.ts b/src/daemon/deferred-interaction-outcome.ts index 0eec0230cd..ab26ba344f 100644 --- a/src/daemon/deferred-interaction-outcome.ts +++ b/src/daemon/deferred-interaction-outcome.ts @@ -1,4 +1,5 @@ import type { CommandFlags } from '@agent-device/contracts/command'; +import type { PostOpenObservation } from '@agent-device/contracts/application-lifecycle-runtime'; import type { SnapshotCaptureAnnotations } from '@agent-device/contracts/capture'; import { isApplePlatform, isMobilePlatform } from '@agent-device/kernel/device'; import type { SnapshotState } from '@agent-device/kernel/snapshot'; @@ -29,7 +30,8 @@ import type { SessionState } from './session-state.ts'; * The deferred interaction outcome: the daemon's answer to "did that mutation * actually take effect?", produced after the mutation's own response has been * sent. This module is its one interface — every mutating route marks through - * `markDeferredInteractionOutcome` right after dispatch, and every snapshot + * `markDeferredInteractionOutcome` right after dispatch (an app open also marks + * `markPostOpenStabilization` once its session is published), and every snapshot * capture resolves through `resolveDeferredInteractionOutcome`. The two * SessionState fields stay with their owner modules; this module is itself the * `postGestureStabilization` owner (R7), so hosting the interface here adds no @@ -106,6 +108,31 @@ function markPostGestureStabilization( }; } +/** + * The one post-open verdict that observed the launch still in flight: the bridge kept answering with + * a launch-transition failure until its window closed, and the open returned on a fixed settle. + * `observable` read the app; `probe-failed`, `app-unidentified`, and `not-eligible` learned nothing + * about the layout, so they keep the fixed settle alone. + */ +const POST_OPEN_IN_FLIGHT_OBSERVATION: PostOpenObservation = 'unobservable'; + +/** + * Open-side marking, called once an app open has published its session. An Apple app open whose + * observation could not read the launched app returns before anything proved its layout at rest, so + * the session's next capture runs the same quiet-window loop a scroll's does: a selector resolved + * from a list still sliding into place taps where its target used to be (#3354). No baseline is + * recorded: the pre-open tree is a different surface, so the loop never claims the open had no effect. + */ +export function markPostOpenStabilization( + session: SessionState, + observation: PostOpenObservation | undefined, +): void { + if (!isApplePlatform(session.device.platform)) return; + if (!supportsPostGestureStabilization(session.device)) return; + if (observation !== POST_OPEN_IN_FLIGHT_OBSERVATION) return; + session.postGestureStabilization = { action: 'open', positionals: [], markedAt: Date.now() }; +} + function clearPostGestureStabilization(session: SessionState | undefined): void { if (!session?.postGestureStabilization) return; session.postGestureStabilization = undefined; diff --git a/src/daemon/session-lifecycle/internal/__tests__/session-open-execution-runtime.test.ts b/src/daemon/session-lifecycle/internal/__tests__/session-open-execution-runtime.test.ts index 8d05648d5d..80cae04eca 100644 --- a/src/daemon/session-lifecycle/internal/__tests__/session-open-execution-runtime.test.ts +++ b/src/daemon/session-lifecycle/internal/__tests__/session-open-execution-runtime.test.ts @@ -5,6 +5,7 @@ import fs from 'node:fs'; import type { ApplicationLifecycleRuntimeOperations, OpenApplicationInput, + PostOpenObservation, } from '@agent-device/contracts/application-lifecycle-runtime'; import type { DaemonRequest } from '../../../daemon-request.ts'; import { @@ -77,6 +78,7 @@ import { resolveAndroidPackageForOpen } from '@agent-device/platform-android/mec import { dispatchApplicationLifecycleEffect } from '../../../__tests__/application-lifecycle-runtime-fixture.ts'; import { makeAndroidEmulator, + makeAppleSimulator, makeSession, makeSessionStore, noopInvoke, @@ -443,6 +445,58 @@ test('open reports the launch confirmation its platform answered', async () => { if (response?.ok) expect(response.data?.launchConfirmation).toBe('accepted'); }); +async function openReportingObservation( + sessionName: string, + postOpenObservation: PostOpenObservation, +) { + const sessionStore = makeSessionStore(); + mockResolveTargetDevice.mockResolvedValue(makeAppleSimulator()); + const bindDefault = mockBindDeviceRuntime.getMockImplementation(); + if (!bindDefault) throw new Error('the harness binds a default runtime'); + mockBindDeviceRuntime.mockImplementationOnce(async (device, use) => { + const binding = await bindDefault(device, use); + const operations = binding.operations as ApplicationLifecycleRuntimeOperations; + return { + ...binding, + operations: { + ...binding.operations, + openApplication: async (input: OpenApplicationInput) => { + const outcome = await operations.openApplication(input); + return { ...outcome, timing: { ...outcome.timing, postOpenObservation } }; + }, + }, + }; + }); + + const response = await handleSessionCommands({ + req: { + token: 't', + session: sessionName, + command: 'open', + positionals: ['Demo'], + flags: { platform: 'ios' }, + }, + sessionName, + logPath: path.join(mkdtempForTestSync('daemon'), 'daemon.log'), + sessionStore, + invoke: noopInvoke, + }); + expect(response?.ok).toBe(true); + return sessionStore.get(sessionName); +} + +test('a fresh iOS open that could not observe its app leaves the next capture to settle', async () => { + const session = await openReportingObservation('unobserved-open', 'unobservable'); + + expect(session?.postGestureStabilization?.action).toBe('open'); +}); + +test('a fresh iOS open that observed its app leaves nothing pending', async () => { + const session = await openReportingObservation('observed-open', 'observable'); + + expect(session?.postGestureStabilization).toBeUndefined(); +}); + function holdNativeOpen() { let entered!: () => void; let release!: () => void; diff --git a/src/daemon/session-lifecycle/internal/session-open-execution.ts b/src/daemon/session-lifecycle/internal/session-open-execution.ts index 5764438cf0..b154161ba0 100644 --- a/src/daemon/session-lifecycle/internal/session-open-execution.ts +++ b/src/daemon/session-lifecycle/internal/session-open-execution.ts @@ -9,6 +9,7 @@ import { } from '@agent-device/device-selection/device-selection-resolver'; import type { BoundDeviceRuntime } from '@agent-device/contracts/platform-runtime'; import type { SessionSurface } from '@agent-device/contracts/session'; +import type { OpenApplicationTiming } from '@agent-device/contracts/application-lifecycle-runtime'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { DaemonRequest, DaemonResponse } from '../../daemon-request.ts'; import type { SessionRef, SessionState } from '../../session-state.ts'; @@ -32,7 +33,10 @@ import { import { STARTUP_SAMPLE_METHOD, type StartupPerfSample } from './session-startup-metrics.ts'; import { buildOpenResult } from './session-open-surface.ts'; import { publishOpenSession, requireOpenSessionAdmission } from './session-open-state.ts'; -import { markDeferredInteractionOutcome } from '../../deferred-interaction-outcome.ts'; +import { + markDeferredInteractionOutcome, + markPostOpenStabilization, +} from '../../deferred-interaction-outcome.ts'; import { emitDiagnostic, getDiagnosticsMeta } from '@agent-device/host-kit/diagnostics'; import { prepareOpenCommandDetails, @@ -69,18 +73,7 @@ export type SessionOpenResult = | Readonly<{ type: 'opened'; response: DaemonResponse; ref: SessionRef }> | Readonly<{ type: 'response'; response: DaemonResponse }>; -type OpenTiming = { - totalDurationMs?: number; - relaunchCloseDurationMs?: number; - runtimeHintsDurationMs?: number; - runnerPrewarmKind?: 'session' | 'xctestrun'; - runnerPrewarmScheduled?: boolean; - runnerPrewarmWaited?: boolean; - runnerPrewarmDurationMs?: number; - openDispatchDurationMs?: number; - launchUrlDurationMs?: number; - postOpenSettleDurationMs?: number; -}; +type OpenTiming = OpenApplicationTiming & { totalDurationMs?: number }; type NewSessionOpenEffects = { mayHaveStarted: boolean }; @@ -322,6 +315,7 @@ export async function completeOpenCommand(params: { deviceClaim, }); const nextSession = sessionStore.requireCurrent(nextRef); + markPostOpenStabilization(nextSession, timing.postOpenObservation); if (req.runtime !== undefined) setSessionRuntimeHintsForOpen(sessionStore, sessionName, runtimeHints); applyOrdinaryScriptRecordingOpenOutcome({ diff --git a/src/daemon/session-state.ts b/src/daemon/session-state.ts index bf5ee40bb8..fcc6a88398 100644 --- a/src/daemon/session-state.ts +++ b/src/daemon/session-state.ts @@ -75,7 +75,7 @@ export type InteractionSurfaceEntry = { export type PostGestureStabilization = { action: string; /** The gesture's own positionals — wording input for the #1600 no-effect - * warning; never re-dispatched. Always set by the only writer. */ + * warning; never re-dispatched. Always set by every writer. */ positionals: string[]; markedAt: number; /** diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts index 26f1fc0ef3..4bedebf839 100644 --- a/test/integration/provider-scenarios/ios-world.ts +++ b/test/integration/provider-scenarios/ios-world.ts @@ -40,6 +40,11 @@ export async function createIosSettingsWorld(): Promise { request: { command: 'uptime' }, result: { uptimeMs: 42 }, }, + // The scripted simulator runs no app process, so every `open` observes it `unobservable` and + // the first capture after it settles over a quiet pair of reads: `capture settings snapshot` + // and `refresh snapshot after install` each take the extra read below. + runnerSnapshot(), + runnerSnapshot(), runnerSnapshot(), { command: 'ios.runner.tap', @@ -269,17 +274,17 @@ type IosBottomTabsSnapshotWorld = { }; export async function createIosBottomTabsSnapshotWorld(): Promise { - const runnerTranscript = createProviderTranscript([ - { - command: 'ios.runner.snapshot', - deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, - platform: 'apple', - result: { - nodes: bottomTabsContactSnapshotNodes(), - truncated: false, - }, + const bottomTabsSnapshot = { + command: 'ios.runner.snapshot', + deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + platform: 'apple' as const, + result: { + nodes: bottomTabsContactSnapshotNodes(), + truncated: false, }, - ]); + }; + // The snapshot is the first capture after an `unobservable` open, so it settles over a quiet pair. + const runnerTranscript = createProviderTranscript([bottomTabsSnapshot, bottomTabsSnapshot]); const appleRunnerProvider = createAppleRunnerProviderFromTranscript( runnerTranscript, 'ios.runner', diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 762457b9ab..fb3eb7f3eb 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -93,6 +93,7 @@ agent-device fold open - Android: `shutdown --platform android --device ` stops a running emulator. - `open [app|url] [url]` boots and activates the selected target when needed. - `open --wait ` waits up to that budget for a device another session is holding instead of failing at once, then fails with `DEVICE_IN_USE` if the budget runs out. Only session contention is waited for: a device claim held by another workspace's daemon is never retriable and returns its recovery command immediately. The wait extends the command's timeout envelope, so a long budget does not need a longer `--timeout`. See [Sessions](/docs/sessions#wait-for-a-slow-or-busy-device) for queuing behavior. +- On iOS, when `open` could not read the launched app before it returned, the next command that reads the screen keeps reading until two reads in a row agree, within a short bounded wait, before it acts. A `click` right after such an open then targets where the element comes to rest, not where it was mid-launch. If the screen is still moving after that wait, the command proceeds and its response warns that the surface was still changing after `open`. - `open ` deep links are supported on Android and iOS. - On an iOS Simulator, handing the URL to the device is one bounded `simctl openurl`: when CoreSimulator never answers it, the open fails within 20 seconds with `details.reason: ios-simulator-openurl-timeout` and kills the child, instead of the request being cancelled around a still-running `openurl`. The Simulator may still have received the URL, so retry the open or answer the prompt with `alert accept`. - `open ` opens a deep link on iOS. From ce8fbbed9d089c9336b4fe495a5ecc7e9a5159e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 9 Oct 2026 21:19:55 +0200 Subject: [PATCH 2/3] docs(open): name the unobservable verdict that triggers the post-open settle Co-Authored-By: Claude Opus 5.5 --- website/docs/docs/commands.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index fb3eb7f3eb..59992868ea 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -93,7 +93,7 @@ agent-device fold open - Android: `shutdown --platform android --device ` stops a running emulator. - `open [app|url] [url]` boots and activates the selected target when needed. - `open --wait ` waits up to that budget for a device another session is holding instead of failing at once, then fails with `DEVICE_IN_USE` if the budget runs out. Only session contention is waited for: a device claim held by another workspace's daemon is never retriable and returns its recovery command immediately. The wait extends the command's timeout envelope, so a long budget does not need a longer `--timeout`. See [Sessions](/docs/sessions#wait-for-a-slow-or-busy-device) for queuing behavior. -- On iOS, when `open` could not read the launched app before it returned, the next command that reads the screen keeps reading until two reads in a row agree, within a short bounded wait, before it acts. A `click` right after such an open then targets where the element comes to rest, not where it was mid-launch. If the screen is still moving after that wait, the command proceeds and its response warns that the surface was still changing after `open`. +- On an iOS Simulator, `open` watches the launch for a bounded time. When the app was still unreadable when that watch ended (`timing.postOpenObservation: "unobservable"` in the open response), the next command that reads the screen keeps reading until two reads in a row agree, within a short bounded wait, before it acts. A `click` right after such an open then targets where the element comes to rest, not where it was mid-launch. If the screen is still moving after that wait, the command proceeds and its response warns that the surface was still changing after `open`. Any other `postOpenObservation` value, including `probe-failed`, leaves the next command reading the screen once. - `open ` deep links are supported on Android and iOS. - On an iOS Simulator, handing the URL to the device is one bounded `simctl openurl`: when CoreSimulator never answers it, the open fails within 20 seconds with `details.reason: ios-simulator-openurl-timeout` and kills the child, instead of the request being cancelled around a still-running `openurl`. The Simulator may still have received the URL, so retry the open or answer the prompt with `alert accept`. - `open ` opens a deep link on iOS. From 601fff53a0b4a14b7cbd035bddc886053307d907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 09:20:08 +0200 Subject: [PATCH 3/3] refactor(daemon): inline the post-open verdict and name the open-time mark in the tap gap Co-Authored-By: Claude Opus 5.5 --- .../contracts/src/interaction-guarantees.ts | 2 +- src/daemon/deferred-interaction-outcome.ts | 19 +++++-------------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 314e195ad4..731cd1c55c 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -166,7 +166,7 @@ const SHARED_RESPONSE_CONSTRUCTION: GuaranteeEnforcement = { const TAP_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { kind: 'waived', reason: - 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response, and the iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', + 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS Simulator open whose launch watch ended unobservable marks the next capture in that session to wait for a quiet surface (markPostOpenStabilization); that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE, }; diff --git a/src/daemon/deferred-interaction-outcome.ts b/src/daemon/deferred-interaction-outcome.ts index ab26ba344f..f3799c8f11 100644 --- a/src/daemon/deferred-interaction-outcome.ts +++ b/src/daemon/deferred-interaction-outcome.ts @@ -109,19 +109,10 @@ function markPostGestureStabilization( } /** - * The one post-open verdict that observed the launch still in flight: the bridge kept answering with - * a launch-transition failure until its window closed, and the open returned on a fixed settle. - * `observable` read the app; `probe-failed`, `app-unidentified`, and `not-eligible` learned nothing - * about the layout, so they keep the fixed settle alone. - */ -const POST_OPEN_IN_FLIGHT_OBSERVATION: PostOpenObservation = 'unobservable'; - -/** - * Open-side marking, called once an app open has published its session. An Apple app open whose - * observation could not read the launched app returns before anything proved its layout at rest, so - * the session's next capture runs the same quiet-window loop a scroll's does: a selector resolved - * from a list still sliding into place taps where its target used to be (#3354). No baseline is - * recorded: the pre-open tree is a different surface, so the loop never claims the open had no effect. + * Called once an app open has published its session. `unobservable` means the launch was still in + * flight or no app process was found when the open returned, so the next capture runs the + * quiet-window loop before a selector resolves against a layout that may still be moving (#3354). No baseline: the pre-open tree is a + * different surface, so the open is never reported as having no effect. */ export function markPostOpenStabilization( session: SessionState, @@ -129,7 +120,7 @@ export function markPostOpenStabilization( ): void { if (!isApplePlatform(session.device.platform)) return; if (!supportsPostGestureStabilization(session.device)) return; - if (observation !== POST_OPEN_IN_FLIGHT_OBSERVATION) return; + if (observation !== 'unobservable') return; session.postGestureStabilization = { action: 'open', positionals: [], markedAt: Date.now() }; }