diff --git a/packages/capture-kit/src/post-gesture-stability.test.ts b/packages/capture-kit/src/post-gesture-stability.test.ts new file mode 100644 index 0000000000..03ee9dc592 --- /dev/null +++ b/packages/capture-kit/src/post-gesture-stability.test.ts @@ -0,0 +1,120 @@ +import assert from 'node:assert/strict'; +import { beforeEach, test, vi } from 'vitest'; +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import type { ObservationClock } from './observe-until.ts'; +import { + runPostGestureStabilityLoop, + type PostGestureStabilityHooks, +} from './post-gesture-stability.ts'; + +vi.mock('@agent-device/host-kit/diagnostics', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, emitDiagnostic: vi.fn() }; +}); + +beforeEach(() => { + vi.mocked(emitDiagnostic).mockClear(); +}); + +type Surface = Readonly<{ signature: readonly string[]; costMs: number }>; + +function fakeClock(): ObservationClock & { advance(ms: number): void } { + let nowMs = 0; + return { + now: () => nowMs, + advance: (ms) => { + nowMs += ms; + }, + sleep: async (ms) => { + nowMs += ms; + }, + }; +} + +/** Captures that each cost their own `costMs` of clock time and ignore any signal. */ +function hooksFor( + clock: ReturnType, + surfaces: readonly Surface[], +): PostGestureStabilityHooks { + let calls = 0; + const same = (a: readonly string[], b: readonly string[]) => a.join() === b.join(); + return { + capture: async () => { + const surface = surfaces[calls]; + calls += 1; + if (!surface) throw new Error('the loop captured more surfaces than the case supplied'); + clock.advance(surface.costMs); + return surface; + }, + readSurface: (value) => ({ signature: value.signature, backend: 'xctest' }), + signaturesStable: same, + classifyBaselineEvidence: (baseline, quiet) => + same(baseline, quiet) ? 'unchanged' : 'changed', + surfacesIdentical: same, + summarizeDivergence: () => ({}), + }; +} + +const PENDING = { action: 'scroll', positionals: ['down'] }; + +function timeoutWarnings(): unknown[] { + return vi + .mocked(emitDiagnostic) + .mock.calls.filter(([event]) => event.phase === 'post_gesture_snapshot_stabilization_timeout'); +} + +test('a capture that runs past the remaining budget is judged and ends unsettled, not thrown', async () => { + const clock = fakeClock(); + const late: Surface = { signature: ['b'], costMs: 2_000 }; + + const outcome = await runPostGestureStabilityLoop({ + pending: PENDING, + needsBaselineDistrust: false, + hooks: hooksFor(clock, [{ signature: ['a'], costMs: 0 }, late]), + clock, + }); + + assert.equal(outcome.value, late); + assert.deepEqual(outcome.postGestureOutcome, { + kind: 'unsettled', + gesture: { action: 'scroll', positionals: ['down'] }, + }); + assert.equal(timeoutWarnings().length, 1); +}); + +test('a first capture slower than the whole budget still forms a quiet pair', async () => { + const clock = fakeClock(); + const quiet: Surface = { signature: ['a'], costMs: 0 }; + + const outcome = await runPostGestureStabilityLoop({ + pending: PENDING, + needsBaselineDistrust: false, + hooks: hooksFor(clock, [{ signature: ['a'], costMs: 2_000 }, quiet]), + clock, + }); + + assert.equal(outcome.value, quiet); + assert.equal(outcome.postGestureOutcome, undefined); + assert.equal(timeoutWarnings().length, 0); +}); + +test('a capture error ends the loop by rethrowing that error', async () => { + const clock = fakeClock(); + const failure = new Error('capture failed'); + const hooks = hooksFor(clock, []); + + await assert.rejects( + runPostGestureStabilityLoop({ + pending: PENDING, + needsBaselineDistrust: false, + hooks: { + ...hooks, + capture: async () => { + throw failure; + }, + }, + clock, + }), + (error) => error === failure, + ); +}); diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index 5ce237f1ce..3204a84e7f 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -1,21 +1,31 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; -import { sleep } from '@agent-device/host-kit/retry'; import type { PostGestureAction, PostGestureOutcome } from '@agent-device/kernel/snapshot'; +import { observeUntil, type ObservationClock, type ObservationSchedule } from './observe-until.ts'; /** - * Pure post-gesture stability mechanics: the quiet-window polling loop and the - * baseline-distrust verdict, parameterized over the capture value and the - * signature comparators. Deliberately a leaf — it imports no cycle owners and - * no `SessionState`, so it stays outside the R9 type cycle while the - * deferred-interaction-outcome owner (which holds the pending record and the - * session mutation) stays the one seam callers see. The owner supplies the - * comparators from interaction-outcome-policy; their semantics (subset - * tolerance, identity keying, discriminating entries) are documented there. + * Pure post-gesture stability mechanics: the quiet-window verdict over the shared + * `observeUntil` engine, plus the baseline-distrust decision, parameterized over + * the capture value and the signature comparators. Deliberately a leaf — it + * imports no cycle owners and no `SessionState`, so it stays outside the R9 + * type cycle while the deferred-interaction-outcome owner (which holds the + * pending record and the session mutation) stays the one seam callers see. The + * owner supplies the comparators from interaction-outcome-policy; their + * semantics (subset tolerance, identity keying, discriminating entries) are + * documented there. */ -const STABILIZATION_DEADLINE_MS = 1_500; -const STABILIZATION_INTERVAL_MS = 200; -const STABILIZATION_MIN_ATTEMPTS = 2; +/** + * Cadence and budget for the quiet-window loop: poll every 200ms, allow 1.5s + * before an unsettled surface times out, and always complete two observations + * (an `initial` counts) so a quiet pair can form even under a tight budget. + * No per-capture deadline: `hooks.capture` takes no signal, so a late capture + * is judged when it returns. + */ +const POST_GESTURE_STABILITY_SCHEDULE: ObservationSchedule = { + intervalMs: 200, + budgetMs: 1_500, + minPolls: 2, +}; /** * Defect 2 (#1542): a bounded extra budget used ONLY when a quiet signature @@ -30,7 +40,7 @@ const STABILIZATION_MIN_ATTEMPTS = 2; * settle-zero-margin-flake, a week-long contention-flake root cause), so this * cap is sized to never come close to that trap. */ -const STABILIZATION_DISTRUST_DEADLINE_MS = STABILIZATION_DEADLINE_MS + 2_000; +const STABILIZATION_DISTRUST_DEADLINE_MS = POST_GESTURE_STABILITY_SCHEDULE.budgetMs + 2_000; export type BaselineSurfaceEvidence = 'changed' | 'unchanged' | 'ambiguous'; @@ -116,38 +126,52 @@ export function decidePostGestureStabilityVerdict( } /** - * The quiet-window stability loop: poll until two consecutive captures agree - * and the verdict accepts the agreement, or the (possibly distrust-extended) - * deadline expires. Session state never enters here — the caller owns the - * pending record's lifecycle and clears it when this returns. + * The quiet-window stability verdict over the shared observation engine: keep + * polling until two consecutive captures agree and the decision accepts the + * agreement, or the (possibly distrust-extended) budget expires. Session + * state never enters here — the caller owns the pending record's lifecycle + * and clears it when this returns. */ export async function runPostGestureStabilityLoop(params: { pending: PostGestureStabilityPending; needsBaselineDistrust: boolean; initial?: T; hooks: PostGestureStabilityHooks; + clock?: ObservationClock; }): Promise> { - const { pending, needsBaselineDistrust, hooks } = params; - const startedAt = Date.now(); - let attempts = 1; - let previous = await captureSurface(hooks, params.initial); + const { pending, needsBaselineDistrust, hooks, clock } = params; + const now = () => clock?.now() ?? Date.now(); + const startedAt = now(); + let attempts = 0; let baselineSignature = pending.baselineSignature; let baselineBackend = pending.baselineBackend; let baselineRebased = false; - // Extended past STABILIZATION_DEADLINE_MS only when the distrust verdict - // fires below; the ordinary (non-distrust) timeout path is unaffected. - let effectiveDeadlineMs = STABILIZATION_DEADLINE_MS; // A rebase or a distrust verdict keeps polling on a pair that DID agree, so - // the deadline can expire on a surface that is already at rest. + // the budget can expire on a surface that is already at rest. let lastPairAgreed = false; + let surfaceCache: CapturedSurface | undefined; + + const surfaceOf = (value: T): CapturedSurface => { + if (surfaceCache?.value === value) return surfaceCache; + surfaceCache = { value, ...hooks.readSurface(value) }; + return surfaceCache; + }; - while (attempts < STABILIZATION_MIN_ATTEMPTS || Date.now() - startedAt < effectiveDeadlineMs) { - await sleep(STABILIZATION_INTERVAL_MS); - attempts += 1; - const current = await captureSurface(hooks); - lastPairAgreed = hooks.signaturesStable(previous.signature, current.signature); - if (lastPairAgreed) { - const elapsedMs = Date.now() - startedAt; + const observed = await observeUntil>({ + ...(params.initial !== undefined ? { initial: params.initial } : {}), + capture: () => hooks.capture(), + schedule: POST_GESTURE_STABILITY_SCHEDULE, + ...(clock ? { clock } : {}), + verdict: (latest, previousValue) => { + attempts += 1; + const current = surfaceOf(latest); + if (previousValue === undefined) return { kind: 'continue' }; + + const previous = surfaceOf(previousValue); + lastPairAgreed = hooks.signaturesStable(previous.signature, current.signature); + if (!lastPairAgreed) return { kind: 'continue' }; + + const elapsedMs = now() - startedAt; // A capture plan may fall back or be pre-empted by the XCTest-channel // penalty at any time, so the backend can change mid-poll. Backends do // not agree on which nodes exist, so this pair says nothing about the @@ -162,8 +186,7 @@ export async function runPostGestureStabilityLoop = { @@ -204,14 +232,6 @@ type CapturedSurface = { backend: string | undefined; }; -async function captureSurface( - hooks: PostGestureStabilityHooks, - initial?: T, -): Promise> { - const value = initial ?? (await hooks.capture()); - return { value, ...hooks.readSurface(value) }; -} - function emitSettleDiagnostic( verdict: 'trust' | 'accept-stale', action: string, diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 57b56a54a9..e80127d94b 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -78,6 +78,11 @@ export const INTERACTION_GUARANTEES = [ // dispatch time. Distinct from occlusion/offscreen/nonHittable, which judge a target already // found; this is about whether one is found at all. 'targetReadiness', + // What the path observes after dispatch to back its success claim, and whether that observation + // is advisory (a hint the caller may ignore) or can change the response (a corroborated failure, + // a settled diff). Distinct from the opt-in verifyEvidence/settleObservation features; this is the + // path's baseline, always-on claim. + 'outcomeObservation', ] as const; export type InteractionGuarantee = (typeof INTERACTION_GUARANTEES)[number]; @@ -154,6 +159,16 @@ const SHARED_RESPONSE_CONSTRUCTION: GuaranteeEnforcement = { via: 'src/daemon/interaction/internal/interaction-touch-response.ts#buildInteractionResponseData', }; +// runtime-selector, runtime-ref, and coordinate share this waiver: all three finalize through +// finalizeTouchInteraction, which calls markDeferredInteractionOutcome with the tap point, and none +// observes the outcome in its own response. +const TAP_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { + kind: 'waived', + reason: + 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, the no-change tap retry when the request sets interactionOutcome.retryOnNoChange, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response, and the iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', + trackingIssue: GAPS_UMBRELLA_ISSUE, +}; + // Both Maestro-compatible fast paths (src/daemon/interaction/internal/interaction-touch-direct-ios.ts) // dispatch ONE fused XCTest runner request (`command: 'tap'` with a selectorKey/selectorValue) built // in packages/platform-apple/src/interactions.ts#tapElementSelector — not the separate @@ -165,6 +180,19 @@ const DIRECT_IOS_SINGLE_QUERY_READINESS: GuaranteeEnforcement = { "Intentional: the fused runner request performs a single XCTest selector/frame query per dispatch and does not itself retry for an as-yet-nonexistent target; a miss is a runner failure (see errorTaxonomy) rather than a wait, and delegation-on-error (ADR 0011) is a different path's guarantee, not a retry of this one.", }; +// Shares the SAME reasoning as TAP_OUTCOME_NOT_OBSERVED_GAP: the tap outcome is not observed on the +// success path here either. The shared ambiguous-XCTest-failure corroboration +// (src/daemon/interaction/internal/interaction-ios-tap-outcome.ts#corroborateIosTapFailure) is +// invoked identically from this fast path and from the tree-based runtime dispatch, but only ever +// reconsiders a THROWN error — it never validates a call that returned success, so it is not a +// baseline outcome claim for either path. +const DIRECT_IOS_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { + kind: 'waived', + reason: + "gap: this replay-only route observes no outcome beyond the runner's own report; --verify/--settle do not apply here (see the inapplicable cells on this row), and the shared ambiguous-failure corroboration only reconsiders a thrown error, never a successful dispatch.", + trackingIssue: GAPS_UMBRELLA_ISSUE, +}; + // The two runtime tree paths (selector and ref resolution) run the SAME shared // guard/observation implementations; only how the target is found // (disambiguation) and how failures are described (errorTaxonomy) differ. @@ -262,6 +290,7 @@ export const INTERACTION_DISPATCH_PATHS: Record { // updates it here with a linked issue. It is the diffable debt list // (umbrella: https://github.com/callstack/agent-device/issues/1081). assert.deepEqual(gaps.sort(), [ + 'coordinate/outcomeObservation', 'maestro-direct-selector/errorTaxonomy', + 'maestro-direct-selector/outcomeObservation', 'maestro-direct-selector/parentOwnedTouchPoint', 'maestro-direct-selector/responseIdentity', 'maestro-non-hittable-fallback/errorTaxonomy', + 'maestro-non-hittable-fallback/outcomeObservation', 'maestro-non-hittable-fallback/parentOwnedTouchPoint', + 'native-ref/outcomeObservation', + 'runtime-ref/outcomeObservation', + 'runtime-selector/outcomeObservation', + 'target-drag/outcomeObservation', ]); }); diff --git a/src/daemon/__tests__/scroll-movement.test.ts b/src/daemon/__tests__/scroll-movement.test.ts index 5756f614d0..5eb444ac58 100644 --- a/src/daemon/__tests__/scroll-movement.test.ts +++ b/src/daemon/__tests__/scroll-movement.test.ts @@ -6,6 +6,7 @@ import { AppError } from '@agent-device/kernel/errors'; import type { CommandFlags } from '@agent-device/contracts/command'; import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import { readScrollEdgeState } from '@agent-device/capture-kit/scroll-edge-state'; import { IOS_SIMULATOR, MACOS_DEVICE } from '../../__tests__/test-utils/device-fixtures.ts'; import { makeSession } from '../../__tests__/test-utils/session-factories.ts'; import { expireRefFrame } from '../ref-frame.ts'; @@ -26,6 +27,12 @@ vi.mock('@agent-device/host-kit/diagnostics', async (importOriginal) => { return { ...actual, emitDiagnostic: vi.fn() }; }); +vi.mock('@agent-device/capture-kit/scroll-edge-state', async (importOriginal) => { + const actual = + await importOriginal(); + return { ...actual, readScrollEdgeState: vi.fn(actual.readScrollEdgeState) }; +}); + const loggedDiagnostics = vi.mocked(emitDiagnostic); function movementDiagnostic(): { phase: string; level: string; data: Record } { @@ -177,6 +184,58 @@ test('a surface that no longer holds the pre-gesture content answers moved on th assert.equal(spy.calls(), 1); }); +/** A clock that moves only when the loop sleeps or a capture spends its own time. */ +function steppedClock() { + let nowMs = 0; + return { + now: () => nowMs, + advance: (ms: number) => { + nowMs += ms; + }, + sleep: async (ms: number) => { + nowMs += ms; + }, + }; +} + +/** Observes under the real 1.5 s budget and 200 ms cadence, with captures that cost clock time. */ +function observeTimed(frames: ReadonlyArray<{ nodes: SnapshotNode[]; costMs: number }>) { + const clock = steppedClock(); + let calls = 0; + loggedDiagnostics.mockClear(); + const observation = observeScrollMovement({ + direction: 'down', + baseline: baselineOf(screen(0)), + swipe: { midpoint: SWIPE_MIDPOINT, pixels: REQUESTED_PIXELS }, + capture: async (): Promise => { + const frame = frames[calls]; + calls += 1; + if (!frame) throw new Error('the observation captured more times than the case supplied'); + clock.advance(frame.costMs); + return { nodes: frame.nodes, backend: 'xctest', producer: 'apple-runner' }; + }, + clock, + }); + return { observation, calls: () => calls }; +} + +test('a post-scroll capture slower than the whole budget that shows movement answers moved', async () => { + const { observation, calls } = observeTimed([{ nodes: screen(-300), costMs: 2_000 }]); + + assert.equal(await observation, 'moved'); + assert.equal(calls(), 1); +}); + +test('a later poll that outlives the remaining budget is judged, not dropped', async () => { + const { observation, calls } = observeTimed([ + { nodes: screen(0), costMs: 0 }, + { nodes: screen(-300), costMs: 2_000 }, + ]); + + assert.equal(await observation, 'moved'); + assert.equal(calls(), 2); +}); + /** * At the end of a list iOS rubber-bands past the edge: the first capture lands mid-bounce with every row * shifted, then the content springs back to exactly the pre-gesture tree (#2884). A baseline that already @@ -309,6 +368,21 @@ test('a refusal without gesture coordinates does not recommend a swipe', async ( ); }); +test('an untouched surface never asks whether the baseline ended in the scrolled direction', async () => { + const baseline = baselineOf(screen(0, false)); + vi.mocked(readScrollEdgeState).mockClear(); + const { observation } = observe({ + baseline, + screens: [screen(0, false), screen(0, false)], + }); + + assert.equal(await observation, 'at-edge'); + const askedOfBaseline = vi + .mocked(readScrollEdgeState) + .mock.calls.filter(([nodes]) => nodes === baseline.nodes); + assert.deepEqual(askedOfBaseline, []); +}); + test('a surface that never shifted with nothing left to reveal answers at-edge, not a refusal', async () => { const { observation } = observe({ baseline: baselineOf(screen(0)), diff --git a/src/daemon/scroll-movement.ts b/src/daemon/scroll-movement.ts index 7bc96f6ca0..cd70a3d3d2 100644 --- a/src/daemon/scroll-movement.ts +++ b/src/daemon/scroll-movement.ts @@ -14,7 +14,11 @@ import { containsPoint } from '@agent-device/kernel/rect'; import { AppError } from '@agent-device/kernel/errors'; import type { Point, Rect, SnapshotState } from '@agent-device/kernel/snapshot'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; -import { sleep } from '@agent-device/host-kit/retry'; +import { + observeUntil, + type ObservationClock, + type ObservationSchedule, +} from '@agent-device/capture-kit/observe-until'; import { areInteractionSurfaceSignaturesStable, buildInteractionSurfaceSignature, @@ -61,9 +65,14 @@ import type { SessionState } from './session-state.ts'; * lineages can never answer either way and the claim is withheld rather than re-based. */ -/** How long a scroll keeps asking whether an untouched surface is really untouched (#1542's window). */ -const MOVEMENT_VERDICT_BUDGET_MS = 1_500; -const MOVEMENT_POLL_MS = 200; +/** + * How long a scroll keeps asking whether an untouched surface is really untouched (#1542's window). + * No per-capture deadline: the scroll capture takes no signal, so a late capture is judged. + */ +const SCROLL_MOVEMENT_SCHEDULE: ObservationSchedule = { + intervalMs: 200, + budgetMs: 1_500, +}; /** The pre-gesture surface, already in hand: no capture is spent to produce it. */ export type ScrollSurfaceBaseline = Readonly<{ @@ -187,6 +196,7 @@ export async function observeScrollMovement(params: { /** The same two overrides `pollForScrollRest` takes: how long to ask, and how often. */ budgetMs?: number; pollMs?: number; + clock?: ObservationClock; }): Promise { const { direction, baseline, swipe } = params; const verdict = await pollForSurfaceVerdict(baseline, params); @@ -217,6 +227,12 @@ type SurfaceVerdict = startedAt: number; }>; +/** What one capture's verdict decides, before the loop's own timing is stitched back on. */ +type SurfaceJudgement = + | Readonly<{ kind: 'blind'; reason: SurfaceBlindReason }> + | Readonly<{ kind: 'moved'; observed: ObservedSurface }> + | Readonly<{ kind: 'settled'; observed: ObservedSurface; evidence: InteractionSurfaceChange }>; + /** Why this capture cannot be compared against the pre-gesture tree at all, movement included. */ type SurfaceBlindReason = 'capture-unreadable' | 'surface-unsettled' | ScrollSurfacePairDrift; @@ -255,32 +271,43 @@ async function pollForSurfaceVerdict( budgetMs?: number; pollMs?: number; swipe: ScrollSwipeEvidence; + clock?: ObservationClock; }, ): Promise { - const startedAt = Date.now(); - const deadline = startedAt + (params.budgetMs ?? MOVEMENT_VERDICT_BUDGET_MS); let previous: InteractionSurfaceSignature | undefined; - let attempts = 0; let changeNeedsRest: boolean | undefined; + const observed = await observeUntil({ + capture: async () => { + const reading = await readOneCapture(baseline, params.capture); + if (reading.kind === 'changed') { + changeNeedsRest ??= await baselineEndsInDirection(baseline, params.direction, params.swipe); + } + return reading; + }, + schedule: { + intervalMs: params.pollMs ?? SCROLL_MOVEMENT_SCHEDULE.intervalMs, + budgetMs: params.budgetMs ?? SCROLL_MOVEMENT_SCHEDULE.budgetMs, + }, + verdict: (latest) => { + if (latest.kind === 'blind') + return { kind: 'done', result: { kind: 'blind', reason: latest.reason } }; + const judged = settledVerdict(latest, { + previous, + changeNeedsRest: changeNeedsRest === true, + }); + previous = latest.observed.signature; + return judged ? { kind: 'done', result: judged } : { kind: 'continue' }; + }, + ...(params.clock ? { clock: params.clock } : {}), + }); - while (true) { - const reading = await readOneCapture(baseline, params.capture); - attempts += 1; - if (reading.kind === 'blind') return { kind: 'blind', reason: reading.reason }; - if (reading.kind === 'changed') { - changeNeedsRest ??= await baselineEndsInDirection(baseline, params.direction, params.swipe); - } - const verdict = settledVerdict(reading, { - previous, - changeNeedsRest: changeNeedsRest === true, - attempts, - startedAt, - }); - if (verdict) return verdict; - if (Date.now() >= deadline) return budgetExpiredVerdict(params, attempts, startedAt); - previous = reading.observed.signature; - await sleep(params.pollMs ?? MOVEMENT_POLL_MS); - } + const attempts = observed.polls.length; + if (observed.kind === 'failed') throw observed.error; + if (observed.kind !== 'done') return budgetExpiredVerdict(params, attempts, observed.waitedMs); + const startedAt = Date.now() - observed.waitedMs; + return observed.result.kind === 'blind' + ? observed.result + : { ...observed.result, attempts, startedAt }; } /** @@ -293,26 +320,17 @@ function settledVerdict( poll: { previous: InteractionSurfaceSignature | undefined; changeNeedsRest: boolean; - attempts: number; - startedAt: number; }, -): SurfaceVerdict | undefined { +): SurfaceJudgement | undefined { const atRest = surfaceIsAtRest(poll.previous, reading.observed.signature); - const { attempts, startedAt } = poll; if (reading.kind === 'changed') { if (!poll.changeNeedsRest || atRest) { - return { kind: 'moved', observed: reading.observed, attempts, startedAt }; + return { kind: 'moved', observed: reading.observed }; } return undefined; } if (!atRest) return undefined; - return { - kind: 'settled', - observed: reading.observed, - evidence: reading.evidence, - attempts, - startedAt, - }; + return { kind: 'settled', observed: reading.observed, evidence: reading.evidence }; } /** @@ -379,7 +397,7 @@ function budgetExpiredVerdict( swipe: ScrollSwipeEvidence; }, attempts: number, - startedAt: number, + durationMs: number, ): SurfaceVerdict { emitDiagnostic({ level: 'warn', @@ -387,7 +405,7 @@ function budgetExpiredVerdict( data: { direction: params.direction, attempts, - durationMs: Date.now() - startedAt, + durationMs, ...(params.swipe.pixels === undefined ? {} : { requestedPixels: params.swipe.pixels }), }, }); diff --git a/test/integration/provider-scenarios/scroll-movement-observation.test.ts b/test/integration/provider-scenarios/scroll-movement-observation.test.ts new file mode 100644 index 0000000000..1a71568b22 --- /dev/null +++ b/test/integration/provider-scenarios/scroll-movement-observation.test.ts @@ -0,0 +1,233 @@ +import assert from 'node:assert/strict'; +import { test, vi } from 'vitest'; +import type { AppleToolProvider } from '@agent-device/platform-apple/tool-provider'; +import type { SimulatorSnapshotSource } from '@agent-device/platform-apple/snapshot-source'; +import type { ExecOptions, ExecResult } from '@agent-device/host-kit/command'; +import { assertRpcError, assertRpcOk } from './assertions.ts'; +import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; +import { + createAppleRunnerProviderFromTranscript, + createRecordingAppleToolProvider, + simctlDeviceLifecycleHandler, +} from './providers.ts'; +import { createProviderTranscript, type ProviderScenarioProviderEntry } from './transcript.ts'; + +// The Simulator AX bridge runs on the host toolchain, outside every provider this scenario scripts: +// on a host with Xcode it builds, spawns, and connects for real before failing, which costs each +// test seconds of wall time. Here it reports unavailable at once, so every capture takes the +// scripted runner, as it does on a host without Xcode. +vi.mock( + '@agent-device/platform-apple/snapshot-source', + (): { createSimulatorSnapshotSource: () => SimulatorSnapshotSource } => ({ + createSimulatorSnapshotSource: () => ({ + acquire: async () => ({ + stage: 'failed', + failure: { kind: 'unsupported', code: 'provider-scenario-no-bridge' }, + }), + close: async () => {}, + }), + }), +); + +const APP = 'com.example.app'; +const DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id; + +// Directional-scroll observation on `observeUntil` (packages/capture-kit/src/observe-until.ts): +// `scroll down` gates its own `movement` claim on the tree it held before the gesture against one +// (or more, while the surface still looks untouched) post-gesture capture. The unit-level coverage +// in src/daemon/__tests__/scroll-movement.test.ts pins the verdict against a scripted capture +// function directly; these two scenarios drive the SAME loop end to end through the daemon's real +// command routing and a scripted Apple runner, the way `settle-observation.test.ts` drives `--settle`. + +// Fixed tab-bar-free list: a ScrollView reporting hidden content below, holding two rows. `rowOffset` +// moves the rows to model a scroll that actually shifted content; `hiddenBelow` toggles whether the +// container still reports more to reveal, which is what the no-progress refusal keys on. +const CONTAINER = { x: 18, y: 178, width: 366, height: 662 }; + +function screen(rowOffset: number, hiddenBelow: boolean) { + return [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 393, height: 852 }, + }, + { + index: 1, + parentIndex: 0, + type: 'ScrollView', + identifier: 'lab-list', + rect: CONTAINER, + ...(hiddenBelow ? { hiddenContentBelow: true } : {}), + }, + { + index: 2, + parentIndex: 1, + type: 'StaticText', + label: 'Row one', + // 300 (+ up to -100 offset) stays inside CONTAINER's clip (y 178..840): the presentation + // validator refuses a child frame that escapes its container's cumulative clip. + rect: { x: 24, y: 300 + rowOffset, width: 300, height: 20 }, + }, + { + index: 3, + parentIndex: 1, + type: 'StaticText', + label: 'Row two', + rect: { x: 24, y: 360 + rowOffset, width: 300, height: 20 }, + }, + ]; +} + +function snapshotEntry(nodes: readonly unknown[]): ProviderScenarioProviderEntry { + return { + command: 'ios.runner.snapshot', + deviceId: DEVICE_ID, + platform: 'apple', + result: { nodes, truncated: false }, + }; +} + +// A UI that never moves: every post-gesture capture returns the same tree, so the loop can never +// see a quiet-then-still-hidden pair on a fixed count. How many captures it takes to prove that is +// wall-clock (200ms poll floor), so the count is not scripted — a repeat entry serves every call. +function repeatSnapshotEntry(nodes: readonly unknown[]): ProviderScenarioProviderEntry { + return { ...snapshotEntry(nodes), repeat: true }; +} + +// Midpoint (201, 437) sits inside CONTAINER, matching the swipe fixture +// `src/daemon/__tests__/scroll-movement.test.ts` uses for the same geometry. +function scrollEntry(): ProviderScenarioProviderEntry { + return { + command: 'ios.runner.scroll', + deviceId: DEVICE_ID, + platform: 'apple', + result: { x: 201, y: 487, x2: 201, y2: 387 }, + }; +} + +/** + * The route ahead of a directional scroll's own capture resolves a live simulator target (a real + * host process lookup) before it ever reaches the scripted runner, so a bare `simctlDeviceLifecycleHandler` + * leaves every capture with a fresh, per-call "unknown generation" identity — comparable to nothing, + * which is `capture-lineage-drift` on the very first post-gesture read. Naming one fixed, stable + * "running app" job (a `launchctl list` line) and a fixed process start time (`ps -p -o lstart=`) + * lets the route resolve ONE identity and reuse it every capture, exactly like a real simulator whose + * app process never restarts mid-scroll — the AX bridge itself still isn't modeled, so every capture + * still falls back to the scripted runner, carrying that one stable identity instead of a random one. + */ +function iosSimulatorTool(): { provider: AppleToolProvider } { + const baseSimctl = simctlDeviceLifecycleHandler('com.apple.CoreSimulator.SimRuntime.iOS-18-0', [ + { name: PROVIDER_SCENARIO_IOS_SIMULATOR.name, udid: DEVICE_ID }, + ]); + const recorded = createRecordingAppleToolProvider({ + simctl: async (args, options) => { + if (args[0] === 'spawn' && args[1] === DEVICE_ID && args[2] === 'launchctl') { + return { stdout: `4242\t0\tUIKitApplication:${APP}[0x1]\n`, stderr: '', exitCode: 0 }; + } + return await baseSimctl(args, options); + }, + }); + const runCommand = async ( + cmd: string, + args: string[], + options?: ExecOptions, + ): Promise => { + // The system-surface presence probe runs before target resolution: report every + // registered host absent so the route proceeds to resolve the (scripted) app target below, + // instead of reading the probe as 'unknown' and falling back with a fresh random identity. + if (cmd === 'pgrep') return { stdout: '', stderr: '', exitCode: 1 }; + if (cmd === 'ps' && args[0] === '-p') { + return { stdout: 'Thu Jan 1 00:00:00 1970\n', stderr: '', exitCode: 0 }; + } + return await recorded.provider.runCommand(cmd, args, options); + }; + return { provider: { ...recorded.provider, runCommand } }; +} + +test('Provider-backed integration scroll down answers moved on the first post-gesture capture', async () => { + const runnerTranscript = createProviderTranscript([ + snapshotEntry(screen(0, true)), // pre-scroll baseline + scrollEntry(), + snapshotEntry(screen(-100, true)), // post-gesture: rows shifted into view on the first read + ]); + const appleRunnerProvider = createAppleRunnerProviderFromTranscript( + runnerTranscript, + 'ios.runner', + ); + const appleTool = iosSimulatorTool(); + + await withProviderScenarioResource( + async () => + await createProviderScenarioHarness({ + appleRunnerProvider: () => appleRunnerProvider, + appleToolProvider: () => appleTool.provider, + deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + }), + async (daemon) => { + assertRpcOk(await daemon.callCommand('open', [APP], { platform: 'ios', udid: DEVICE_ID })); + assertRpcOk(await daemon.callCommand('snapshot')); + + const callsBeforeScroll = runnerTranscript.calls.length; + const scroll = await daemon.callCommand('scroll', ['down']); + const scrollData = assertRpcOk<{ movement?: string }>(scroll); + + assert.equal(scrollData.movement, 'moved'); + // Gesture first, then exactly one post-gesture snapshot: a scroll that worked is confirmed by + // the cheapest possible read, and the order proves the read followed the gesture rather than + // racing it. + assert.deepEqual( + runnerTranscript.calls.slice(callsBeforeScroll).map((call) => call.command), + ['ios.runner.scroll', 'ios.runner.snapshot'], + ); + + runnerTranscript.assertComplete(); + }, + ); +}); + +test('Provider-backed integration scroll down that never moves a hidden-content container refuses with scroll_no_progress', async () => { + const runnerTranscript = createProviderTranscript([ + snapshotEntry(screen(0, true)), // pre-scroll baseline + scrollEntry(), + // Every post-gesture read is byte-identical to the baseline and the container still reports + // hidden content below: the gesture never reached it. + repeatSnapshotEntry(screen(0, true)), + ]); + const appleRunnerProvider = createAppleRunnerProviderFromTranscript( + runnerTranscript, + 'ios.runner', + ); + const appleTool = iosSimulatorTool(); + + await withProviderScenarioResource( + async () => + await createProviderScenarioHarness({ + appleRunnerProvider: () => appleRunnerProvider, + appleToolProvider: () => appleTool.provider, + deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + }), + async (daemon) => { + assertRpcOk(await daemon.callCommand('open', [APP], { platform: 'ios', udid: DEVICE_ID })); + assertRpcOk(await daemon.callCommand('snapshot')); + + const callsBeforeScroll = runnerTranscript.calls.length; + const scroll = await daemon.callCommand('scroll', ['down']); + const errorData = assertRpcError(scroll, 'COMMAND_FAILED', /scroll down moved nothing/); + assert.equal( + errorData.details && (errorData.details as Record).reason, + 'scroll_no_progress', + ); + + const callsDuringScroll = runnerTranscript.calls.slice(callsBeforeScroll); + // Gesture first, then at least the quiet-pair minimum of post-gesture snapshots — an exact + // count would assert the loop's polling speed rather than the refusal itself. + assert.equal(callsDuringScroll[0]?.command, 'ios.runner.scroll'); + assert.ok( + callsDuringScroll.filter((call) => call.command === 'ios.runner.snapshot').length >= 2, + `expected at least two post-gesture captures, saw ${callsDuringScroll.length - 1}`, + ); + }, + ); +});