refactor(capture): carry the gesture no-effect fact on the snapshot tree

CHANGELOG.md

@@ -2,14 +2,19 @@
 
 ## Unreleased
 
+- Fixed (mobile): the warning for a `scroll`, `swipe`, or `gesture swipe` that had no visible effect
+  now reaches `is`, `get`, `find`, `wait`, and interactions, not only `snapshot`. The capture it
+  was proven on carries `gestureNoEffect` (`{ action, positionals }`), and those commands report it
+  in `data` or `error.details` with the appended warning.
 - Fixed (mobile): a read taken right after a `scroll`, `swipe`, or `gesture swipe` no longer reports
   a definite miss when the surface never settled. When post-gesture stabilization ran out of budget
   on a surface still moving, `is visible` answered a plain `selector_not_found` and `is absent`
-  passed. That capture now carries `unsettledGesture`: `is`, `get`, `find`, and `wait` report it (in
-  `error.details` or `data`) with an appended warning, `snapshot` appends the warning, `is absent`
-  refuses with `observation: "unsettled"`, `wait absent` keeps polling, and the next read captures
-  afresh. Click, press, and fill by selector do not disclose it yet. A failed read now also carries
-  `targetActivation` in `error.details`, the same place as `unsettledGesture`.
+  passed. That capture now carries `unsettledGesture`: `is`, `get`, `find`, `wait`, and every
+  interaction that captured it (`click`, `press`, `fill`, and the other touch and gesture commands)
+  report it (in `error.details` or `data`) with an appended warning, `snapshot` appends the warning,
+  `is absent` refuses with `observation: "unsettled"`, `wait absent` keeps polling, and the next read
+  captures afresh. A failed read now also carries `targetActivation` in `error.details`, the same
+  place as `unsettledGesture`.
 - Fixed (ios): `open` on a local Simulator now waits for the launched app's discovery before it
   decides whether the app is observable. On a loaded host `simctl spawn launchctl list` outlasts one
   1.5 s discovery wait slice, and the launch observation read that slice as an unobservable app, so

packages/capture-kit/src/post-gesture-stability.ts

@@ -83,9 +83,9 @@ function describePostGestureAction(gesture: PostGestureAction): string {
  * moved a stuck list when synthesized scrolls did not (#1600, element-18:
  * raw `swipe` worked where scroll/fling/pan all silently no-opped).
  */
-export function formatGestureNoEffectWarning(action: string, positionals: string[]): string {
+export function formatGestureNoEffectWarning(gesture: PostGestureAction): string {
   return (
-    `${describePostGestureAction({ action, positionals })} produced no visible change: the tree still matches its pre-gesture state. ` +
+    `${describePostGestureAction(gesture)} produced no visible change: the tree still matches its pre-gesture state. ` +
     'Either the container is already at its edge, or it ignores synthesized scrolls — ' +
     'a raw drag moves such lists: swipe x1 y1 x2 y2 (start inside the list).'
   );

packages/kernel/src/snapshot.ts

@@ -502,6 +502,8 @@ export type SnapshotState = {
   targetActivation?: IosTargetActivation;
   /** The gesture whose surface was still changing when stabilization gave up on this capture. */
   unsettledGesture?: PostGestureAction;
+  /** The gesture this capture proved had no visible effect (#1600). */
+  gestureNoEffect?: PostGestureAction;
 } & SnapshotStateProvenance;
 
 /** The gesture a post-gesture outcome fact names: the command and its positionals. */

src/commands/capture/runtime/snapshot.test.ts

@@ -13,7 +13,10 @@ import {
   type CommandSessionStore,
 } from '../../../runtime.ts';
 import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures';
-import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability';
+import {
+  formatGestureNoEffectWarning,
+  formatGestureUnsettledWarning,
+} from '@agent-device/capture-kit/post-gesture-stability';
 
 test('runtime snapshot captures nodes and updates the session baseline', async () => {
   let stored: Parameters<CommandSessionStore['set']>[0] | undefined;
@@ -803,3 +806,19 @@ test('runtime snapshot warns when its tree was read on a surface still moving af
 
   assert.deepEqual(result.warnings, [formatGestureUnsettledWarning(gesture)]);
 });
+
+test('runtime snapshot warns when its tree proved the gesture before it had no effect', async () => {
+  const gesture = { action: 'scroll', positionals: ['down'] };
+  const device = createSnapshotOnlyDevice({
+    snapshot: {
+      ...makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], {
+        backend: 'xctest',
+      }),
+      gestureNoEffect: gesture,
+    },
+  });
+
+  const result = await device.capture.snapshot({ session: 'default' });
+
+  assert.deepEqual(result.warnings, [formatGestureNoEffectWarning(gesture)]);
+});

src/commands/capture/runtime/snapshot.ts

@@ -26,7 +26,10 @@ import {
   renderSnapshotQualityWarnings,
   truncatedCaptureWarning,
 } from '@agent-device/capture-kit/quality-warnings';
-import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability';
+import {
+  formatGestureNoEffectWarning,
+  formatGestureUnsettledWarning,
+} from '@agent-device/capture-kit/post-gesture-stability';
 import { buildSnapshotVisibility } from '@agent-device/capture-kit/snapshot-visibility';
 import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '@agent-device/contracts/android-system-surface-disclosure';
 import { formatReactNativeOverlayWarning } from '../../react-native/overlay.ts';
@@ -264,6 +267,9 @@ function buildSnapshotWarnings(params: {
   if (params.snapshot.unsettledGesture) {
     warnings.push(formatGestureUnsettledWarning(params.snapshot.unsettledGesture));
   }
+  if (params.snapshot.gestureNoEffect) {
+    warnings.push(formatGestureNoEffectWarning(params.snapshot.gestureNoEffect));
+  }
   warnings.push(...buildEmptyAndroidInteractiveWarnings(params));
   if (!params.annotations.quality) {
     // Legacy runners without a structured verdict keep the old daemon-side heuristics.

src/daemon/__tests__/capture-disclosure-target-activation.test.ts

@@ -77,7 +77,7 @@ test('a failure does not borrow a repair the stored snapshot happens to carry',
   const response = withCaptureDisclosures({
     response: failed,
     consumedTree: { targetActivation: FACT },
-    activationProof: {},
+    captureProof: {},
   });
 
   assert.equal(response, failed);
@@ -95,7 +95,7 @@ test('surface and foreground disclosures ride one response together', () => {
       iosSystemSurfaceBundleId: 'com.apple.SafariViewService',
       targetActivation: FACT,
     },
-    activationProof: { state: { targetActivation: FACT } },
+    captureProof: { targetActivation: FACT },
   });
   const data = dataOf(response);
   assert.match(String(data.warning), /system web sign-in sheet/);
@@ -117,17 +117,17 @@ test('a repair that passes through two wrappers is named once in the failure hin
       details: { hint: 'Use snapshot to see the current tree.' },
     },
   };
-  const proof = { state: { targetActivation: FACT } };
+  const proof = { targetActivation: FACT };
 
   const once = withCaptureDisclosures({
     response: missed,
     consumedTree: { targetActivation: FACT },
-    activationProof: proof,
+    captureProof: proof,
   });
   const twice = withCaptureDisclosures({
     response: once,
     consumedTree: { targetActivation: FACT },
-    activationProof: proof,
+    captureProof: proof,
   });
 
   assert.equal(twice.ok, false);

src/daemon/__tests__/deferred-interaction-outcome.test.ts

@@ -280,7 +280,7 @@ test('a pending stabilization resolves through the quiet-window loop and clears
   assert.equal(result?.warnings, undefined);
 });
 
-test('a proven no-effect gesture surfaces its warning on the resolved capture (iOS accept-stale)', async () => {
+test('a proven no-effect gesture is stamped on the resolved capture tree (iOS accept-stale)', async () => {
   vi.useFakeTimers();
   const session = makeSession('ios');
   session.snapshot = pickupSnapshot();
@@ -299,8 +299,7 @@ test('a proven no-effect gesture surfaces its warning on the resolved capture (i
   }
   const result = await pendingResult;
 
-  assert.equal(result?.warnings?.length, 1);
-  assert.match(result?.warnings?.[0] ?? '', /produced no visible change/);
+  assert.deepEqual(result?.snapshot.gestureNoEffect, { action: 'scroll', positionals: ['down'] });
   assert.equal(isPostGestureStabilizationPending(session), false);
 });
 

src/daemon/__tests__/is-runtime.test.ts

@@ -13,7 +13,10 @@ import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fix
 import type { DaemonRequest } from '../daemon-request.ts';
 import { selectorCaptureFixture } from './selector-capture-fixture.ts';
 import { markDeferredInteractionOutcome } from '../deferred-interaction-outcome.ts';
-import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability';
+import {
+  formatGestureNoEffectWarning,
+  formatGestureUnsettledWarning,
+} from '@agent-device/capture-kit/post-gesture-stability';
 
 const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ mockRunAppleRunnerCommand: vi.fn() }));
 
@@ -468,3 +471,36 @@ test('a miss on a surface that never settled carries the unsettled fact, and the
   expect(fixture.captures.length).toBe(captures + 1);
   expect(reread?.ok === false && reread.error.details?.unsettledGesture).toBeUndefined();
 });
+
+test('a read after a scroll that moved nothing carries the no-effect fact', async () => {
+  vi.useFakeTimers();
+  const row = {
+    index: 0,
+    type: 'Cell',
+    identifier: 'row',
+    rect: { x: 0, y: 200, width: 390, height: 60 },
+  };
+  const fixture = selectorCaptureFixture({
+    snapshot: () => ({ nodes: [row], backend: 'xctest', producer: 'apple-runner' }),
+  });
+  const sessionStore = makeSessionStore();
+  const session = makeIosAppSession('is-no-effect', { snapshot: makeSnapshotState([row]) });
+  markDeferredInteractionOutcome({ session, command: 'scroll', positionals: ['down'], flags: {} });
+  sessionStore.set('is-no-effect', session);
+  const pending = dispatchIsViaRuntime({
+    req: isRequest('is-no-effect', ['visible', 'id=row']),
+    sessionName: 'is-no-effect',
+    sessionStore,
+    inspectFacts: fixture.inspectFacts,
+    bindDevice: fixture.bindDevice,
+  });
+  // A tree that still matches its pre-gesture baseline is distrusted up to the 3.5s cap.
+  await vi.advanceTimersByTimeAsync(3_700);
+  const gesture = { action: 'scroll', positionals: ['down'] };
+
+  const response = await pending;
+  expect(response?.ok && response.data).toMatchObject({
+    gestureNoEffect: gesture,
+    warnings: [formatGestureNoEffectWarning(gesture)],
+  });
+});

src/daemon/__tests__/post-gesture-no-effect-claim.test.ts

@@ -362,19 +362,28 @@ test('summarizeDiscriminatingSurfaceDivergence counts one-sided keys and moved r
 test('formatGestureNoEffectWarning names the gesture and the raw-drag escape hatch', () => {
   // Positionals echo verbatim: the warning names the gesture the agent issued,
   // and `scroll down 1` is what they issued.
-  const scrollWarning = formatGestureNoEffectWarning('scroll', ['down', '1']);
+  const scrollWarning = formatGestureNoEffectWarning({
+    action: 'scroll',
+    positionals: ['down', '1'],
+  });
   assert.match(scrollWarning, /scroll down 1 produced no visible change/);
   assert.match(scrollWarning, /swipe x1 y1 x2 y2/);
   assert.match(scrollWarning, /already at its edge/);
 
-  const gestureWarning = formatGestureNoEffectWarning('gesture', ['swipe', 'left']);
+  const gestureWarning = formatGestureNoEffectWarning({
+    action: 'gesture',
+    positionals: ['swipe', 'left'],
+  });
   assert.match(gestureWarning, /gesture swipe left produced no visible change/);
 
-  const bareWarning = formatGestureNoEffectWarning('swipe', []);
+  const bareWarning = formatGestureNoEffectWarning({ action: 'swipe', positionals: [] });
   assert.match(bareWarning, /swipe produced no visible change/);
 
   // The regression the deleted heuristic caused: every positional of a swipe is
   // a coordinate, so "drop anything numeric-looking" left a contentless "swipe".
-  const swipeWarning = formatGestureNoEffectWarning('swipe', ['10', '20', '30', '40']);
+  const swipeWarning = formatGestureNoEffectWarning({
+    action: 'swipe',
+    positionals: ['10', '20', '30', '40'],
+  });
   assert.match(swipeWarning, /^swipe 10 20 30 40 produced no visible change/);
 });

src/daemon/__tests__/selector-capture-runtime.test.ts

@@ -9,7 +9,7 @@ import {
   type SnapshotState,
 } from '@agent-device/kernel/snapshot';
 import type { DaemonResponse } from '../daemon-request.ts';
-import { type RequestActivationProof, withCaptureDisclosures } from '../capture-disclosure.ts';
+import { type RequestCaptureProof, withCaptureDisclosures } from '../capture-disclosure.ts';
 import { makeIosSession } from '../../__tests__/test-utils/session-factories.ts';
 import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts';
 import { createSelectorCaptureRuntime } from '../selector-capture-runtime.ts';
@@ -223,14 +223,14 @@ function proofRuntime(params: {
       : {}),
   } as never);
   const consumedSnapshot: { state?: SnapshotState } = {};
-  const activationProof: RequestActivationProof = {};
+  const captureProof: RequestCaptureProof = {};
   const runtime = createSelectorCaptureRuntime({
     device: session.device,
     session,
     sessionStore,
     sessionName: params.sessionName,
     consumedSnapshot,
-    activationProof,
+    captureProof,
     capture: boundCapture,
     req: {
       token: 't',
@@ -240,7 +240,7 @@ function proofRuntime(params: {
       flags: {},
     },
   });
-  return { runtime, consumedSnapshot, activationProof };
+  return { runtime, consumedSnapshot, captureProof };
 }
 
 /**
@@ -264,12 +264,12 @@ test('a session-snapshot cache hit consumes a repaired tree without earning the
 
   expect(boundCapture).not.toHaveBeenCalled();
   expect(holders.consumedSnapshot.state?.targetActivation).toEqual(REPAIR);
-  expect(holders.activationProof.state).toBeUndefined();
+  expect(holders.captureProof.targetActivation).toBeUndefined();
 
   const response = withCaptureDisclosures({
     response: { ok: true, data: { nodes: [] } } as DaemonResponse,
     consumedTree: holders.consumedSnapshot.state,
-    activationProof: holders.activationProof,
+    captureProof: holders.captureProof,
   });
   expect(response.ok).toBe(true);
   if (response.ok) {
@@ -287,7 +287,7 @@ test('a capture the request took itself earns the repair proof', async () => {
   await holders.runtime.capture({ flags: {}, cache: { useSessionSnapshot: true } });
 
   expect(boundCapture).toHaveBeenCalledTimes(1);
-  expect(holders.activationProof.state?.targetActivation).toEqual(REPAIR);
+  expect(holders.captureProof.targetActivation).toEqual(REPAIR);
 });
 
 /**
@@ -310,7 +310,7 @@ test('a later fact-less capture does not erase an earlier repair proof', async (
   await holders.runtime.capture({ flags: {}, cache: { forceFresh: true } });
 
   expect(boundCapture).toHaveBeenCalledTimes(2);
-  expect(holders.activationProof.state?.targetActivation).toEqual(REPAIR);
+  expect(holders.captureProof.targetActivation).toEqual(REPAIR);
 });
 
 function makeCaptureRuntime(sessionName: string) {