fix(web): place nodes inside iframes in page coordinates

examples/test-app/README.md

@@ -25,7 +25,10 @@ Navigation uses Expo Router native bottom tabs, so the tab bar itself is also pa
 
 The deterministic WebView fixture is the stable accessibility oracle. On iOS, an interactive
 snapshot should expose its root as `webview`, both titles as `heading`, paragraph and label content
-as `text`, and the form controls as `text-field`, `switch`, and `button`. The live-site buttons are
+as `text`, and the form controls as `text-field`, `switch`, and `button`. Its `Embedded frames`
+section adds a same-origin `srcdoc` frame, an opaque-origin `data:` frame, and a cross-site frame
+(`https://example.com/`, needs network); each frame's content appears inline in the snapshot on
+iOS, Android, and web. The live-site buttons are
 exploratory smoke coverage for real-world WebKit trees, not stable assertion targets.
 Use an unscoped snapshot for this oracle: XCTest can detach a scoped WebKit document subtree from
 its `WebView` ancestor, leaving insufficient evidence for safe semantic projection.

examples/test-app/src/screens/WebViewScreen.tsx

@@ -116,6 +116,9 @@ export function WebViewScreen(props: { onClose: () => void }) {
             setLoadState((current) => (current.kind === 'error' ? current : { kind: 'loaded' }))
           }
           onLoadStart={() => setLoadState({ kind: 'loading' })}
+          // The default whitelist also gates iframe loads, which blocks the fixture's
+          // `about:srcdoc` and `data:` frames.
+          originWhitelist={['http://*', 'https://*', 'about:srcdoc', 'data:*']}
           source={site.source}
           startInLoadingState
           style={styles.webView}

examples/test-app/src/webview-accessibility-fixture.ts

@@ -66,6 +66,16 @@ export const WEBVIEW_ACCESSIBILITY_FIXTURE = `
         alt="Blue accessibility test square"
         src="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='48' height='48'%3E%3Crect width='48' height='48' fill='%230078d4'/%3E%3C/svg%3E"
       />
+      <h2>Embedded frames</h2>
+      <iframe
+        title="Same-origin frame"
+        srcdoc="<h3>Same-origin frame heading</h3><button>Same-origin frame button</button>"
+      ></iframe>
+      <iframe
+        title="Opaque-origin frame"
+        src="data:text/html,%3Ch3%3EOpaque-origin frame heading%3C/h3%3E%3Cbutton%3EOpaque-origin frame button%3C/button%3E"
+      ></iframe>
+      <iframe title="Cross-site frame" src="https://example.com/"></iframe>
     </main>
     <footer>Fixture footer text</footer>
   </body>

packages/capture-kit/src/snapshot-rect-projection.ts

@@ -5,7 +5,7 @@ import {
   isViewportRootNode,
   normalizeType,
 } from '@agent-device/contracts/snapshot';
-import { isPositiveFiniteRect, rectArea } from '@agent-device/kernel/rect';
+import { intersectRects, isPositiveFiniteRect, rectArea } from '@agent-device/kernel/rect';
 import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot';
 
 /**
@@ -71,12 +71,7 @@ export function intersectScreenshotRect(
   imageWidth: number,
   imageHeight: number,
 ): Rect | null {
-  const x = Math.max(rect.x, 0);
-  const y = Math.max(rect.y, 0);
-  const right = Math.min(rect.x + rect.width, imageWidth);
-  const bottom = Math.min(rect.y + rect.height, imageHeight);
-  if (right <= x || bottom <= y) return null;
-  return { x, y, width: right - x, height: bottom - y };
+  return intersectRects(rect, { x: 0, y: 0, width: imageWidth, height: imageHeight }) ?? null;
 }
 
 /**

packages/kernel/src/rect.test.ts

@@ -3,6 +3,7 @@ import { test } from 'vitest';
 import type { Rect } from './snapshot.ts';
 import {
   containsPoint,
+  intersectRects,
   isGeometricallyActionable,
   isPositiveFiniteRect,
   isRectVisibleInViewport,
@@ -221,3 +222,20 @@ test('isGeometricallyActionable matches the Swift runner predicate', () => {
     'a center on the bottom edge is not hittable',
   );
 });
+
+test('intersectRects returns the shared area and nothing for rects that only touch', () => {
+  assert.deepEqual(intersectRects({ x: 10, y: 20, width: 100, height: 50 }, VIEWPORT), {
+    x: 10,
+    y: 20,
+    width: 100,
+    height: 50,
+  });
+  assert.deepEqual(intersectRects({ x: -20, y: 480, width: 40, height: 40 }, VIEWPORT), {
+    x: 0,
+    y: 480,
+    width: 20,
+    height: 20,
+  });
+  assert.equal(intersectRects({ x: 300, y: 0, width: 10, height: 10 }, VIEWPORT), undefined);
+  assert.equal(intersectRects({ x: 0, y: -10, width: 10, height: 10 }, VIEWPORT), undefined);
+});

packages/kernel/src/rect.ts

@@ -91,6 +91,15 @@ export function rectContains(container: Rect, nested: Rect): boolean {
   );
 }
 
+/** The overlap of two rects, or nothing when they share no area. */
+export function intersectRects(left: Rect, right: Rect): Rect | undefined {
+  const x = Math.max(left.x, right.x);
+  const y = Math.max(left.y, right.y);
+  const width = Math.min(left.x + left.width, right.x + right.width) - x;
+  const height = Math.min(left.y + left.height, right.y + right.height) - y;
+  return width > 0 && height > 0 ? { x, y, width, height } : undefined;
+}
+
 export function rectArea(rect: Rect): number {
   return rect.width * rect.height;
 }

packages/maestro/src/internal/runtime-port-geometry.ts

@@ -1,6 +1,6 @@
 import { AppError } from '@agent-device/kernel/errors';
 import { buildInPageSwipeGesturePlan } from '@agent-device/contracts/scroll-gesture';
-import { isPositiveFiniteRect, pickLargestRect } from '@agent-device/kernel/rect';
+import { intersectRects, isPositiveFiniteRect, pickLargestRect } from '@agent-device/kernel/rect';
 import type { Rect, SnapshotState } from '@agent-device/kernel/snapshot';
 import { pointInsideRect } from './shared.ts';
 import {
@@ -123,14 +123,6 @@ function findLargestPositiveRect(
   return largest;
 }
 
-function intersectRects(left: Rect, right: Rect): Rect | undefined {
-  const x = Math.max(left.x, right.x);
-  const y = Math.max(left.y, right.y);
-  const width = Math.min(left.x + left.width, right.x + right.width) - x;
-  const height = Math.min(left.y + left.height, right.y + right.height) - y;
-  return width > 0 && height > 0 ? { x, y, width, height } : undefined;
-}
-
 function compareViewportAreaDescending(
   left: { readonly viewport: Rect },
   right: { readonly viewport: Rect },

packages/platform-web/src/__tests__/agent-browser-snapshot-refless.test.ts

@@ -282,15 +282,15 @@ describe('normalizeAgentBrowserSnapshot ref-less content', () => {
   });
 
   test('attaches geometry only to the nodes the backend can measure', async () => {
-    const result = await normalizeAgentBrowserSnapshot(
-      FORM_AFTER_SUBMIT,
-      async (refs) =>
+    const result = await normalizeAgentBrowserSnapshot(FORM_AFTER_SUBMIT, {
+      rects: async (refs) =>
         new Map(
           refs.flatMap((ref) =>
             ref === 'e2' ? [[ref, { x: 1, y: 2, width: 10, height: 20 }] as const] : [],
           ),
         ),
-    );
+      frameContentInsets: async () => new Map(),
+    });
 
     expect(result.nodes.find((node) => node.ref === 'e2')?.rect).toEqual({
       x: 1,

packages/platform-web/src/agent-browser-provider-rects.test.ts

@@ -436,6 +436,101 @@ test('agent-browser provider asks the backend nothing for a capture of only ref-
   });
 });
 
+test('agent-browser provider places nodes inside nested iframes in page coordinates', async () => {
+  // Measured on agent-browser 0.39: a node inside an iframe reports its box against that frame's
+  // viewport, so a press at the reported point lands on the page around the frame.
+  await withWebProvider(async (provider) => {
+    const calls: AgentBrowserCall[] = [];
+    const snapshot = await withCommandExecutorOverride(
+      frameGeometryExecutor(calls),
+      async () => await provider.snapshot({ includeRects: true }),
+    );
+
+    const stylesBatch = calls.find((call) => call.stdin?.includes('"styles"'));
+    assert.deepEqual(readBatchCommands(stylesBatch?.stdin), [
+      ['get', 'styles', '@e2'],
+      ['get', 'styles', '@e4'],
+    ]);
+    assert.deepEqual(snapshotRects(snapshot), {
+      e1: { x: 8, y: 400, width: 640, height: 44 },
+      e2: { x: 24, y: 770, width: 304, height: 154 },
+      e3: { x: 24 + 2 + 8, y: 770 + 2 + 59, width: 166, height: 21 },
+      e4: { x: 24 + 2 + 10, y: 770 + 2 + 90, width: 200, height: 50 },
+      e5: { x: 36 + 1 + 5 + 4, y: 862 + 1 + 5 + 6, width: 80, height: 20 },
+    });
+  });
+});
+
+test('agent-browser provider leaves a frame without a measurable inset with unplaced content', async () => {
+  await withWebProvider(async (provider) => {
+    const snapshot = await withCommandExecutorOverride(
+      frameGeometryExecutor([], { styles: { ...FRAME_STYLES, '@e4': null } }),
+      async () => await provider.snapshot({ includeRects: true }),
+    );
+
+    const rects = snapshotRects(snapshot);
+    // The nested frame itself sits in a measured frame; only the content it renders stays unplaced.
+    assert.deepEqual(rects.e4, { x: 36, y: 862, width: 200, height: 50 });
+    assert.equal(rects.e5, undefined);
+    assert.deepEqual(rects.e3, { x: 34, y: 831, width: 166, height: 21 });
+  });
+});
+
+test('agent-browser provider leaves a frame without a measurable box with unplaced content', async () => {
+  await withWebProvider(async (provider) => {
+    const snapshot = await withCommandExecutorOverride(
+      frameGeometryExecutor([], { boxes: { ...FRAME_BOXES, '@e2': undefined } }),
+      async () => await provider.snapshot({ includeRects: true }),
+    );
+
+    const rects = snapshotRects(snapshot);
+    assert.deepEqual(rects.e1, { x: 8, y: 400, width: 640, height: 44 });
+    assert.equal(rects.e2, undefined);
+    assert.equal(rects.e3, undefined);
+    assert.equal(rects.e4, undefined);
+    assert.equal(rects.e5, undefined);
+  });
+});
+
+test('agent-browser provider clips frame content to the frame viewport', async () => {
+  // A frame scrolled by its own document reports content above or below its viewport. The page
+  // around the frame renders at those points, so a press there must not target the content.
+  await withWebProvider(async (provider) => {
+    const snapshot = await withCommandExecutorOverride(
+      frameGeometryExecutor([], {
+        boxes: {
+          ...FRAME_BOXES,
+          '@e3': { x: 8, y: -40, width: 166, height: 21 },
+          '@e4': { x: 10, y: 130, width: 200, height: 50 },
+        },
+      }),
+      async () => await provider.snapshot({ includeRects: true }),
+    );
+
+    const rects = snapshotRects(snapshot);
+    assert.equal(rects.e3, undefined, 'content scrolled above the frame has no reachable point');
+    // The outer frame's viewport ends at 770 + 154 - 2: the visible part of the nested frame.
+    assert.deepEqual(rects.e4, { x: 36, y: 902, width: 200, height: 20 });
+  });
+});
+
+test('agent-browser provider places content of a clipped nested frame from its unclipped viewport', async () => {
+  await withWebProvider(async (provider) => {
+    const snapshot = await withCommandExecutorOverride(
+      // The outer frame is scrolled so the nested frame starts 20px above the outer viewport.
+      frameGeometryExecutor([], {
+        boxes: { ...FRAME_BOXES, '@e4': { x: 10, y: -20, width: 200, height: 50 } },
+      }),
+      async () => await provider.snapshot({ includeRects: true }),
+    );
+
+    const rects = snapshotRects(snapshot);
+    assert.deepEqual(rects.e4, { x: 36, y: 772, width: 200, height: 30 });
+    // The nested viewport starts at 772 - 20 + 6, so the button's top 8px of 20 are hidden.
+    assert.deepEqual(rects.e5, { x: 46, y: 772, width: 80, height: 12 });
+  });
+});
+
 async function withWebProvider(
   testFn: (provider: ReturnType<typeof createAgentBrowserWebProvider>) => void | Promise<void>,
 ): Promise<void> {
@@ -460,30 +555,113 @@ async function withWebProvider(
  * box. `@e1`/`@e2`/`@e4` report the boxes `boxRect` describes.
  */
 function geometryExecutor(calls: AgentBrowserCall[]) {
+  return batchExecutor(calls, snapshotPayload, ([, , ref]) =>
+    ref === '@e3' ? undefined : boxRect(ref),
+  );
+}
+
+/**
+ * A backend that answers `snapshot` with `snapshot()` and each batch command with `answer`; a
+ * command `answer` returns nothing for comes back as a failed entry.
+ */
+function batchExecutor(
+  calls: AgentBrowserCall[],
+  snapshot: () => ExecResult,
+  answer: (command: string[]) => unknown,
+) {
   return async (
     cmd: string,
     args: readonly string[],
     options: { allowFailure?: boolean; stdin?: string | Buffer },
   ): Promise<ExecResult> => {
     const cliArgs = recordCall(calls, cmd, args, options.stdin);
-    if (cliArgs[0] === 'snapshot') return snapshotPayload();
+    if (cliArgs[0] === 'snapshot') return snapshot();
     if (cliArgs[0] === 'batch') {
       const commands = readBatchCommands(options.stdin);
       assert.equal(options.allowFailure, true);
       assert.ok(!cliArgs.includes('--bail'), 'a bailed batch drops the refs after a failed one');
       return jsonResult(
-        commands.map((command) =>
-          command.slice(0, 3).join(' ') === 'get box @e3'
-            ? { command, success: false, error: 'No box for element', result: null }
-            : { command, success: true, error: null, result: boxRect(command[2]) },
-        ),
+        commands.map((command) => {
+          const result = answer(command);
+          return result === undefined
+            ? { command, success: false, error: 'No element', result: null }
+            : { command, success: true, error: null, result };
+        }),
         1,
       );
     }
     return jsonResult({ success: true, data: {} });
   };
 }
 
+type BackendBox = { x: number; y: number; width: number; height: number };
+
+/** Each frame's computed styles, as `get styles` reports them; `null` is a frame it cannot read. */
+type FrameStyles = Record<string, Record<string, string> | null>;
+
+const FRAME_STYLES: FrameStyles = {
+  '@e2': frameStyles({ border: 2, padding: 0 }),
+  '@e4': frameStyles({ border: 1, padding: 5 }),
+};
+
+function frameStyles({ border, padding }: { border: number; padding: number }) {
+  const styles: Record<string, string> = {};
+  for (const side of ['left', 'top', 'right', 'bottom']) {
+    styles[`border-${side}-width`] = `${border}px`;
+    styles[`padding-${side}`] = `${padding}px`;
+  }
+  return styles;
+}
+
+/** Boxes as the backend reports them: `@e3`-`@e5` against the viewport of the frame they render in. */
+const FRAME_BOXES: Record<string, BackendBox | undefined> = {
+  '@e1': { x: 8, y: 400, width: 640, height: 44 },
+  '@e2': { x: 24, y: 770, width: 304, height: 154 },
+  '@e3': { x: 8, y: 59, width: 166, height: 21 },
+  '@e4': { x: 10, y: 90, width: 200, height: 50 },
+  '@e5': { x: 4, y: 6, width: 80, height: 20 },
+};
+
+function frameGeometryExecutor(
+  calls: AgentBrowserCall[],
+  {
+    styles = FRAME_STYLES,
+    boxes = FRAME_BOXES,
+  }: { styles?: FrameStyles; boxes?: typeof FRAME_BOXES } = {},
+) {
+  return batchExecutor(calls, frameSnapshotPayload, ([, property, ref]) => {
+    if (property === 'box') return boxes[ref!];
+    const frameStyles = styles[ref!];
+    return frameStyles ? { styles: frameStyles } : undefined;
+  });
+}
+
+function frameSnapshotPayload(): ExecResult {
+  return jsonResult({
+    success: true,
+    data: {
+      refs: {
+        e1: { role: 'button', name: 'Page button' },
+        e2: { role: 'Iframe', name: 'Outer frame' },
+        e3: { role: 'button', name: 'Outer frame button' },
+        e4: { role: 'Iframe', name: 'Inner frame' },
+        e5: { role: 'button', name: 'Inner frame button' },
+      },
+      snapshot: [
+        '- button "Page button" [ref=e1]',
+        '- Iframe "Outer frame" [ref=e2]',
+        '  - button "Outer frame button" [ref=e3]',
+        '  - Iframe "Inner frame" [ref=e4]',
+        '    - button "Inner frame button" [ref=e5]',
+      ].join('\n'),
+    },
+  });
+}
+
+function snapshotRects(snapshot: WebSnapshotResult) {
+  return Object.fromEntries(snapshot.nodes.map((node) => [node.ref, node.rect]));
+}
+
 function assertGeometryRects(snapshot: WebSnapshotResult): void {
   assert.deepEqual(
     snapshot.nodes.map((node) => node.rect),