diff --git a/examples/test-app/README.md b/examples/test-app/README.md index 46f606a3aa..a7652b8295 100644 --- a/examples/test-app/README.md +++ b/examples/test-app/README.md @@ -25,7 +25,10 @@ Navigation uses Expo Router native bottom tabs, so the tab bar itself is also pa The deterministic WebView fixture is the stable accessibility oracle. On iOS, an interactive snapshot should expose its root as `webview`, both titles as `heading`, paragraph and label content -as `text`, and the form controls as `text-field`, `switch`, and `button`. The live-site buttons are +as `text`, and the form controls as `text-field`, `switch`, and `button`. Its `Embedded frames` +section adds a same-origin `srcdoc` frame, an opaque-origin `data:` frame, and a cross-site frame +(`https://example.com/`, needs network); each frame's content appears inline in the snapshot on +iOS, Android, and web. The live-site buttons are exploratory smoke coverage for real-world WebKit trees, not stable assertion targets. Use an unscoped snapshot for this oracle: XCTest can detach a scoped WebKit document subtree from its `WebView` ancestor, leaving insufficient evidence for safe semantic projection. diff --git a/examples/test-app/src/screens/WebViewScreen.tsx b/examples/test-app/src/screens/WebViewScreen.tsx index 0d56a34921..c157f6aa51 100644 --- a/examples/test-app/src/screens/WebViewScreen.tsx +++ b/examples/test-app/src/screens/WebViewScreen.tsx @@ -116,6 +116,9 @@ export function WebViewScreen(props: { onClose: () => void }) { setLoadState((current) => (current.kind === 'error' ? current : { kind: 'loaded' })) } onLoadStart={() => setLoadState({ kind: 'loading' })} + // The default whitelist also gates iframe loads, which blocks the fixture's + // `about:srcdoc` and `data:` frames. + originWhitelist={['http://*', 'https://*', 'about:srcdoc', 'data:*']} source={site.source} startInLoadingState style={styles.webView} diff --git a/examples/test-app/src/webview-accessibility-fixture.ts b/examples/test-app/src/webview-accessibility-fixture.ts index 84f3820756..1a8a125693 100644 --- a/examples/test-app/src/webview-accessibility-fixture.ts +++ b/examples/test-app/src/webview-accessibility-fixture.ts @@ -66,6 +66,16 @@ export const WEBVIEW_ACCESSIBILITY_FIXTURE = ` alt="Blue accessibility test square" src="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='48' height='48'%3E%3Crect width='48' height='48' fill='%230078d4'/%3E%3C/svg%3E" /> +

Embedded frames

+ + + diff --git a/packages/capture-kit/src/snapshot-rect-projection.ts b/packages/capture-kit/src/snapshot-rect-projection.ts index 0d3b48717e..3e056679ca 100644 --- a/packages/capture-kit/src/snapshot-rect-projection.ts +++ b/packages/capture-kit/src/snapshot-rect-projection.ts @@ -5,7 +5,7 @@ import { isViewportRootNode, normalizeType, } from '@agent-device/contracts/snapshot'; -import { isPositiveFiniteRect, rectArea } from '@agent-device/kernel/rect'; +import { intersectRects, isPositiveFiniteRect, rectArea } from '@agent-device/kernel/rect'; import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; /** @@ -71,12 +71,7 @@ export function intersectScreenshotRect( imageWidth: number, imageHeight: number, ): Rect | null { - const x = Math.max(rect.x, 0); - const y = Math.max(rect.y, 0); - const right = Math.min(rect.x + rect.width, imageWidth); - const bottom = Math.min(rect.y + rect.height, imageHeight); - if (right <= x || bottom <= y) return null; - return { x, y, width: right - x, height: bottom - y }; + return intersectRects(rect, { x: 0, y: 0, width: imageWidth, height: imageHeight }) ?? null; } /** diff --git a/packages/kernel/src/rect.test.ts b/packages/kernel/src/rect.test.ts index 63362aee31..73e3837c29 100644 --- a/packages/kernel/src/rect.test.ts +++ b/packages/kernel/src/rect.test.ts @@ -3,6 +3,7 @@ import { test } from 'vitest'; import type { Rect } from './snapshot.ts'; import { containsPoint, + intersectRects, isGeometricallyActionable, isPositiveFiniteRect, isRectVisibleInViewport, @@ -221,3 +222,20 @@ test('isGeometricallyActionable matches the Swift runner predicate', () => { 'a center on the bottom edge is not hittable', ); }); + +test('intersectRects returns the shared area and nothing for rects that only touch', () => { + assert.deepEqual(intersectRects({ x: 10, y: 20, width: 100, height: 50 }, VIEWPORT), { + x: 10, + y: 20, + width: 100, + height: 50, + }); + assert.deepEqual(intersectRects({ x: -20, y: 480, width: 40, height: 40 }, VIEWPORT), { + x: 0, + y: 480, + width: 20, + height: 20, + }); + assert.equal(intersectRects({ x: 300, y: 0, width: 10, height: 10 }, VIEWPORT), undefined); + assert.equal(intersectRects({ x: 0, y: -10, width: 10, height: 10 }, VIEWPORT), undefined); +}); diff --git a/packages/kernel/src/rect.ts b/packages/kernel/src/rect.ts index deaf8fa2a9..b1b82f9ed8 100644 --- a/packages/kernel/src/rect.ts +++ b/packages/kernel/src/rect.ts @@ -91,6 +91,15 @@ export function rectContains(container: Rect, nested: Rect): boolean { ); } +/** The overlap of two rects, or nothing when they share no area. */ +export function intersectRects(left: Rect, right: Rect): Rect | undefined { + const x = Math.max(left.x, right.x); + const y = Math.max(left.y, right.y); + const width = Math.min(left.x + left.width, right.x + right.width) - x; + const height = Math.min(left.y + left.height, right.y + right.height) - y; + return width > 0 && height > 0 ? { x, y, width, height } : undefined; +} + export function rectArea(rect: Rect): number { return rect.width * rect.height; } diff --git a/packages/maestro/src/internal/runtime-port-geometry.ts b/packages/maestro/src/internal/runtime-port-geometry.ts index 16a39017a5..708f8fdc56 100644 --- a/packages/maestro/src/internal/runtime-port-geometry.ts +++ b/packages/maestro/src/internal/runtime-port-geometry.ts @@ -1,6 +1,6 @@ import { AppError } from '@agent-device/kernel/errors'; import { buildInPageSwipeGesturePlan } from '@agent-device/contracts/scroll-gesture'; -import { isPositiveFiniteRect, pickLargestRect } from '@agent-device/kernel/rect'; +import { intersectRects, isPositiveFiniteRect, pickLargestRect } from '@agent-device/kernel/rect'; import type { Rect, SnapshotState } from '@agent-device/kernel/snapshot'; import { pointInsideRect } from './shared.ts'; import { @@ -123,14 +123,6 @@ function findLargestPositiveRect( return largest; } -function intersectRects(left: Rect, right: Rect): Rect | undefined { - const x = Math.max(left.x, right.x); - const y = Math.max(left.y, right.y); - const width = Math.min(left.x + left.width, right.x + right.width) - x; - const height = Math.min(left.y + left.height, right.y + right.height) - y; - return width > 0 && height > 0 ? { x, y, width, height } : undefined; -} - function compareViewportAreaDescending( left: { readonly viewport: Rect }, right: { readonly viewport: Rect }, diff --git a/packages/platform-web/src/__tests__/agent-browser-snapshot-refless.test.ts b/packages/platform-web/src/__tests__/agent-browser-snapshot-refless.test.ts index 52e4f7d9cc..759de154ed 100644 --- a/packages/platform-web/src/__tests__/agent-browser-snapshot-refless.test.ts +++ b/packages/platform-web/src/__tests__/agent-browser-snapshot-refless.test.ts @@ -282,15 +282,15 @@ describe('normalizeAgentBrowserSnapshot ref-less content', () => { }); test('attaches geometry only to the nodes the backend can measure', async () => { - const result = await normalizeAgentBrowserSnapshot( - FORM_AFTER_SUBMIT, - async (refs) => + const result = await normalizeAgentBrowserSnapshot(FORM_AFTER_SUBMIT, { + rects: async (refs) => new Map( refs.flatMap((ref) => ref === 'e2' ? [[ref, { x: 1, y: 2, width: 10, height: 20 }] as const] : [], ), ), - ); + frameContentInsets: async () => new Map(), + }); expect(result.nodes.find((node) => node.ref === 'e2')?.rect).toEqual({ x: 1, diff --git a/packages/platform-web/src/agent-browser-provider-rects.test.ts b/packages/platform-web/src/agent-browser-provider-rects.test.ts index 609604b1a1..aebc2df069 100644 --- a/packages/platform-web/src/agent-browser-provider-rects.test.ts +++ b/packages/platform-web/src/agent-browser-provider-rects.test.ts @@ -436,6 +436,101 @@ test('agent-browser provider asks the backend nothing for a capture of only ref- }); }); +test('agent-browser provider places nodes inside nested iframes in page coordinates', async () => { + // Measured on agent-browser 0.39: a node inside an iframe reports its box against that frame's + // viewport, so a press at the reported point lands on the page around the frame. + await withWebProvider(async (provider) => { + const calls: AgentBrowserCall[] = []; + const snapshot = await withCommandExecutorOverride( + frameGeometryExecutor(calls), + async () => await provider.snapshot({ includeRects: true }), + ); + + const stylesBatch = calls.find((call) => call.stdin?.includes('"styles"')); + assert.deepEqual(readBatchCommands(stylesBatch?.stdin), [ + ['get', 'styles', '@e2'], + ['get', 'styles', '@e4'], + ]); + assert.deepEqual(snapshotRects(snapshot), { + e1: { x: 8, y: 400, width: 640, height: 44 }, + e2: { x: 24, y: 770, width: 304, height: 154 }, + e3: { x: 24 + 2 + 8, y: 770 + 2 + 59, width: 166, height: 21 }, + e4: { x: 24 + 2 + 10, y: 770 + 2 + 90, width: 200, height: 50 }, + e5: { x: 36 + 1 + 5 + 4, y: 862 + 1 + 5 + 6, width: 80, height: 20 }, + }); + }); +}); + +test('agent-browser provider leaves a frame without a measurable inset with unplaced content', async () => { + await withWebProvider(async (provider) => { + const snapshot = await withCommandExecutorOverride( + frameGeometryExecutor([], { styles: { ...FRAME_STYLES, '@e4': null } }), + async () => await provider.snapshot({ includeRects: true }), + ); + + const rects = snapshotRects(snapshot); + // The nested frame itself sits in a measured frame; only the content it renders stays unplaced. + assert.deepEqual(rects.e4, { x: 36, y: 862, width: 200, height: 50 }); + assert.equal(rects.e5, undefined); + assert.deepEqual(rects.e3, { x: 34, y: 831, width: 166, height: 21 }); + }); +}); + +test('agent-browser provider leaves a frame without a measurable box with unplaced content', async () => { + await withWebProvider(async (provider) => { + const snapshot = await withCommandExecutorOverride( + frameGeometryExecutor([], { boxes: { ...FRAME_BOXES, '@e2': undefined } }), + async () => await provider.snapshot({ includeRects: true }), + ); + + const rects = snapshotRects(snapshot); + assert.deepEqual(rects.e1, { x: 8, y: 400, width: 640, height: 44 }); + assert.equal(rects.e2, undefined); + assert.equal(rects.e3, undefined); + assert.equal(rects.e4, undefined); + assert.equal(rects.e5, undefined); + }); +}); + +test('agent-browser provider clips frame content to the frame viewport', async () => { + // A frame scrolled by its own document reports content above or below its viewport. The page + // around the frame renders at those points, so a press there must not target the content. + await withWebProvider(async (provider) => { + const snapshot = await withCommandExecutorOverride( + frameGeometryExecutor([], { + boxes: { + ...FRAME_BOXES, + '@e3': { x: 8, y: -40, width: 166, height: 21 }, + '@e4': { x: 10, y: 130, width: 200, height: 50 }, + }, + }), + async () => await provider.snapshot({ includeRects: true }), + ); + + const rects = snapshotRects(snapshot); + assert.equal(rects.e3, undefined, 'content scrolled above the frame has no reachable point'); + // The outer frame's viewport ends at 770 + 154 - 2: the visible part of the nested frame. + assert.deepEqual(rects.e4, { x: 36, y: 902, width: 200, height: 20 }); + }); +}); + +test('agent-browser provider places content of a clipped nested frame from its unclipped viewport', async () => { + await withWebProvider(async (provider) => { + const snapshot = await withCommandExecutorOverride( + // The outer frame is scrolled so the nested frame starts 20px above the outer viewport. + frameGeometryExecutor([], { + boxes: { ...FRAME_BOXES, '@e4': { x: 10, y: -20, width: 200, height: 50 } }, + }), + async () => await provider.snapshot({ includeRects: true }), + ); + + const rects = snapshotRects(snapshot); + assert.deepEqual(rects.e4, { x: 36, y: 772, width: 200, height: 30 }); + // The nested viewport starts at 772 - 20 + 6, so the button's top 8px of 20 are hidden. + assert.deepEqual(rects.e5, { x: 46, y: 772, width: 80, height: 12 }); + }); +}); + async function withWebProvider( testFn: (provider: ReturnType) => void | Promise, ): Promise { @@ -460,23 +555,38 @@ async function withWebProvider( * box. `@e1`/`@e2`/`@e4` report the boxes `boxRect` describes. */ function geometryExecutor(calls: AgentBrowserCall[]) { + return batchExecutor(calls, snapshotPayload, ([, , ref]) => + ref === '@e3' ? undefined : boxRect(ref), + ); +} + +/** + * A backend that answers `snapshot` with `snapshot()` and each batch command with `answer`; a + * command `answer` returns nothing for comes back as a failed entry. + */ +function batchExecutor( + calls: AgentBrowserCall[], + snapshot: () => ExecResult, + answer: (command: string[]) => unknown, +) { return async ( cmd: string, args: readonly string[], options: { allowFailure?: boolean; stdin?: string | Buffer }, ): Promise => { const cliArgs = recordCall(calls, cmd, args, options.stdin); - if (cliArgs[0] === 'snapshot') return snapshotPayload(); + if (cliArgs[0] === 'snapshot') return snapshot(); if (cliArgs[0] === 'batch') { const commands = readBatchCommands(options.stdin); assert.equal(options.allowFailure, true); assert.ok(!cliArgs.includes('--bail'), 'a bailed batch drops the refs after a failed one'); return jsonResult( - commands.map((command) => - command.slice(0, 3).join(' ') === 'get box @e3' - ? { command, success: false, error: 'No box for element', result: null } - : { command, success: true, error: null, result: boxRect(command[2]) }, - ), + commands.map((command) => { + const result = answer(command); + return result === undefined + ? { command, success: false, error: 'No element', result: null } + : { command, success: true, error: null, result }; + }), 1, ); } @@ -484,6 +594,74 @@ function geometryExecutor(calls: AgentBrowserCall[]) { }; } +type BackendBox = { x: number; y: number; width: number; height: number }; + +/** Each frame's computed styles, as `get styles` reports them; `null` is a frame it cannot read. */ +type FrameStyles = Record | null>; + +const FRAME_STYLES: FrameStyles = { + '@e2': frameStyles({ border: 2, padding: 0 }), + '@e4': frameStyles({ border: 1, padding: 5 }), +}; + +function frameStyles({ border, padding }: { border: number; padding: number }) { + const styles: Record = {}; + for (const side of ['left', 'top', 'right', 'bottom']) { + styles[`border-${side}-width`] = `${border}px`; + styles[`padding-${side}`] = `${padding}px`; + } + return styles; +} + +/** Boxes as the backend reports them: `@e3`-`@e5` against the viewport of the frame they render in. */ +const FRAME_BOXES: Record = { + '@e1': { x: 8, y: 400, width: 640, height: 44 }, + '@e2': { x: 24, y: 770, width: 304, height: 154 }, + '@e3': { x: 8, y: 59, width: 166, height: 21 }, + '@e4': { x: 10, y: 90, width: 200, height: 50 }, + '@e5': { x: 4, y: 6, width: 80, height: 20 }, +}; + +function frameGeometryExecutor( + calls: AgentBrowserCall[], + { + styles = FRAME_STYLES, + boxes = FRAME_BOXES, + }: { styles?: FrameStyles; boxes?: typeof FRAME_BOXES } = {}, +) { + return batchExecutor(calls, frameSnapshotPayload, ([, property, ref]) => { + if (property === 'box') return boxes[ref!]; + const frameStyles = styles[ref!]; + return frameStyles ? { styles: frameStyles } : undefined; + }); +} + +function frameSnapshotPayload(): ExecResult { + return jsonResult({ + success: true, + data: { + refs: { + e1: { role: 'button', name: 'Page button' }, + e2: { role: 'Iframe', name: 'Outer frame' }, + e3: { role: 'button', name: 'Outer frame button' }, + e4: { role: 'Iframe', name: 'Inner frame' }, + e5: { role: 'button', name: 'Inner frame button' }, + }, + snapshot: [ + '- button "Page button" [ref=e1]', + '- Iframe "Outer frame" [ref=e2]', + ' - button "Outer frame button" [ref=e3]', + ' - Iframe "Inner frame" [ref=e4]', + ' - button "Inner frame button" [ref=e5]', + ].join('\n'), + }, + }); +} + +function snapshotRects(snapshot: WebSnapshotResult) { + return Object.fromEntries(snapshot.nodes.map((node) => [node.ref, node.rect])); +} + function assertGeometryRects(snapshot: WebSnapshotResult): void { assert.deepEqual( snapshot.nodes.map((node) => node.rect), diff --git a/packages/platform-web/src/agent-browser-provider.ts b/packages/platform-web/src/agent-browser-provider.ts index e8bee2f14d..4914d4e332 100644 --- a/packages/platform-web/src/agent-browser-provider.ts +++ b/packages/platform-web/src/agent-browser-provider.ts @@ -15,10 +15,11 @@ import { normalizeAgentBrowserAudioProbeResult, } from './agent-browser-audio-probe.ts'; import { normalizeAgentBrowserNetworkRequests } from './agent-browser-network.ts'; -import { normalizeAgentBrowserSnapshot } from './agent-browser-snapshot.ts'; +import { normalizeAgentBrowserSnapshot, type FrameContentInset } from './agent-browser-snapshot.ts'; import { isJsonObject, readNumberProperty, + readProperty, readStringProperty, type JsonObject, } from './json-utils.ts'; @@ -265,13 +266,19 @@ async function movePointer(send: AgentBrowserSend, x: number, y: number): Promis } async function captureAgentBrowserSnapshot( - runJson: (args: string[], run?: AgentBrowserRunOptions) => Promise, + runJson: AgentBrowserRunJson, options: WebSnapshotOptions | undefined, ): Promise { const data = await runJson(buildSnapshotArgs(options)); return await normalizeAgentBrowserSnapshot( data, - options?.includeRects ? (refs) => fetchRefRects(runJson, refs) : undefined, + options?.includeRects + ? { + rects: (refs) => fetchRefRects(runJson, refs), + frameContentInsets: (frameRefs) => + readRefsInBatches(runJson, frameRefs, 'styles', parseFrameContentInset), + } + : undefined, ); } @@ -297,29 +304,31 @@ function buildSnapshotArgs(options: WebSnapshotOptions | undefined): string[] { * is a DOM read of an already-loaded page rather than an interpreter boot. Raising the number is safe * only while that ratio holds — the stdin payload itself (~23 bytes per ref) is not the constraint. */ -const MAX_REFS_PER_RECT_BATCH = 200; +const MAX_REFS_PER_BATCH = 200; + +type AgentBrowserRunJson = (args: string[], run?: AgentBrowserRunOptions) => Promise; /** - * The bounds of every ref of the capture, in `batch` round trips: the backend measures them inside + * One `get ` read for every ref, in `batch` round trips: the backend reads them inside * one process, where the per-ref alternative pays one spawn per ref (measured on a 408-ref page: one * capture cost 385 spawns and 90 s, and 4 spawns and 0.3 s batched). * - * A ref the backend could not measure comes back as a failed entry and simply gets no rect — an - * element that is absent or has no box is an ordinary answer, and the entry's own `success` field is - * what distinguishes it, never the error wording. `--bail` is deliberately not requested: it stops - * at the first failed entry, which would silently drop the geometry of every ref after it. + * A ref the backend could not read comes back as a failed entry and is simply absent from the map — + * an element that is absent or has no box is an ordinary answer, and the entry's own `success` field + * is what distinguishes it, never the error wording. `--bail` is deliberately not requested: it stops + * at the first failed entry, which would silently drop the answer for every ref after it. */ -async function fetchRefRects( - runJson: (args: string[], run?: AgentBrowserRunOptions) => Promise, +async function readRefsInBatches( + runJson: AgentBrowserRunJson, refs: readonly string[], -): Promise> { - const rects = new Map(); - // A capture with no addressable node is an empty group list, so the loop below never runs and the - // backend is never called. That is the one place this seam declines to ask, and it is also what - // keeps such a capture from tripping the refusal: nothing failed, there was nothing to measure. - for (let start = 0; start < refs.length; start += MAX_REFS_PER_RECT_BATCH) { - const group = refs.slice(start, start + MAX_REFS_PER_RECT_BATCH); - const commands = group.map((ref) => ['get', 'box', browserRefSelector(ref)]); + property: 'box' | 'styles', + parse: (result: unknown) => T | undefined, +): Promise> { + const values = new Map(); + // No refs is an empty group list, so the loop below never runs and the backend is never called. + for (let start = 0; start < refs.length; start += MAX_REFS_PER_BATCH) { + const group = refs.slice(start, start + MAX_REFS_PER_BATCH); + const commands = group.map((ref) => ['get', property, browserRefSelector(ref)]); const entries = parseBatchEntries( await runJson(['batch'], { stdin: JSON.stringify(commands) }), commands, @@ -327,10 +336,20 @@ async function fetchRefRects( ); for (const [index, entry] of entries.entries()) { const ref = group[index]; - const rect = entry.success === true ? parseRect(entry.result) : undefined; - if (rect && ref !== undefined) rects.set(ref, rect); + const value = entry.success === true ? parse(entry.result) : undefined; + if (value !== undefined && ref !== undefined) values.set(ref, value); } } + return values; +} + +async function fetchRefRects( + runJson: AgentBrowserRunJson, + refs: readonly string[], +): Promise> { + // A capture with no addressable node asks nothing, which is also what keeps it from tripping the + // refusal below: nothing failed, there was nothing to measure. + const rects = await readRefsInBatches(runJson, refs, 'box', parseRect); if (refs.length > 0 && rects.size === 0) { // An element the backend CAN see measures as a box even when it is hidden or zero-sized, so a // capture whose every ref failed answered to a ref namespace the snapshot does not publish — @@ -353,6 +372,34 @@ async function fetchRefRects( return rects; } +/** + * The backend reports the box of a node inside an iframe against the viewport that frame renders + * into. The frame's border and padding are what separate that viewport from the frame's own box. + */ +function parseFrameContentInset(data: unknown): FrameContentInset | undefined { + const styles = readProperty(data, 'styles'); + if (!isJsonObject(styles)) return undefined; + const left = sumPixelStyles(styles, ['border-left-width', 'padding-left']); + const top = sumPixelStyles(styles, ['border-top-width', 'padding-top']); + const right = sumPixelStyles(styles, ['border-right-width', 'padding-right']); + const bottom = sumPixelStyles(styles, ['border-bottom-width', 'padding-bottom']); + return left === undefined || top === undefined || right === undefined || bottom === undefined + ? undefined + : { left, top, right, bottom }; +} + +/** Computed border widths and paddings resolve to non-negative pixel lengths. */ +function sumPixelStyles(styles: JsonObject, keys: readonly string[]): number | undefined { + let total = 0; + for (const key of keys) { + const value = styles[key]; + const pixels = typeof value === 'string' ? value.match(/^(\d+(?:\.\d+)?)px$/)?.[1] : undefined; + if (pixels === undefined) return undefined; + total += Number(pixels); + } + return total; +} + type AgentBrowserBatchEntry = { command?: unknown; success?: unknown; diff --git a/packages/platform-web/src/agent-browser-snapshot.ts b/packages/platform-web/src/agent-browser-snapshot.ts index d6f6ff2cfc..769a24a5e0 100644 --- a/packages/platform-web/src/agent-browser-snapshot.ts +++ b/packages/platform-web/src/agent-browser-snapshot.ts @@ -1,3 +1,4 @@ +import { intersectRects } from '@agent-device/kernel/rect'; import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; import { isJsonObject, @@ -46,15 +47,31 @@ const LABEL_ELEMENT_ROLES = new Set(['labeltext']); */ export type SnapshotRectFetch = (refs: readonly string[]) => Promise>; +/** The border and padding between an iframe's border box and the viewport its document renders into. */ +export type FrameContentInset = { left: number; top: number; right: number; bottom: number }; + +/** + * The content inset of every iframe ref asked for, in one call. A frame the backend could not + * measure is absent from the returned map. + */ +type FrameContentInsetFetch = ( + frameRefs: readonly string[], +) => Promise>; + +type SnapshotGeometryFetch = { + rects: SnapshotRectFetch; + frameContentInsets: FrameContentInsetFetch; +}; + export async function normalizeAgentBrowserSnapshot( data: unknown, - fetchRects?: SnapshotRectFetch, + geometry?: SnapshotGeometryFetch, ): Promise { const snapshotText = readStringProperty(data, 'snapshot') ?? ''; const refs = collectSnapshotRefs(readProperty(data, 'refs')); const drafts = parseSnapshotDraftNodes(snapshotText, refs); - if (fetchRects) await attachDraftRects(drafts, fetchRects); + if (geometry) await attachDraftRects(drafts, geometry); return { // Preserve each draft's backend ref as the node's `ref`. agent-browser resolves @@ -77,20 +94,91 @@ export async function normalizeAgentBrowserSnapshot( async function attachDraftRects( drafts: SnapshotDraftNode[], - fetchRects: SnapshotRectFetch, + geometry: SnapshotGeometryFetch, ): Promise { // One fetch for the whole capture. A capture of nothing addressable still comes through, and the // fetcher decides not to call the backend for it: that decision belongs with the refusal below. - const rects = await fetchRects( + const rects = await geometry.rects( drafts.flatMap((draft) => (draft.ref === undefined ? [] : [draft.ref])), ); - for (const draft of drafts) { - const ref = draft.ref; - const rect = ref === undefined ? undefined : rects.get(ref); - if (rect) draft.node.rect = rect; + const frameIndexes = drafts.map((draft) => + findAncestorIndex(drafts, draft.node.parentIndex, isFrameDraft), + ); + const frameRefs = [...new Set(frameIndexes.flatMap((index) => drafts[index ?? -1]?.ref ?? []))]; + const insets = + frameRefs.length > 0 + ? await geometry.frameContentInsets(frameRefs) + : new Map(); + // Drafts list a parent before its children, so each frame's viewport is known before its + // content is placed. + const frameViewports = new Map(); + for (const [index, draft] of drafts.entries()) { + const measured = draft.ref === undefined ? undefined : rects.get(draft.ref); + const placed = measured && placeRect(measured, frameViewports, frameIndexes[index]); + if (!placed) continue; + draft.node.rect = placed.visible; + const inset = draft.ref === undefined ? undefined : insets.get(draft.ref); + const viewport = inset && frameViewport(placed, inset); + if (viewport) frameViewports.set(index, viewport); } } +/** Where a node sits on the page, and the part of it the page shows. */ +type PlacedRect = { rect: Rect; visible: Rect }; + +/** Where a frame's document renders on the page, and the part of it the page shows. */ +type FrameViewport = { origin: Rect; visible: Rect }; + +/** The backend measures a node inside an iframe against that frame's viewport. */ +function placeRect( + measured: Rect, + frameViewports: ReadonlyMap, + frameIndex: number | undefined, +): PlacedRect | undefined { + if (frameIndex === undefined) return { rect: measured, visible: measured }; + // A frame whose position cannot be read leaves its content without a rect rather than with a + // wrong one. + const frame = frameViewports.get(frameIndex); + if (!frame) return undefined; + const rect = { ...measured, x: measured.x + frame.origin.x, y: measured.y + frame.origin.y }; + // Content scrolled out of its frame has no point a press could reach: the page around the + // frame renders there instead. + const visible = intersectRects(rect, frame.visible); + return visible && { rect, visible }; +} + +/** Content lines up with the frame's unclipped viewport and shows only where the frame does. */ +function frameViewport(frame: PlacedRect, inset: FrameContentInset): FrameViewport | undefined { + const origin = { + x: frame.rect.x + inset.left, + y: frame.rect.y + inset.top, + width: frame.rect.width - inset.left - inset.right, + height: frame.rect.height - inset.top - inset.bottom, + }; + const visible = intersectRects(origin, frame.visible); + return visible && { origin, visible }; +} + +function isFrameDraft(draft: SnapshotDraftNode): boolean { + return draft.node.type?.toLowerCase() === 'iframe'; +} + +/** The index of the nearest kept draft, from `parentIndex` upwards, that matches. */ +function findAncestorIndex( + drafts: readonly SnapshotDraftNode[], + parentIndex: number | undefined, + matches: (draft: SnapshotDraftNode) => boolean, +): number | undefined { + let index = parentIndex; + while (index !== undefined) { + const draft = drafts[index]; + if (!draft) return undefined; + if (matches(draft)) return index; + index = draft.node.parentIndex; + } + return undefined; +} + /** One snapshot line, with the node it reports when the backend issued a ref for it. */ type SnapshotLine = { line: string; @@ -204,14 +292,8 @@ function findAddressedAncestor( drafts: readonly SnapshotDraftNode[], parentIndex: number | undefined, ): SnapshotDraftNode | undefined { - let index = parentIndex; - while (index !== undefined) { - const draft = drafts[index]; - if (!draft) return undefined; - if (draft.ref !== undefined) return draft; - index = draft.node.parentIndex; - } - return undefined; + const index = findAncestorIndex(drafts, parentIndex, (draft) => draft.ref !== undefined); + return index === undefined ? undefined : drafts[index]; } /** The node a line reports: the addressable one parsed up front, or the ref-less text it carries. */ diff --git a/packages/selectors/src/interaction-touch-point.ts b/packages/selectors/src/interaction-touch-point.ts index cdea66bc9e..3505590783 100644 --- a/packages/selectors/src/interaction-touch-point.ts +++ b/packages/selectors/src/interaction-touch-point.ts @@ -4,7 +4,7 @@ import { type Rect, type SnapshotNode, } from '@agent-device/kernel/snapshot'; -import { containsPoint } from '@agent-device/kernel/rect'; +import { containsPoint, intersectRects } from '@agent-device/kernel/rect'; import { areRectsApproximatelyEqual, normalizeRect, @@ -42,8 +42,8 @@ export function resolveInteractionTouchPoint( return { kind: 'resolved', point: center, strategy: 'center' }; } - const searchRect = (options.bounds ?? []).reduce( - (current, bound) => (current ? intersectRects(current, bound) : null), + const searchRect = (options.bounds ?? []).reduce( + (current, bound) => (current ? intersectRects(current, bound) : undefined), targetRect, ); const point = searchRect @@ -81,9 +81,7 @@ function collectInteractiveDescendantRects( const rect = normalizeRect(candidate.rect); if (!rect || areRectsApproximatelyEqual(rect, targetRect)) return []; const clipped = intersectRects(rect, targetRect); - return clipped && clipped.width > 0 && clipped.height > 0 - ? [{ ref: candidate.ref, index: candidate.index, rect: clipped }] - : []; + return clipped ? [{ ref: candidate.ref, index: candidate.index, rect: clipped }] : []; }) .sort((left, right) => left.index - right.index) ); @@ -106,15 +104,6 @@ function isDescendantOf( return false; } -function intersectRects(left: Rect, right: Rect): Rect | null { - const x = Math.max(left.x, right.x); - const y = Math.max(left.y, right.y); - const maxX = Math.min(left.x + left.width, right.x + right.width); - const maxY = Math.min(left.y + left.height, right.y + right.height); - if (maxX <= x || maxY <= y) return null; - return { x, y, width: maxX - x, height: maxY - y }; -} - /** * Finds the smallest deterministic move away from the historical center that * leaves enough clearance from the parent edge and every independently