diff --git a/.env.example b/.env.example index 813969f..c6fa5eb 100644 --- a/.env.example +++ b/.env.example @@ -12,10 +12,9 @@ # Port the daemon's WebSocket + HTTP server binds on. The extension connects # here. Override only if something else owns 7225 on your machine. # WS_PORT=7225 - -# Interface to bind. 127.0.0.1 keeps the daemon loopback-only (recommended); -# do NOT set 0.0.0.0 unless you understand the exposure. -# WS_HOST=127.0.0.1 +# The daemon always binds its HTTP/WebSocket control plane to 127.0.0.1. +# There is intentionally no WS_HOST override: remote callers (including n8n) +# need a separately authenticated transport, not a public browser socket. # --- Daemon state -------------------------------------------------------- diff --git a/README.md b/README.md index 4d69a27..d94f647 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@

CI - v2.3.0 + v2.4.0 License: MIT Node >= 20 TypeScript strict @@ -34,7 +34,7 @@ It already has your browser open right there. It just can't see it. - **Multiple agents at once.** Cursor can drive tab 10 while Claude drives tab 11 — both through one shared daemon, neither blocking the other. - **Tab targeting, not "the active tab."** Every action names a `tabId`. Move your mouse, switch tabs, watch YouTube — the agent keeps working on the tab you told it to. It never hijacks the page you're reading. - **Per-tab isolation.** Element refs, console logs, and network buffers are scoped per tab. A ref from tab 10 can never click something in tab 20. -- **Per-tab concurrency.** Two actions on the *same* tab serialize (no races); actions on *different* tabs run in parallel. +- **Per-tab concurrency.** Two actions on the _same_ tab serialize (no races); actions on _different_ tabs run in parallel. - **Tab locking.** An agent can claim a tab so others queue behind it instead of racing (`browser_tabs { action: "lock" }`). Locks survive Chrome's service-worker recycling (`chrome.storage.session`). - **Agent-control shield.** While an agent works on a tab you see a translucent blue inner frame and your input on that tab is blocked (mouse, keyboard, wheel) — the badge shows `agent controlling the tab` and disappears when the action finishes. Locking a tab keeps a plain frame for the lock's lifetime. - **Same-origin iframe piercing.** Legacy/enterprise UIs that live inside iframes (e.g. an ONT console in `iframe#mainFrame`) are reachable: all locator tools search iframe documents, and `find`/`click_text` walk every frame. @@ -44,13 +44,19 @@ It already has your browser open right there. It just can't see it. - **Real, trusted input.** Clicks, typing and key presses go through the Chrome DevTools Protocol, so the page sees `isTrusted` events, focus really moves, default actions run (Tab moves focus, Enter submits, arrows drive autocomplete menus) and focus/blur fire even while the window is in the background — legacy grids and lookup widgets behave as they do for a person. One debugger session per tab is reused and detached after 30 s idle (the yellow "being debugged" banner shows only while it is attached). Pass `trusted: false` for the old synthetic events with no banner; they are also the automatic fallback when the debugger can't attach. - **Batches.** `browser_batch` runs a list of tool calls in one round-trip and stops at the first failure — a click → type → Tab → wait → read sequence is one call instead of five. - **Console-style JavaScript.** `browser_evaluate` accepts code as you'd type it in DevTools: top-level `await`, several statements, the last expression's value is returned, DOM nodes come back as readable descriptions — and page CSP doesn't block it. +- **Refs that stay right.** Snapshot/find refs resolve through one shared page runtime: the registered element first, then the first *visible* selector match (across open and closed shadow roots and same-origin iframes), then a verified fallback that only re-binds when role, tag and name identify one element — an ambiguous match is reported as gone instead of clicked. Hidden duplicates are skipped. +- **Shadow DOM everywhere.** `snapshot`, `text`, `find`, `click_text`, `wait` and every locator see web components (open and closed roots, slots), so sites like caniuse read like any other page. +- **Frozen tabs don't freeze the agent.** Every page call has an 8 s budget; a tab that stops answering is reported as `TAB_WEDGED` in seconds, later calls fail fast after a 1.5 s probe, and `browser_navigate` / `browser_tabs reload` replace the frozen tab in place (the result carries the new `tabId`). +- **Coordinates when you need them.** Click, hover and wheel-scroll at `x`/`y`, triple-click, ctrl/shift-click, key sequences with `repeat`, and zoomed `region` screenshots that tell you how image pixels map to those coordinates. +- **Record and replay.** `browser_gif` records a flow as an animated GIF (clicks marked) for the user; `browser_shortcuts` saves a flow with `{{variables}}` and replays it in one call. +- **Several browsers.** Every connected Chrome profile is its own connection; `browser_list_browsers` / `browser_select_browser` pick one per session (one browser behaves exactly as before). - **Honest errors.** Every tool failure reaches your agent as a real `isError` result with the full payload — no "success" responses hiding failures mid-workflow. --- ## How it works -Three pieces, all on your machine. Nothing leaves localhost. +Three pieces, all on your machine. Nothing leaves localhost. The daemon control plane is hard-bound to `127.0.0.1`; there is no supported remote/n8n listener. ```mermaid flowchart LR @@ -139,7 +145,11 @@ By default the daemon names each connection after its parent IDE ("Cursor", "Cla "mcpServers": { "browser-controller": { "command": "node", - "args": ["/path/to/browser-controller/mcp-server/dist/index.js", "--agent", "My Project Agent"] + "args": [ + "/path/to/browser-controller/mcp-server/dist/index.js", + "--agent", + "My Project Agent" + ] } } } @@ -165,6 +175,45 @@ Green dot = you're connected. Your agent can now see your browser. > These secrets prevent any other local process from opening a WebSocket and driving your authenticated browser sessions. To rotate them, stop your MCP clients, delete the folder, and the next run recreates both secrets. See [SECURITY.md](SECURITY.md) for the full threat model. +### Runtime lifecycle (authoritative daemon) + +The **daemon** is the only process that owns the extension-facing runtime. MCP +clients are thin stdio adapters and may start it automatically, but deployment +scripts should use the lifecycle commands below so there is one restart owner. +Do not run a second `daemon.js` or install a launch supervisor that competes for +port `7225`. + +```bash +npm run build +npm run daemon:start # start, or report the already-running PID +npm run daemon:status # JSON health/runtime information +npm run daemon:stop # graceful SIGTERM; removes runtime metadata +npm run daemon:restart # stop, then start; preserves token/enrollment +``` + +The daemon survives browser/Chrome restarts: token, enrollment secret, and +pairing remain in `~/.browser-controller/` (override with `BC_STATE_DIR`), and +the extension reconnects to the same endpoint. The lifecycle wrapper reads the +`daemon.json` PID and is safe to run repeatedly; a stale lock is removed only +when its recorded PID is not alive. + +### Expected endpoints + +- **MCP endpoint:** stdio, launched as `node mcp-server/dist/index.js` (the + standard `mcpServers` command/args form is shown above). +- **Extension runtime:** `ws://127.0.0.1:7225` by default, with HTTP + `/pair`, `/status`, and `/kill?sessionId=...` on the same port. These are + daemon/popup endpoints, not an MCP HTTP transport. +- **MCP client IPC:** `~/.browser-controller/daemon.sock` on Unix or + `\\.\pipe\browser-controller` on Windows. It is internal and token-authenticated. +- **State:** `~/.browser-controller/{daemon.json,daemon.lock,token.json, + enrollment.json,daemon.log}`. `WS_PORT`, `WS_HOST`, and `BC_STATE_DIR` are the + supported configuration overrides. + +This preserves the real Chrome session workflow: no Playwright/headless browser +is launched, and the existing Chrome profile, cookies, logins, and tabs remain +the browser being controlled. + ### Permission and trust boundary The unpacked extension deliberately requests Chrome's powerful `debugger`, `scripting`, `webRequest`, and `` permissions. They are what let it inspect network activity, inject functions, upload files through CDP, and automate any normal web tab you select. They also mean a connected MCP agent can read and change sensitive pages in your signed-in browser. Install the extension only from source you trust, pair it only with a trusted local daemon, and do not expose the daemon port beyond localhost. Chrome-protected pages such as `chrome://`, the Web Store, and DevTools remain inaccessible. @@ -173,7 +222,7 @@ The unpacked extension deliberately requests Chrome's powerful `debugger`, `scri ## Using it -The model is **tab-first**: the agent always says *which* tab to act on. It never assumes "the active tab." +The model is **tab-first**: the agent always says _which_ tab to act on. It never assumes "the active tab." ### Basic workflow @@ -262,6 +311,7 @@ npm run setup:cursor # or: node mcp-server/dist/index.js --setup cursor ``` This installs: + - `~/.cursor/rules/browser-controller.mdc` — the tab-targeting workflow, dropdown handling, when to lock tabs - `~/.cursor/commands/check-browser.md` — adds `/check-browser` to your Cursor chat @@ -284,35 +334,36 @@ See [`agent-config/`](agent-config/) for manual installation or to customize the ## What It Can Do -25 tools. Every page-interaction tool takes a **`tabId`** (the one exception is `browser_navigate`, where it's optional). +30 tools. Every page-interaction tool takes a **`tabId`** (the one exception is `browser_navigate`, where it's optional). **See** | Tool | What it does | |------|-------------| | `browser_observe` | Compact atomic semantic observation with snapshot/document identity, geometry, state, and dynamic allowed actions | -| `browser_snapshot` | Accessibility tree with element refs. Compact mode (default) returns only interactive elements. Traverses open shadow DOM + same-origin iframes. | -| `browser_screenshot` | Capture a tab as an image over CDP — `maxWidth` / `scale` / `jpeg` to cut tokens, `fullPage` for the whole page | -| `browser_text` | Extract raw text from page or element | -| `browser_find` | Query elements by natural language — walks same-origin iframes too | +| `browser_snapshot` | Accessibility tree with element refs. Compact mode (default) returns only interactive elements; `filter` / `depth` / `ref` (subtree) / `maxChars` (default 20k) keep it small. Traverses open + closed shadow DOM, slots and same-origin iframes. | +| `browser_screenshot` | Capture a tab as an image over CDP — `maxWidth` / `scale` / `jpeg` to cut tokens, `fullPage` for the whole page, `region` to zoom; reports the pixel → x/y mapping | +| `browser_text` | Extract text from page or element (incl. shadow DOM); `mode:"article"` = main content only; `offset` paging | +| `browser_find` | Query elements by natural language ("search input", "Save button") — tokenized, role-aware, shadow DOM + same-origin iframes | **Interact** | Tool | What it does | |------|-------------| | `browser_act` | Safely click/type/select/focus/hover/keypress/scroll/upload against a `browser_observe` snapshot | -| `browser_click` | Real (trusted) click by ref or CSS selector — pierces same-origin iframes | -| `browser_click_text` | Click by visible text. Works through React portals and overlays | -| `browser_type` | Real key presses into inputs and contenteditable fields; returns the resulting value | -| `browser_press_key` | Real key presses and combos (`Enter`, `Tab`, `ctrl+a`) | +| `browser_click` | Real (trusted) click by ref, CSS selector or `x`/`y` — `clickCount` 1-3, `modifiers`; pierces same-origin iframes and shadow DOM | +| `browser_click_text` | Click by visible text (case-insensitive, shadow DOM too) with a real click on the owning control. Works through React portals and overlays | +| `browser_type` | Real key presses into inputs and contenteditable fields (or the focused field); returns the resulting value | +| `browser_press_key` | Real key presses, combos (`Enter`, `Tab`, `ctrl+a`), sequences (`"ArrowDown ArrowDown Enter"`) and `repeat` | | `browser_batch` | Run several tool calls in one round-trip; stops at the first failure | -| `browser_scroll` | Scroll pages and virtual containers | -| `browser_hover` | Trigger tooltips and dropdowns | +| `browser_shortcuts` | Save a flow with `{{variables}}`, replay it in one call | +| `browser_scroll` | Scroll pages and virtual containers, or wheel-scroll at `x`/`y` | +| `browser_hover` | Trigger tooltips and dropdowns (ref, selector or `x`/`y`) | | `browser_select` | Pick from native `` (uses CDP, strict-CSP safe) | +| `browser_upload_file` | Upload files through `` (CDP, strict-CSP safe), or bytes / a screenshot into an input or a drop zone |

Uploading files — no file dialog @@ -332,32 +383,36 @@ Paths are absolute and local to the machine running the browser. Omit `ref`/`sel | Tool | What it does | |------|-------------| -| `browser_navigate` | Go to a URL in a tab (`tabId` optional, defaults to active) | -| `browser_tabs` | List / create / close / focus / **lock** / **unlock** tabs | +| `browser_navigate` | Go to a URL in a tab (`tabId` optional, defaults to active); replaces a frozen tab | +| `browser_tabs` | List / create (`active:false` for background) / close / focus / reload / **lock** / **unlock** tabs | +| `browser_resize_window` | Resize or maximize the window holding a tab (responsive testing) | +| `browser_list_browsers` / `browser_select_browser` | See the connected browsers (profiles) and route this session to one | **Debug & Advanced** | Tool | What it does | |------|-------------| -| `browser_console` | Console output (log, warn, error) — per-tab, capped at 200 entries | -| `browser_network` | XHR/fetch requests with status codes — per-tab, optional `limit` | +| `browser_console` | The page's console output (log, info, warn, error, debug, uncaught errors) — per-tab, capped at 200 entries; `pattern` / `level` / `limit` | +| `browser_intercept` | Block, redirect or set request headers per tab (Chrome session rules); captures + HAR export. `mock`/`log` rules are ledger-only | +| `browser_network` | Requests with status codes and failures — per-tab; `urlPattern`, regex `filter`, `failed`, `limit` | +| `browser_gif` | Record the agent's actions in a tab as an animated GIF (clicks marked); export writes the file | | `browser_evaluate` | Run JavaScript like the DevTools console: top-level `await`, last value returned, not blocked by CSP | | `browser_handle_dialog` | Dismiss/accept an open alert/confirm/prompt via CDP (works on frozen pages) | -| `browser_run_action` | Run a self-contained JS action object via CDP | +| `browser_run_action` | Run a self-contained JS action object via CDP | --- ## How Others Compare -| | Browser Controller | Playwright MCP | Chrome DevTools MCP | -|---|---|---|---| -| Uses your existing browser | Yes | No, launches new | Partial, needs debug port | -| Sessions and cookies | Already there | Fresh profile | Manual setup | -| Works behind corporate SSO | Yes | No | Depends | -| Multiple agents, multiple tabs | Yes | No | No | -| Tab-targeting (won't hijack active tab) | Yes | N/A | No | -| Authenticated local connection | Yes | N/A | No | -| Setup | Build from source + extension | Headless browser | Chrome with `--remote-debugging-port` | +| | Browser Controller | Playwright MCP | Chrome DevTools MCP | +| --------------------------------------- | ----------------------------- | ---------------- | ------------------------------------- | +| Uses your existing browser | Yes | No, launches new | Partial, needs debug port | +| Sessions and cookies | Already there | Fresh profile | Manual setup | +| Works behind corporate SSO | Yes | No | Depends | +| Multiple agents, multiple tabs | Yes | No | No | +| Tab-targeting (won't hijack active tab) | Yes | N/A | No | +| Authenticated local connection | Yes | N/A | No | +| Setup | Build from source + extension | Headless browser | Chrome with `--remote-debugging-port` | --- @@ -366,20 +421,20 @@ Paths are absolute and local to the machine running the browser. Omit `ref`/`sel | Env var | Default | What it does | |---------|---------|-------------| | `WS_PORT` | `7225` | WebSocket port the daemon uses for the extension connection | -| `BROWSER_CONTROLLER_PROGRESSIVE` | (unset) | Set to `1` to enable progressive tool disclosure: only the `browser_tools` meta tool is visible at startup (~150 tokens instead of loading all 25 definitions). The agent discovers tools via `browser_tools {action:"list"/"search"}` and activates them with `{action:"details", tool:"…"}`. Default (unset) shows all tools upfront — safe for agents whose instructions call tools directly. | +| `BROWSER_CONTROLLER_PROGRESSIVE` | (unset) | Set to `1` to enable progressive tool disclosure: only the `browser_tools` meta tool is visible at startup (~150 tokens instead of loading all 26 definitions). The agent discovers tools via `browser_tools {action:"list"/"search"}` and activates them with `{action:"details", tool:"…"}`. Default (unset) shows all tools upfront — safe for agents whose instructions call tools directly. | | `MCP_AGENT_NAME` | (auto: IDE name) | Override the agent name shown in the popup (same as `--agent`) | ### Daemon state files The daemon keeps everything in `~/.browser-controller/` (Windows: `%USERPROFILE%\.browser-controller\`): -| File | Purpose | -|------|---------| -| `enrollment.json` | One-time pairing secret for the extension (mode `0600`) | -| `token.json` | Auth token the extension must present on every WebSocket connection (mode `0600`) | -| `daemon.sock` | The IPC socket thin clients connect to (AF_UNIX on mac/linux; named pipe on Windows) | -| `daemon.json` | Daemon metadata (pid, port, start time) — used to detect a running daemon | -| `daemon.log` | Daemon stdout/stderr when spawned by a client | +| File | Purpose | +| ----------------- | ------------------------------------------------------------------------------------ | +| `enrollment.json` | One-time pairing secret for the extension (mode `0600`) | +| `token.json` | Auth token the extension must present on every WebSocket connection (mode `0600`) | +| `daemon.sock` | The IPC socket thin clients connect to (AF_UNIX on mac/linux; named pipe on Windows) | +| `daemon.json` | Daemon metadata (pid, port, start time) — used to detect a running daemon | +| `daemon.log` | Daemon stdout/stderr when spawned by a client | To fully reset: stop your MCP clients, delete the folder, and the next run recreates it with fresh secrets. diff --git a/SECURITY.md b/SECURITY.md index 14c4f1a..4e59aa6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -28,7 +28,8 @@ Report vulnerabilities through: ## Security Model -- **Local-only communication**: WebSocket between extension and server runs on localhost only. +- **Local-only communication**: the daemon's HTTP/WebSocket control plane is hard-bound to IPv4 loopback (`127.0.0.1`), not configurable through `WS_HOST`. WebSocket traffic between the extension and daemon remains local. MCP clients use stdio and then an authenticated local IPC socket; they do not get a TCP listener. +- **Remote callers and n8n**: Browser Controller has no supported remote or n8n ingress protocol. Do not publish port 7225 or change the listener to a wildcard address. A remote integration should run on the same host and reach the MCP client through its approved local process boundary, or be given a separately authenticated transport; browser-control credentials must not be reused as a network API key. - **Origin validation (exact-match on a pinned extension ID)**: the daemon pins the extension's `chrome-extension://` Origin on first contact, then rejects every later request whose Origin is not an exact match. This applies to BOTH the WebSocket upgrade and the HTTP endpoints (`/pair`, `/status`, `/kill`) through one shared gate — a web page and a co-installed hostile extension (which carries its own Origin and cannot forge ours) are both rejected. The browser sets the `Origin` header; it cannot be forged from page JS. - **Token auth on the control plane**: the WebSocket upgrade additionally requires the daemon's auth token (sent out-of-band via `Sec-WebSocket-Protocol` subprotocol, with a `?token=` legacy fallback). HTTP endpoints cannot require that token because `/pair` is how it is obtained; they instead require the browser-asserted pinned Origin plus the separate `X-BC-Enrollment` secret. - **Versioned capability handshake**: transport authentication is necessary but not sufficient to mark the extension ready. The daemon and extension advertise a protocol major, application version, and capabilities before tool traffic is routed. An explicitly incompatible protocol major is rejected; application-version differences alone are diagnostic because compatible patch releases can share the same wire contract. Peers that omit the protocol field are identified as legacy during the migration window rather than being silently mistaken for a current peer. diff --git a/deploy/systemd/browser-controller-daemon.service b/deploy/systemd/browser-controller-daemon.service new file mode 100644 index 0000000..4c0c229 --- /dev/null +++ b/deploy/systemd/browser-controller-daemon.service @@ -0,0 +1,17 @@ +# Example systemd user unit. Adjust the checkout path (and node location if +# `node` is not on the user manager's PATH) to match your installation. +[Unit] +Description=Browser Controller shared daemon +After=graphical-session.target + +[Service] +Type=simple +WorkingDirectory=%h/browser-controller +Environment=HOME=%h +ExecStart=/usr/bin/env node %h/browser-controller/mcp-server/dist/daemon.js +Restart=always +RestartSec=2 +NoNewPrivileges=true + +[Install] +WantedBy=default.target diff --git a/eslint.config.js b/eslint.config.js index 3ae527d..0406d3f 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -27,7 +27,8 @@ export default [ parserOptions: { requireConfigFile: false, babelOptions: { - presets: [['@babel/preset-typescript', { allowDeclareFields: true }]], + // Babel 8 always allows `declare` fields (the option was removed). + presets: ['@babel/preset-typescript'], }, }, }, diff --git a/extension/console-main.js b/extension/console-main.js new file mode 100644 index 0000000..3cc1301 --- /dev/null +++ b/extension/console-main.js @@ -0,0 +1,36 @@ +/** + * MAIN-world console capture. content.js runs in the extension's isolated + * world, where patching `console` only sees the extension's own calls — the + * page's console.log/warn/info/debug/error never reached browser_console. + * This script patches the PAGE's console and hands each entry to content.js + * as a JSON string on a private DOM event (object details don't cross + * worlds). Uncaught errors / rejections are still captured by content.js. + */ +(function () { + 'use strict'; + if (window.__bcConsoleMain) return; + Object.defineProperty(window, '__bcConsoleMain', { value: true }); + const EVENT = '__bc_console_entry'; + const LEVELS = ['log', 'info', 'warn', 'error', 'debug']; + const fmt = (a) => { + if (typeof a === 'string') return a; + if (a instanceof Error) return `${a.name}: ${a.message}`; + if (a && typeof a === 'object') { + try { return JSON.stringify(a); } catch { return Object.prototype.toString.call(a); } + } + return String(a); + }; + for (const level of LEVELS) { + const orig = console[level]; + if (typeof orig !== 'function') continue; + const patched = function (...args) { + try { + let text = args.map(fmt).join(' '); + if (text.length > 2000) text = text.slice(0, 2000) + '…[truncated]'; + document.dispatchEvent(new CustomEvent(EVENT, { detail: JSON.stringify({ level, text }) })); + } catch { /* never break the page's logging */ } + return orig.apply(this, args); + }; + try { console[level] = patched; } catch { /* frozen console */ } + } +})(); diff --git a/extension/content.js b/extension/content.js index 8e38748..95daab3 100644 --- a/extension/content.js +++ b/extension/content.js @@ -37,6 +37,15 @@ console.info = (...a) => capture('info', ...a); console.debug = (...a) => capture('debug', ...a); + // Page console entries from console-main.js (MAIN world), JSON on a DOM event. + document.addEventListener('__bc_console_entry', (e) => { + let entry; + try { entry = JSON.parse(e.detail); } catch { return; } + if (!entry || typeof entry.text !== 'string') return; + if (entry.text.indexOf('ResizeObserver loop') !== -1) return; + try { chrome.runtime.sendMessage({ type: 'console', level: String(entry.level || 'log'), text: entry.text.slice(0, 2100) }); } catch {} + }); + window.addEventListener('error', (e) => { // Silence the well-known ResizeObserver loop warning: it's a benign browser // notice (element resized during its own observation callback), not a real diff --git a/extension/events.js b/extension/events.js index 58fadf2..d4dc8a6 100644 --- a/extension/events.js +++ b/extension/events.js @@ -13,9 +13,10 @@ import { persistSessionState, dropTabState, dropDocumentState, -} from './lib/state.js'; -import { showLockShield, hideLockShield } from './lib/overlay.js'; -import { lockTabUi, releaseTabUi } from './lib/lock-ops.js'; +} from "./lib/state.js"; +import { showLockShield, hideLockShield } from "./lib/overlay.js"; +import { enrichCapture } from "./handlers/intercept.js"; +import { lockTabUi, releaseTabUi } from "./lib/lock-ops.js"; import { getOpenTabs, buildStatusPayload, @@ -23,7 +24,7 @@ import { applyPort, applyToken, applyEnrollment, -} from './lib/connection.js'; +} from "./lib/connection.js"; export function registerEventListeners() { chrome.runtime.onMessage.addListener((msg, sender, respond) => { @@ -32,12 +33,17 @@ export function registerEventListeners() { // before that, Chrome logs "message channel closed before a response was // received". Every branch below responds synchronously (or is fire-and-forget), // so we return false (or nothing) — Chrome handles it without the warning. - if (msg.type === 'console' && sender.tab?.id != null) { + if (msg.type === "console" && sender.tab?.id != null) { const buf = getTabBuffer(consoleByTab, sender.tab.id); - pushCapped(buf, { level: msg.level, text: msg.text, timestamp: Date.now(), url: sender.tab.url }); + pushCapped(buf, { + level: msg.level, + text: msg.text, + timestamp: Date.now(), + url: sender.tab.url, + }); return false; // fire-and-forget; no response expected } - if (msg.type === 'getStatus') { + if (msg.type === "getStatus") { // Async: fetch tabs before responding so the popup gets a full snapshot // (connection + locks + open tabs) in one message. Returning true signals // Chrome we'll call respond() asynchronously. @@ -46,15 +52,15 @@ export function registerEventListeners() { }); return true; // async response } - if (msg.type === 'setPort') { + if (msg.type === "setPort") { respond(applyPort(msg.port)); return false; } - if (msg.type === 'setToken') { + if (msg.type === "setToken") { respond(applyToken(msg.token)); return false; } - if (msg.type === 'setEnrollment') { + if (msg.type === "setEnrollment") { // The popup owns the user-facing entry of the enrollment secret. Persist, // re-pair, reconnect — all inside connection.js. The onMessage listener is // NOT async, so we .then() and return true (Chrome keeps the respond() @@ -64,38 +70,40 @@ export function registerEventListeners() { }); return true; // async response — respond() fires from the .then() } - if (msg.type === 'unlockAll') { + if (msg.type === "unlockAll") { // Snapshot BEFORE unlockAll() — unlockAll clears the map, so reading after // would lose the list of tabs whose shields need removing. const prev = tabLocks.snapshot(); tabLocks.unlockAll(); persistSessionState(); for (const { tabId } of prev) hideLockShield(tabId); - broadcastStatus('All tab locks cleared'); + broadcastStatus("All tab locks cleared"); respond({ success: true }); return false; } - if (msg.type === 'lockTab') { + if (msg.type === "lockTab") { const owner = msg.sessionId; if (msg.tabId == null || !owner) { - respond({ success: false, error: 'tabId and sessionId required' }); + respond({ success: false, error: "tabId and sessionId required" }); return false; } // lockTabUi is async (it awaits the shield injection) — keep Chrome's // respond() channel open for the async reply. lockTabUi(msg.tabId, owner, `Tab ${msg.tabId} pinned to ${owner}`) .then((shielded) => respond({ success: true, shielded })) - .catch((err) => respond({ success: false, error: err?.message || String(err) })); + .catch((err) => + respond({ success: false, error: err?.message || String(err) }), + ); return true; } - if (msg.type === 'unlockTab') { + if (msg.type === "unlockTab") { // { tabId } — release one tab's lock (vs unlockAll which clears all). if (msg.tabId == null) { - respond({ success: false, error: 'tabId required' }); + respond({ success: false, error: "tabId required" }); return false; } const was = releaseTabUi(msg.tabId); - broadcastStatus(`Tab ${msg.tabId} unpinned (was ${was || '-'})`); + broadcastStatus(`Tab ${msg.tabId} unpinned (was ${was || "-"})`); respond({ success: true, previousSession: was || null }); return false; } @@ -106,12 +114,34 @@ export function registerEventListeners() { (details) => { if (details.tabId == null || details.tabId < 0) return; // not a real tab const buf = getTabBuffer(networkByTab, details.tabId); - pushCapped(buf, { + const entry = { method: details.method, url: details.url, status: details.statusCode, type: details.type, timestamp: details.timeStamp, + }; + pushCapped(buf, entry); + // Intercept ledger enrichment (best-effort; never breaks capture). + try { + enrichCapture(details.tabId, entry); + } catch {} + }, + { urls: [""] }, + ); + + // Requests that never completed (DNS failure, blocked, aborted, CORS…): + // onCompleted never fires for them, so without this they were invisible. + chrome.webRequest.onErrorOccurred.addListener( + (details) => { + if (details.tabId == null || details.tabId < 0) return; + const buf = getTabBuffer(networkByTab, details.tabId); + pushCapped(buf, { + method: details.method, + url: details.url, + error: details.error, + type: details.type, + timestamp: details.timeStamp, }); }, { urls: [''] }, @@ -136,11 +166,11 @@ export function registerEventListeners() { // from the short-lived per-call listener inside handleNavigate — they share no // state and Chrome supports multiple onUpdated listeners (review NOTE 7c). chrome.tabs.onUpdated.addListener((tabId, changeInfo) => { - if (changeInfo.status === 'loading') { + if (changeInfo.status === "loading") { dropDocumentState(tabId); persistSessionState(); } - if (changeInfo.status === 'complete' && tabLocks.owner(tabId)) { + if (changeInfo.status === "complete" && tabLocks.owner(tabId)) { showLockShield(tabId); } }); diff --git a/extension/handlers/cdp.js b/extension/handlers/cdp.js index cf1a3d7..457b7bd 100644 --- a/extension/handlers/cdp.js +++ b/extension/handlers/cdp.js @@ -3,9 +3,10 @@ * upload_file — the two tools that cannot be implemented with * chrome.scripting (CSP bypass / DOM.setFileInputFiles). */ -import { resolveTab, safeExec } from '../lib/page-exec.js'; +import { resolveTab, execDom, getFallback } from '../lib/page-exec.js'; import { MAX_RESULT_CHARS } from '../lib/state.js'; import { ensureCdp } from '../lib/cdp-session.js'; +import { handleScreenshot } from './tabs.js'; export async function handleRunAction(params, _sessionId, _agentName, signal) { const { tabId, code, actionParams = {} } = params; @@ -67,71 +68,135 @@ export async function handleRunAction(params, _sessionId, _agentName, signal) { } } +/** Main-world expression returning the node marked data-bc-upload=token (pierces open shadow roots / same-origin frames). */ +export function findMarkedExpression(token) { + return `(() => { const s = '[data-bc-upload="${token}"]'; + const q = (root, d) => { const hit = root.querySelector(s); if (hit || d > 6) return hit; + for (const el of root.querySelectorAll('*')) { + if (el.shadowRoot) { const h = q(el.shadowRoot, d + 1); if (h) return h; } + if (el.tagName === 'IFRAME') { try { const h = el.contentDocument && q(el.contentDocument, d + 1); if (h) return h; } catch (e) {} } + } + return null; }; + return q(document, 0); })()`; +} + +/** + * Page-side: build a File from base64 bytes and hand it to the page — into an + * (files + input/change events) or, for any other target, + * as a drag-and-drop (dragenter/dragover/drop with a DataTransfer), which is + * what upload drop zones listen for. No temp files, no file dialog. + */ +function pagePutFile(ref, sel, fb, b64, mime, name, x, y) { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + let target; + if (ref || sel) target = D.resolve(ref, sel, fb).el; + else if (Number.isFinite(x) && Number.isFinite(y)) target = D.elementAt(x, y); + else target = (D.queryAll('input[type="file"]', true) || [])[0] || null; + if (!target) return { success: false, error: 'Upload target not found' }; + let bytes; + try { + const bin = atob(b64); + bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); + } catch { return { success: false, error: 'imageBase64 is not valid base64' }; } + const file = new File([bytes], name, { type: mime }); + const dt = new DataTransfer(); + dt.items.add(file); + if (target.tagName === 'INPUT' && target.type === 'file') { + target.files = dt.files; + target.dispatchEvent(new Event('input', { bubbles: true })); + target.dispatchEvent(new Event('change', { bubbles: true })); + return { success: true, mode: 'input', file: name, size: file.size }; + } + const r = target.getBoundingClientRect(); + const init = { bubbles: true, cancelable: true, composed: true, dataTransfer: dt, clientX: r.left + r.width / 2, clientY: r.top + r.height / 2 }; + for (const type of ['dragenter', 'dragover', 'drop']) target.dispatchEvent(new DragEvent(type, init)); + return { success: true, mode: 'drop', file: name, size: file.size, target: D.describe(target) }; +} + +/** Upload bytes (base64 or a fresh screenshot) instead of a local path. */ +async function uploadBytes(tab, params) { + let b64 = params.imageBase64 || null; + let mime = params.mimeType || 'image/png'; + let name = params.fileName || 'image.png'; + if (params.fromScreenshot) { + const shot = await handleScreenshot({ + tabId: params.screenshotTabId ?? tab.id, format: 'png', + ...(params.region ? { region: params.region } : {}), + }); + if (!shot?.data) throw new Error('Screenshot for upload returned no data'); + b64 = shot.data; + mime = 'image/png'; + name = params.fileName || 'screenshot.png'; + } + if (!b64) throw new Error('imageBase64 or fromScreenshot required'); + const res = await execDom(tab.id, pagePutFile, [ + params.ref || null, params.selector || null, getFallback(tab.id, params.ref), + b64, mime, name, params.x ?? null, params.y ?? null, + ]); + return res; +} + export async function handleUploadFile(params) { const { tabId, ref, selector, filePath, files: fileList } = params; const tab = await resolveTab(tabId); + if (params.imageBase64 || params.fromScreenshot) return uploadBytes(tab, params); const filePaths = fileList || (filePath ? [filePath] : []); - if (filePaths.length === 0) throw new Error('filePath or files required'); + if (filePaths.length === 0) throw new Error('filePath, files, imageBase64 or fromScreenshot required'); - let sel = 'input[type="file"]'; - if (ref) sel = `[data-mcp-ref="${ref}"]`; - else if (selector) sel = selector; - - // Verify the target BEFORE the CDP round-trip: CDP's DOM.querySelector - // happily resolves any node, and DOM.setFileInputFiles on a non-file input - // fails with an opaque protocol error (or worse, on some Chrome versions, - // appears to succeed). React onChange handlers also require a change/input - // event after the files are set — CDP doesn't fire one. - const check = await safeExec(tab.id, (s) => { - const el = document.querySelector(s); + // Resolve in the page with the shared resolver (ref registry, visible-first + // selector across shadow roots / same-origin frames, verified fallback), then + // hand the node to CDP through a one-shot marker attribute. + const sel = selector || (ref ? null : 'input[type="file"]'); + const what = selector || (ref ? `ref ${ref}` : 'input[type="file"]'); + const token = `u${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`; + const check = await execDom(tab.id, (_ref, _sel, _fb, _token) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const el = D.resolve(_ref, _sel, _fb).el; if (!el) return { found: false }; + el.setAttribute('data-bc-upload', _token); return { found: true, isFileInput: el.tagName === 'INPUT' && el.type === 'file', multiple: !!el.multiple, }; - }, [sel]).catch(() => null); - if (check && check.found) { - if (!check.isFileInput) throw new Error(`Element matching ${sel} is not an .`); - if (filePaths.length > 1 && !check.multiple) { - throw new Error(`File input matching ${sel} does not accept multiple files.`); - } + }, [ref || null, sel, getFallback(tab.id, ref), token]).catch(() => null); + if (!check || !check.found) throw new Error(`File input not found: ${what}`); + if (!check.isFileInput) throw new Error(`Element matching ${what} is not an .`); + if (filePaths.length > 1 && !check.multiple) { + throw new Error(`File input matching ${what} does not accept multiple files.`); } // upload_file stays on CDP (DOM.setFileInputFiles is CDP-only). let uploaded = false; try { const send = await ensureCdp(tab.id); - await send('DOM.enable'); - const { root } = await send('DOM.getDocument'); - - const { nodeId } = await send('DOM.querySelector', { - nodeId: root.nodeId, - selector: sel, - }); - - if (!nodeId) throw new Error(`File input not found with selector: ${sel}`); - - await send('DOM.setFileInputFiles', { - files: filePaths, - nodeId, - }); + // Find the marked node wherever it lives (open shadow roots, same-origin frames). + const { result } = await send('Runtime.evaluate', { expression: findMarkedExpression(token) }); + if (!result || !result.objectId) throw new Error(`File input not found: ${what}`); + await send('DOM.setFileInputFiles', { files: filePaths, objectId: result.objectId }); uploaded = true; } finally { // Fire the events React/Vue file inputs listen for after a successful set, - // and always remove the short-lived Observation V2 handoff marker. + // and always remove the one-shot marker (and the Observation V2 handoff marker). try { - await safeExec(tab.id, (s, notify) => { - const el = document.querySelector(s); - if (!el) return; + await execDom(tab.id, (_token, notify) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const el = (D.queryAll(`[data-bc-upload="${_token}"]`, true) || [])[0]; + if (!el) return null; if (notify) { el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true })); } + el.removeAttribute('data-bc-upload'); el.removeAttribute('data-bc-v2-upload'); - }, [sel, uploaded]); + return null; + }, [token, uploaded]); } catch { /* page changed — CDP outcome still determines the tool result */ } } - return { success: true, files: filePaths, selector: sel }; + return { success: true, files: filePaths, selector: what }; } diff --git a/extension/handlers/find.js b/extension/handlers/find.js new file mode 100644 index 0000000..d93aca0 --- /dev/null +++ b/extension/handlers/find.js @@ -0,0 +1,128 @@ +/** + * browser_find: natural-language element search (tokenized, role-aware, + * shadow DOM + same-origin iframes, wrapper/echo suppression). Refs go into + * the shared page registry so every ref tool can use them. + */ +import { safeExec, execDom, resolveTab } from '../lib/page-exec.js'; +import { fallbackByTab, persistSessionState, nextRefPrefix } from '../lib/state.js'; +import { PAGE_FALLBACK_INSTALL } from '../utils/smart-selector.js'; + +export async function handleFind(params) { + const { tabId, query, limit = 10, role } = params; + await resolveTab(tabId); + await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); + const refPrefix = nextRefPrefix('f'); + + return execDom(tabId, (_q, _lim, _refPrefix, _role) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const genFallback = (globalThis.__browserControllerFallbackRuntime || {}).generateFallback || null; + const fallbacks = {}; + + // Words that describe the KIND of element, mapped to the roles they mean. + const ROLE_WORDS = { + button: ['button'], btn: ['button'], link: ['link'], anchor: ['link'], + input: ['textbox', 'searchbox', 'combobox', 'spinbutton'], field: ['textbox', 'searchbox', 'combobox', 'spinbutton'], + textbox: ['textbox', 'searchbox'], box: ['textbox', 'searchbox', 'combobox', 'checkbox'], textarea: ['textbox'], + searchbox: ['searchbox'], checkbox: ['checkbox'], check: ['checkbox'], radio: ['radio'], + dropdown: ['combobox', 'listbox', 'button'], select: ['combobox', 'listbox'], combobox: ['combobox'], + tab: ['tab'], menu: ['menu', 'menubar', 'button'], menuitem: ['menuitem'], option: ['option'], + heading: ['heading'], title: ['heading'], image: ['img'], img: ['img'], icon: ['img', 'button'], + dialog: ['dialog', 'alertdialog'], modal: ['dialog', 'alertdialog'], switch: ['switch'], toggle: ['switch', 'button', 'checkbox'], + slider: ['slider'], list: ['list', 'listbox'], table: ['table', 'grid'], row: ['row'], cell: ['cell', 'gridcell'], + }; + const STOP = new Set(['the', 'a', 'an', 'to', 'of', 'for', 'on', 'in', 'with', 'and', 'that', 'this', 'element', 'please']); + const words = String(_q).toLowerCase().split(/[^\p{L}\p{N}_-]+/u).filter((w) => w && !STOP.has(w)); + const roleHints = new Set(); + const content = []; + for (const w of words) { + if (ROLE_WORDS[w]) ROLE_WORDS[w].forEach((r) => roleHints.add(r)); + else content.push(w); + } + // "search" names the purpose AND a role. + if (words.includes('search')) roleHints.add('searchbox'); + const phrase = content.join(' '); + const wantRole = _role ? String(_role).toLowerCase() : null; + + const SKIP = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'META', 'LINK', 'HEAD', 'HTML', 'BODY', 'BR', 'PATH']); + const cands = []; + for (const root of D.allRoots(true)) { + let els = []; + try { els = root.querySelectorAll('*'); } catch {} + for (const el of els) { + if (SKIP.has(el.tagName)) continue; + const r = D.roleOf(el); + if (r === 'none') continue; + if (wantRole && r !== wantRole) continue; + const name = D.nameOf(el).toLowerCase(); + const attrs = [el.id, D.attr(el, 'name'), D.attr(el, 'type'), D.attr(el, 'placeholder'), D.attr(el, 'data-testid'), + D.attr(el, 'title'), typeof el.className === 'string' ? el.className : ''].join(' ').toLowerCase(); + const interactive = D.isInteractive(el); + let score = 0; + let covered = 0; + for (const w of content) { + const inName = name.includes(w); + const inAttr = attrs.includes(w); + if (inName) score += new RegExp(`(^|[^\\p{L}\\p{N}])${w.replace(/[.*+?^${}()|[\]\\-]/g, '\\$&')}([^\\p{L}\\p{N}]|$)`, 'u').test(name) ? 6 : 4; + else if (inAttr) score += 3; + if (inName || inAttr) covered++; + } + if (content.length && covered === 0) continue; + if (phrase && name === phrase) score += 12; + else if (phrase && content.length > 1 && name.includes(phrase)) score += 6; + if (roleHints.size) { + if (roleHints.has(r) || (roleHints.has('searchbox') && /search/.test(attrs) && ['textbox', 'searchbox', 'combobox'].includes(r))) score += 8; + else if (!content.length) continue; + else score -= 2; + } + if (interactive) score += 4; + else if (r === 'generic') score -= 3; + // A container whose text merely CONTAINS the words is a weak match. + if (name.length > 120) score -= 4; + const coverage = content.length ? covered / content.length : 1; + if (coverage < 0.5) continue; + score = Math.round(score * coverage * 10) / 10; + if (score <= 0) continue; + cands.push({ el, r, name, score, interactive }); + } + } + cands.sort((a, b) => b.score - a.score); + // Visibility is the expensive check: only for the best-scoring pool. + const pool = []; + for (const c of cands) { + if (pool.length >= _lim * 6) break; + if (D.isVisible(c.el)) pool.push(c); + } + // Drop wrappers (an ancestor scoring no better than a descendant) and echoes + // (a descendant repeating the name of the control that contains it). + const kept = pool.filter((c) => !pool.some((o) => o !== c && ( + (o.score >= c.score && D.composedContains(c.el, o.el)) + || (o.interactive && !c.interactive && o.score >= c.score && o.name === c.name && D.composedContains(o.el, c.el))))); + + const matches = []; + kept.slice(0, _lim).forEach((c, i) => { + const ref = `${_refPrefix}${i}`; + D.registry.set(ref, c.el); + try { if (genFallback) fallbacks[ref] = genFallback(c.el); } catch {} + const rect = D.centerOf(c.el).rect; + matches.push({ + ref, role: c.r, name: D.nameOf(c.el).slice(0, 80), tag: c.el.tagName.toLowerCase(), score: c.score, + bounds: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, + }); + }); + return { + success: true, query: _q, matches, + ...(matches.length === 0 ? { hint: 'No match. Try fewer/other words, a role filter, browser_snapshot, or browser_text.' } : {}), + __fallbacks: fallbacks, + }; + }, [query, limit, refPrefix, role || null]).then((res) => { + if (res && res.__fallbacks) { + const map = fallbackByTab.get(tabId) || new Map(); + for (const [ref, fbEntry] of Object.entries(res.__fallbacks)) map.set(ref, fbEntry); + fallbackByTab.set(tabId, map); + delete res.__fallbacks; + persistSessionState(); + } + return res; + }); +} diff --git a/extension/handlers/gif.js b/extension/handlers/gif.js new file mode 100644 index 0000000..a376c36 --- /dev/null +++ b/extension/handlers/gif.js @@ -0,0 +1,154 @@ +/** + * browser_gif: record what the agent does in a tab as an animated GIF + * (Claude-in-Chrome gif_creator). While recording, the router captures a + * downscaled frame after every page-changing action; export encodes the + * frames (lib/gif-encoder.js) with a red ring where clicks landed. The MCP + * server writes the file — the GIF bytes never go to the agent. + */ +import { resolveTab } from '../lib/page-exec.js'; +import { encodeGif, drawMarker } from '../lib/gif-encoder.js'; +import { handleScreenshot } from './tabs.js'; + +/** Tools after which a frame is captured. Reads (text/snapshot/find…) don't change the page. */ +export const GIF_FRAME_TOOLS = new Set([ + 'browser_navigate', 'browser_click', 'browser_type', 'browser_press_key', 'browser_scroll', 'browser_hover', + 'browser_select', 'browser_click_text', 'browser_drag', 'browser_fill_form', 'browser_act', 'browser_upload_file', + 'browser_handle_dialog', 'browser_run_action', 'browser_evaluate', 'browser_wait', +]); + +const MAX_FRAMES_CAP = 500; +/** Export travels in parts: the daemon's WebSocket frames are capped at 1 MB. */ +export const GIF_PART_BYTES = 600_000; +/** tabId -> { frames, width, maxFrames, activate, recording, skipped, startedAt } */ +const recordings = new Map(); + +export function isRecording(tabId) { + const r = recordings.get(tabId); + return !!(r && r.recording); +} + +/** Capture one frame (after an action). Never throws: a failed frame is just skipped. */ +export async function recordFrame(tabId, label, result) { + const rec = recordings.get(tabId); + if (!rec || !rec.recording) return; + if (rec.frames.length >= rec.maxFrames) { rec.skipped++; return; } + try { + const tab = await chrome.tabs.get(tabId); + // Hidden tabs don't paint; activating one flashes it for ~150 ms (like browser_screenshot). + if (!tab.active && !rec.activate) { rec.skipped++; return; } + const shot = await handleScreenshot({ tabId, format: 'jpeg', quality: 70, maxWidth: rec.width }); + if (!shot?.data) { rec.skipped++; return; } + const f = shot.frame; + const at = result && result.at && f && !f.page + ? [(result.at.x - f.origin[0]) * f.scale, (result.at.y - f.origin[1]) * f.scale] + : null; + rec.frames.push({ data: shot.data, t: Date.now(), label, ...(at ? { at } : {}) }); + } catch { + rec.skipped++; + } +} + +function b64ToBytes(b64) { + const bin = atob(b64); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} + +function bytesToB64(bytes) { + let s = ''; + for (let i = 0; i < bytes.length; i += 0x8000) s += String.fromCharCode.apply(null, bytes.subarray(i, i + 0x8000)); + return btoa(s); +} + +async function encodeRecording(rec) { + const decoded = []; + let W = 0; + let H = 0; + for (const fr of rec.frames) { + const bmp = await createImageBitmap(new Blob([b64ToBytes(fr.data)], { type: 'image/jpeg' })); + if (!W) { W = bmp.width; H = bmp.height; } + const canvas = new OffscreenCanvas(W, H); + const ctx = canvas.getContext('2d'); + ctx.drawImage(bmp, 0, 0, W, H); + bmp.close?.(); + const rgba = ctx.getImageData(0, 0, W, H).data; + if (fr.at) drawMarker(rgba, W, H, fr.at[0] * (W / (bmp.width || W)), fr.at[1] * (H / (bmp.height || H))); + decoded.push({ rgba, t: fr.t }); + } + const frames = decoded.map((d, i) => ({ + rgba: d.rgba, + // Real pacing, clamped so the GIF is watchable: 0.4 s … 2.5 s, 1.5 s on the last frame. + delayMs: i + 1 < decoded.length ? Math.min(2500, Math.max(400, decoded[i + 1].t - d.t)) : 1500, + })); + return { bytes: encodeGif(W, H, frames), width: W, height: H }; +} + +export async function handleGif(params) { + const { tabId, action } = params; + await resolveTab(tabId); + switch (action) { + case 'start': { + const rec = { + frames: [], + width: Math.min(Math.max(Number(params.width) || 800, 200), 1600), + maxFrames: Math.min(Math.max(Number(params.maxFrames) || 300, 1), MAX_FRAMES_CAP), + activate: params.activate !== false, + recording: true, + skipped: 0, + startedAt: Date.now(), + }; + recordings.set(tabId, rec); + await recordFrame(tabId, 'start', null); // the first frame: how the page looked + return { success: true, recording: true, frames: rec.frames.length, width: rec.width, maxFrames: rec.maxFrames }; + } + case 'frame': { + const rec = recordings.get(tabId); + if (!rec) throw new Error('Not recording this tab — call browser_gif action:"start" first.'); + const was = rec.recording; + rec.recording = true; + await recordFrame(tabId, 'frame', null); + rec.recording = was; + return { success: true, frames: rec.frames.length }; + } + case 'stop': { + const rec = recordings.get(tabId); + if (!rec) throw new Error('Not recording this tab.'); + rec.recording = false; + return { success: true, recording: false, frames: rec.frames.length, skipped: rec.skipped, seconds: Math.round((Date.now() - rec.startedAt) / 1000) }; + } + case 'status': { + const rec = recordings.get(tabId); + return rec + ? { success: true, recording: rec.recording, frames: rec.frames.length, skipped: rec.skipped } + : { success: true, recording: false, frames: 0 }; + } + case 'clear': { + recordings.delete(tabId); + return { success: true, cleared: true }; + } + case 'export': { + const rec = recordings.get(tabId); + if (!rec || (rec.frames.length === 0 && !rec.encoded)) throw new Error('No frames recorded for this tab.'); + rec.recording = false; + // Encode once (part 0), then hand the bytes out part by part. + const part = Number.isInteger(params.part) && params.part > 0 ? params.part : 0; + if (part === 0 || !rec.encoded) rec.encoded = await encodeRecording(rec); + const { bytes, width, height } = rec.encoded; + const parts = Math.max(1, Math.ceil(bytes.length / GIF_PART_BYTES)); + if (part >= parts) throw new Error(`part ${part} out of range (${parts} parts)`); + const chunk = bytes.subarray(part * GIF_PART_BYTES, (part + 1) * GIF_PART_BYTES); + const frames = rec.frames.length; + if (part === parts - 1) { + rec.encoded = null; + if (params.clear !== false) recordings.delete(tabId); + } + return { success: true, frames, width, height, bytes: bytes.length, part, parts, gifBase64: bytesToB64(chunk) }; + } + default: + throw new Error(`Unknown action: ${action}`); + } +} + +/** Test hook. */ +export function _recordings() { return recordings; } diff --git a/extension/handlers/inspection.js b/extension/handlers/inspection.js index 3bea899..f189c06 100644 --- a/extension/handlers/inspection.js +++ b/extension/handlers/inspection.js @@ -2,15 +2,18 @@ * Inspection handlers (extracted from background.js): wait, scroll, snapshot, * find, text, evaluate — the read side of the toolset. */ -import { safeExec, resolveTab, getFallback } from '../lib/page-exec.js'; -import { fallbackByTab, lastSnapshotFingerprints, MAX_RESULT_CHARS, persistSessionState } from '../lib/state.js'; +import { safeExec, execDom, resolveTab, getFallback, assertResponsive, hasPoint } from '../lib/page-exec.js'; +import { trustedSender, pointInfo, releaseShield } from '../lib/trusted-input.js'; +import { fallbackByTab, lastSnapshotFingerprints, MAX_RESULT_CHARS, persistSessionState, nextRefPrefix } from '../lib/state.js'; import { PAGE_FALLBACK_INSTALL } from '../utils/smart-selector.js'; -import { PAGE_LEGACY_REF_INSTALL } from '../utils/legacy-refs.js'; import { withCdp } from '../lib/cdp-session.js'; import { cdpEvaluate } from '../lib/cdp-evaluate.js'; +/** Default output cap for snapshots (chars of serialized tree). */ +export const SNAPSHOT_MAX_CHARS = 20_000; + export async function handleWait(params, _sessionId, _agentName, signal) { - const { tabId, selector, state = 'visible', timeout = 10000, delay } = params; + const { tabId, selector, state = 'visible', timeout = 10000, delay, text, urlIncludes } = params; // A promise that rejects when this call is cancelled (client gone / bridge // timeout forwarded). Long waits race against it so a cancelled call releases @@ -22,6 +25,7 @@ export async function handleWait(params, _sessionId, _agentName, signal) { }) : null; + const hasCondition = !!selector || text != null || !!urlIncludes; if (delay) { const sleep = new Promise((r) => setTimeout(r, Math.min(delay, 30000))); try { @@ -29,55 +33,94 @@ export async function handleWait(params, _sessionId, _agentName, signal) { } catch { return { success: false, error: 'aborted', waited: 0 }; } - return { success: true, waited: delay }; + return { success: true, waited: delay }; // documented: a delay ignores the conditions } - if (!selector) return { success: false, error: 'Need selector or delay' }; + if (!hasCondition) return { success: false, error: 'Need selector, text, urlIncludes or delay' }; await resolveTab(tabId); const start = Date.now(); + const what = selector || (text != null ? `text "${text}"` : `url containing "${urlIncludes}"`); while (Date.now() - start < timeout) { // Bail the moment the caller is gone so we don't pin the tab mutex for the // full timeout window after the originating agent was evicted (consistent // with handleNavigate / handleRunAction). if (signal?.aborted) return { success: false, error: 'aborted', selector, state }; - const found = await safeExec(tabId, (_sel, _state) => { - const el = document.querySelector(_sel); - if (_state === 'hidden') return !el || el.offsetParent === null; - if (_state === 'attached') return !!el; - return el && el.offsetParent !== null; - }, [selector, state]); - - if (found) return { success: true, selector, state, elapsed: Date.now() - start }; + let found; + try { + found = await execDom(tabId, (_sel, _state, _text, _url) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const hidden = _state === 'hidden'; + if (_url != null && !location.href.includes(_url)) return false; + if (_text != null) { + const has = D.pageText(document.body).toLowerCase().includes(String(_text).toLowerCase()); + if (hidden ? has : !has) return false; + } + if (_sel) { + // Every match across shadow roots / same-origin frames, not just the first. + const all = D.queryAll(_sel, true); + if (all === null) return { error: `Invalid CSS selector: ${_sel}` }; + if (_state === 'attached') return all.length > 0; + const anyVisible = all.some((el) => D.isVisible(el)); + return hidden ? !anyVisible : anyVisible; + } + return true; + }, [selector ?? null, state, text ?? null, urlIncludes ?? null]); + } catch { found = false; /* navigating: the next document isn't ready yet */ } + if (found && found.error) return { success: false, error: found.error }; + + if (found === true) { + return { + success: true, + ...(selector ? { selector } : {}), + ...(text != null ? { text } : {}), + ...(urlIncludes ? { urlIncludes } : {}), + state, + elapsed: Date.now() - start, + }; + } await new Promise((r) => setTimeout(r, 200)); } - return { success: false, error: `Timeout waiting for ${selector} to be ${state}` }; + return { success: false, error: `Timeout waiting for ${what} to be ${state}` }; } export async function handleScroll(params) { const { tabId, direction = 'down', amount = 500, selector, toElement, position } = params; await resolveTab(tabId); + // x/y: a real mouse-wheel event at that point — scrolls whatever is under + // it (inner panels, maps, virtual lists) exactly like a user's wheel. + if (hasPoint(params) && !toElement && !position && !selector) { + const send = await trustedSender(tabId, true); + if (!send) throw new Error(`Scrolling at x/y needs the debugger (CDP), which could not attach to tab ${tabId}. Use selector/toElement instead.`); + const deltaX = direction === 'right' ? amount : direction === 'left' ? -amount : 0; + const deltaY = direction === 'down' ? amount : direction === 'up' ? -amount : 0; + const info = await pointInfo(tabId, params.x, params.y); + try { + await send('Input.dispatchMouseEvent', { type: 'mouseWheel', x: params.x, y: params.y, deltaX, deltaY }); + } finally { + await releaseShield(tabId); + } + return { success: true, input: 'cdp', at: { x: params.x, y: params.y }, deltaX, deltaY, refsMayBeStale: true, ...(info.hit ? { over: info.hit } : {}) }; + } const fb = getFallback(tabId, toElement); - if (fb) await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); - await safeExec(tabId, PAGE_LEGACY_REF_INSTALL, []); - return safeExec(tabId, (_dir, _amt, _sel, _toEl, _pos, _fb) => { + return execDom(tabId, (_dir, _amt, _sel, _toEl, _pos, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; if (_toEl) { - const resolveFallback = (globalThis.__browserControllerFallbackRuntime || {}).resolveFallback || null; - const resolveRef = (globalThis.__browserControllerLegacyRefRuntime || {}).resolveRef || null; - const el = (resolveRef ? resolveRef(_toEl) : null) || - document.querySelector(`[data-mcp-ref="${_toEl}"]`) || - document.querySelector(_toEl) || - (_fb && resolveFallback ? resolveFallback(_fb) : null); + // toElement accepts a ref or a CSS selector (first visible match). + let el = D.resolve(_toEl, null, _fb).el; + if (!el) { try { el = D.resolve(null, _toEl, null).el; } catch { el = null; } } if (el) { - el.scrollIntoView({ behavior: 'smooth', block: 'center' }); + el.scrollIntoView({ behavior: 'instant', block: 'center' }); return { success: true, scrolledTo: 'element' }; } return { success: false, error: 'Element not found' }; } - const target = _sel ? document.querySelector(_sel) : window; + const target = _sel ? D.resolve(null, _sel, null).el : window; if (!target) return { success: false, error: 'Scroll container not found' }; if (_pos === 'top') { @@ -118,7 +161,9 @@ export async function handleScroll(params) { * DOM with permanent data-mcp-ref attributes. */ export async function handleSnapshot(params) { - const { tabId, selector, compact = true } = params; + const { tabId, selector, ref: rootRef, depth, maxChars = SNAPSHOT_MAX_CHARS } = params; + // filter:"interactive"|"all" (Claude-in-Chrome naming) is an alias of compact. + const compact = params.filter === 'all' ? false : params.filter === 'interactive' ? true : params.compact !== false; await resolveTab(tabId); // Install the fallback page runtime first (v2 install-once pattern): the @@ -127,196 +172,185 @@ export async function handleSnapshot(params) { // extension CSP (script-src 'self', no unsafe-eval) throws in every // isolated world, which silently killed fallback capture before this fix. await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); - await safeExec(tabId, PAGE_LEGACY_REF_INSTALL, []); // isNew feature: pass the fingerprints seen in the PREVIOUS snapshot so the // page function can mark newly-appeared elements. Array is serializable. - const prevFingerprints = lastSnapshotFingerprints.get(tabId) || []; - const refPrefix = `e-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}-`; + const prevFingerprints = lastSnapshotFingerprints.get(tabId) || null; + const refPrefix = nextRefPrefix('s'); - return safeExec(tabId, (_sel, _compact, _prevFingerprints, _refPrefix) => { + return execDom(tabId, (_sel, _compact, _prevFingerprints, _refPrefix, _rootRef, _depth, _maxChars) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; let refCount = 0; /** @type {Record} ref -> fallback, returned to background */ const fallbacks = {}; /** @type {string[]} fingerprints of THIS snapshot (role|name), returned to background */ const fingerprints = []; - const prevSet = new Set(_prevFingerprints); + // No previous snapshot → nothing is "new" (marking every node wasted tokens). + const prevSet = _prevFingerprints ? new Set(_prevFingerprints) : null; // Descriptor generator comes from the pre-installed page runtime. const genFallback = (globalThis.__browserControllerFallbackRuntime || {}).generateFallback || null; - const registerRef = (globalThis.__browserControllerLegacyRefRuntime || {}).registerRef || null; const skipTags = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'SVG', 'PATH', 'BR', 'HR', 'WBR', 'META', 'LINK']); + const maxDepth = Number.isInteger(_depth) && _depth >= 0 ? _depth : Infinity; + // Output budget: stop emitting nodes once the serialized size reaches it. + let budget = Number.isInteger(_maxChars) && _maxChars > 0 ? _maxChars : Infinity; + let truncated = false; + // 'show' = render normally, 'pass' = no box of its own (display:contents, + // slots) but its children may render, false = hidden subtree. function vis(el) { - const s = getComputedStyle(el); - if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) === 0) return false; + const s = D.styleOf(el); + if (!s || s.display === 'none') return false; + if (s.display === 'contents' || el.tagName === 'SLOT') return 'pass'; + if (s.visibility === 'hidden' || s.visibility === 'collapse' || parseFloat(s.opacity) === 0) { + // visibility is inherited but can be re-enabled below; keep walking. + return 'pass'; + } const r = el.getBoundingClientRect(); - return r.width > 0 && r.height > 0; + if (r.width > 0 && r.height > 0) return 'show'; + // Zero-size wrappers (custom-element hosts, overflow containers) can still hold visible children. + return el.childElementCount > 0 || D.shadowOf(el) ? 'pass' : false; } - function role(el) { - const r = el.getAttribute('role'); - if (r) return r; - const map = { - A: 'link', BUTTON: 'button', SELECT: 'combobox', TEXTAREA: 'textbox', IMG: 'img', - H1: 'heading', H2: 'heading', H3: 'heading', H4: 'heading', H5: 'heading', H6: 'heading', - NAV: 'navigation', MAIN: 'main', HEADER: 'banner', FOOTER: 'contentinfo', FORM: 'form', - TABLE: 'table', UL: 'list', OL: 'list', LI: 'listitem', - }; - if (el.tagName === 'INPUT') { - const t = el.type?.toLowerCase(); - if (t === 'checkbox') return 'checkbox'; - if (t === 'radio') return 'radio'; - return 'textbox'; - } - return map[el.tagName] || 'generic'; - } + const role = (el) => D.roleOf(el); + // Landmarks/regions are named only by an explicit label: their text is just + // their children's names again (token noise). + const elName = (el, r) => (landmarkRoles.has(r) && r !== 'dialog' + ? D.clean(D.attr(el, 'aria-label') || D.attr(el, 'title')) + : D.nameOf(el)).slice(0, 80); + const isInteractive = (el) => D.isInteractive(el); - function elName(el) { - const raw = ( - el.getAttribute('aria-label') || el.getAttribute('alt') || - el.getAttribute('title') || el.getAttribute('placeholder') || - '' - ).trim(); - if (raw) return raw.slice(0, 80); - const text = el.innerText; - if (!text) return ''; - const first = text.split('\n')[0].trim(); - return first.slice(0, 80); - } + const landmarkRoles = new Set(['navigation', 'main', 'banner', 'contentinfo', 'form', 'search', 'complementary', 'region', 'dialog']); - function isInteractive(el) { - const tags = ['A', 'BUTTON', 'INPUT', 'SELECT', 'TEXTAREA']; - return tags.includes(el.tagName) || el.onclick || el.getAttribute('tabindex') !== null || - el.getAttribute('role') === 'button' || el.getAttribute('role') === 'link' || - el.getAttribute('role') === 'tab' || el.getAttribute('role') === 'menuitem' || - el.getAttribute('role') === 'option' || el.getAttribute('role') === 'switch' || - el.getAttribute('contenteditable') === 'true'; + // Flat-tree children: open AND closed shadow roots, slotted content, + // same-origin iframe bodies (lib/page-dom.js flatChildren). + function childrenOf(el) { + return D.flatChildren(el).filter((c) => c.nodeType === 1); } - const landmarkRoles = new Set(['navigation', 'main', 'banner', 'contentinfo', 'form', 'search', 'complementary', 'region']); + const origin = location.origin; + function hrefOf(el) { + const h = el.href; + if (!h || typeof h !== 'string') return null; + if (h.startsWith(origin + '/')) return h.slice(origin.length); // same-origin: path only + return h; + } - // Children including shadow DOM (open roots) and same-origin iframes. - function childrenOf(el) { - const out = []; - for (const c of el.children) out.push(c); - if (el.shadowRoot) { - for (const c of el.shadowRoot.children) out.push(c); - } - // same-origin iframes: expose their document body children too. - if (el.tagName === 'IFRAME') { - try { - const doc = el.contentDocument; - if (doc && doc.body) for (const c of doc.body.children) out.push(c); - } catch { /* cross-origin: skip */ } - } - return out; + function emit(el, r, n, extra, isNewCheck) { + const ref = `${_refPrefix}${refCount++}`; + D.registry.set(ref, el); + try { if (genFallback) fallbacks[ref] = genFallback(el); } catch {} + const fp = `${r}|${n}`; + fingerprints.push(fp); + const node = { ref, role: r, ...extra }; + if (n) node.name = n; + if (isNewCheck && prevSet && !prevSet.has(fp)) node.isNew = true; + if (el.value !== undefined && el.value !== '' && typeof el.value !== 'object') node.value = String(el.value).slice(0, 200); + if (el.tagName === 'INPUT' && (el.type === 'checkbox' || el.type === 'radio')) node.checked = el.checked; + else if (D.attr(el, 'aria-checked')) node.checked = D.attr(el, 'aria-checked') === 'true'; + if (D.attr(el, 'aria-expanded')) node.expanded = D.attr(el, 'aria-expanded') === 'true'; + if (D.attr(el, 'aria-selected') === 'true') node.selected = true; + if (el.disabled) node.disabled = true; + if (el.tagName === 'A') { const h = hrefOf(el); if (h) node.href = h; } + budget -= JSON.stringify(node).length + 16; + return node; } - function buildCompact(el) { + function buildCompact(el, d) { if (!el || el.nodeType !== 1) return null; if (skipTags.has(el.tagName)) return null; - if (!vis(el)) return null; + if (budget <= 0) { truncated = true; return null; } + const v = vis(el); + if (!v) return null; - const ia = isInteractive(el); + const ia = v === 'show' && isInteractive(el); const r = role(el); - const isLandmark = landmarkRoles.has(r); + const isLandmark = v === 'show' && (landmarkRoles.has(r) || (r === 'heading')); + const own = ia || isLandmark; - const kids = []; - for (const c of childrenOf(el)) { - const cn = buildCompact(c); - if (cn) Array.isArray(cn) ? kids.push(...cn) : kids.push(cn); + let node = null; + if (own) { + if (d > maxDepth) { truncated = true; return null; } + node = emit(el, r, elName(el, r), {}, true); } + const kids = []; + if (!(own && d >= maxDepth)) { + for (const c of childrenOf(el)) { + const cn = buildCompact(c, own ? d + 1 : d); + if (cn) Array.isArray(cn) ? kids.push(...cn) : kids.push(cn); + } + } else if (childrenOf(el).length) truncated = true; - if (!ia && !isLandmark && r !== 'heading') { - return kids.length === 0 ? null : kids.length === 1 ? kids[0] : kids; - } - - const ref = `${_refPrefix}${refCount++}`; - if (registerRef) registerRef(ref, el); - const n = elName(el); - try { if (genFallback) fallbacks[ref] = genFallback(el); } catch {} - - // isNew: mark elements whose (role|name) wasn't in the previous snapshot. - const fp = `${r}|${n}`; - fingerprints.push(fp); - const isNew = !prevSet.has(fp); - - const node = { ref, role: r }; - if (n) node.name = n; - if (isNew) node.isNew = true; - if (el.value !== undefined && el.value !== '') node.value = String(el.value); - if (el.checked !== undefined) node.checked = el.checked; - if (el.disabled) node.disabled = true; - if (el.href && el.tagName === 'A') node.href = el.href; + if (!own) return kids.length === 0 ? null : kids.length === 1 ? kids[0] : kids; if (kids.length) node.children = kids; - return node; } - function buildFull(el, depth) { + function buildFull(el, d) { if (!el || el.nodeType !== 1) return null; if (skipTags.has(el.tagName)) return null; - if (!vis(el)) return null; + if (budget <= 0) { truncated = true; return null; } + const v = vis(el); + if (!v) return null; const r = role(el); - const n = elName(el); - const ia = isInteractive(el); + const ia = v === 'show' && isInteractive(el); + const n = v === 'show' ? elName(el, r) : ''; - if (r === 'generic' && !n && !ia && depth > 1) { + if (v !== 'show' || (r === 'generic' && !n && !ia && d > 1)) { const kids = []; for (const c of childrenOf(el)) { - const cn = buildFull(c, depth + 1); + const cn = buildFull(c, d + (v === 'show' ? 1 : 0)); if (cn) Array.isArray(cn) ? kids.push(...cn) : kids.push(cn); } return kids.length === 0 ? null : kids.length === 1 ? kids[0] : kids; } + if (d > maxDepth) { truncated = true; return null; } - const ref = `${_refPrefix}${refCount++}`; - if (registerRef) registerRef(ref, el); - try { if (genFallback) fallbacks[ref] = genFallback(el); } catch {} - - // isNew: mark elements whose (role|name) wasn't in the previous snapshot. - const fp = `${r}|${n}`; - fingerprints.push(fp); - const isNew = !prevSet.has(fp); - - const node = { ref, role: r }; - if (r === 'generic') node.tag = el.tagName.toLowerCase(); - if (n) node.name = n; - if (isNew) node.isNew = true; - if (el.value !== undefined && el.value !== '') node.value = String(el.value); - if (el.checked !== undefined) node.checked = el.checked; - if (el.disabled) node.disabled = true; - if (el.href && el.tagName === 'A') node.href = el.href; - + const node = emit(el, r, n, r === 'generic' ? { tag: el.tagName.toLowerCase() } : {}, true); const kids = []; for (const c of childrenOf(el)) { - const cn = buildFull(c, depth + 1); + const cn = buildFull(c, d + 1); if (cn) Array.isArray(cn) ? kids.push(...cn) : kids.push(cn); } if (kids.length) node.children = kids; - return node; } - const root = _sel ? document.querySelector(_sel) : document.body; + let root = document.body; + if (_rootRef) { + root = D.registry.get(_rootRef); + if (!D.connected(root)) return { success: false, error: `ref ${_rootRef} is gone — take a new snapshot` }; + } else if (_sel) { + const hit = D.resolve(null, _sel, null); + if (hit.error === 'INVALID_SELECTOR') return { success: false, error: `Invalid CSS selector: ${_sel}` }; + root = hit.el; + } if (!root) return { success: false, error: 'Root element not found' }; - const tree = _compact ? buildCompact(root) : buildFull(root, 0); + const tree = _compact ? buildCompact(root, 0) : buildFull(root, 0); return { success: true, url: location.href, title: document.title, compact: _compact, tree, + ...(truncated ? { + truncated: true, + hint: 'Output capped (maxChars/depth). Scope it with selector or ref (a subtree), or raise maxChars.', + } : {}), // internal: background stores these per-tab; never sent to the agent. __fallbacks: fallbacks, __fingerprints: fingerprints, }; - }, [selector, compact, prevFingerprints, refPrefix]).then((res) => { + }, [selector ?? null, compact, prevFingerprints, refPrefix, rootRef ?? null, depth ?? null, maxChars]).then((res) => { // Store the fallbacks per-tab so click/type can resolve stale refs, and - // persist them across service-worker recycles (MV3 lifetime). + // persist them across service-worker recycles (MV3 lifetime). Merged, not + // replaced: a scoped snapshot must not invalidate refs from the full one. if (res && res.__fallbacks) { - const map = new Map(Object.entries(res.__fallbacks)); + const map = fallbackByTab.get(tabId) || new Map(); + for (const [ref, fbEntry] of Object.entries(res.__fallbacks)) map.set(ref, fbEntry); + // Bound the map: keep the most recent entries. + while (map.size > 3000) map.delete(map.keys().next().value); fallbackByTab.set(tabId, map); delete res.__fallbacks; // keep it out of the agent-visible payload persistSessionState(); @@ -330,104 +364,49 @@ export async function handleSnapshot(params) { }); } -export async function handleFind(params) { - const { tabId, query, limit = 10 } = params; - await resolveTab(tabId); - await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); - await safeExec(tabId, PAGE_LEGACY_REF_INSTALL, []); - const refPrefix = `f-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}-`; - - return safeExec(tabId, (_q, _lim, _refPrefix) => { - const qLow = _q.toLowerCase(); - const matches = []; - const fallbacks = {}; - const genFallback = (globalThis.__browserControllerFallbackRuntime || {}).generateFallback || null; - const registerRef = (globalThis.__browserControllerLegacyRefRuntime || {}).registerRef || null; - - function aName(el) { - return (el.getAttribute('aria-label') || el.getAttribute('alt') || el.getAttribute('title') || - el.getAttribute('placeholder') || el.innerText?.slice(0, 200) || '').trim(); - } - - function aRole(el) { - const r = el.getAttribute('role'); - if (r) return r; - const map = { A: 'link', BUTTON: 'button', INPUT: 'input', SELECT: 'combobox', TEXTAREA: 'textbox', IMG: 'image' }; - return map[el.tagName] || el.tagName.toLowerCase(); - } - - // Same-origin iframe piercing (field report: legacy UIs live entirely - // inside #mainFrame — the top-document walk saw none of it). - const roots = [document.body]; - (function collectFrames(doc, depth) { - if (depth >= 3) return; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d && d.body) { roots.push(d.body); collectFrames(d, depth + 1); } } catch {} - } - })(document, 0); - let rc = 0; - let node; - for (const root of roots) { - const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT); - while ((node = walker.nextNode()) && matches.length < _lim * 3) { - const s = getComputedStyle(node); - const rect = node.getBoundingClientRect(); - if (s.display === 'none' || s.visibility === 'hidden' || rect.width === 0) continue; - - const n = aName(node).toLowerCase(); - const r = aRole(node).toLowerCase(); - const id = (node.id || '').toLowerCase(); - let score = 0; - if (n.includes(qLow)) score += 10; - if (r.includes(qLow)) score += 5; - if (id.includes(qLow)) score += 3; - if (score === 0) continue; - - const ref = `${_refPrefix}${rc++}`; - if (registerRef) registerRef(ref, node); - try { if (genFallback) fallbacks[ref] = genFallback(node); } catch {} - matches.push({ - ref, role: r, name: n.slice(0, 100), tag: node.tagName.toLowerCase(), score, - bounds: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, - }); - } - } - - matches.sort((a, b) => b.score - a.score); - return { success: true, query: _q, matches: matches.slice(0, _lim), __fallbacks: fallbacks }; - }, [query, limit, refPrefix]).then((res) => { - if (res && res.__fallbacks) { - const map = fallbackByTab.get(tabId) || new Map(); - for (const [ref, fbEntry] of Object.entries(res.__fallbacks)) map.set(ref, fbEntry); - fallbackByTab.set(tabId, map); - delete res.__fallbacks; - persistSessionState(); - } - return res; - }); -} - export async function handleGetPageText(params) { // Default must match the MCP schema (text.ts: maxLength .default(5000)) — // it drifted 10x here once, so direct-WS callers got 50000 while MCP callers // got 5000 from the same knob. - const { tabId, selector, maxLength = 5000 } = params; + const { tabId, selector, maxLength = 5000, mode = 'all', offset = 0 } = params; await resolveTab(tabId); - const args = selector === undefined ? [null, maxLength] : [selector, maxLength]; - - return safeExec(tabId, (_sel, _max) => { - const root = _sel ? document.querySelector(_sel) : document.body; + const max = Math.min(Math.max(1, Number(maxLength) || 5000), 100_000); + const from = Math.max(0, Number(offset) || 0); + + return execDom(tabId, (_sel, _max, _mode, _from) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const article = _mode === 'article'; + let root = document.body; + if (_sel) { + const hit = D.resolve(null, _sel, null); + if (hit.error === 'INVALID_SELECTOR') return { success: false, error: `Invalid CSS selector: ${_sel}` }; + root = hit.el; + } else if (article) { + root = D.articleRoot(); + } if (!root) return { success: false, error: 'Element not found' }; - let text = root.innerText || root.textContent || ''; - text = text.replace(/\t/g, ' ').replace(/\n\s*\n/g, '\n\n').replace(/ +/g, ' ').trim(); + // Composed text: includes open/closed shadow roots and same-origin frames + // (innerText alone misses web-component content such as caniuse's tables). + let text = D.pageText(root, { article, max: _from + _max + 1000 }); + const total = text.length; + if (_from) text = text.slice(_from); const truncated = text.length > _max; if (truncated) text = text.slice(0, _max) + '...'; - return { success: true, url: location.href, title: document.title, text, length: text.length, truncated }; - }, args); + return { + success: true, url: location.href, title: document.title, text, length: text.length, truncated, + ...(_from ? { offset: _from } : {}), + ...(truncated ? { nextOffset: _from + _max } : {}), + ...(article ? { mode: 'article' } : {}), + ...(total && _from >= total ? { note: `offset ${_from} is past the end (${total} chars)` } : {}), + }; + }, [selector ?? null, max, mode, from]); } +export { handleFind } from './find.js'; + /** * evaluate (task 1.5): runs in the page's MAIN world via chrome.scripting — no * chrome.debugger, so no yellow "is being debugged" banner. Replaces the old @@ -444,6 +423,7 @@ export async function handleEvaluate(params, _sessionId, _agentName, signal) { // Default: REPL semantics over CDP (top-level await, last expression is the // result, not blocked by CSP). mode:"scripting" (or no debugger available) // keeps the banner-free chrome.scripting path below. + await assertResponsive(tabId); if (mode !== 'scripting') { let attached = false; try { diff --git a/extension/handlers/interaction-advanced.js b/extension/handlers/interaction-advanced.js index 6fe73ef..61c94fa 100644 --- a/extension/handlers/interaction-advanced.js +++ b/extension/handlers/interaction-advanced.js @@ -3,7 +3,7 @@ * orchestration. Kept separate from the common pointer/keyboard handlers so * each module stays focused and reviewable. */ -import { resolveTab, safeExec } from '../lib/page-exec.js'; +import { resolveTab, safeExec, execDom, getFallback } from '../lib/page-exec.js'; import { withCdp } from '../lib/cdp-session.js'; import { openShield, releaseShield } from '../lib/trusted-input.js'; @@ -64,32 +64,19 @@ export async function handleDrag(params) { let sx = startX, sy = startY, ex = endX, ey = endY; if (sx == null || sy == null || ex == null || ey == null) { - const coords = await safeExec(tabId, (_sRef, _sSel, _eRef, _eSel) => { - function deepQuery(sel) { - const query = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const frame of doc.querySelectorAll('iframe')) { - try { - const child = frame.contentDocument; - if (child) { const el = query(child, depth + 1); if (el) return el; } - } catch {} - } - return null; - }; - return query(document, 0); - } - - function find(ref, selector) { - let el = ref ? deepQuery(`[data-mcp-ref="${ref}"]`) : null; - if (!el && selector) el = deepQuery(selector); + const coords = await execDom(tabId, (_sRef, _sSel, _eRef, _eSel, _sFb, _eFb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + function find(ref, selector, fb) { + if (!ref && !selector) return null; + const el = D.resolve(ref, selector, fb).el; if (!el) return null; el.scrollIntoView({ behavior: 'instant', block: 'center' }); - const rect = el.getBoundingClientRect(); - return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }; + const { x, y } = D.centerOf(el); + return { x, y }; } - return { start: find(_sRef, _sSel), end: find(_eRef, _eSel) }; - }, [startRef, startSelector, endRef, endSelector]); + return { start: find(_sRef, _sSel, _sFb), end: find(_eRef, _eSel, _eFb) }; + }, [startRef, startSelector, endRef, endSelector, getFallback(tabId, startRef), getFallback(tabId, endRef)]); if (coords.start) { sx = coords.start.x; sy = coords.start.y; } if (coords.end) { ex = coords.end.x; ey = coords.end.y; } @@ -127,21 +114,12 @@ export async function handleFillForm(params) { } await resolveTab(tabId); - return safeExec(tabId, (_fields, _submit) => { - function deepQuery(sel) { - const query = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const frame of doc.querySelectorAll('iframe')) { - try { - const child = frame.contentDocument; - if (child) { const el = query(child, depth + 1); if (el) return el; } - } catch {} - } - return null; - }; - return query(document, 0); - } + // Attach each ref's snapshot descriptor so stale refs re-resolve (verified) in the page. + const withFb = fields.map((f) => (f && f.ref ? { ...f, fb: getFallback(tabId, f.ref) } : f)); + + return execDom(tabId, (_fields, _submit) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; const setNativeValue = (target, nextValue) => { const prototype = target instanceof HTMLTextAreaElement @@ -154,9 +132,8 @@ export async function handleFillForm(params) { const results = []; let containingForm = null; for (const field of _fields) { - const { ref, selector, value, clear } = field; - let el = ref ? deepQuery(`[data-mcp-ref="${ref}"]`) : null; - if (!el && selector) el = deepQuery(selector); + const { ref, selector, value, clear, fb } = field; + const el = D.resolve(ref, selector, fb).el; if (!el) { results.push({ selector: selector || ref, success: false, error: 'Not found' }); continue; @@ -164,20 +141,27 @@ export async function handleFillForm(params) { el.focus(); if (el.form && !containingForm) containingForm = el.form; - if (clear !== false) { + const isChoice = el.tagName === 'SELECT' || el.type === 'checkbox' || el.type === 'radio'; + if (clear !== false && !isChoice) { if (el.isContentEditable) el.textContent = ''; else setNativeValue(el, ''); el.dispatchEvent(new Event('input', { bubbles: true })); } if (el.tagName === 'SELECT') { - const option = Array.from(el.options).find((candidate) => candidate.value === String(value)); + // Match the option's value first, then its visible label. + const want = String(value); + const options = Array.from(el.options); + const option = options.find((candidate) => candidate.value === want) + || options.find((candidate) => candidate.textContent.trim() === want.trim()) + || options.find((candidate) => candidate.textContent.trim().toLowerCase() === want.trim().toLowerCase()); if (!option) { results.push({ selector: selector || ref, success: false, error: `Option "${value}" not found` }); continue; } - setNativeValue(el, String(value)); - el.dispatchEvent(new Event('change', { bubbles: true })); + const setter = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value')?.set; + if (setter) setter.call(el, option.value); else el.value = option.value; + el.dispatchEvent(new Event('input', { bubbles: true })); } else if (el.type === 'checkbox' || el.type === 'radio') { const checked = value === true || value === 'true'; if (el.checked !== checked) el.click(); @@ -205,5 +189,5 @@ export async function handleFillForm(params) { return failed === 0 ? { success: true, fields: results } : { success: false, error: `${failed} of ${results.length} fields failed`, fields: results }; - }, [fields, submit]); + }, [withFb, submit]); } diff --git a/extension/handlers/interaction.js b/extension/handlers/interaction.js index af074db..386c6ba 100644 --- a/extension/handlers/interaction.js +++ b/extension/handlers/interaction.js @@ -3,15 +3,19 @@ * hover, select, click_text, dialog, drag, fill_form — the write side that * drives the page's event system (synthetic events) or CDP when required. */ -import { resolveTab, requireTarget, safeExec, getFallback } from '../lib/page-exec.js'; +import { resolveTab, requireTarget, hasPoint, execDom, getFallback } from '../lib/page-exec.js'; import { autoReSnapshot } from './inspection.js'; -import { PAGE_FALLBACK_INSTALL } from '../utils/smart-selector.js'; -import { trustedSender, locateTarget, releaseShield, cdpClickAt, cdpKeyPress, cdpTypeText, keyDefinition } from '../lib/trusted-input.js'; +import { trustedSender, locateTarget, releaseShield, cdpClickAt, cdpKeyPress, cdpTypeText, keyDefinition, modifierBits, pointInfo } from '../lib/trusted-input.js'; export { handleDialog, handleDrag, handleFillForm } from './interaction-advanced.js'; /** Shared REF_GONE recovery: re-snapshot and hand fresh refs back (no auto-retry). */ -async function refGone(tabId, res, ref) { +async function refGone(tabId, res, ref, selector) { + // A selector that matches nothing is usually the wrong page (navigation, + // postback), not a virtualized feed — say which locator failed. + if (!(res._ref || ref) && selector) { + return { success: false, error: `No element matches selector ${selector} on the current page (${res.url || 'navigated?'}).` }; + } const fresh = await autoReSnapshot(tabId); return { success: false, @@ -21,25 +25,63 @@ async function refGone(tabId, res, ref) { } const BUTTONS = new Set(['left', 'right', 'middle']); +const MODS = new Set(['ctrl', 'alt', 'shift', 'meta']); + +/** clickCount from the params (1–3; doubleClick = 2). */ +function clickCountOf(params) { + const n = Number(params.clickCount); + if (Number.isInteger(n) && n >= 1) return Math.min(n, 3); + return params.doubleClick ? 2 : 1; +} + +/** Modifier names held during a click ("ctrl+click" opens links in a new tab). */ +function clickModifiers(params) { + const mods = Array.isArray(params.modifiers) ? params.modifiers.filter((m) => MODS.has(m)) : []; + return modifierBits(mods); +} + +/** Coordinate actions need CDP: there is no element to dispatch synthetic events on. */ +async function requireCdp(tabId, what) { + const send = await trustedSender(tabId, true); + if (!send) throw new Error(`${what} at x/y needs the debugger (CDP), which could not attach to tab ${tabId}. Use ref or selector instead.`); + return send; +} + +/** Real mouse click at viewport coordinates (the same CSS-pixel frame as browser_screenshot). */ +async function clickAtPoint(tabId, params) { + const { x, y, button = 'left' } = params; + if (!BUTTONS.has(button)) throw new Error(`Unknown button ${button}`); + const send = await requireCdp(tabId, 'Clicking'); + const info = await pointInfo(tabId, x, y); + try { + await cdpClickAt(send, x, y, { button, clickCount: clickCountOf(params), modifiers: clickModifiers(params) }); + } finally { + await releaseShield(tabId); + } + return { + success: true, input: 'cdp', at: { x, y }, + ...(info.hit ? { hit: info.hit } : {}), + ...(info.inView === false ? { warning: 'point is outside the viewport' } : {}), + }; +} export async function handleClick(params) { const { tabId, ref, selector, button = 'left', doubleClick = false, trusted } = params; await resolveTab(tabId); - requireTarget(params); + requireTarget(params, { allowPoint: true }); + if (!ref && !selector) return clickAtPoint(tabId, params); + // Snapshot-time descriptor used by the shared resolver when the ref is stale. const fb = getFallback(tabId, ref); - // Install the fallback page runtime only when a descriptor exists (v2 - // install-once pattern — eval rebuilding is impossible under MV3 CSP). - if (fb) await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); // Trusted path: a real mouse click at the element's centre over CDP, so // focus moves, default actions run and the page sees isTrusted:true. const send = await trustedSender(tabId, trusted); if (send && BUTTONS.has(button)) { const loc = await locateTarget(tabId, { ref, selector, fb }); - if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref); + if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref, selector); if (loc?.success && loc.visible) { try { - await cdpClickAt(send, loc.x, loc.y, { button, clickCount: doubleClick ? 2 : 1 }); + await cdpClickAt(send, loc.x, loc.y, { button, clickCount: clickCountOf(params), modifiers: clickModifiers(params) }); } finally { await releaseShield(tabId); } @@ -54,33 +96,19 @@ export async function handleClick(params) { // Zero-size element: no point to hit — fall through to the synthetic path. } - const res = await safeExec(tabId, async (_ref, _sel, _btn, _dbl, _fb) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + const res = await execDom(tabId, async (_ref, _sel, _btn, _dbl, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; - let el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : null; - let via = 'ref'; - if (!el && _sel) { el = deepQuery(_sel); via = 'selector'; } - // Resolver comes from the pre-installed page runtime (no eval). - const resolveFallback = (globalThis.__browserControllerFallbackRuntime || {}).resolveFallback || null; - // Smart-selector fallback (plan task 3): ref broke → try robust selector, - // then text+role+tag scan. The agent doesn't request this; it's automatic. - if (!el && _fb && resolveFallback) { el = resolveFallback(_fb); if (el) via = 'fallback'; } + // ref registry → first visible selector match → verified fallback (lib/page-dom.js). + const found = D.resolve(_ref, _sel, _fb); + if (found.error === 'INVALID_SELECTOR') return { success: false, error: `Invalid CSS selector: ${_sel}` }; + let el = found.el || null; + const via = found.via || 'ref'; if (!el) { // Element is gone (likely virtualized away on scroll). Abort WITHOUT // clicking — the background auto-re-snapshots and embeds fresh refs. - return { success: false, error: 'REF_GONE', _ref }; + return { success: false, error: 'REF_GONE', _ref, url: location.href }; } el.scrollIntoView({ behavior: 'instant', block: 'center' }); @@ -95,7 +123,7 @@ export async function handleClick(params) { if (!visible0) { await new Promise((r) => setTimeout(r, 200)); // re-resolve the element (it may have been re-rendered with a new node) - el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : el; + el = D.resolve(_ref, _sel, _fb).el || el; if (el) el.scrollIntoView({ behavior: 'instant', block: 'center' }); } if (!el) return { success: false, error: 'REF_GONE', _ref }; @@ -133,26 +161,31 @@ export async function handleClick(params) { // Auto-re-snapshot and embed fresh refs so the agent retries in ONE step. // We do NOT auto-retry the click: it's non-idempotent and the element that // re-appears may be a different post after the scroll shifted the feed. - if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref); + if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref, selector); return res; } export async function handleType(params) { const { tabId, ref, selector, text, clear = false, trusted } = params; await resolveTab(tabId); - requireTarget(params); + // No ref/selector: type into the element that has focus (like a user + // typing after clicking a field). + const focusedOnly = !ref && !selector; + // Snapshot-time descriptor used by the shared resolver when the ref is stale. const fb = getFallback(tabId, ref); - // Install the fallback page runtime only when a descriptor exists (v2 - // install-once pattern — eval rebuilding is impossible under MV3 CSP). - if (fb) await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); // Trusted path: focus the field, then real key presses over CDP (keydown / // keypress / input / keyup per character). Like a user, this does NOT fire // `change` until focus leaves the field — press Tab to commit. const send = await trustedSender(tabId, trusted); if (send) { - const loc = await locateTarget(tabId, { ref, selector, fb, mode: clear ? 'clear' : 'focus' }); - if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref); + const mode = focusedOnly ? (clear ? 'focused-clear' : 'focused') : clear ? 'clear' : 'focus'; + const loc = await locateTarget(tabId, { ref, selector, fb, mode }); + if (loc && loc.error === 'NO_FOCUS') { + await releaseShield(tabId); + return { success: false, error: 'No field has focus: pass ref/selector, or click the field first.' }; + } + if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref, selector); if (loc?.success && (loc.focused || loc.visible)) { let after; try { @@ -175,30 +208,25 @@ export async function handleType(params) { await releaseShield(tabId); } - const res = await safeExec(tabId, (_ref, _sel, _text, _clear, _fb) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + const res = await execDom(tabId, (_ref, _sel, _text, _clear, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; - let el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : null; - let via = 'ref'; - if (!el && _sel) { el = deepQuery(_sel); via = 'selector'; } - const resolveFallback = (globalThis.__browserControllerFallbackRuntime || {}).resolveFallback || null; - if (!el && _fb && resolveFallback) { el = resolveFallback(_fb); if (el) via = 'fallback'; } + let found; + if (!_ref && !_sel) { + const a = document.activeElement; + if (!a || a === document.body) return { success: false, error: 'No field has focus: pass ref/selector, or click the field first.' }; + found = { el: a, via: 'active' }; + } else { + found = D.resolve(_ref, _sel, _fb); + } + if (found.error === 'INVALID_SELECTOR') return { success: false, error: `Invalid CSS selector: ${_sel}` }; + const el = found.el || null; + const via = found.via || 'ref'; if (!el) { // Element gone (virtualized feed) — abort WITHOUT typing; background // auto-re-snapshots and embeds fresh refs for a one-step retry. - return { success: false, error: 'REF_GONE', _ref }; + return { success: false, error: 'REF_GONE', _ref, url: location.href }; } el.focus(); @@ -235,7 +263,7 @@ export async function handleType(params) { // Virtualization recovery (same as click): type target is gone, so // auto-re-snapshot and embed fresh refs. No auto-retry (non-idempotent). - if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref); + if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref, selector); return res; } @@ -258,55 +286,54 @@ export async function handlePressKey(params) { const { tabId, ref, selector, trusted } = params; const { key, mods: modifiers } = parseKeyCombo(params.key, params.modifiers || []); await resolveTab(tabId); + const fb = getFallback(tabId, ref); + + // "ArrowDown ArrowDown Enter" / "ctrl+a Backspace": a space-separated key + // sequence; `repeat` presses the whole sequence N times. + const raw = String(params.key ?? ''); + const seq = raw.length > 1 && /\s/.test(raw.trim()) ? raw.trim().split(/\s+/) : [raw]; + const combos = seq.map((k) => parseKeyCombo(k, params.modifiers || [])); + const repeat = Math.min(Math.max(1, Number.isInteger(params.repeat) ? params.repeat : 1), 100); // Trusted path: a real key press, so default actions run (Tab moves focus // and fires blur/focusout, Enter submits, arrows drive autocomplete menus). let knownKey = true; - try { keyDefinition(key); } catch { knownKey = false; } + for (const c of combos) { try { keyDefinition(c.key); } catch { knownKey = false; } } + if (!knownKey && (combos.length > 1 || repeat > 1)) throw new Error(`Unknown key in "${raw}"`); const send = knownKey ? await trustedSender(tabId, trusted) : null; if (send) { - const loc = await locateTarget(tabId, { ref, selector, mode: ref || selector ? 'focus' : 'active' }); + const loc = await locateTarget(tabId, { ref, selector, fb, mode: ref || selector ? 'focus' : 'active' }); if (!loc || loc.success === false) { await releaseShield(tabId); if (ref || selector) return { success: false, error: `Element ${ref ? `with ref ${ref}` : `with selector ${selector}`} not found` }; } let after; try { - await cdpKeyPress(send, key, modifiers); + for (let r = 0; r < repeat; r++) { + for (const c of combos) await cdpKeyPress(send, c.key, c.mods); + } } finally { after = await releaseShield(tabId); } - return { success: true, key, ...(modifiers.length ? { modifiers } : {}), input: 'cdp', ...(after?.focusedTag ? { focused: after.focusedTag } : {}) }; + return { + success: true, key: combos.length > 1 ? raw : key, ...(modifiers.length && combos.length === 1 ? { modifiers } : {}), + ...(repeat > 1 ? { repeat } : {}), input: 'cdp', ...(after?.focusedTag ? { focused: after.focusedTag } : {}), + }; } - return safeExec(tabId, (_key, _mods, _ref, _sel) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + if (combos.length > 1 || repeat > 1) throw new Error('Key sequences and repeat need the debugger (CDP); press keys one at a time with trusted:false.'); + return execDom(tabId, (_key, _mods, _ref, _sel, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; let target = document.activeElement || document.body; // When the caller names a target, an unresolved ref/selector must FAIL — // silently falling back to activeElement sent Enter to the wrong control // with a success result. (Omitting both is still legitimate: intentional // activeElement targeting.) - if (_ref) { - const el = deepQuery(`[data-mcp-ref="${_ref}"]`); - if (!el) return { success: false, error: `Element with ref ${_ref} not found` }; - el.focus(); - target = el; - } else if (_sel) { - const el = deepQuery(_sel); - if (!el) return { success: false, error: `Element with selector ${_sel} not found` }; + if (_ref || _sel) { + const el = D.resolve(_ref, _sel, _fb).el; + if (!el) return { success: false, error: _ref ? `Element with ref ${_ref} not found` : `Element with selector ${_sel} not found` }; el.focus(); target = el; } @@ -327,17 +354,28 @@ export async function handlePressKey(params) { target.dispatchEvent(new KeyboardEvent('keyup', init)); return { success: true, key: _key }; - }, [key, modifiers, ref, selector]); + }, [key, modifiers, ref, selector, fb]); } export async function handleHover(params) { const { tabId, ref, selector, trusted } = params; await resolveTab(tabId); - requireTarget(params); + requireTarget(params, { allowPoint: true }); + if (!ref && !selector && hasPoint(params)) { + const sendAt = await requireCdp(tabId, 'Hovering'); + const info = await pointInfo(tabId, params.x, params.y); + try { + await sendAt('Input.dispatchMouseEvent', { type: 'mouseMoved', x: params.x, y: params.y }); + } finally { + await releaseShield(tabId); + } + return { success: true, input: 'cdp', at: { x: params.x, y: params.y }, ...(info.hit ? { hit: info.hit } : {}) }; + } + const fb = getFallback(tabId, ref); const send = await trustedSender(tabId, trusted); if (send) { - const loc = await locateTarget(tabId, { ref, selector }); + const loc = await locateTarget(tabId, { ref, selector, fb }); if (loc?.success && loc.visible) { try { await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: loc.x, y: loc.y }); @@ -350,23 +388,11 @@ export async function handleHover(params) { if (loc && loc.success === false) return { success: false, error: 'Element not found' }; } - return safeExec(tabId, (_ref, _sel) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + return execDom(tabId, (_ref, _sel, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; - let el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : null; - if (!el && _sel) el = deepQuery(_sel); + const el = D.resolve(_ref, _sel, _fb).el; if (!el) return { success: false, error: 'Element not found' }; el.scrollIntoView({ behavior: 'instant', block: 'center' }); @@ -380,7 +406,7 @@ export async function handleHover(params) { el.dispatchEvent(new MouseEvent('mousemove', init)); return { success: true }; - }, [ref, selector]); + }, [ref, selector, fb]); } export async function handleSelect(params) { @@ -390,24 +416,13 @@ export async function handleSelect(params) { if (value === undefined && label === undefined && index === undefined) { throw new Error('One of value, label, or index is required to pick an option.'); } + const fb = getFallback(tabId, ref); - return safeExec(tabId, (_ref, _sel, _val, _lbl, _idx) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + return execDom(tabId, (_ref, _sel, _val, _lbl, _idx, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; - let el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : null; - if (!el && _sel) el = deepQuery(_sel); + const el = D.resolve(_ref, _sel, _fb).el; if (!el) return { success: false, error: 'Element not found' }; if (el.tagName !== 'SELECT') return { success: false, error: 'Not a select element' }; @@ -424,69 +439,98 @@ export async function handleSelect(params) { el.dispatchEvent(new Event('change', { bubbles: true })); el.dispatchEvent(new Event('input', { bubbles: true })); return { success: true, selected: el.value }; - }, [ref, selector, value, label, index]); + }, [ref, selector, value, label, index, fb]); } export async function handleClickByText(params) { - const { tabId, text, index = 0, exact = false } = params; + const { tabId, text, index = 0, exact = false, trusted } = params; await resolveTab(tabId); - - return safeExec(tabId, (_text, _index, _exact) => { - const textLower = _text.toLowerCase(); - const candidates = []; - // Same-origin iframe piercing — walk every frame body, not just the top. - const roots = [document.body]; - (function collectFrames(doc, depth) { - if (depth >= 3) return; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d && d.body) { roots.push(d.body); collectFrames(d, depth + 1); } } catch {} - } - })(document, 0); - let node; - for (const root of roots) { - const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT); - while ((node = walker.nextNode())) { - const s = getComputedStyle(node); - if (s.display === 'none' || s.visibility === 'hidden') continue; - const r = node.getBoundingClientRect(); - if (r.width === 0 || r.height === 0) continue; - - const nodeText = (node.innerText || node.textContent || '').trim(); - const firstLine = nodeText.split('\n')[0].trim(); - const match = _exact - ? firstLine === _text - : firstLine.toLowerCase().includes(textLower); - - if (match) { - candidates.push({ el: node, text: firstLine, depth: getDepth(node) }); + const tempRef = `t${Date.now().toString(36)}${Math.random().toString(36).slice(2, 5)}`; + + // Page side: find the element by accessible name / composed text (shadow + // roots + same-origin frames), climb to the control that owns it, and park + // it in the ref registry so the click itself goes through the normal path. + const found = await execDom(tabId, (_text, _index, _exact, _ref) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const want = D.clean(_text).toLowerCase(); + if (!want) return { success: false, error: 'text is required' }; + const SKIP = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'HEAD', 'HTML', 'BODY', 'META', 'LINK']); + const hits = []; + const seen = new Set(); + const matches = (s) => { + const t = D.clean(s).toLowerCase(); + if (!t) return false; + return _exact ? t === want : t.includes(want); + }; + for (const root of D.allRoots(true)) { + let els = []; + try { els = root.querySelectorAll('*'); } catch {} + for (const el of els) { + if (SKIP.has(el.tagName)) continue; + const own = D.isInteractive(el) ? D.nameOf(el) : D.composedText(el, 200); + // aria-label / title / value also count as the element's text. + if (!matches(own) && !matches(D.attr(el, 'aria-label')) && !matches(D.attr(el, 'title')) + && !(el.tagName === 'INPUT' && matches(el.value))) continue; + // Climb to the control that owns this text (MUI: inside

not clicked

`); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const page = `http://127.0.0.1:${server.address().port}/`; + const managedDaemon = await hasManagedDaemon(); + const wasRunning = await daemonAlive(); + if (managedDaemon) { + if (!wasRunning) await startManagedDaemon(); + log(wasRunning ? 'systemd daemon already running; restarting service during test' : 'started systemd daemon for test'); + } else if (!wasRunning) { + startDaemon(); + startedByTest = true; + } else { + log('unmanaged daemon already running; restarting it during test'); + } + await waitForDaemon(); + client = await connect(); + await call('browser_tabs', { action: 'list' }); + client.close(); + if (managedDaemon) { + await restartManagedDaemon(); + } else { + await stopDaemon(); + if (!(await daemonAlive())) startDaemon(); + await waitForDaemon(); + } + client = await connect(); + assert(Array.isArray((await call('browser_tabs', { action: 'list' }))?.tabs), 'tab list failed after daemon restart'); + log('daemon restart/reconnect verified'); + + const created = await call('browser_tabs', { action: 'create', url: page }); + tabId = created.tabId ?? created.tabs?.[0]?.id; + assert(Number.isInteger(tabId), `could not determine created tabId: ${JSON.stringify(created)}`); + assert(Array.isArray((await call('browser_tabs', { action: 'list' }))?.tabs), 'tab list failed'); + const snapshot = await call('browser_snapshot', { tabId, compact: false }); + assert(textOf(snapshot).includes('Smoke page'), 'snapshot did not contain smoke page'); + const buttonRef = refsIn(snapshot, 'button')[0]; + const inputRef = refsIn(snapshot, 'textbox')[0]; + assert(buttonRef && inputRef, 'snapshot did not provide button/textbox refs'); + log(`snapshot ok (tab ${tabId})`); + await call('browser_click', { tabId, ref: buttonRef }); + const clicked = await call('browser_evaluate', { tabId, expression: 'document.querySelector("#result").textContent' }); + assert(textOf(clicked).includes('clicked'), `click verification failed: ${textOf(clicked)}`); + await call('browser_type', { tabId, ref: inputRef, text: 'typed by smoke', clear: true }); + const value = await call('browser_evaluate', { tabId, expression: 'document.querySelector("#text").value' }); + assert(textOf(value).includes('typed by smoke'), `type verification failed: ${textOf(value)}`); + log('click and type ok'); + const consoleResult = await call('browser_console', { tabId }); + assert(textOf(consoleResult).includes('SMOKE_CONSOLE_MARKER'), 'console marker was not captured'); + const networkResult = await call('browser_network', { tabId, filter: '/api' }); + assert(textOf(networkResult).includes('/api'), 'network request was not captured'); + log('console and network ok'); + await call('browser_upload_file', { tabId, selector: '#file', filePath: UPLOAD }); + const files = await call('browser_evaluate', { tabId, expression: 'document.querySelector("#file").files[0]?.name' }); + assert(textOf(files).includes('smoke-upload.txt'), `upload verification failed: ${textOf(files)}`); + log('upload ok'); + await call('browser_tabs', { action: 'close', tabId }); tabId = undefined; + log('PASS: real Chrome profile, reconnect, and all requested tools verified'); +} + +main().catch(error => { console.error('[smoke] FAIL:', error.message); process.exitCode = 1; }).finally(async () => { + if (client) client.close(); + if (tabId !== undefined) { try { await call('browser_tabs', { action: 'close', tabId }); } catch {} } + if (server) await new Promise(resolve => server.close(resolve)); + if (startedByTest) await stopDaemon(); + try { fs.unlinkSync(UPLOAD); } catch {} +}); diff --git a/tests/bridge-multibrowser.test.ts b/tests/bridge-multibrowser.test.ts new file mode 100644 index 0000000..49d198e --- /dev/null +++ b/tests/bridge-multibrowser.test.ts @@ -0,0 +1,107 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { WebSocket } from 'ws'; +import { ExtensionBridge } from '../mcp-server/src/bridge.js'; +import { buildExtensionHelloAck } from '../mcp-server/src/protocol.js'; +import { buildExtensionHelloAck as extensionHelloAck } from '../extension/lib/protocol.js'; + +let port = 26_000 + (process.pid % 3_000); +const sockets: WebSocket[] = []; +const bridges: ExtensionBridge[] = []; + +afterEach(async () => { + sockets.forEach((s) => { try { s.close(); } catch { /* closed */ } }); + sockets.length = 0; + bridges.forEach((b) => b.stop()); + bridges.length = 0; + await new Promise((r) => setTimeout(r, 30)); +}); + +async function startBridge() { + const bridge = new ExtensionBridge({ port: ++port, maxRetries: 0, pingIntervalMs: 60_000, handshakeGraceMs: 500 }); + bridges.push(bridge); + await bridge.start(); + return { bridge, port }; +} + +/** A fake extension: answers the hello with its browser identity and echoes tool calls with its name. */ +async function fakeBrowser(p: number, browserId: string, opts: { silent?: boolean } = {}) { + const ws = new WebSocket(`ws://localhost:${p}`); + sockets.push(ws); + const calls: string[] = []; + ws.on('message', (data) => { + const msg = JSON.parse(data.toString()); + if (msg.type === 'hello') ws.send(JSON.stringify({ ...buildExtensionHelloAck('test'), browserId, browserLabel: `Chrome ${browserId}` })); + if (msg.tool) { + calls.push(msg.tool); + if (!opts.silent) ws.send(JSON.stringify({ id: msg.id, success: true, result: { from: browserId } })); + } + }); + await new Promise((resolve, reject) => { ws.on('open', () => resolve()); ws.on('error', reject); }); + await new Promise((r) => setTimeout(r, 60)); + return { ws, calls }; +} + +describe('multi-browser bridge', () => { + it('keeps several browsers connected; the newest is the default', async () => { + const { bridge, port: p } = await startBridge(); + await fakeBrowser(p, 'work'); + await fakeBrowser(p, 'home'); + const list = bridge.callTool('browser_list_browsers', {}, 's1') as Promise; + const { browsers } = await list; + expect(browsers.map((b: any) => b.browserId).sort()).toEqual(['home', 'work']); + expect(browsers.find((b: any) => b.default).browserId).toBe('home'); + expect(await bridge.callTool('browser_tabs', { action: 'list' }, 's1')).toEqual({ from: 'home' }); + }); + + it('routes a session to the browser it selected, other sessions keep the default', async () => { + const { bridge, port: p } = await startBridge(); + await fakeBrowser(p, 'work'); + await fakeBrowser(p, 'home'); + expect(await bridge.callTool('browser_select_browser', { browserId: 'Chrome work' }, 's1')).toMatchObject({ selected: 'work' }); + expect(await bridge.callTool('browser_tabs', { action: 'list' }, 's1')).toEqual({ from: 'work' }); + expect(await bridge.callTool('browser_tabs', { action: 'list' }, 's2')).toEqual({ from: 'home' }); + await bridge.callTool('browser_select_browser', { browserId: 'auto' }, 's1'); + expect(await bridge.callTool('browser_tabs', { action: 'list' }, 's1')).toEqual({ from: 'home' }); + await expect(bridge.callTool('browser_select_browser', { browserId: 'nope' }, 's1')).rejects.toThrow(/No connected browser/); + }); + + it('a reconnect of the same browser replaces its old socket; a different browser is untouched', async () => { + const { bridge, port: p } = await startBridge(); + const first = await fakeBrowser(p, 'work'); + await fakeBrowser(p, 'home'); + const firstClosed = new Promise((r) => first.ws.once('close', () => r())); + await fakeBrowser(p, 'work'); + await firstClosed; + const { browsers } = await (bridge.callTool('browser_list_browsers', {}, 's1') as Promise); + expect(browsers.map((b: any) => b.browserId).sort()).toEqual(['home', 'work']); + }); + + it('one browser disconnecting fails only its own calls', async () => { + const { bridge, port: p } = await startBridge(); + const work = await fakeBrowser(p, 'work', { silent: true }); + await fakeBrowser(p, 'home'); + await bridge.callTool('browser_select_browser', { browserId: 'work' }, 's1'); + const hanging = bridge.callTool('browser_wait', { delay: 10 }, 's1'); + await new Promise((r) => setTimeout(r, 50)); + work.ws.close(); + await expect(hanging).rejects.toThrow(/disconnected/); + expect(await bridge.callTool('browser_tabs', { action: 'list' }, 's2')).toEqual({ from: 'home' }); + // The session that picked the gone browser gets a clear error, not the wrong browser. + await expect(bridge.callTool('browser_tabs', { action: 'list' }, 's1')).rejects.toThrow(/not connected/); + }); + + it('releasing a session forgets its browser choice', async () => { + const { bridge, port: p } = await startBridge(); + await fakeBrowser(p, 'work'); + await fakeBrowser(p, 'home'); + await bridge.callTool('browser_select_browser', { browserId: 'work' }, 's1'); + bridge.sendControl('releaseSession', { sessionId: 's1' }); + expect(await bridge.callTool('browser_tabs', { action: 'list' }, 's1')).toEqual({ from: 'home' }); + }); + + it('the extension announces its browser identity in helloAck', () => { + expect(extensionHelloAck('2.4.0', { browserId: 'abc123', browserLabel: 'Chrome on Windows (abc1)' })) + .toMatchObject({ type: 'helloAck', browserId: 'abc123', browserLabel: 'Chrome on Windows (abc1)' }); + expect(extensionHelloAck('2.4.0')).not.toHaveProperty('browserId'); + }); +}); diff --git a/tests/daemon-config.test.ts b/tests/daemon-config.test.ts index ad539c6..d360266 100644 --- a/tests/daemon-config.test.ts +++ b/tests/daemon-config.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, afterEach } from 'vitest'; -import { envInt } from '../mcp-server/src/daemon-config.js'; +import { envInt, DEFAULT_WS_HOST } from '../mcp-server/src/daemon-config.js'; /** * envInt (critical audit #9): bare `parseInt(process.env.X || '…')` yielded @@ -11,6 +11,11 @@ import { envInt } from '../mcp-server/src/daemon-config.js'; const SET_KEYS = ['BC_TEST_INT', 'WS_PORT'] as const; describe('envInt', () => { + it('keeps the control plane loopback-only', () => { + expect(DEFAULT_WS_HOST).toBe('127.0.0.1'); + }); + + afterEach(() => { for (const k of SET_KEYS) delete process.env[k]; }); diff --git a/tests/daemon-lifecycle-identity.test.ts b/tests/daemon-lifecycle-identity.test.ts new file mode 100644 index 0000000..27d5cb8 --- /dev/null +++ b/tests/daemon-lifecycle-identity.test.ts @@ -0,0 +1,49 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); +const SCRIPT = path.join(ROOT, 'scripts', 'daemon.mjs'); +const helpers: ChildProcess[] = []; + +afterEach(() => { + for (const h of helpers) { try { h.kill(); } catch { /* gone */ } } + helpers.length = 0; +}); + +const alive = (pid: number) => { try { process.kill(pid, 0); return true; } catch { return false; } }; + +/** A state dir whose daemon.json names a live process that is NOT a daemon. */ +function impostorState() { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bc-lifecycle-')); + const helper = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1e9)'], { stdio: 'ignore' }); + helpers.push(helper); + const socket = process.platform === 'win32' ? `\\\\.\\pipe\\bc-test-none-${process.pid}-${Date.now()}` : path.join(stateDir, 'none.sock'); + fs.writeFileSync(path.join(stateDir, 'daemon.json'), JSON.stringify({ pid: helper.pid, socket, port: 1, host: '127.0.0.1', startedAt: Date.now() })); + return { stateDir, helper }; +} + +const run = (cmd: string, stateDir: string) => spawnSync(process.execPath, [SCRIPT, cmd], { + env: { ...process.env, BC_STATE_DIR: stateDir }, encoding: 'utf8', timeout: 20_000, +}); + +describe('scripts/daemon.mjs verifies daemon identity before trusting a pid', () => { + it('status does not report an unrelated live pid as a running daemon', () => { + const { stateDir, helper } = impostorState(); + const res = run('status', stateDir); + expect(res.status).toBe(1); + expect(res.stdout).toMatch(/not running/); + expect(res.stdout).toContain(`pid ${helper.pid}`); + }); + + it('stop never signals a process that is not the daemon', () => { + const { stateDir, helper } = impostorState(); + const res = run('stop', stateDir); + expect(res.stdout).toMatch(/not running/); + expect(alive(helper.pid!)).toBe(true); + expect(fs.existsSync(path.join(stateDir, 'daemon.json'))).toBe(false); // stale metadata cleaned + }); +}); diff --git a/tests/extension-agent-api.test.ts b/tests/extension-agent-api.test.ts index dec7bd2..3e2156f 100644 --- a/tests/extension-agent-api.test.ts +++ b/tests/extension-agent-api.test.ts @@ -32,6 +32,8 @@ let queryNodeId = 2; debuggerCommands.push(method); if (method === 'DOM.getDocument') return { root: { nodeId: 1 } }; if (method === 'DOM.querySelector') return { nodeId: queryNodeId }; + // upload_file finds the marked input in the page and hands CDP its objectId. + if (method === 'Runtime.evaluate') return { result: queryNodeId ? { objectId: 'obj-1' } : { type: 'object', subtype: 'null' } }; return {}; }, }, @@ -196,7 +198,7 @@ describe('observe/act extension handlers', () => { }, 'session-a'); expect(result).toMatchObject({ success: true, ok: true, action: 'upload', files: ['/tmp/resume.pdf'] }); - expect(debuggerCommands.filter((m) => m.startsWith('DOM.'))).toEqual(['DOM.enable', 'DOM.getDocument', 'DOM.querySelector', 'DOM.setFileInputFiles']); + expect(debuggerCommands.filter((m) => m.startsWith('DOM.'))).toEqual(['DOM.setFileInputFiles']); expect(result.metrics).toMatchObject({ protocolCalls: 8 }); }); diff --git a/tests/extension-intercept-dnr.test.ts b/tests/extension-intercept-dnr.test.ts new file mode 100644 index 0000000..4727b40 --- /dev/null +++ b/tests/extension-intercept-dnr.test.ts @@ -0,0 +1,111 @@ +import { beforeEach, describe, expect, it } from "vitest"; + +/** + * Enforcement against a STATEFUL declarativeNetRequest mock: what Chrome + * actually has installed must match what browser_intercept reports. + */ +type DnrRule = { id: number; action: Record; condition: Record }; +const sessionRules = new Map(); +const dynamicRules = new Map(); + +(globalThis as unknown as { chrome: unknown }).chrome = { + tabs: { get: async (id: number) => ({ id, url: "https://x.test/", windowId: 1 }), query: async () => [] }, + scripting: { executeScript: async () => [{ result: null }] }, + storage: { session: { get: async () => ({}), set: async () => {} } }, + declarativeNetRequest: { + getSessionRules: async () => [...sessionRules.values()], + updateSessionRules: async ({ removeRuleIds = [], addRules = [] }: { removeRuleIds?: number[]; addRules?: DnrRule[] }) => { + for (const id of removeRuleIds) sessionRules.delete(id); + for (const r of addRules) { + if (sessionRules.has(r.id)) throw new Error(`Rule with id ${r.id} does not have a unique ID.`); + sessionRules.set(r.id, r); + } + }, + getDynamicRules: async () => [...dynamicRules.values()], + updateDynamicRules: async ({ removeRuleIds = [], addRules = [] }: { removeRuleIds?: number[]; addRules?: DnrRule[] }) => { + for (const id of removeRuleIds) dynamicRules.delete(id); + for (const r of addRules) dynamicRules.set(r.id, r); + }, + }, +}; + +const { handleIntercept, rulesByScope, enrichCapture } = await import("../extension/handlers/intercept.js"); + +describe("browser_intercept enforcement (stateful DNR)", () => { + beforeEach(async () => { + sessionRules.clear(); + dynamicRules.clear(); + rulesByScope.clear(); + }); + + it("a tab-scoped block is installed as a tab-scoped session rule with its resource types", async () => { + const res = await handleIntercept({ + action: "set-rules", tabId: 15, + rules: [{ id: "api", match: "https://api\\.x\\.test/", action: "block", types: ["xmlhttprequest"] }], + }); + expect(res).toMatchObject({ success: true, enforcement: "full", scope: "tabs:15", enforced: 1 }); + expect(dynamicRules.size).toBe(0); // never browser-wide dynamic rules + const [rule] = [...sessionRules.values()]; + expect(rule.condition).toEqual({ regexFilter: "https://api\\.x\\.test/", tabIds: [15], resourceTypes: ["xmlhttprequest"] }); + }); + + it("clearing removes the rules from Chrome too (not just from the list)", async () => { + await handleIntercept({ action: "set-rules", tabId: 15, rules: [{ match: "https://ads\\.test/", action: "block" }] }); + await handleIntercept({ action: "set-rules", rules: [{ match: "https://old\\.test/", action: "redirect", redirectUrl: "https://new.test/" }] }); + expect(sessionRules.size).toBe(2); + await handleIntercept({ action: "clear-rules", tabId: 15 }); + expect([...sessionRules.values()].map((r) => r.condition.regexFilter)).toEqual(["https://old\\.test/"]); + await handleIntercept({ action: "clear-rules" }); + expect(sessionRules.size).toBe(0); + expect((await handleIntercept({ action: "list-rules" })).rules).toEqual([]); + }); + + it("replacing a tab's rules replaces them (scope keys are deduplicated)", async () => { + await handleIntercept({ + action: "set-rules", tabId: 15, + rules: [{ id: "a", match: "https://a\\.test/", action: "block" }, { id: "b", match: "https://b\\.test/", action: "block" }], + }); + expect([...rulesByScope.keys()]).toEqual(["tabs:15"]); + await handleIntercept({ action: "set-rules", tabId: 15, rules: [{ id: "c", match: "https://c\\.test/", action: "block" }] }); + expect([...rulesByScope.keys()]).toEqual(["tabs:15"]); + expect([...sessionRules.values()].map((r) => r.condition.regexFilter)).toEqual(["https://c\\.test/"]); + }); + + it("header rules are enforced as request-header modifications", async () => { + const res = await handleIntercept({ + action: "set-rules", tabId: 15, + rules: [{ id: "h", match: "https://api\\.x\\.test/", action: "header", headers: { "X-Test": "1" } }], + }); + expect(res).toMatchObject({ enforcement: "full", enforced: 1 }); + expect([...sessionRules.values()][0].action).toEqual({ + type: "modifyHeaders", requestHeaders: [{ header: "X-Test", operation: "set", value: "1" }], + }); + }); + + it("mock rules are not claimed as enforced", async () => { + const res = await handleIntercept({ + action: "set-rules", tabId: 15, + rules: [ + { id: "m", match: "https://api\\.x\\.test/v1", action: "mock", mockStatus: 200, mockBody: "{}" }, + { id: "b", match: "https://ads\\.test/", action: "block" }, + ], + }); + expect(res).toMatchObject({ enforcement: "partial", enforced: 1 }); + expect(res.unsupported).toEqual([expect.objectContaining({ id: "m", action: "mock" })]); + expect(sessionRules.size).toBe(1); + const entry = enrichCapture(15, { url: "https://api.x.test/v1/users", type: "xmlhttprequest", method: "GET" }) as { intercept: { applied: Array<{ ruleId: string; outcome: string }> } }; + expect(entry.intercept.applied).toEqual([{ ruleId: "m", outcome: "ledger-only" }]); + const blocked = enrichCapture(15, { url: "https://ads.test/x.js", type: "script", method: "GET" }) as { intercept: { applied: Array<{ outcome: string }> } }; + expect(blocked.intercept.applied[0].outcome).toBe("enforced"); + const listed = await handleIntercept({ action: "list-rules", tabId: 15 }); + expect(listed.enforcement).toBe("partial"); + expect(listed.rules.map((r: { id: string; enforced: boolean }) => [r.id, r.enforced])).toEqual([["m", false], ["b", true]]); + }); + + it("rules installed by a previous service worker are removed on the next sync", async () => { + sessionRules.set(1003, { id: 1003, action: { type: "block" }, condition: { regexFilter: "stale" } }); + sessionRules.set(42, { id: 42, action: { type: "block" }, condition: { regexFilter: "not ours" } }); + await handleIntercept({ action: "set-rules", tabId: 15, rules: [{ match: "https://a\\.test/", action: "block" }] }); + expect([...sessionRules.keys()].sort((a, b) => a - b)).toEqual([42, 1000]); + }); +}); diff --git a/tests/extension-intercept.test.ts b/tests/extension-intercept.test.ts new file mode 100644 index 0000000..71c9bea --- /dev/null +++ b/tests/extension-intercept.test.ts @@ -0,0 +1,309 @@ +import { vi, describe, it, expect, beforeEach } from "vitest"; + +/** + * Intercept plane: pure engine (S2) + handler CRUD/degrade/HAR (S3/S4). + * chrome mock has NO declarativeNetRequest -> every enforcement assertion + * must expect capture-only degradation, never a throw. + */ + +const tabStore = new Map< + number, + { id: number; windowId: number; url: string; title: string; active: boolean } +>(); +(globalThis as unknown as { chrome: unknown }).chrome = { + tabs: { + get: async (id: number) => + tabStore.get(id) ?? Promise.reject(new Error(`No tab ${id}`)), + query: async () => [], + update: async () => ({}), + remove: async () => ({}), + create: async () => ({}), + }, + scripting: { executeScript: async () => [{ result: null }] }, + storage: { session: { get: async () => ({}), set: async () => {} } }, +}; + +const { validateRule, validateRuleSet, matchRule, evaluateRules } = + await import("../extension/lib/intercept.js"); +const { handleIntercept, enrichCapture, rulesByScope, interceptLedgerByTab } = + await import("../extension/handlers/intercept.js"); +const { networkByTab } = await import("../extension/lib/state.js"); + +describe("pure engine (S2)", () => { + it("rejects invalid regex", () => { + expect(() => validateRule({ match: "([", action: "log" }, 0)).toThrow( + /invalid match regex/, + ); + }); + + it("rejects match-all patterns", () => { + expect(() => validateRule({ match: ".*", action: "block" }, 0)).toThrow( + /match-all/, + ); + expect(() => validateRule({ match: ".+", action: "block" }, 0)).toThrow( + /match-all/, + ); + }); + + it("rejects empty match and unknown action", () => { + expect(() => validateRule({ match: "", action: "log" }, 0)).toThrow( + /match must be/, + ); + expect(() => + validateRule({ match: "https://x\\.com/", action: "nuke" }, 0), + ).toThrow(/action must be/); + }); + + it("rejects redirect without redirectUrl and oversize mock", () => { + expect(() => + validateRule({ match: "https://x\\.com/", action: "redirect" }, 0), + ).toThrow(/redirectUrl/); + expect(() => + validateRule( + { + match: "https://x\\.com/", + action: "mock", + mockBody: "x".repeat(20_001), + }, + 0, + ), + ).toThrow(/mockBody exceeds/); + }); + + it("rejects 51 rules and duplicate ids", () => { + const many = Array.from({ length: 51 }, (_, i) => ({ + id: `r${i}`, + match: `https://x${i}\\.com/`, + action: "log", + })); + expect(() => validateRuleSet(many)).toThrow(/Too many rules/); + expect(() => + validateRuleSet([ + { id: "dup", match: "https://a\\.com/", action: "log" }, + { id: "dup", match: "https://b\\.com/", action: "log" }, + ]), + ).toThrow(/Duplicate rule id/); + }); + + it("match respects regex + type + tab scope; disabled never matches", () => { + const rule = { + id: "r1", + match: "api\\.example\\.com", + types: ["xmlhttprequest"], + tabIds: [15], + action: "log", + }; + expect( + matchRule(rule, { + url: "https://api.example.com/v1", + type: "xmlhttprequest", + tabId: 15, + }), + ).toBe(true); + expect( + matchRule(rule, { + url: "https://api.example.com/v1", + type: "image", + tabId: 15, + }), + ).toBe(false); + expect( + matchRule(rule, { + url: "https://api.example.com/v1", + type: "xmlhttprequest", + tabId: 16, + }), + ).toBe(false); + expect( + matchRule( + { ...rule, enabled: false }, + { + url: "https://api.example.com/v1", + type: "xmlhttprequest", + tabId: 15, + }, + ), + ).toBe(false); + expect( + evaluateRules([rule], { + url: "https://other.com/", + type: "xmlhttprequest", + tabId: 15, + }), + ).toEqual([]); + }); +}); + +describe("handler CRUD + degrade (S3)", () => { + beforeEach(() => { + rulesByScope.clear(); + interceptLedgerByTab.clear(); + networkByTab.clear(); + tabStore.clear(); + tabStore.set(15, { + id: 15, + windowId: 1, + url: "https://app.example.com/", + title: "App", + active: true, + }); + }); + + it("set/list/clear round-trips with capture-only enforcement (no DNR in tests)", async () => { + const set = await handleIntercept({ + action: "set-rules", + tabId: 15, + rules: [{ match: "tracker\\.example\\.com", action: "block" }], + }); + expect(set.success).toBe(true); + expect(set.enforcement).toBe("capture-only"); + expect(set.ruleCount).toBe(1); + + const list = await handleIntercept({ action: "list-rules", tabId: 15 }); + expect(list.rules.length).toBe(1); + + const clear = await handleIntercept({ action: "clear-rules", tabId: 15 }); + expect(clear.success).toBe(true); + const after = await handleIntercept({ action: "list-rules", tabId: 15 }); + expect(after.rules.length).toBe(0); + }); + + it("rejects invalid rules without persisting", async () => { + await expect( + handleIntercept({ + action: "set-rules", + rules: [{ match: "([", action: "log" }], + }), + ).rejects.toThrow(/invalid match regex/); + const list = await handleIntercept({ action: "list-rules" }); + expect(list.rules.length).toBe(0); + }); + + it("unknown action throws honestly", async () => { + await expect(handleIntercept({ action: "nuke" })).rejects.toThrow( + /Unknown intercept action/, + ); + }); +}); + +describe("capture + HAR (S4)", () => { + beforeEach(() => { + rulesByScope.clear(); + interceptLedgerByTab.clear(); + networkByTab.clear(); + tabStore.clear(); + tabStore.set(15, { + id: 15, + windowId: 1, + url: "https://app.example.com/", + title: "App", + active: true, + }); + }); + + it("enrichCapture marks matched entries and ledgers them", async () => { + await handleIntercept({ + action: "set-rules", + rules: [{ id: "blk-track", match: "tracker\\.example", action: "block" }], + }); + const entry: Record = { + method: "GET", + url: "https://tracker.example/ping", + status: 200, + type: "script", + }; + const out = enrichCapture(15, entry); + expect( + (out.intercept as { matchedRuleIds: string[] }).matchedRuleIds, + ).toContain("blk-track"); + expect(interceptLedgerByTab.get(15)?.length).toBe(1); + }); + + it("list-captures rejects invalid filter regex honestly", async () => { + await expect( + handleIntercept({ action: "list-captures", tabId: 15, filter: "([" }), + ).rejects.toThrow(/Invalid filter regex/); + }); + + it("export-har returns valid HAR 1.2 with redaction flag", async () => { + networkByTab.set(15, [ + { + method: "GET", + url: "https://app.example.com/", + status: 200, + type: "main_frame", + timestamp: Date.now(), + }, + ]); + const res = await handleIntercept({ action: "export-har", tabId: 15 }); + expect(res.success).toBe(true); + expect(res.har.log.version).toBe("1.2"); + expect(res.har.log.creator.name).toBe("browser-controller"); + expect(res.entries).toBe(1); + expect(res.har.log.entries[0].response._redacted).toBe(true); + }); + + it("list-captures requires tabId", async () => { + await expect(handleIntercept({ action: "list-captures" })).rejects.toThrow( + /tabId required/, + ); + }); +}); + +describe("capture dedupe (fix)", () => { + beforeEach(() => { + rulesByScope.clear(); + interceptLedgerByTab.clear(); + networkByTab.clear(); + tabStore.clear(); + tabStore.set(15, { + id: 15, + windowId: 1, + url: "https://app.example.com/", + title: "App", + active: true, + }); + }); + + it("a matched request appears once in list-captures (not ledger + buffer)", async () => { + await handleIntercept({ + action: "set-rules", + rules: [{ id: "blk-track", match: "tracker\\.example", action: "block" }], + }); + // Simulate the events.js flow: buffer push, then enrich (mutates + ledgers). + const buf = networkByTab.get(15) ?? []; + networkByTab.set(15, buf); + const entry = { + method: "GET", + url: "https://tracker.example/ping", + status: 200, + type: "script", + timestamp: 1234567890, + }; + buf.push(entry); + enrichCapture(15, entry); + const res = await handleIntercept({ action: "list-captures", tabId: 15 }); + expect(res.captures.length).toBe(1); + expect(res.captures[0].intercept.matchedRuleIds).toContain("blk-track"); + }); + + it("export-har counts each request once", async () => { + await handleIntercept({ + action: "set-rules", + rules: [{ id: "blk-track", match: "tracker\\.example", action: "block" }], + }); + const buf: Record[] = []; + networkByTab.set(15, buf); + const entry = { + method: "GET", + url: "https://tracker.example/ping", + status: 200, + type: "script", + timestamp: 1234567890, + }; + buf.push(entry); + enrichCapture(15, entry); + const res = await handleIntercept({ action: "export-har", tabId: 15 }); + expect(res.entries).toBe(1); + }); +}); diff --git a/tests/extension-router.test.ts b/tests/extension-router.test.ts index 1a973ce..c87fcfc 100644 --- a/tests/extension-router.test.ts +++ b/tests/extension-router.test.ts @@ -1,4 +1,4 @@ -import { vi, describe, it, expect, beforeEach } from 'vitest'; +import { vi, describe, it, expect, beforeEach } from "vitest"; /** * First behavior tests for the extension side (architecture item: background @@ -9,8 +9,10 @@ import { vi, describe, it, expect, beforeEach } from 'vitest'; const sent = vi.hoisted(() => [] as Array>); -vi.mock('../extension/lib/connection.js', () => ({ - sendJson: (obj: Record) => { sent.push(obj); }, +vi.mock("../extension/lib/connection.js", () => ({ + sendJson: (obj: Record) => { + sent.push(obj); + }, updateBadge: () => {}, broadcastStatus: async () => {}, isWsConnected: () => true, @@ -19,10 +21,14 @@ vi.mock('../extension/lib/connection.js', () => ({ // chrome.* mock — must exist before any handler runs (module evaluation of the // handler modules never touches chrome; only function bodies do). -const tabStore = new Map(); +const tabStore = new Map< + number, + { id: number; windowId: number; url: string; title: string; active: boolean } +>(); (globalThis as unknown as { chrome: unknown }).chrome = { tabs: { - get: async (id: number) => tabStore.get(id) ?? Promise.reject(new Error(`No tab ${id}`)), + get: async (id: number) => + tabStore.get(id) ?? Promise.reject(new Error(`No tab ${id}`)), query: async () => [] as unknown[], update: async () => ({}), remove: async () => ({}), @@ -34,14 +40,21 @@ const tabStore = new Map ({}), set: async () => {} }, local: { get: async () => ({}), set: async () => {} }, }, - runtime: { sendMessage: async () => {}, onMessage: { addListener: () => {} } }, + runtime: { + sendMessage: async () => {}, + onMessage: { addListener: () => {} }, + }, alarms: { create: () => {}, onAlarm: { addListener: () => {} } }, webRequest: { onCompleted: { addListener: () => {} } }, - debugger: { attach: async () => {}, detach: async () => {}, sendCommand: async () => ({}) }, + debugger: { + attach: async () => {}, + detach: async () => {}, + sendCommand: async () => ({}), + }, }; const { handleMessage, dispatchedTools } = await import('../extension/lib/router.js'); -const { tabLocks, observationSnapshots } = await import('../extension/lib/state.js'); +const { tabLocks, observationSnapshots, wedgedTabs } = await import('../extension/lib/state.js'); const { allTools } = await import('../mcp-server/src/tools/index.js'); function lastFrame(): Record { @@ -54,7 +67,7 @@ async function flush(ms = 60): Promise { await new Promise((r) => setTimeout(r, ms)); } -describe('extension router (handleMessage)', () => { +describe("extension router (handleMessage)", () => { beforeEach(() => { sent.length = 0; tabStore.clear(); @@ -64,52 +77,103 @@ describe('extension router (handleMessage)', () => { tabStore.set(3, { id: 3, windowId: 1, url: 'https://example.com/page', title: 'Page', active: true }); }); - it('answers an unknown tool with a wire-level error', async () => { - await handleMessage({ id: 'n1', tool: 'browser_nope', params: {} }); - expect(lastFrame()).toMatchObject({ id: 'n1', success: false }); - expect((lastFrame().error as string)).toContain('Unknown tool'); + it("answers an unknown tool with a wire-level error", async () => { + await handleMessage({ id: "n1", tool: "browser_nope", params: {} }); + expect(lastFrame()).toMatchObject({ id: "n1", success: false }); + expect(lastFrame().error as string).toContain("Unknown tool"); }); - it('marks in-band {success:false} results as wire failures WITH the payload (unified error channel)', async () => { + it("marks in-band {success:false} results as wire failures WITH the payload (unified error channel)", async () => { // browser_wait without selector/delay returns in-band {success:false} — // the router must send success:false + the payload as `result`, not wrap // it in a success envelope. - await handleMessage({ id: 'w1', tool: 'browser_wait', params: { tabId: 3 }, sessionId: 's1' }); + await handleMessage({ + id: "w1", + tool: "browser_wait", + params: { tabId: 3 }, + sessionId: "s1", + }); await flush(); const frame = lastFrame(); - expect(frame.id).toBe('w1'); + expect(frame.id).toBe("w1"); expect(frame.success).toBe(false); - expect(frame.error).toBe('Need selector or delay'); - expect(frame.result).toEqual({ success: false, error: 'Need selector or delay' }); + expect(frame.error).toBe('Need selector, text, urlIncludes or delay'); + expect(frame.result).toEqual({ success: false, error: 'Need selector, text, urlIncludes or delay' }); + }); + + it('frozen-tab navigate cannot replace a tab locked by another session', async () => { + const created: unknown[] = []; + const removed: number[] = []; + (globalThis as any).chrome.tabs.create = async (o: unknown) => { created.push(o); return { id: 99 }; }; + (globalThis as any).chrome.tabs.remove = async (id: number) => { removed.push(id); }; + tabLocks.lock(3, 'session-a'); + wedgedTabs.set(3, Date.now()); + await handleMessage({ id: 'fz1', tool: 'browser_navigate', params: { tabId: 3, url: 'https://example.com/x', snapshot: false }, sessionId: 'session-b' }); + await flush(); + expect(lastFrame()).toMatchObject({ id: 'fz1', success: false }); + expect(String(lastFrame().error)).toMatch(/locked by session-a/); + expect(created).toEqual([]); + expect(removed).toEqual([]); + expect(tabLocks.owner(3)).toBe('session-a'); + wedgedTabs.clear(); }); - it('converts a THROWN handler error into a wire-level error', async () => { + it('the lock owner recovering its own frozen tab keeps the lock on the replacement', async () => { + const { replaceFrozenTab } = await import('../extension/lib/page-exec.js'); + (globalThis as any).chrome.tabs.create = async () => ({ id: 99 }); + (globalThis as any).chrome.tabs.remove = async () => {}; + tabLocks.lock(3, 'session-a'); + wedgedTabs.set(3, Date.now()); + await expect(replaceFrozenTab({ id: 3, windowId: 1, index: 0, active: true }, null, 'session-b')).rejects.toThrow(/locked by session-a/); + const fresh = await replaceFrozenTab({ id: 3, windowId: 1, index: 0, active: true }, null, 'session-a'); + expect(fresh.id).toBe(99); + expect(tabLocks.owner(99)).toBe('session-a'); + expect(tabLocks.owner(3)).toBeUndefined(); + wedgedTabs.clear(); + }); + + it("converts a THROWN handler error into a wire-level error", async () => { // click with neither ref nor selector throws in requireTarget. - await handleMessage({ id: 'c1', tool: 'browser_click', params: { tabId: 3 }, sessionId: 's1' }); + await handleMessage({ + id: "c1", + tool: "browser_click", + params: { tabId: 3 }, + sessionId: "s1", + }); await flush(); const frame = lastFrame(); - expect(frame.id).toBe('c1'); + expect(frame.id).toBe("c1"); expect(frame.success).toBe(false); - expect((frame.error as string)).toContain('ref or selector is required'); + expect(frame.error as string).toContain("ref or selector is required"); }); - it('routes a successful tool through the mutex and replies success', async () => { - await handleMessage({ id: 'k1', tool: 'browser_console', params: { tabId: 3 }, sessionId: 's1' }); + it("routes a successful tool through the mutex and replies success", async () => { + await handleMessage({ + id: "k1", + tool: "browser_console", + params: { tabId: 3 }, + sessionId: "s1", + }); await flush(); const frame = lastFrame(); - expect(frame.id).toBe('k1'); + expect(frame.id).toBe("k1"); expect(frame.success).toBe(true); expect((frame.result as { messages: unknown[] }).messages).toEqual([]); }); - it('honors tab-lock ownership: a non-owner call waits instead of running (TOCTOU fix, end-to-end)', async () => { - tabLocks.lock(3, 'ownerA'); - await handleMessage({ id: 'k2', tool: 'browser_console', params: { tabId: 3 }, sessionId: 'sessionB' }); + it("honors tab-lock ownership: a non-owner call waits instead of running (TOCTOU fix, end-to-end)", async () => { + tabLocks.lock(3, "ownerA"); + await handleMessage({ + id: "k2", + tool: "browser_console", + params: { tabId: 3 }, + sessionId: "sessionB", + }); // No reply yet — B is queued behind owner A's lock. - expect(sent.filter((f) => f.id === 'k2')).toEqual([]); - tabLocks.unlock(3, 'ownerA'); + expect(sent.filter((f) => f.id === "k2")).toEqual([]); + tabLocks.unlock(3, "ownerA"); await new Promise((r) => setTimeout(r, 120)); - const frame = sent.find((f) => f.id === 'k2'); + const frame = sent.find((f) => f.id === "k2"); expect(frame?.success).toBe(true); }); }); @@ -239,7 +303,8 @@ describe('observe/act concurrency integration', () => { describe('dispatch registry ↔ MCP tool registry (drift guard)', () => { // Server-local tools never reach the extension: browser_batch runs other // tools' handlers in the MCP process (the meta tool isn't in allTools). - const wireTools = allTools.filter((t) => t.name !== 'browser_batch'); + // browser_batch runs in the MCP process; browser selection is answered by the bridge. + const wireTools = allTools.filter((t) => !['browser_batch', 'browser_shortcuts', 'browser_list_browsers', 'browser_select_browser'].includes(t.name)); it('every registered MCP tool has an extension handler', () => { for (const tool of wireTools) { diff --git a/tests/gif-encoder.test.ts b/tests/gif-encoder.test.ts new file mode 100644 index 0000000..ec980f0 --- /dev/null +++ b/tests/gif-encoder.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'vitest'; +import { encodeGif, lzwEncode, indexPixels, buildPalette, drawMarker } from '../extension/lib/gif-encoder.js'; + +/** Reference GIF LZW decoder (spec algorithm) for round-trip checks. */ +function lzwDecode(data: number[]): number[] { + const min = data[0]; + const bytes: number[] = []; + let i = 1; + while (data[i] !== 0) { const n = data[i]; bytes.push(...data.slice(i + 1, i + 1 + n)); i += n + 1; } + const clear = 1 << min; + const eoi = clear + 1; + let size = min + 1; + let dict: number[][] = []; + const reset = () => { dict = []; for (let k = 0; k < clear; k++) dict[k] = [k]; dict[clear] = []; dict[eoi] = []; size = min + 1; }; + reset(); + const out: number[] = []; + let bitPos = 0; + const read = () => { + let code = 0; + for (let b = 0; b < size; b++) { + const byte = bytes[(bitPos + b) >> 3]; + if (((byte >> ((bitPos + b) & 7)) & 1) === 1) code |= 1 << b; + } + bitPos += size; + return code; + }; + let prev: number[] | null = null; + for (;;) { + const code = read(); + if (code === clear) { reset(); prev = null; continue; } + if (code === eoi) break; + let entry: number[]; + if (dict[code]) entry = dict[code]; + else if (prev) entry = [...prev, prev[0]]; + else throw new Error('bad code'); + out.push(...entry); + if (prev) { + dict.push([...prev, entry[0]]); + if (dict.length === (1 << size) && size < 12) size++; + } + prev = entry; + } + return out; +} + +describe('GIF encoder', () => { + it('LZW round-trips short, repetitive and dictionary-overflowing data', () => { + const cases = [ + [5], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + Array.from({ length: 5000 }, (_, i) => (i * 7) % 256), + Array.from({ length: 60000 }, (_, i) => ((i * 2654435761) >>> 24) & 0xff), // forces dictionary resets + ]; + for (const c of cases) expect(lzwDecode(lzwEncode(Uint8Array.from(c)))).toEqual(c); + }); + + it('maps greys to the grey ramp and colours to the cube', () => { + const pal = buildPalette(); + const idx = indexPixels(Uint8Array.from([255, 255, 255, 255, 0, 0, 0, 255, 255, 0, 0, 255, 128, 128, 128, 255]), 4); + expect([pal[idx[0] * 3], pal[idx[0] * 3 + 1]]).toEqual([255, 255]); + expect(pal[idx[1] * 3]).toBe(0); + expect([pal[idx[2] * 3], pal[idx[2] * 3 + 1], pal[idx[2] * 3 + 2]]).toEqual([255, 0, 0]); + expect(idx[3]).toBeGreaterThanOrEqual(216); + }); + + it('writes a well-formed looping GIF89a with one image per frame', () => { + const w = 4; const h = 3; + const frame = (v: number) => ({ rgba: new Uint8Array(w * h * 4).fill(v), delayMs: 400 }); + const f2 = frame(200); + drawMarker(f2.rgba, w, h, 1, 1, 1); + const gif = encodeGif(w, h, [frame(0), f2]); + const text = String.fromCharCode(...gif.slice(0, 6)); + expect(text).toBe('GIF89a'); + expect(gif[6] | (gif[7] << 8)).toBe(w); + expect(gif[8] | (gif[9] << 8)).toBe(h); + expect(String.fromCharCode(...gif.slice(13 + 768 + 3, 13 + 768 + 14))).toBe('NETSCAPE2.0'); + // one graphic-control extension (21 F9 04) per frame + expect(gif.filter((b, i) => b === 0x21 && gif[i + 1] === 0xf9 && gif[i + 2] === 0x04).length).toBe(2); + expect(gif[gif.length - 1]).toBe(0x3b); + }); +}); diff --git a/tests/helpers/fake-dom.ts b/tests/helpers/fake-dom.ts new file mode 100644 index 0000000..9842048 --- /dev/null +++ b/tests/helpers/fake-dom.ts @@ -0,0 +1,173 @@ +/** + * Tiny DOM for exercising the injected page runtime (extension/lib/page-dom.js) + * in node: elements, text nodes, open shadow roots, a handful of selector + * forms, visibility via a `hidden` flag / inline display, fixed-size layout. + * Deliberately small — enough for resolver / find / click_text semantics. + */ +export class FakeText { + nodeType = 3; + parentNode: FakeNode | null = null; + constructor(public nodeValue: string) {} + get textContent() { return this.nodeValue; } +} + +type FakeNode = FakeElement | FakeShadowRoot; + +export class FakeShadowRoot { + nodeType = 11; + childNodes: Array = []; + constructor(public host: FakeElement) {} + get children() { return this.childNodes.filter((c): c is FakeElement => c instanceof FakeElement); } + append(...nodes: Array) { for (const n of nodes) { n.parentNode = this; if (n instanceof FakeElement) n.parentElement = null; this.childNodes.push(n); } return this; } + querySelectorAll(sel: string) { return this.children.flatMap((c) => c.selfAndDescendants()).filter((e) => e.matches(sel)); } + querySelector(sel: string) { return this.querySelectorAll(sel)[0] ?? null; } + getElementById(id: string) { return this.querySelectorAll(`#${id}`)[0] ?? null; } + get activeElement() { return null; } + elementFromPoint() { return null; } +} + +export class FakeElement { + nodeType = 1; + tagName: string; + attrs = new Map(); + childNodes: Array = []; + parentElement: FakeElement | null = null; + parentNode: FakeNode | null = null; + shadowRoot: FakeShadowRoot | null = null; + hidden = false; + value: string | undefined; + type: string | undefined; + disabled = false; + isConnected = true; + onclick = null; + isContentEditable = false; + events: string[] = []; + ownerDocument: FakeDocument; + + constructor(doc: FakeDocument, tag: string, attrs: Record = {}) { + this.ownerDocument = doc; + this.tagName = tag.toUpperCase(); + for (const [k, v] of Object.entries(attrs)) this.setAttribute(k, v); + } + + get children() { return this.childNodes.filter((c): c is FakeElement => c instanceof FakeElement); } + get childElementCount() { return this.children.length; } + get id() { return this.attrs.get('id') ?? ''; } + get className() { return this.attrs.get('class') ?? ''; } + get textContent(): string { return this.childNodes.map((c) => c.textContent).join(''); } + get innerText(): string { return this.textContent; } + get labels() { return []; } + get multiple() { return false; } + + append(...nodes: Array) { + for (const raw of nodes) { + const n = typeof raw === 'string' ? new FakeText(raw) : raw; + n.parentNode = this; + if (n instanceof FakeElement) n.parentElement = this; + this.childNodes.push(n); + } + return this; + } + attachShadow() { this.shadowRoot = new FakeShadowRoot(this); return this.shadowRoot; } + getAttribute(n: string) { if (n === 'type' && this.type) return this.type; return this.attrs.get(n) ?? null; } + setAttribute(n: string, v: string) { this.attrs.set(n, v); if (n === 'type') this.type = v; if (n === 'value') this.value = v; } + removeAttribute(n: string) { this.attrs.delete(n); } + hasAttribute(n: string) { return this.attrs.has(n); } + getRootNode(): unknown { + let cur: FakeElement = this; + while (cur.parentElement) cur = cur.parentElement; + if (cur.parentNode instanceof FakeShadowRoot) return cur.parentNode; + return this.ownerDocument; + } + closest() { return null; } + get isHiddenInTree(): boolean { + let cur: FakeElement | null = this; + while (cur) { + if (cur.hidden || cur.attrs.get('style')?.includes('display:none')) return true; + const p: FakeNode | null = cur.parentNode; + cur = cur.parentElement ?? (p instanceof FakeShadowRoot ? p.host : null); + } + return false; + } + getBoundingClientRect() { + const w = this.isHiddenInTree ? 0 : 100; + return { x: 10, y: 10, left: 10, top: 10, width: w, height: w ? 20 : 0, right: 10 + w, bottom: 30 }; + } + checkVisibility() { return !this.isHiddenInTree; } + scrollIntoView() {} + focus() { this.ownerDocument.activeElement = this; } + dispatchEvent(e: { type: string }) { this.events.push(e.type); return true; } + selfAndDescendants(): FakeElement[] { + return [this, ...this.children.flatMap((c) => c.selfAndDescendants())]; + } + querySelectorAll(sel: string) { return this.children.flatMap((c) => c.selfAndDescendants()).filter((e) => e.matches(sel)); } + querySelector(sel: string) { return this.querySelectorAll(sel)[0] ?? null; } + /** Supports: *, tag, #id, .cls, tag.cls, [attr], [attr="v"], tag[attr="v"], comma lists. */ + matches(sel: string): boolean { + return sel.split(',').some((part) => { + const s = part.trim(); + if (s === '*') return true; + const m = s.match(/^([a-zA-Z0-9-]*)((?:[#.][\w-]+)*)((?:\[[^\]]+\])*)$/); + if (!m) throw new Error(`fake-dom: unsupported selector ${s}`); + const [, tag, idcls, attrPart] = m; + if (tag && tag.toUpperCase() !== this.tagName) return false; + for (const t of idcls.match(/[#.][\w-]+/g) ?? []) { + if (t[0] === '#' && this.id !== t.slice(1)) return false; + if (t[0] === '.' && !this.className.split(/\s+/).includes(t.slice(1))) return false; + } + for (const a of attrPart.match(/\[[^\]]+\]/g) ?? []) { + const am = a.match(/^\[([\w-]+)(?:=["']?([^"'\]]*)["']?)?\]$/); + if (!am) return false; + const have = this.getAttribute(am[1]); + if (have == null) return false; + if (am[2] !== undefined && have !== am[2]) return false; + } + return true; + }); + } +} + +export class FakeDocument { + nodeType = 9; + title = 'Fixture'; + activeElement: FakeElement | null = null; + body: FakeElement; + defaultView: { getComputedStyle: (el: FakeElement) => Record; frameElement: FakeElement | null } = { getComputedStyle: (el: FakeElement) => ({ display: el.attrs.get('style')?.includes('display:contents') ? 'contents' : el.isHiddenInTree ? 'none' : 'block', visibility: 'visible', opacity: '1' }), frameElement: null }; + constructor() { this.body = new FakeElement(this, 'BODY'); } + el(tag: string, attrs: Record = {}, ...kids: Array) { + return new FakeElement(this, tag, attrs).append(...kids); + } + querySelectorAll(sel: string) { return this.body.selfAndDescendants().filter((e) => e.matches(sel)); } + querySelector(sel: string) { return this.querySelectorAll(sel)[0] ?? null; } + getElementById(id: string) { return this.querySelector(`#${id}`); } + elementFromPoint() { return null; } + createTreeWalker() { throw new Error('fake-dom: tree walkers are not supported'); } +} + +/** Give an