From 5dc744bfb08e5fdaad1be4e6b358256d2705adeb Mon Sep 17 00:00:00 2001 From: bzcasper Date: Thu, 24 Sep 2026 17:26:58 -0700 Subject: [PATCH 01/26] Harden browser controller ingress --- .env.example | 7 +++---- README.md | 2 +- SECURITY.md | 3 ++- extension/manifest.json | 3 +-- mcp-server/src/daemon-config.ts | 10 +++++++++- tests/daemon-config.test.ts | 7 ++++++- 6 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 813969f..c6fa5eb 100644 --- a/.env.example +++ b/.env.example @@ -12,10 +12,9 @@ # Port the daemon's WebSocket + HTTP server binds on. The extension connects # here. Override only if something else owns 7225 on your machine. # WS_PORT=7225 - -# Interface to bind. 127.0.0.1 keeps the daemon loopback-only (recommended); -# do NOT set 0.0.0.0 unless you understand the exposure. -# WS_HOST=127.0.0.1 +# The daemon always binds its HTTP/WebSocket control plane to 127.0.0.1. +# There is intentionally no WS_HOST override: remote callers (including n8n) +# need a separately authenticated transport, not a public browser socket. # --- Daemon state -------------------------------------------------------- diff --git a/README.md b/README.md index ec92ec9..12f477d 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ It already has your browser open right there. It just can't see it. ## How it works -Three pieces, all on your machine. Nothing leaves localhost. +Three pieces, all on your machine. Nothing leaves localhost. The daemon control plane is hard-bound to `127.0.0.1`; there is no supported remote/n8n listener. ``` Agent (Cursor / Claude / Windsurf) ── other agents connect too ──┐ diff --git a/SECURITY.md b/SECURITY.md index f7ff3a2..ffc787d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -28,7 +28,8 @@ Report vulnerabilities through: ## Security Model -- **Local-only communication**: WebSocket between extension and server runs on localhost only. +- **Local-only communication**: the daemon's HTTP/WebSocket control plane is hard-bound to IPv4 loopback (`127.0.0.1`), not configurable through `WS_HOST`. WebSocket traffic between the extension and daemon remains local. MCP clients use stdio and then an authenticated local IPC socket; they do not get a TCP listener. +- **Remote callers and n8n**: Browser Controller has no supported remote or n8n ingress protocol. Do not publish port 7225 or change the listener to a wildcard address. A remote integration should run on the same host and reach the MCP client through its approved local process boundary, or be given a separately authenticated transport; browser-control credentials must not be reused as a network API key. - **Origin validation (exact-match on a pinned extension ID)**: the daemon pins the extension's `chrome-extension://` Origin on first contact, then rejects every later request whose Origin is not an exact match. This applies to BOTH the WebSocket upgrade and the HTTP endpoints (`/pair`, `/status`, `/kill`) through one shared gate — a web page and a co-installed hostile extension (which carries its own Origin and cannot forge ours) are both rejected. The browser sets the `Origin` header; it cannot be forged from page JS. - **Token auth on the control plane**: the WebSocket upgrade additionally requires the daemon's auth token (sent out-of-band via `Sec-WebSocket-Protocol` subprotocol, with a `?token=` legacy fallback). The HTTP endpoints do not require the token — they rely on the Origin gate instead, because `/pair` is itself how the token is first obtained (a chicken-and-egg a token gate would break). - **No data exfiltration**: Nothing leaves your machine. No cloud, no telemetry, no analytics. diff --git a/extension/manifest.json b/extension/manifest.json index 9be8d17..aa29be8 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -19,8 +19,7 @@ ], "host_permissions": [ "", - "http://127.0.0.1:7225/*", - "http://localhost:7225/*" + "http://127.0.0.1:7225/*" ], "background": { "service_worker": "background.js", diff --git a/mcp-server/src/daemon-config.ts b/mcp-server/src/daemon-config.ts index 61987a1..d8a0189 100644 --- a/mcp-server/src/daemon-config.ts +++ b/mcp-server/src/daemon-config.ts @@ -48,7 +48,15 @@ export function envInt(name: string, def: number, min = 1, max?: number): number } export const DEFAULT_WS_PORT = envInt('WS_PORT', 7225, 1, 65535); -export const DEFAULT_WS_HOST = process.env.WS_HOST || '127.0.0.1'; +/** + * The daemon's HTTP/WebSocket control plane is intentionally loopback-only. + * + * Do not make this configurable to 0.0.0.0: the extension-facing token gate is + * useful for local pairing, but it is not a replacement for a network trust + * boundary. Remote callers (including n8n) require a separately authenticated + * transport and must not be given direct access to this browser socket. + */ +export const DEFAULT_WS_HOST = '127.0.0.1'; /** * Directory under the user's home where daemon state lives (token, socket, diff --git a/tests/daemon-config.test.ts b/tests/daemon-config.test.ts index ad539c6..d360266 100644 --- a/tests/daemon-config.test.ts +++ b/tests/daemon-config.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, afterEach } from 'vitest'; -import { envInt } from '../mcp-server/src/daemon-config.js'; +import { envInt, DEFAULT_WS_HOST } from '../mcp-server/src/daemon-config.js'; /** * envInt (critical audit #9): bare `parseInt(process.env.X || '…')` yielded @@ -11,6 +11,11 @@ import { envInt } from '../mcp-server/src/daemon-config.js'; const SET_KEYS = ['BC_TEST_INT', 'WS_PORT'] as const; describe('envInt', () => { + it('keeps the control plane loopback-only', () => { + expect(DEFAULT_WS_HOST).toBe('127.0.0.1'); + }); + + afterEach(() => { for (const k of SET_KEYS) delete process.env[k]; }); From 82601953c22af158e6d0704c562065b2beab71c3 Mon Sep 17 00:00:00 2001 From: bzcasper Date: Thu, 24 Sep 2026 17:27:45 -0700 Subject: [PATCH 02/26] Make daemon deployment reproducible and restart-safe --- README.md | 39 ++++++++++++++++++ mcp-server/src/daemon-config.ts | 42 ++++++++++++++++--- mcp-server/src/daemon.ts | 58 +++++++++++++------------- package.json | 4 ++ scripts/daemon.mjs | 73 +++++++++++++++++++++++++++++++++ 5 files changed, 180 insertions(+), 36 deletions(-) create mode 100644 scripts/daemon.mjs diff --git a/README.md b/README.md index 12f477d..827c6e8 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,45 @@ Green dot = you're connected. Your agent can now see your browser. > These secrets prevent any other local process from opening a WebSocket and driving your authenticated browser sessions. To rotate them, stop your MCP clients, delete the folder, and the next run recreates both secrets. See [SECURITY.md](SECURITY.md) for the full threat model. +### Runtime lifecycle (authoritative daemon) + +The **daemon** is the only process that owns the extension-facing runtime. MCP +clients are thin stdio adapters and may start it automatically, but deployment +scripts should use the lifecycle commands below so there is one restart owner. +Do not run a second `daemon.js` or install a launch supervisor that competes for +port `7225`. + +```bash +npm run build +npm run daemon:start # start, or report the already-running PID +npm run daemon:status # JSON health/runtime information +npm run daemon:stop # graceful SIGTERM; removes runtime metadata +npm run daemon:restart # stop, then start; preserves token/enrollment +``` + +The daemon survives browser/Chrome restarts: token, enrollment secret, and +pairing remain in `~/.browser-controller/` (override with `BC_STATE_DIR`), and +the extension reconnects to the same endpoint. The lifecycle wrapper reads the +`daemon.json` PID and is safe to run repeatedly; a stale lock is removed only +when its recorded PID is not alive. + +### Expected endpoints + +- **MCP endpoint:** stdio, launched as `node mcp-server/dist/index.js` (the + standard `mcpServers` command/args form is shown above). +- **Extension runtime:** `ws://127.0.0.1:7225` by default, with HTTP + `/pair`, `/status`, and `/kill?sessionId=...` on the same port. These are + daemon/popup endpoints, not an MCP HTTP transport. +- **MCP client IPC:** `~/.browser-controller/daemon.sock` on Unix or + `\\.\pipe\browser-controller` on Windows. It is internal and token-authenticated. +- **State:** `~/.browser-controller/{daemon.json,daemon.lock,token.json, + enrollment.json,daemon.log}`. `WS_PORT`, `WS_HOST`, and `BC_STATE_DIR` are the + supported configuration overrides. + +This preserves the real Chrome session workflow: no Playwright/headless browser +is launched, and the existing Chrome profile, cookies, logins, and tabs remain +the browser being controlled. + --- ## Using it diff --git a/mcp-server/src/daemon-config.ts b/mcp-server/src/daemon-config.ts index d8a0189..937b80a 100644 --- a/mcp-server/src/daemon-config.ts +++ b/mcp-server/src/daemon-config.ts @@ -58,11 +58,7 @@ export const DEFAULT_WS_PORT = envInt('WS_PORT', 7225, 1, 65535); */ export const DEFAULT_WS_HOST = '127.0.0.1'; -/** - * Directory under the user's home where daemon state lives (token, socket, - * daemon.json, daemon.log). Override with BC_STATE_DIR for isolated tests so - * the suite never touches the real ~/.browser-controller. - */ +/** Directory under the user's home where daemon state lives. */ export const STATE_DIR = process.env.BC_STATE_DIR || path.join(os.homedir(), '.browser-controller'); /** Local IPC socket the daemon listens on (thin clients connect here). */ @@ -71,9 +67,43 @@ export const IPC_SOCKET_PATH = ? '\\\\.\\pipe\\browser-controller' : path.join(STATE_DIR, 'daemon.sock'); -/** Daemon metadata file: { pid, socket, port, startedAt }. */ +/** Daemon metadata file. */ export const DAEMON_INFO_FILE = path.join(STATE_DIR, 'daemon.json'); +/** Daemon ownership lock; a live PID identifies the authoritative runtime. */ +export const DAEMON_LOCK_FILE = path.join(STATE_DIR, 'daemon.lock'); + +/** Acquire the runtime lock, removing only a demonstrably stale lock. */ +export function acquireDaemonLock(): void { + fs.mkdirSync(STATE_DIR, { recursive: true, mode: 0o700 }); + for (;;) { + try { + const fd = fs.openSync(DAEMON_LOCK_FILE, 'wx', 0o600); + fs.writeFileSync(fd, JSON.stringify({ pid: process.pid, startedAt: Date.now() })); + fs.closeSync(fd); + return; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err; + let ownerPid = 0; + try { ownerPid = JSON.parse(fs.readFileSync(DAEMON_LOCK_FILE, 'utf8')).pid ?? 0; } catch { /* stale/corrupt */ } + let alive = false; + if (ownerPid > 0) { + try { process.kill(ownerPid, 0); alive = true; } catch { /* not running */ } + } + if (alive) throw new Error(`Browser Controller daemon lock is held by live process ${ownerPid}`); + try { fs.unlinkSync(DAEMON_LOCK_FILE); } catch { /* raced with cleanup */ } + } + } +} + +export function releaseDaemonLock(): void { + try { + if (JSON.parse(fs.readFileSync(DAEMON_LOCK_FILE, 'utf8')).pid === process.pid) { + fs.unlinkSync(DAEMON_LOCK_FILE); + } + } catch { /* already gone */ } +} + /** Auth token file (3.1): both IPC clients and the extension must present it. */ export const TOKEN_FILE = path.join(STATE_DIR, 'token.json'); diff --git a/mcp-server/src/daemon.ts b/mcp-server/src/daemon.ts index f238431..07a1778 100644 --- a/mcp-server/src/daemon.ts +++ b/mcp-server/src/daemon.ts @@ -28,6 +28,8 @@ import { DEFAULT_WS_HOST, DEFAULT_WS_PORT, DAEMON_INFO_FILE, + acquireDaemonLock, + releaseDaemonLock, ENROLLMENT_FILE, IPC_SOCKET_PATH, STATE_DIR, @@ -116,38 +118,33 @@ class Daemon { async start(): Promise { fs.mkdirSync(STATE_DIR, { recursive: true, mode: 0o700 }); + acquireDaemonLock(); - // 1) extension-facing WS server (task 1.0). The bridge handles the stale- - // port eviction already (lsof replaced by net-based probe in bridge). - await this.bridge.start(); - - // 1b) HTTP endpoints on the SAME port (bridge shares it). The popup uses - // these to auto-pair the token and to show connected agents — without - // them the extension (no fs access) could never read the token. - this.bridge.registerHttpHandler((req, url) => this.handleHttp(req, url)); - - // 2) IPC server for thin MCP clients. - this.ipcServer = this.createIpcServer(); - - // 2b) heartbeat: evict half-open IPC sockets so the popup's "Connected - // Agents" list doesn't accumulate zombies from killed IDE processes. - this.heartbeatTimer = this.startHeartbeat(); - - // 3) write daemon info so thin clients can find / healthcheck us. - this.writeDaemonInfo(); - - console.error(`[${SERVER_NAME}] listening. WS=${DEFAULT_WS_HOST}:${DEFAULT_WS_PORT} IPC=${IPC_SOCKET_PATH}`); - console.error(`[${SERVER_NAME}] auth token at ${TOKEN_FILE}`); - console.error(`[${SERVER_NAME}] enrollment secret stored at ${ENROLLMENT_FILE}`); - - // graceful shutdown - const shutdown = (sig: string) => { - console.error(`[${SERVER_NAME}] ${sig} received, shutting down`); + try { + // 1) extension-facing WS server (task 1.0). + await this.bridge.start(); + this.bridge.registerHttpHandler((req, url) => this.handleHttp(req, url)); + + // 2) IPC server for thin MCP clients. + this.ipcServer = this.createIpcServer(); + this.heartbeatTimer = this.startHeartbeat(); + this.writeDaemonInfo(); + + console.error(`[${SERVER_NAME}] listening. WS=${DEFAULT_WS_HOST}:${DEFAULT_WS_PORT} IPC=${IPC_SOCKET_PATH}`); + console.error(`[${SERVER_NAME}] auth token at ${TOKEN_FILE}`); + console.error(`[${SERVER_NAME}] enrollment secret stored at ${ENROLLMENT_FILE}`); + + const shutdown = (sig: string) => { + console.error(`[${SERVER_NAME}] ${sig} received, shutting down`); + this.stop(); + process.exit(0); + }; + process.on('SIGINT', () => shutdown('SIGINT')); + process.on('SIGTERM', () => shutdown('SIGTERM')); + } catch (err) { this.stop(); - process.exit(0); - }; - process.on('SIGINT', () => shutdown('SIGINT')); - process.on('SIGTERM', () => shutdown('SIGTERM')); + throw err; + } } private createIpcServer(): net.Server { @@ -492,6 +489,7 @@ class Daemon { try { fs.unlinkSync(DAEMON_INFO_FILE); } catch {} + releaseDaemonLock(); } } diff --git a/package.json b/package.json index 65f5d7e..cc9232d 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,10 @@ "build": "tsc -p mcp-server/tsconfig.json", "dev": "tsc -p mcp-server/tsconfig.json --watch", "start": "node mcp-server/dist/index.js", + "daemon:start": "node scripts/daemon.mjs start", + "daemon:stop": "node scripts/daemon.mjs stop", + "daemon:restart": "node scripts/daemon.mjs restart", + "daemon:status": "node scripts/daemon.mjs status", "test": "vitest run", "test:watch": "vitest", "typecheck": "tsc -p mcp-server/tsconfig.json --noEmit", diff --git a/scripts/daemon.mjs b/scripts/daemon.mjs new file mode 100644 index 0000000..1aa4442 --- /dev/null +++ b/scripts/daemon.mjs @@ -0,0 +1,73 @@ +#!/usr/bin/env node +/** + * Explicit lifecycle CLI for the single authoritative Browser Controller daemon. + * MCP clients may still auto-start it, but deployment/restart should use these + * commands so only this process manager is responsible for the runtime. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); +const entry = path.join(root, 'mcp-server', 'dist', 'daemon.js'); +const stateDir = process.env.BC_STATE_DIR || path.join(os.homedir(), '.browser-controller'); +const infoFile = path.join(stateDir, 'daemon.json'); +const lockFile = path.join(stateDir, 'daemon.lock'); +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + +function readInfo() { + try { return JSON.parse(fs.readFileSync(infoFile, 'utf8')); } catch { return null; } +} +function live(info) { + if (!info?.pid) return false; + try { process.kill(info.pid, 0); return true; } catch { return false; } +} +async function waitForStart(timeoutMs = 8000) { + const end = Date.now() + timeoutMs; + while (Date.now() < end) { + const info = readInfo(); + if (live(info)) return info; + await sleep(100); + } + throw new Error('daemon did not become ready; see daemon.log'); +} +async function start() { + if (!fs.existsSync(entry)) throw new Error('daemon build missing; run `npm run build` first'); + const existing = readInfo(); + if (live(existing)) { + console.log(`daemon already running (pid ${existing.pid}) on ${existing.host}:${existing.port}`); + return; + } + fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); + const log = fs.openSync(path.join(stateDir, 'daemon.log'), 'a', 0o600); + const child = spawn(process.execPath, [entry], { detached: true, stdio: ['ignore', log, log], env: process.env }); + child.unref(); + const info = await waitForStart(); + console.log(`daemon started (pid ${info.pid})`); + console.log(`MCP stdio entry: ${path.join(root, 'mcp-server', 'dist', 'index.js')}`); + console.log(`daemon endpoints: http/ws://${info.host}:${info.port} (WS + /pair /status /kill); IPC ${info.socket}`); +} +async function stop() { + const info = readInfo(); + if (!live(info)) { + for (const file of [infoFile, lockFile]) { try { fs.unlinkSync(file); } catch {} } + console.log('daemon is not running'); + return; + } + process.kill(info.pid, 'SIGTERM'); + const end = Date.now() + 5000; + while (Date.now() < end && live(info)) await sleep(100); + if (live(info)) throw new Error(`daemon pid ${info.pid} did not stop`); + console.log('daemon stopped'); +} +const command = process.argv[2] || 'status'; +if (command === 'start') await start(); +else if (command === 'stop') await stop(); +else if (command === 'restart') { await stop(); await start(); } +else if (command === 'status') { + const info = readInfo(); + if (!live(info)) { console.log('daemon is not running'); process.exitCode = 1; } + else console.log(JSON.stringify({ ...info, running: true }, null, 2)); +} else throw new Error(`unknown command: ${command}`); From 22690fc504fe1d62ab587fb8369aefb36a9a5430 Mon Sep 17 00:00:00 2001 From: bzcasper Date: Thu, 24 Sep 2026 17:29:15 -0700 Subject: [PATCH 03/26] test: add real Chrome reconnect smoke test --- package.json | 1 + scripts/smoke-real-chrome.mjs | 166 ++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 scripts/smoke-real-chrome.mjs diff --git a/package.json b/package.json index cc9232d..3ed1791 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "daemon:status": "node scripts/daemon.mjs status", "test": "vitest run", "test:watch": "vitest", + "smoke:real-chrome": "node scripts/smoke-real-chrome.mjs", "typecheck": "tsc -p mcp-server/tsconfig.json --noEmit", "setup:cursor": "node agent-config/setup.mjs cursor", "setup:claude": "node agent-config/setup.mjs claude", diff --git a/scripts/smoke-real-chrome.mjs b/scripts/smoke-real-chrome.mjs new file mode 100644 index 0000000..9bb8165 --- /dev/null +++ b/scripts/smoke-real-chrome.mjs @@ -0,0 +1,166 @@ +#!/usr/bin/env node +/** + * Real-profile smoke test for extension reconnect and core tools. + * Requires the unpacked extension enrolled in the user's normal Chrome profile. + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import net from 'node:net'; +import http from 'node:http'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const STATE = path.join(os.homedir(), '.browser-controller'); +const SOCKET = path.join(STATE, 'daemon.sock'); +const TOKEN = JSON.parse(fs.readFileSync(path.join(STATE, 'token.json'), 'utf8')).token; +const DAEMON = path.join(ROOT, 'mcp-server', 'dist', 'daemon.js'); +const UPLOAD = path.join(STATE, 'smoke-upload.txt'); +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +const log = (...args) => console.log('[smoke]', ...args); +let server, daemon, tabId, client, startedByTest = false; + +function textOf(result) { + if (result?.content) return result.content.map(x => x.text || '').join('\n'); + return typeof result === 'string' ? result : JSON.stringify(result); +} +function parseText(result) { try { return JSON.parse(textOf(result)); } catch { return null; } } +function refsIn(result, role) { + const refs = []; + const walk = node => { + if (!node || typeof node !== 'object') return; + if (node.role === role && node.ref) refs.push(node.ref); + for (const child of node.children || []) walk(child); + if (node.tree) walk(node.tree); + }; + walk(parseText(result)); + return refs; +} +function assert(value, message) { if (!value) throw new Error(message); } +function daemonAlive() { + return new Promise(resolve => { + const s = net.createConnection(SOCKET); + s.once('connect', () => { s.destroy(); resolve(true); }); + s.once('error', () => resolve(false)); + }); +} +function startDaemon() { + if (!fs.existsSync(DAEMON)) throw new Error(`Run npm run build first (missing ${DAEMON})`); + daemon = spawn(process.execPath, [DAEMON], { cwd: ROOT, stdio: 'ignore', detached: false }); + daemon.on('error', error => console.error('[daemon]', error.message)); +} +async function waitForDaemon() { + for (let i = 0; i < 300; i++) { if (await daemonAlive()) return; await sleep(100); } + throw new Error(`daemon did not become ready: ${SOCKET}`); +} +async function stopDaemon() { + if (!daemon) { + try { + const info = JSON.parse(fs.readFileSync(path.join(STATE, 'daemon.json'), 'utf8')); + daemon = { kill: () => process.kill(info.pid, 'SIGTERM'), once: (_event, done) => done() }; + } catch { return; } + } + const exited = new Promise(resolve => daemon.once('exit', resolve)); + daemon.kill('SIGTERM'); + await Promise.race([exited, sleep(1500)]); + daemon = undefined; + for (let i = 0; i < 30 && await daemonAlive(); i++) await sleep(100); +} +async function call(tool, params) { return client.call(tool, params); } + + +async function connect() { + const socket = net.createConnection(SOCKET); + socket.setEncoding('utf8'); + let buffer = '', sequence = 0; + const pending = new Map(); + let readyResolve, readyReject; + const ready = new Promise((resolve, reject) => { readyResolve = resolve; readyReject = reject; }); + socket.on('connect', () => socket.write(JSON.stringify({ kind: 'hello', token: TOKEN, agentName: 'RealChromeSmoke' }) + '\n')); + socket.on('data', chunk => { + buffer += chunk; + let end; + while ((end = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, end).trim(); buffer = buffer.slice(end + 1); + if (!line) continue; + const msg = JSON.parse(line); + if (msg.kind === 'welcome') readyResolve(msg); + if (msg.kind === 'ping') socket.write(JSON.stringify({ kind: 'pong' }) + '\n'); + if (msg.kind === 'result' && pending.has(msg.id)) { + const done = pending.get(msg.id); pending.delete(msg.id); done(msg); + } + } + }); + socket.on('error', error => { readyReject(error); }); + await ready; + return { + socket, + call(tool, params) { + return new Promise((resolve, reject) => { + const id = String(++sequence); + const timer = setTimeout(() => { pending.delete(id); reject(new Error(`${tool} timed out`)); }, 30000); + pending.set(id, msg => { clearTimeout(timer); msg.success ? resolve(msg.result) : reject(new Error(msg.error || tool)); }); + socket.write(JSON.stringify({ kind: 'call', id, tool, params }) + '\n'); + }); + }, + close() { socket.destroy(); }, + }; +} + +async function main() { + fs.writeFileSync(UPLOAD, 'browser-controller smoke upload\n'); + server = http.createServer((req, res) => { + if (req.url.startsWith('/api')) { res.writeHead(200, { 'content-type': 'application/json' }); res.end('{"ok":true}'); return; } + res.writeHead(200, { 'content-type': 'text/html' }); + res.end(`Browser Controller Smoke

Smoke page

not clicked

`); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const page = `http://127.0.0.1:${server.address().port}/`; + if (await daemonAlive()) log('daemon already running; restarting it during test'); else { startDaemon(); startedByTest = true; } + await waitForDaemon(); + client = await connect(); + await call('browser_tabs', { action: 'list' }); + await stopDaemon(); + if (!(await daemonAlive())) startDaemon(); + await waitForDaemon(); client.close(); client = await connect(); + assert(Array.isArray((await call('browser_tabs', { action: 'list' }))?.tabs), 'tab list failed after daemon restart'); + log('daemon restart/reconnect verified'); + + const created = await call('browser_tabs', { action: 'create', url: page }); + tabId = created.tabId ?? created.tabs?.[0]?.id; + assert(Number.isInteger(tabId), `could not determine created tabId: ${JSON.stringify(created)}`); + assert(Array.isArray((await call('browser_tabs', { action: 'list' }))?.tabs), 'tab list failed'); + const snapshot = await call('browser_snapshot', { tabId, compact: false }); + assert(textOf(snapshot).includes('Smoke page'), 'snapshot did not contain smoke page'); + const buttonRef = refsIn(snapshot, 'button')[0]; + const inputRef = refsIn(snapshot, 'textbox')[0]; + assert(buttonRef && inputRef, 'snapshot did not provide button/textbox refs'); + log(`snapshot ok (tab ${tabId})`); + await call('browser_click', { tabId, ref: buttonRef }); + const clicked = await call('browser_evaluate', { tabId, expression: 'document.querySelector("#result").textContent' }); + assert(textOf(clicked).includes('clicked'), `click verification failed: ${textOf(clicked)}`); + await call('browser_type', { tabId, ref: inputRef, text: 'typed by smoke', clear: true }); + const value = await call('browser_evaluate', { tabId, expression: 'document.querySelector("#text").value' }); + assert(textOf(value).includes('typed by smoke'), `type verification failed: ${textOf(value)}`); + log('click and type ok'); + const consoleResult = await call('browser_console', { tabId }); + assert(textOf(consoleResult).includes('SMOKE_CONSOLE_MARKER'), 'console marker was not captured'); + const networkResult = await call('browser_network', { tabId, filter: '/api' }); + assert(textOf(networkResult).includes('/api'), 'network request was not captured'); + log('console and network ok'); + await call('browser_upload_file', { tabId, selector: '#file', filePath: UPLOAD }); + const files = await call('browser_evaluate', { tabId, expression: 'document.querySelector("#file").files[0]?.name' }); + assert(textOf(files).includes('smoke-upload.txt'), `upload verification failed: ${textOf(files)}`); + log('upload ok'); + await call('browser_tabs', { action: 'close', tabId }); tabId = undefined; + log('PASS: real Chrome profile, reconnect, and all requested tools verified'); +} + +main().catch(error => { console.error('[smoke] FAIL:', error.message); process.exitCode = 1; }).finally(async () => { + if (client) client.close(); + if (tabId !== undefined) { try { await call('browser_tabs', { action: 'close', tabId }); } catch {} } + if (server) await new Promise(resolve => server.close(resolve)); + if (startedByTest) await stopDaemon(); + try { fs.unlinkSync(UPLOAD); } catch {} +}); From 829e72f169607ebd0beca611bf8c2092ee281c24 Mon Sep 17 00:00:00 2001 From: bzcasper Date: Thu, 24 Sep 2026 17:49:34 -0700 Subject: [PATCH 04/26] fix: capture page console from main world --- extension/content.js | 57 +++++++------------------------ extension/manifest.json | 19 +++++++++-- extension/page-console.js | 45 ++++++++++++++++++++++++ tests/page-console-bridge.test.ts | 21 ++++++++++++ 4 files changed, 96 insertions(+), 46 deletions(-) create mode 100644 extension/page-console.js create mode 100644 tests/page-console-bridge.test.ts diff --git a/extension/content.js b/extension/content.js index 8e38748..db41fff 100644 --- a/extension/content.js +++ b/extension/content.js @@ -3,49 +3,18 @@ if (window.__browserControllerInjected) return; window.__browserControllerInjected = true; - const orig = { - log: console.log, - warn: console.warn, - error: console.error, - info: console.info, - debug: console.debug, - }; - - function capture(level, ...args) { - orig[level].apply(console, args); - const text = args.map(a => { - if (typeof a === 'object') { try { return JSON.stringify(a); } catch { return String(a); } } - return String(a); - }).join(' '); - // Drop well-known benign warnings so they don't spam the captured log: - // - ResizeObserver loop (element resized during its own RO callback) - // - "message channel closed before a response was received" — happens on - // heavy SPA pages (Meta.ai etc.) when an async onMessage sender closes - // before the listener replies. Harmless; nothing our code can do about it. - if (text.indexOf('ResizeObserver loop') !== -1) return; - if (text.indexOf('message channel closed before a response was received') !== -1) return; - // Cap the entry BEFORE shipping it: the per-tab buffer only caps the ENTRY - // COUNT (200), so one console.log(hugeString) would otherwise pin the full - // payload in service-worker memory and return all of it to the agent. - const capped = text.length > 2000 ? text.slice(0, 2000) + '…[truncated]' : text; - try { chrome.runtime.sendMessage({ type: 'console', level, text: capped }); } catch {} - } - - console.log = (...a) => capture('log', ...a); - console.warn = (...a) => capture('warn', ...a); - console.error = (...a) => capture('error', ...a); - console.info = (...a) => capture('info', ...a); - console.debug = (...a) => capture('debug', ...a); - - window.addEventListener('error', (e) => { - // Silence the well-known ResizeObserver loop warning: it's a benign browser - // notice (element resized during its own observation callback), not a real - // error. Every RO-based UI triggers it; capturing it just spams the console. - if (e && typeof e.message === 'string' && e.message.indexOf('ResizeObserver loop') !== -1) return; - capture('error', `Uncaught: ${e.message} at ${e.filename}:${e.lineno}`); - }); - - window.addEventListener('unhandledrejection', (e) => { - capture('error', `Unhandled rejection: ${e.reason}`); + window.addEventListener('message', (event) => { + if (event.source !== window) return; + const data = event.data; + if (!data || data.__browserControllerConsole !== true) return; + if (!['log', 'warn', 'error', 'info', 'debug'].includes(data.level)) return; + if (typeof data.text !== 'string') return; + try { + chrome.runtime.sendMessage({ + type: 'console', + level: data.level, + text: data.text.length > 2000 ? data.text.slice(0, 2000) + '…[truncated]' : data.text, + }); + } catch {} }); })(); diff --git a/extension/manifest.json b/extension/manifest.json index aa29be8..2c0dcde 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -27,8 +27,23 @@ }, "content_scripts": [ { - "matches": [""], - "js": ["content.js"], + "matches": [ + "" + ], + "js": [ + "page-console.js" + ], + "run_at": "document_start", + "all_frames": true, + "world": "MAIN" + }, + { + "matches": [ + "" + ], + "js": [ + "content.js" + ], "run_at": "document_start", "all_frames": true } diff --git a/extension/page-console.js b/extension/page-console.js new file mode 100644 index 0000000..b61c58c --- /dev/null +++ b/extension/page-console.js @@ -0,0 +1,45 @@ +(function () { + 'use strict'; + if (window.__browserControllerPageConsoleInjected) return; + window.__browserControllerPageConsoleInjected = true; + + const levels = ['log', 'warn', 'error', 'info', 'debug']; + const original = Object.fromEntries(levels.map((level) => [level, console[level]])); + + function serialize(args) { + const text = args.map((value) => { + if (typeof value === 'object' && value !== null) { + try { return JSON.stringify(value); } catch { return String(value); } + } + return String(value); + }).join(' '); + return text.length > 2000 ? text.slice(0, 2000) + '…[truncated]' : text; + } + + function emit(level, args) { + const text = serialize(args); + if (text.includes('ResizeObserver loop')) return; + if (text.includes('message channel closed before a response was received')) return; + window.postMessage({ + __browserControllerConsole: true, + level, + text, + }, window.location.origin === 'null' ? '*' : window.location.origin); + } + + for (const level of levels) { + console[level] = (...args) => { + original[level].apply(console, args); + try { emit(level, args); } catch {} + }; + } + + window.addEventListener('error', (event) => { + if (typeof event?.message === 'string' && event.message.includes('ResizeObserver loop')) return; + try { emit('error', [`Uncaught: ${event?.message || 'Unknown error'} at ${event?.filename || ''}:${event?.lineno || 0}`]); } catch {} + }); + + window.addEventListener('unhandledrejection', (event) => { + try { emit('error', [`Unhandled rejection: ${String(event?.reason)}`]); } catch {} + }); +})(); diff --git a/tests/page-console-bridge.test.ts b/tests/page-console-bridge.test.ts new file mode 100644 index 0000000..627d27a --- /dev/null +++ b/tests/page-console-bridge.test.ts @@ -0,0 +1,21 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const root = path.resolve(import.meta.dirname, '..'); + +describe('page console bridge', () => { + it('loads a MAIN-world page bridge before the isolated relay', () => { + const manifest = JSON.parse(fs.readFileSync(path.join(root, 'extension/manifest.json'), 'utf8')); + expect(manifest.content_scripts[0]).toMatchObject({ js: ['page-console.js'], world: 'MAIN', run_at: 'document_start' }); + expect(manifest.content_scripts[1]).toMatchObject({ js: ['content.js'], run_at: 'document_start' }); + }); + + it('uses a tagged window message contract and validates it in the relay', () => { + const page = fs.readFileSync(path.join(root, 'extension/page-console.js'), 'utf8'); + const relay = fs.readFileSync(path.join(root, 'extension/content.js'), 'utf8'); + expect(page).toContain('__browserControllerConsole: true'); + expect(relay).toContain('data.__browserControllerConsole !== true'); + expect(relay).toContain('event.source !== window'); + }); +}); From 5e4d829bca81860dc415c8cd4e9ab5ce3ce3082d Mon Sep 17 00:00:00 2001 From: bzcasper Date: Thu, 24 Sep 2026 17:53:07 -0700 Subject: [PATCH 05/26] fix: preserve systemd daemon ownership in smoke test --- scripts/smoke-real-chrome.mjs | 55 +++++++++++++++++++++++++++++++---- tests/smoke-lifecycle.test.ts | 13 +++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 tests/smoke-lifecycle.test.ts diff --git a/scripts/smoke-real-chrome.mjs b/scripts/smoke-real-chrome.mjs index 9bb8165..e06f072 100644 --- a/scripts/smoke-real-chrome.mjs +++ b/scripts/smoke-real-chrome.mjs @@ -8,7 +8,8 @@ import os from 'node:os'; import path from 'node:path'; import net from 'node:net'; import http from 'node:http'; -import { spawn } from 'node:child_process'; +import { spawn, execFile } from 'node:child_process'; +import { promisify } from 'node:util'; import { fileURLToPath } from 'node:url'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); @@ -18,6 +19,15 @@ const TOKEN = JSON.parse(fs.readFileSync(path.join(STATE, 'token.json'), 'utf8') const DAEMON = path.join(ROOT, 'mcp-server', 'dist', 'daemon.js'); const UPLOAD = path.join(STATE, 'smoke-upload.txt'); const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +const execFileAsync = promisify(execFile); +const SYSTEMD_SERVICE = 'browser-controller-daemon.service'; +const userSystemdEnv = { + ...process.env, + XDG_RUNTIME_DIR: process.env.XDG_RUNTIME_DIR || `/run/user/${process.getuid?.() ?? 1000}`, +}; +if (!userSystemdEnv.DBUS_SESSION_BUS_ADDRESS) { + userSystemdEnv.DBUS_SESSION_BUS_ADDRESS = `unix:path=${userSystemdEnv.XDG_RUNTIME_DIR}/bus`; +} const log = (...args) => console.log('[smoke]', ...args); let server, daemon, tabId, client, startedByTest = false; @@ -69,6 +79,25 @@ async function stopDaemon() { } async function call(tool, params) { return client.call(tool, params); } +async function hasManagedDaemon() { + try { + const { stdout } = await execFileAsync('systemctl', ['--user', 'show', '--property=LoadState', '--value', SYSTEMD_SERVICE], { env: userSystemdEnv }); + return stdout.trim() === 'loaded'; + } catch { + return false; + } +} + +async function restartManagedDaemon() { + await execFileAsync('systemctl', ['--user', 'restart', SYSTEMD_SERVICE], { env: userSystemdEnv }); + await waitForDaemon(); +} + +async function startManagedDaemon() { + await execFileAsync('systemctl', ['--user', 'start', SYSTEMD_SERVICE], { env: userSystemdEnv }); + await waitForDaemon(); +} + async function connect() { const socket = net.createConnection(SOCKET); @@ -117,13 +146,29 @@ async function main() { }); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); const page = `http://127.0.0.1:${server.address().port}/`; - if (await daemonAlive()) log('daemon already running; restarting it during test'); else { startDaemon(); startedByTest = true; } + const managedDaemon = await hasManagedDaemon(); + const wasRunning = await daemonAlive(); + if (managedDaemon) { + if (!wasRunning) await startManagedDaemon(); + log(wasRunning ? 'systemd daemon already running; restarting service during test' : 'started systemd daemon for test'); + } else if (!wasRunning) { + startDaemon(); + startedByTest = true; + } else { + log('unmanaged daemon already running; restarting it during test'); + } await waitForDaemon(); client = await connect(); await call('browser_tabs', { action: 'list' }); - await stopDaemon(); - if (!(await daemonAlive())) startDaemon(); - await waitForDaemon(); client.close(); client = await connect(); + client.close(); + if (managedDaemon) { + await restartManagedDaemon(); + } else { + await stopDaemon(); + if (!(await daemonAlive())) startDaemon(); + await waitForDaemon(); + } + client = await connect(); assert(Array.isArray((await call('browser_tabs', { action: 'list' }))?.tabs), 'tab list failed after daemon restart'); log('daemon restart/reconnect verified'); diff --git a/tests/smoke-lifecycle.test.ts b/tests/smoke-lifecycle.test.ts new file mode 100644 index 0000000..590277e --- /dev/null +++ b/tests/smoke-lifecycle.test.ts @@ -0,0 +1,13 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = fs.readFileSync(path.resolve(import.meta.dirname, '../scripts/smoke-real-chrome.mjs'), 'utf8'); + +describe('real Chrome smoke lifecycle', () => { + it('uses the systemd user service when it owns the daemon', () => { + expect(script).toContain("SYSTEMD_SERVICE = 'browser-controller-daemon.service'"); + expect(script).toContain("['--user', 'restart', SYSTEMD_SERVICE]"); + expect(script).toContain('if (managedDaemon)'); + }); +}); From 8d2831105b150886cff157149fb8b05ac6d458f8 Mon Sep 17 00:00:00 2001 From: bzcasper Date: Thu, 24 Sep 2026 17:55:29 -0700 Subject: [PATCH 06/26] fix: separate managed daemon lifecycle from bridge --- .../systemd/browser-controller-bridge.service | 17 +++++++++++++++ .../systemd/browser-controller-daemon.service | 15 +++++++++++++ mcp-server/src/index.ts | 19 ++++++++++++++--- tests/managed-daemon-mode.test.ts | 21 +++++++++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 deploy/systemd/browser-controller-bridge.service create mode 100644 deploy/systemd/browser-controller-daemon.service create mode 100644 tests/managed-daemon-mode.test.ts diff --git a/deploy/systemd/browser-controller-bridge.service b/deploy/systemd/browser-controller-bridge.service new file mode 100644 index 0000000..1cab1a5 --- /dev/null +++ b/deploy/systemd/browser-controller-bridge.service @@ -0,0 +1,17 @@ +[Unit] +Description=Browser Controller MCP stdio-to-SSE bridge (for n8n/MCPHub) +Requires=browser-controller-daemon.service +After=network.target browser-controller-daemon.service + +[Service] +Environment=PATH=%h/.nvm/versions/node/v24.19.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +Environment=BROWSER_CONTROLLER_DAEMON_MODE=connect +ExecStart=%h/.local/bin/mcp-proxy node %h/projects/browser-controller/mcp-server/dist/index.js --port 9090 --host 0.0.0.0 -e MCP_AGENT_NAME n8n-bridge -e BROWSER_CONTROLLER_DAEMON_MODE connect +Restart=always +RestartSec=3 +StandardOutput=append:%h/.browser-controller/mcp-bridge.log +StandardError=append:%h/.browser-controller/mcp-bridge.log +NoNewPrivileges=true + +[Install] +WantedBy=default.target diff --git a/deploy/systemd/browser-controller-daemon.service b/deploy/systemd/browser-controller-daemon.service new file mode 100644 index 0000000..bc3f0ad --- /dev/null +++ b/deploy/systemd/browser-controller-daemon.service @@ -0,0 +1,15 @@ +[Unit] +Description=Browser Controller shared daemon +After=graphical-session.target + +[Service] +Type=simple +WorkingDirectory=%h/projects/browser-controller +Environment=HOME=%h +ExecStart=%h/.nvm/versions/node/v24.19.0/bin/node %h/projects/browser-controller/mcp-server/dist/daemon.js +Restart=always +RestartSec=2 +NoNewPrivileges=true + +[Install] +WantedBy=default.target diff --git a/mcp-server/src/index.ts b/mcp-server/src/index.ts index 268d5a5..3fb91d4 100644 --- a/mcp-server/src/index.ts +++ b/mcp-server/src/index.ts @@ -69,6 +69,8 @@ const SERVER_VERSION = APP_VERSION; const DAEMON_STARTUP_MS = 8_000; const CONNECT_RETRY_MS = 250; const MAX_CONNECT_TRIES = 32; // ~8s +const DAEMON_MODE = (process.env.BROWSER_CONTROLLER_DAEMON_MODE ?? 'spawn').trim().toLowerCase(); +const CONNECT_ONLY_DAEMON = DAEMON_MODE === 'connect' || DAEMON_MODE === 'managed'; /** * Daemon connection: a line-delimited JSON socket speaking the IPC protocol @@ -395,10 +397,18 @@ async function main(): Promise { console.error(`[${SERVER_NAME}] enrollment secret (enter in the popup once): ${enrollment}`); } - // 2) ensure daemon is up + // 2) ensure daemon is up. Desktop/stdio clients default to self-managed + // spawning. Long-running supervisors (systemd, launchd, containers) can set + // BROWSER_CONTROLLER_DAEMON_MODE=connect so this process never competes for + // daemon lifecycle ownership. if (!(await daemonLooksAlive())) { - spawnDaemon(); - await waitForDaemon(); + if (CONNECT_ONLY_DAEMON) { + console.error(`[${SERVER_NAME}] daemon mode=connect; waiting for managed daemon`); + await waitForDaemon(); + } else { + spawnDaemon(); + await waitForDaemon(); + } } // 3) connect to daemon. Resolve a human-readable agent name for the popup UI. @@ -428,6 +438,9 @@ async function main(): Promise { try { if (await daemonLooksAlive()) { console.error(`[${SERVER_NAME}] daemon connection lost — reconnecting to live daemon`); + } else if (CONNECT_ONLY_DAEMON) { + console.error(`[${SERVER_NAME}] daemon connection lost — waiting for managed daemon`); + await waitForDaemon(); } else { console.error(`[${SERVER_NAME}] daemon connection lost — attempting one respawn`); spawnDaemon(); diff --git a/tests/managed-daemon-mode.test.ts b/tests/managed-daemon-mode.test.ts new file mode 100644 index 0000000..1632a19 --- /dev/null +++ b/tests/managed-daemon-mode.test.ts @@ -0,0 +1,21 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const root = path.resolve(import.meta.dirname, '..'); + +describe('managed daemon deployment contract', () => { + it('supports a connect-only thin-client mode', () => { + const source = fs.readFileSync(path.join(root, 'mcp-server/src/index.ts'), 'utf8'); + expect(source).toContain('BROWSER_CONTROLLER_DAEMON_MODE'); + expect(source).toContain('CONNECT_ONLY_DAEMON'); + expect(source).toContain('waiting for managed daemon'); + }); + + it('configures the bridge to depend on and connect to the systemd daemon', () => { + const unit = fs.readFileSync(path.join(root, 'deploy/systemd/browser-controller-bridge.service'), 'utf8'); + expect(unit).toContain('Requires=browser-controller-daemon.service'); + expect(unit).toContain('After=network.target browser-controller-daemon.service'); + expect(unit).toContain('BROWSER_CONTROLLER_DAEMON_MODE=connect'); + }); +}); From 8a57f17ef08af2f471a927171cd0100197903632 Mon Sep 17 00:00:00 2001 From: noiemany Date: Wed, 30 Sep 2026 01:54:14 +0300 Subject: [PATCH 07/26] fix(errors): name the selector when it matches nothing instead of "Element undefined" A selector that finds nothing (usually the page navigated or posted back) was reported as "Element undefined is gone from the DOM (feed virtualized)" plus a useless re-snapshot. It now names the selector and the page URL. Co-Authored-By: Claude Opus 5.5 --- extension/handlers/interaction.js | 15 ++++++++++----- extension/lib/trusted-input.js | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/extension/handlers/interaction.js b/extension/handlers/interaction.js index af074db..cc11fd1 100644 --- a/extension/handlers/interaction.js +++ b/extension/handlers/interaction.js @@ -11,7 +11,12 @@ import { trustedSender, locateTarget, releaseShield, cdpClickAt, cdpKeyPress, cd export { handleDialog, handleDrag, handleFillForm } from './interaction-advanced.js'; /** Shared REF_GONE recovery: re-snapshot and hand fresh refs back (no auto-retry). */ -async function refGone(tabId, res, ref) { +async function refGone(tabId, res, ref, selector) { + // A selector that matches nothing is usually the wrong page (navigation, + // postback), not a virtualized feed — say which locator failed. + if (!(res._ref || ref) && selector) { + return { success: false, error: `No element matches selector ${selector} on the current page (${res.url || 'navigated?'}).` }; + } const fresh = await autoReSnapshot(tabId); return { success: false, @@ -36,7 +41,7 @@ export async function handleClick(params) { const send = await trustedSender(tabId, trusted); if (send && BUTTONS.has(button)) { const loc = await locateTarget(tabId, { ref, selector, fb }); - if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref); + if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref, selector); if (loc?.success && loc.visible) { try { await cdpClickAt(send, loc.x, loc.y, { button, clickCount: doubleClick ? 2 : 1 }); @@ -133,7 +138,7 @@ export async function handleClick(params) { // Auto-re-snapshot and embed fresh refs so the agent retries in ONE step. // We do NOT auto-retry the click: it's non-idempotent and the element that // re-appears may be a different post after the scroll shifted the feed. - if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref); + if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref, selector); return res; } @@ -152,7 +157,7 @@ export async function handleType(params) { const send = await trustedSender(tabId, trusted); if (send) { const loc = await locateTarget(tabId, { ref, selector, fb, mode: clear ? 'clear' : 'focus' }); - if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref); + if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref, selector); if (loc?.success && (loc.focused || loc.visible)) { let after; try { @@ -235,7 +240,7 @@ export async function handleType(params) { // Virtualization recovery (same as click): type target is gone, so // auto-re-snapshot and embed fresh refs. No auto-retry (non-idempotent). - if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref); + if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref, selector); return res; } diff --git a/extension/lib/trusted-input.js b/extension/lib/trusted-input.js index 1c8fd72..b6d7d22 100644 --- a/extension/lib/trusted-input.js +++ b/extension/lib/trusted-input.js @@ -132,7 +132,7 @@ function pageLocate(ref, sel, fb, mode) { const resolveFallback = (globalThis.__browserControllerFallbackRuntime || {}).resolveFallback || null; if (!el && fb && resolveFallback) { el = resolveFallback(fb); if (el) via = 'fallback'; } if (!el && mode === 'active') { el = document.activeElement; via = 'active'; } - if (!el) return { success: false, error: 'REF_GONE', _ref: ref }; + if (!el) return { success: false, error: 'REF_GONE', _ref: ref, url: location.href }; // Agent input pass-through for the lock shield (see overlay.js). window.__bcAgentInputUntil = Date.now() + 8000; From a268db02d02b8a92ff64ff117f80317af79eaf94 Mon Sep 17 00:00:00 2001 From: noiemany Date: Wed, 30 Sep 2026 03:46:20 +0300 Subject: [PATCH 08/26] fix(targeting): one shared resolver for every tool, find/click_text rewrite, shadow DOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs were looked up by a data-mcp-ref attribute nothing writes any more, so every ref action fell through to the smart-selector fallback, whose first step returned the FIRST querySelector match — clicks "succeeded" on the wrong element, and hover/select/drag/fill_form/upload/press_key could not use snapshot refs at all. - lib/page-dom.js: page runtime (install-once, execDom) with ref registry → first VISIBLE selector match across open+closed shadow roots and same-origin iframes → verified fallback (unique, or nth among exact role/tag/name twins; ambiguous = REF_GONE instead of a guess). Composed hit-testing for occlusion, flat-tree text, CSS-independent names. - All tools use it: click, type, press_key, hover, select, drag, fill_form (select also matches option labels), upload_file (CDP objectId, works in shadow roots/frames), scroll. - browser_find: tokenized scoring over names/labels/attributes, role words ("search input", "... button"), shadow DOM, wrapper/echo suppression, sort before limit, optional role filter, hint when empty. - browser_click_text: case-insensitive name/composed-text match (CSS uppercase no longer breaks exact), shadow DOM, clicks the owning control with a trusted CDP click. - browser_wait: any visible match (not just the first), plus text and urlIncludes conditions. - browser_text: shadow-DOM content, mode:"article", offset paging, 100k cap. - browser_snapshot: flat tree (closed shadow roots, slots, display:contents), short refs, no isNew on the first snapshot, path-only same-origin hrefs, landmarks named by labels only, filter/depth/ref/maxChars (default 20k). - tests: page-dom resolver/find/click_text/wait/snapshot/text suites on a small fake DOM. Co-Authored-By: Claude Opus 5.5 --- extension/handlers/cdp.js | 81 ++-- extension/handlers/inspection.js | 512 ++++++++++++--------- extension/handlers/interaction-advanced.js | 78 ++-- extension/handlers/interaction.js | 273 +++++------ extension/lib/page-dom.js | 433 +++++++++++++++++ extension/lib/page-exec.js | 15 + extension/lib/trusted-input.js | 85 ++-- mcp-server/src/tools/click-text.ts | 7 +- mcp-server/src/tools/find.ts | 3 +- mcp-server/src/tools/snapshot.ts | 6 +- mcp-server/src/tools/text.ts | 6 +- mcp-server/src/tools/wait.ts | 6 +- tests/extension-agent-api.test.ts | 4 +- tests/extension-router.test.ts | 4 +- tests/helpers/fake-dom.ts | 165 +++++++ tests/legacy-ref-registry.test.ts | 7 +- tests/page-dom.test.ts | 217 +++++++++ 17 files changed, 1392 insertions(+), 510 deletions(-) create mode 100644 extension/lib/page-dom.js create mode 100644 tests/helpers/fake-dom.ts create mode 100644 tests/page-dom.test.ts diff --git a/extension/handlers/cdp.js b/extension/handlers/cdp.js index cf1a3d7..408f74b 100644 --- a/extension/handlers/cdp.js +++ b/extension/handlers/cdp.js @@ -3,7 +3,7 @@ * upload_file — the two tools that cannot be implemented with * chrome.scripting (CSP bypass / DOM.setFileInputFiles). */ -import { resolveTab, safeExec } from '../lib/page-exec.js'; +import { resolveTab, execDom, getFallback } from '../lib/page-exec.js'; import { MAX_RESULT_CHARS } from '../lib/state.js'; import { ensureCdp } from '../lib/cdp-session.js'; @@ -67,71 +67,76 @@ export async function handleRunAction(params, _sessionId, _agentName, signal) { } } +/** Main-world expression returning the node marked data-bc-upload=token (pierces open shadow roots / same-origin frames). */ +export function findMarkedExpression(token) { + return `(() => { const s = '[data-bc-upload="${token}"]'; + const q = (root, d) => { const hit = root.querySelector(s); if (hit || d > 6) return hit; + for (const el of root.querySelectorAll('*')) { + if (el.shadowRoot) { const h = q(el.shadowRoot, d + 1); if (h) return h; } + if (el.tagName === 'IFRAME') { try { const h = el.contentDocument && q(el.contentDocument, d + 1); if (h) return h; } catch (e) {} } + } + return null; }; + return q(document, 0); })()`; +} + export async function handleUploadFile(params) { const { tabId, ref, selector, filePath, files: fileList } = params; const tab = await resolveTab(tabId); const filePaths = fileList || (filePath ? [filePath] : []); if (filePaths.length === 0) throw new Error('filePath or files required'); - let sel = 'input[type="file"]'; - if (ref) sel = `[data-mcp-ref="${ref}"]`; - else if (selector) sel = selector; - - // Verify the target BEFORE the CDP round-trip: CDP's DOM.querySelector - // happily resolves any node, and DOM.setFileInputFiles on a non-file input - // fails with an opaque protocol error (or worse, on some Chrome versions, - // appears to succeed). React onChange handlers also require a change/input - // event after the files are set — CDP doesn't fire one. - const check = await safeExec(tab.id, (s) => { - const el = document.querySelector(s); + // Resolve in the page with the shared resolver (ref registry, visible-first + // selector across shadow roots / same-origin frames, verified fallback), then + // hand the node to CDP through a one-shot marker attribute. + const sel = selector || (ref ? null : 'input[type="file"]'); + const what = selector || (ref ? `ref ${ref}` : 'input[type="file"]'); + const token = `u${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`; + const check = await execDom(tab.id, (_ref, _sel, _fb, _token) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const el = D.resolve(_ref, _sel, _fb).el; if (!el) return { found: false }; + el.setAttribute('data-bc-upload', _token); return { found: true, isFileInput: el.tagName === 'INPUT' && el.type === 'file', multiple: !!el.multiple, }; - }, [sel]).catch(() => null); - if (check && check.found) { - if (!check.isFileInput) throw new Error(`Element matching ${sel} is not an .`); - if (filePaths.length > 1 && !check.multiple) { - throw new Error(`File input matching ${sel} does not accept multiple files.`); - } + }, [ref || null, sel, getFallback(tab.id, ref), token]).catch(() => null); + if (!check || !check.found) throw new Error(`File input not found: ${what}`); + if (!check.isFileInput) throw new Error(`Element matching ${what} is not an .`); + if (filePaths.length > 1 && !check.multiple) { + throw new Error(`File input matching ${what} does not accept multiple files.`); } // upload_file stays on CDP (DOM.setFileInputFiles is CDP-only). let uploaded = false; try { const send = await ensureCdp(tab.id); - await send('DOM.enable'); - const { root } = await send('DOM.getDocument'); - - const { nodeId } = await send('DOM.querySelector', { - nodeId: root.nodeId, - selector: sel, - }); - - if (!nodeId) throw new Error(`File input not found with selector: ${sel}`); - - await send('DOM.setFileInputFiles', { - files: filePaths, - nodeId, - }); + // Find the marked node wherever it lives (open shadow roots, same-origin frames). + const { result } = await send('Runtime.evaluate', { expression: findMarkedExpression(token) }); + if (!result || !result.objectId) throw new Error(`File input not found: ${what}`); + await send('DOM.setFileInputFiles', { files: filePaths, objectId: result.objectId }); uploaded = true; } finally { // Fire the events React/Vue file inputs listen for after a successful set, - // and always remove the short-lived Observation V2 handoff marker. + // and always remove the one-shot marker (and the Observation V2 handoff marker). try { - await safeExec(tab.id, (s, notify) => { - const el = document.querySelector(s); - if (!el) return; + await execDom(tab.id, (_token, notify) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const el = (D.queryAll(`[data-bc-upload="${_token}"]`, true) || [])[0]; + if (!el) return null; if (notify) { el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true })); } + el.removeAttribute('data-bc-upload'); el.removeAttribute('data-bc-v2-upload'); - }, [sel, uploaded]); + return null; + }, [token, uploaded]); } catch { /* page changed — CDP outcome still determines the tool result */ } } - return { success: true, files: filePaths, selector: sel }; + return { success: true, files: filePaths, selector: what }; } diff --git a/extension/handlers/inspection.js b/extension/handlers/inspection.js index 3bea899..d67e329 100644 --- a/extension/handlers/inspection.js +++ b/extension/handlers/inspection.js @@ -2,15 +2,26 @@ * Inspection handlers (extracted from background.js): wait, scroll, snapshot, * find, text, evaluate — the read side of the toolset. */ -import { safeExec, resolveTab, getFallback } from '../lib/page-exec.js'; +import { safeExec, execDom, resolveTab, getFallback } from '../lib/page-exec.js'; import { fallbackByTab, lastSnapshotFingerprints, MAX_RESULT_CHARS, persistSessionState } from '../lib/state.js'; import { PAGE_FALLBACK_INSTALL } from '../utils/smart-selector.js'; -import { PAGE_LEGACY_REF_INSTALL } from '../utils/legacy-refs.js'; import { withCdp } from '../lib/cdp-session.js'; import { cdpEvaluate } from '../lib/cdp-evaluate.js'; +/** Default output cap for snapshots (chars of serialized tree). */ +export const SNAPSHOT_MAX_CHARS = 20_000; + +// Short refs ("s4k2-17"): a per-worker salt keeps refs from a recycled service +// worker from colliding with live ones in the page registry. +const REF_SALT = Math.random().toString(36).slice(2, 4); +let refSeq = 0; +export function nextRefPrefix(kind) { + refSeq = (refSeq + 1) % 1296; + return `${kind}${REF_SALT}${refSeq.toString(36)}-`; +} + export async function handleWait(params, _sessionId, _agentName, signal) { - const { tabId, selector, state = 'visible', timeout = 10000, delay } = params; + const { tabId, selector, state = 'visible', timeout = 10000, delay, text, urlIncludes } = params; // A promise that rejects when this call is cancelled (client gone / bridge // timeout forwarded). Long waits race against it so a cancelled call releases @@ -22,6 +33,7 @@ export async function handleWait(params, _sessionId, _agentName, signal) { }) : null; + const hasCondition = !!selector || text != null || !!urlIncludes; if (delay) { const sleep = new Promise((r) => setTimeout(r, Math.min(delay, 30000))); try { @@ -29,55 +41,79 @@ export async function handleWait(params, _sessionId, _agentName, signal) { } catch { return { success: false, error: 'aborted', waited: 0 }; } - return { success: true, waited: delay }; + return { success: true, waited: delay }; // documented: a delay ignores the conditions } - if (!selector) return { success: false, error: 'Need selector or delay' }; + if (!hasCondition) return { success: false, error: 'Need selector, text, urlIncludes or delay' }; await resolveTab(tabId); const start = Date.now(); + const what = selector || (text != null ? `text "${text}"` : `url containing "${urlIncludes}"`); while (Date.now() - start < timeout) { // Bail the moment the caller is gone so we don't pin the tab mutex for the // full timeout window after the originating agent was evicted (consistent // with handleNavigate / handleRunAction). if (signal?.aborted) return { success: false, error: 'aborted', selector, state }; - const found = await safeExec(tabId, (_sel, _state) => { - const el = document.querySelector(_sel); - if (_state === 'hidden') return !el || el.offsetParent === null; - if (_state === 'attached') return !!el; - return el && el.offsetParent !== null; - }, [selector, state]); - - if (found) return { success: true, selector, state, elapsed: Date.now() - start }; + let found; + try { + found = await execDom(tabId, (_sel, _state, _text, _url) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const hidden = _state === 'hidden'; + if (_url != null && !location.href.includes(_url)) return false; + if (_text != null) { + const has = D.pageText(document.body).toLowerCase().includes(String(_text).toLowerCase()); + if (hidden ? has : !has) return false; + } + if (_sel) { + // Every match across shadow roots / same-origin frames, not just the first. + const all = D.queryAll(_sel, true); + if (all === null) return { error: `Invalid CSS selector: ${_sel}` }; + if (_state === 'attached') return all.length > 0; + const anyVisible = all.some((el) => D.isVisible(el)); + return hidden ? !anyVisible : anyVisible; + } + return true; + }, [selector ?? null, state, text ?? null, urlIncludes ?? null]); + } catch { found = false; /* navigating: the next document isn't ready yet */ } + if (found && found.error) return { success: false, error: found.error }; + + if (found === true) { + return { + success: true, + ...(selector ? { selector } : {}), + ...(text != null ? { text } : {}), + ...(urlIncludes ? { urlIncludes } : {}), + state, + elapsed: Date.now() - start, + }; + } await new Promise((r) => setTimeout(r, 200)); } - return { success: false, error: `Timeout waiting for ${selector} to be ${state}` }; + return { success: false, error: `Timeout waiting for ${what} to be ${state}` }; } export async function handleScroll(params) { const { tabId, direction = 'down', amount = 500, selector, toElement, position } = params; await resolveTab(tabId); const fb = getFallback(tabId, toElement); - if (fb) await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); - await safeExec(tabId, PAGE_LEGACY_REF_INSTALL, []); - return safeExec(tabId, (_dir, _amt, _sel, _toEl, _pos, _fb) => { + return execDom(tabId, (_dir, _amt, _sel, _toEl, _pos, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; if (_toEl) { - const resolveFallback = (globalThis.__browserControllerFallbackRuntime || {}).resolveFallback || null; - const resolveRef = (globalThis.__browserControllerLegacyRefRuntime || {}).resolveRef || null; - const el = (resolveRef ? resolveRef(_toEl) : null) || - document.querySelector(`[data-mcp-ref="${_toEl}"]`) || - document.querySelector(_toEl) || - (_fb && resolveFallback ? resolveFallback(_fb) : null); + // toElement accepts a ref or a CSS selector (first visible match). + let el = D.resolve(_toEl, null, _fb).el; + if (!el) { try { el = D.resolve(null, _toEl, null).el; } catch { el = null; } } if (el) { - el.scrollIntoView({ behavior: 'smooth', block: 'center' }); + el.scrollIntoView({ behavior: 'instant', block: 'center' }); return { success: true, scrolledTo: 'element' }; } return { success: false, error: 'Element not found' }; } - const target = _sel ? document.querySelector(_sel) : window; + const target = _sel ? D.resolve(null, _sel, null).el : window; if (!target) return { success: false, error: 'Scroll container not found' }; if (_pos === 'top') { @@ -118,7 +154,9 @@ export async function handleScroll(params) { * DOM with permanent data-mcp-ref attributes. */ export async function handleSnapshot(params) { - const { tabId, selector, compact = true } = params; + const { tabId, selector, ref: rootRef, depth, maxChars = SNAPSHOT_MAX_CHARS } = params; + // filter:"interactive"|"all" (Claude-in-Chrome naming) is an alias of compact. + const compact = params.filter === 'all' ? false : params.filter === 'interactive' ? true : params.compact !== false; await resolveTab(tabId); // Install the fallback page runtime first (v2 install-once pattern): the @@ -127,196 +165,185 @@ export async function handleSnapshot(params) { // extension CSP (script-src 'self', no unsafe-eval) throws in every // isolated world, which silently killed fallback capture before this fix. await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); - await safeExec(tabId, PAGE_LEGACY_REF_INSTALL, []); // isNew feature: pass the fingerprints seen in the PREVIOUS snapshot so the // page function can mark newly-appeared elements. Array is serializable. - const prevFingerprints = lastSnapshotFingerprints.get(tabId) || []; - const refPrefix = `e-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}-`; + const prevFingerprints = lastSnapshotFingerprints.get(tabId) || null; + const refPrefix = nextRefPrefix('s'); - return safeExec(tabId, (_sel, _compact, _prevFingerprints, _refPrefix) => { + return execDom(tabId, (_sel, _compact, _prevFingerprints, _refPrefix, _rootRef, _depth, _maxChars) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; let refCount = 0; /** @type {Record} ref -> fallback, returned to background */ const fallbacks = {}; /** @type {string[]} fingerprints of THIS snapshot (role|name), returned to background */ const fingerprints = []; - const prevSet = new Set(_prevFingerprints); + // No previous snapshot → nothing is "new" (marking every node wasted tokens). + const prevSet = _prevFingerprints ? new Set(_prevFingerprints) : null; // Descriptor generator comes from the pre-installed page runtime. const genFallback = (globalThis.__browserControllerFallbackRuntime || {}).generateFallback || null; - const registerRef = (globalThis.__browserControllerLegacyRefRuntime || {}).registerRef || null; const skipTags = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'SVG', 'PATH', 'BR', 'HR', 'WBR', 'META', 'LINK']); + const maxDepth = Number.isInteger(_depth) && _depth >= 0 ? _depth : Infinity; + // Output budget: stop emitting nodes once the serialized size reaches it. + let budget = Number.isInteger(_maxChars) && _maxChars > 0 ? _maxChars : Infinity; + let truncated = false; + // 'show' = render normally, 'pass' = no box of its own (display:contents, + // slots) but its children may render, false = hidden subtree. function vis(el) { - const s = getComputedStyle(el); - if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) === 0) return false; + const s = D.styleOf(el); + if (!s || s.display === 'none') return false; + if (s.display === 'contents' || el.tagName === 'SLOT') return 'pass'; + if (s.visibility === 'hidden' || s.visibility === 'collapse' || parseFloat(s.opacity) === 0) { + // visibility is inherited but can be re-enabled below; keep walking. + return 'pass'; + } const r = el.getBoundingClientRect(); - return r.width > 0 && r.height > 0; + if (r.width > 0 && r.height > 0) return 'show'; + // Zero-size wrappers (custom-element hosts, overflow containers) can still hold visible children. + return el.childElementCount > 0 || D.shadowOf(el) ? 'pass' : false; } - function role(el) { - const r = el.getAttribute('role'); - if (r) return r; - const map = { - A: 'link', BUTTON: 'button', SELECT: 'combobox', TEXTAREA: 'textbox', IMG: 'img', - H1: 'heading', H2: 'heading', H3: 'heading', H4: 'heading', H5: 'heading', H6: 'heading', - NAV: 'navigation', MAIN: 'main', HEADER: 'banner', FOOTER: 'contentinfo', FORM: 'form', - TABLE: 'table', UL: 'list', OL: 'list', LI: 'listitem', - }; - if (el.tagName === 'INPUT') { - const t = el.type?.toLowerCase(); - if (t === 'checkbox') return 'checkbox'; - if (t === 'radio') return 'radio'; - return 'textbox'; - } - return map[el.tagName] || 'generic'; - } + const role = (el) => D.roleOf(el); + // Landmarks/regions are named only by an explicit label: their text is just + // their children's names again (token noise). + const elName = (el, r) => (landmarkRoles.has(r) && r !== 'dialog' + ? D.clean(D.attr(el, 'aria-label') || D.attr(el, 'title')) + : D.nameOf(el)).slice(0, 80); + const isInteractive = (el) => D.isInteractive(el); - function elName(el) { - const raw = ( - el.getAttribute('aria-label') || el.getAttribute('alt') || - el.getAttribute('title') || el.getAttribute('placeholder') || - '' - ).trim(); - if (raw) return raw.slice(0, 80); - const text = el.innerText; - if (!text) return ''; - const first = text.split('\n')[0].trim(); - return first.slice(0, 80); - } + const landmarkRoles = new Set(['navigation', 'main', 'banner', 'contentinfo', 'form', 'search', 'complementary', 'region', 'dialog']); - function isInteractive(el) { - const tags = ['A', 'BUTTON', 'INPUT', 'SELECT', 'TEXTAREA']; - return tags.includes(el.tagName) || el.onclick || el.getAttribute('tabindex') !== null || - el.getAttribute('role') === 'button' || el.getAttribute('role') === 'link' || - el.getAttribute('role') === 'tab' || el.getAttribute('role') === 'menuitem' || - el.getAttribute('role') === 'option' || el.getAttribute('role') === 'switch' || - el.getAttribute('contenteditable') === 'true'; + // Flat-tree children: open AND closed shadow roots, slotted content, + // same-origin iframe bodies (lib/page-dom.js flatChildren). + function childrenOf(el) { + return D.flatChildren(el).filter((c) => c.nodeType === 1); } - const landmarkRoles = new Set(['navigation', 'main', 'banner', 'contentinfo', 'form', 'search', 'complementary', 'region']); + const origin = location.origin; + function hrefOf(el) { + const h = el.href; + if (!h || typeof h !== 'string') return null; + if (h.startsWith(origin + '/')) return h.slice(origin.length); // same-origin: path only + return h; + } - // Children including shadow DOM (open roots) and same-origin iframes. - function childrenOf(el) { - const out = []; - for (const c of el.children) out.push(c); - if (el.shadowRoot) { - for (const c of el.shadowRoot.children) out.push(c); - } - // same-origin iframes: expose their document body children too. - if (el.tagName === 'IFRAME') { - try { - const doc = el.contentDocument; - if (doc && doc.body) for (const c of doc.body.children) out.push(c); - } catch { /* cross-origin: skip */ } - } - return out; + function emit(el, r, n, extra, isNewCheck) { + const ref = `${_refPrefix}${refCount++}`; + D.registry.set(ref, el); + try { if (genFallback) fallbacks[ref] = genFallback(el); } catch {} + const fp = `${r}|${n}`; + fingerprints.push(fp); + const node = { ref, role: r, ...extra }; + if (n) node.name = n; + if (isNewCheck && prevSet && !prevSet.has(fp)) node.isNew = true; + if (el.value !== undefined && el.value !== '' && typeof el.value !== 'object') node.value = String(el.value).slice(0, 200); + if (el.tagName === 'INPUT' && (el.type === 'checkbox' || el.type === 'radio')) node.checked = el.checked; + else if (D.attr(el, 'aria-checked')) node.checked = D.attr(el, 'aria-checked') === 'true'; + if (D.attr(el, 'aria-expanded')) node.expanded = D.attr(el, 'aria-expanded') === 'true'; + if (D.attr(el, 'aria-selected') === 'true') node.selected = true; + if (el.disabled) node.disabled = true; + if (el.tagName === 'A') { const h = hrefOf(el); if (h) node.href = h; } + budget -= JSON.stringify(node).length + 16; + return node; } - function buildCompact(el) { + function buildCompact(el, d) { if (!el || el.nodeType !== 1) return null; if (skipTags.has(el.tagName)) return null; - if (!vis(el)) return null; + if (budget <= 0) { truncated = true; return null; } + const v = vis(el); + if (!v) return null; - const ia = isInteractive(el); + const ia = v === 'show' && isInteractive(el); const r = role(el); - const isLandmark = landmarkRoles.has(r); - - const kids = []; - for (const c of childrenOf(el)) { - const cn = buildCompact(c); - if (cn) Array.isArray(cn) ? kids.push(...cn) : kids.push(cn); - } + const isLandmark = v === 'show' && (landmarkRoles.has(r) || (r === 'heading')); + const own = ia || isLandmark; - if (!ia && !isLandmark && r !== 'heading') { - return kids.length === 0 ? null : kids.length === 1 ? kids[0] : kids; + let node = null; + if (own) { + if (d > maxDepth) { truncated = true; return null; } + node = emit(el, r, elName(el, r), {}, true); } + const kids = []; + if (!(own && d >= maxDepth)) { + for (const c of childrenOf(el)) { + const cn = buildCompact(c, own ? d + 1 : d); + if (cn) Array.isArray(cn) ? kids.push(...cn) : kids.push(cn); + } + } else if (childrenOf(el).length) truncated = true; - const ref = `${_refPrefix}${refCount++}`; - if (registerRef) registerRef(ref, el); - const n = elName(el); - try { if (genFallback) fallbacks[ref] = genFallback(el); } catch {} - - // isNew: mark elements whose (role|name) wasn't in the previous snapshot. - const fp = `${r}|${n}`; - fingerprints.push(fp); - const isNew = !prevSet.has(fp); - - const node = { ref, role: r }; - if (n) node.name = n; - if (isNew) node.isNew = true; - if (el.value !== undefined && el.value !== '') node.value = String(el.value); - if (el.checked !== undefined) node.checked = el.checked; - if (el.disabled) node.disabled = true; - if (el.href && el.tagName === 'A') node.href = el.href; + if (!own) return kids.length === 0 ? null : kids.length === 1 ? kids[0] : kids; if (kids.length) node.children = kids; - return node; } - function buildFull(el, depth) { + function buildFull(el, d) { if (!el || el.nodeType !== 1) return null; if (skipTags.has(el.tagName)) return null; - if (!vis(el)) return null; + if (budget <= 0) { truncated = true; return null; } + const v = vis(el); + if (!v) return null; const r = role(el); - const n = elName(el); - const ia = isInteractive(el); + const ia = v === 'show' && isInteractive(el); + const n = v === 'show' ? elName(el, r) : ''; - if (r === 'generic' && !n && !ia && depth > 1) { + if (v !== 'show' || (r === 'generic' && !n && !ia && d > 1)) { const kids = []; for (const c of childrenOf(el)) { - const cn = buildFull(c, depth + 1); + const cn = buildFull(c, d + (v === 'show' ? 1 : 0)); if (cn) Array.isArray(cn) ? kids.push(...cn) : kids.push(cn); } return kids.length === 0 ? null : kids.length === 1 ? kids[0] : kids; } + if (d > maxDepth) { truncated = true; return null; } - const ref = `${_refPrefix}${refCount++}`; - if (registerRef) registerRef(ref, el); - try { if (genFallback) fallbacks[ref] = genFallback(el); } catch {} - - // isNew: mark elements whose (role|name) wasn't in the previous snapshot. - const fp = `${r}|${n}`; - fingerprints.push(fp); - const isNew = !prevSet.has(fp); - - const node = { ref, role: r }; - if (r === 'generic') node.tag = el.tagName.toLowerCase(); - if (n) node.name = n; - if (isNew) node.isNew = true; - if (el.value !== undefined && el.value !== '') node.value = String(el.value); - if (el.checked !== undefined) node.checked = el.checked; - if (el.disabled) node.disabled = true; - if (el.href && el.tagName === 'A') node.href = el.href; - + const node = emit(el, r, n, r === 'generic' ? { tag: el.tagName.toLowerCase() } : {}, true); const kids = []; for (const c of childrenOf(el)) { - const cn = buildFull(c, depth + 1); + const cn = buildFull(c, d + 1); if (cn) Array.isArray(cn) ? kids.push(...cn) : kids.push(cn); } if (kids.length) node.children = kids; - return node; } - const root = _sel ? document.querySelector(_sel) : document.body; + let root = document.body; + if (_rootRef) { + root = D.registry.get(_rootRef); + if (!D.connected(root)) return { success: false, error: `ref ${_rootRef} is gone — take a new snapshot` }; + } else if (_sel) { + const hit = D.resolve(null, _sel, null); + if (hit.error === 'INVALID_SELECTOR') return { success: false, error: `Invalid CSS selector: ${_sel}` }; + root = hit.el; + } if (!root) return { success: false, error: 'Root element not found' }; - const tree = _compact ? buildCompact(root) : buildFull(root, 0); + const tree = _compact ? buildCompact(root, 0) : buildFull(root, 0); return { success: true, url: location.href, title: document.title, compact: _compact, tree, + ...(truncated ? { + truncated: true, + hint: 'Output capped (maxChars/depth). Scope it with selector or ref (a subtree), or raise maxChars.', + } : {}), // internal: background stores these per-tab; never sent to the agent. __fallbacks: fallbacks, __fingerprints: fingerprints, }; - }, [selector, compact, prevFingerprints, refPrefix]).then((res) => { + }, [selector ?? null, compact, prevFingerprints, refPrefix, rootRef ?? null, depth ?? null, maxChars]).then((res) => { // Store the fallbacks per-tab so click/type can resolve stale refs, and - // persist them across service-worker recycles (MV3 lifetime). + // persist them across service-worker recycles (MV3 lifetime). Merged, not + // replaced: a scoped snapshot must not invalidate refs from the full one. if (res && res.__fallbacks) { - const map = new Map(Object.entries(res.__fallbacks)); + const map = fallbackByTab.get(tabId) || new Map(); + for (const [ref, fbEntry] of Object.entries(res.__fallbacks)) map.set(ref, fbEntry); + // Bound the map: keep the most recent entries. + while (map.size > 3000) map.delete(map.keys().next().value); fallbackByTab.set(tabId, map); delete res.__fallbacks; // keep it out of the agent-visible payload persistSessionState(); @@ -331,71 +358,114 @@ export async function handleSnapshot(params) { } export async function handleFind(params) { - const { tabId, query, limit = 10 } = params; + const { tabId, query, limit = 10, role } = params; await resolveTab(tabId); await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); - await safeExec(tabId, PAGE_LEGACY_REF_INSTALL, []); - const refPrefix = `f-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}-`; + const refPrefix = nextRefPrefix('f'); - return safeExec(tabId, (_q, _lim, _refPrefix) => { - const qLow = _q.toLowerCase(); - const matches = []; - const fallbacks = {}; + return execDom(tabId, (_q, _lim, _refPrefix, _role) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; const genFallback = (globalThis.__browserControllerFallbackRuntime || {}).generateFallback || null; - const registerRef = (globalThis.__browserControllerLegacyRefRuntime || {}).registerRef || null; + const fallbacks = {}; - function aName(el) { - return (el.getAttribute('aria-label') || el.getAttribute('alt') || el.getAttribute('title') || - el.getAttribute('placeholder') || el.innerText?.slice(0, 200) || '').trim(); + // Words that describe the KIND of element, mapped to the roles they mean. + const ROLE_WORDS = { + button: ['button'], btn: ['button'], link: ['link'], anchor: ['link'], + input: ['textbox', 'searchbox', 'combobox', 'spinbutton'], field: ['textbox', 'searchbox', 'combobox', 'spinbutton'], + textbox: ['textbox', 'searchbox'], box: ['textbox', 'searchbox', 'combobox', 'checkbox'], textarea: ['textbox'], + searchbox: ['searchbox'], checkbox: ['checkbox'], check: ['checkbox'], radio: ['radio'], + dropdown: ['combobox', 'listbox', 'button'], select: ['combobox', 'listbox'], combobox: ['combobox'], + tab: ['tab'], menu: ['menu', 'menubar', 'button'], menuitem: ['menuitem'], option: ['option'], + heading: ['heading'], title: ['heading'], image: ['img'], img: ['img'], icon: ['img', 'button'], + dialog: ['dialog', 'alertdialog'], modal: ['dialog', 'alertdialog'], switch: ['switch'], toggle: ['switch', 'button', 'checkbox'], + slider: ['slider'], list: ['list', 'listbox'], table: ['table', 'grid'], row: ['row'], cell: ['cell', 'gridcell'], + }; + const STOP = new Set(['the', 'a', 'an', 'to', 'of', 'for', 'on', 'in', 'with', 'and', 'that', 'this', 'element', 'please']); + const words = String(_q).toLowerCase().split(/[^\p{L}\p{N}_-]+/u).filter((w) => w && !STOP.has(w)); + const roleHints = new Set(); + const content = []; + for (const w of words) { + if (ROLE_WORDS[w]) ROLE_WORDS[w].forEach((r) => roleHints.add(r)); + else content.push(w); } - - function aRole(el) { - const r = el.getAttribute('role'); - if (r) return r; - const map = { A: 'link', BUTTON: 'button', INPUT: 'input', SELECT: 'combobox', TEXTAREA: 'textbox', IMG: 'image' }; - return map[el.tagName] || el.tagName.toLowerCase(); + // "search" names the purpose AND a role. + if (words.includes('search')) roleHints.add('searchbox'); + const phrase = content.join(' '); + const wantRole = _role ? String(_role).toLowerCase() : null; + + const SKIP = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'META', 'LINK', 'HEAD', 'HTML', 'BODY', 'BR', 'PATH']); + const cands = []; + for (const root of D.allRoots(true)) { + let els = []; + try { els = root.querySelectorAll('*'); } catch {} + for (const el of els) { + if (SKIP.has(el.tagName)) continue; + const r = D.roleOf(el); + if (r === 'none') continue; + if (wantRole && r !== wantRole) continue; + const name = D.nameOf(el).toLowerCase(); + const attrs = [el.id, D.attr(el, 'name'), D.attr(el, 'type'), D.attr(el, 'placeholder'), D.attr(el, 'data-testid'), + D.attr(el, 'title'), typeof el.className === 'string' ? el.className : ''].join(' ').toLowerCase(); + const interactive = D.isInteractive(el); + let score = 0; + let covered = 0; + for (const w of content) { + const inName = name.includes(w); + const inAttr = attrs.includes(w); + if (inName) score += new RegExp(`(^|[^\\p{L}\\p{N}])${w.replace(/[.*+?^${}()|[\]\\-]/g, '\\$&')}([^\\p{L}\\p{N}]|$)`, 'u').test(name) ? 6 : 4; + else if (inAttr) score += 3; + if (inName || inAttr) covered++; + } + if (content.length && covered === 0) continue; + if (phrase && name === phrase) score += 12; + else if (phrase && content.length > 1 && name.includes(phrase)) score += 6; + if (roleHints.size) { + if (roleHints.has(r) || (roleHints.has('searchbox') && /search/.test(attrs) && ['textbox', 'searchbox', 'combobox'].includes(r))) score += 8; + else if (!content.length) continue; + else score -= 2; + } + if (interactive) score += 4; + else if (r === 'generic') score -= 3; + // A container whose text merely CONTAINS the words is a weak match. + if (name.length > 120) score -= 4; + const coverage = content.length ? covered / content.length : 1; + if (coverage < 0.5) continue; + score = Math.round(score * coverage * 10) / 10; + if (score <= 0) continue; + cands.push({ el, r, name, score, interactive }); + } + } + cands.sort((a, b) => b.score - a.score); + // Visibility is the expensive check: only for the best-scoring pool. + const pool = []; + for (const c of cands) { + if (pool.length >= _lim * 6) break; + if (D.isVisible(c.el)) pool.push(c); } + // Drop wrappers (an ancestor scoring no better than a descendant) and echoes + // (a descendant repeating the name of the control that contains it). + const kept = pool.filter((c) => !pool.some((o) => o !== c && ( + (o.score >= c.score && D.composedContains(c.el, o.el)) + || (o.interactive && !c.interactive && o.score >= c.score && o.name === c.name && D.composedContains(o.el, c.el))))); - // Same-origin iframe piercing (field report: legacy UIs live entirely - // inside #mainFrame — the top-document walk saw none of it). - const roots = [document.body]; - (function collectFrames(doc, depth) { - if (depth >= 3) return; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d && d.body) { roots.push(d.body); collectFrames(d, depth + 1); } } catch {} - } - })(document, 0); - let rc = 0; - let node; - for (const root of roots) { - const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT); - while ((node = walker.nextNode()) && matches.length < _lim * 3) { - const s = getComputedStyle(node); - const rect = node.getBoundingClientRect(); - if (s.display === 'none' || s.visibility === 'hidden' || rect.width === 0) continue; - - const n = aName(node).toLowerCase(); - const r = aRole(node).toLowerCase(); - const id = (node.id || '').toLowerCase(); - let score = 0; - if (n.includes(qLow)) score += 10; - if (r.includes(qLow)) score += 5; - if (id.includes(qLow)) score += 3; - if (score === 0) continue; - - const ref = `${_refPrefix}${rc++}`; - if (registerRef) registerRef(ref, node); - try { if (genFallback) fallbacks[ref] = genFallback(node); } catch {} + const matches = []; + kept.slice(0, _lim).forEach((c, i) => { + const ref = `${_refPrefix}${i}`; + D.registry.set(ref, c.el); + try { if (genFallback) fallbacks[ref] = genFallback(c.el); } catch {} + const rect = D.centerOf(c.el).rect; matches.push({ - ref, role: r, name: n.slice(0, 100), tag: node.tagName.toLowerCase(), score, + ref, role: c.r, name: D.nameOf(c.el).slice(0, 80), tag: c.el.tagName.toLowerCase(), score: c.score, bounds: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, }); - } - } - - matches.sort((a, b) => b.score - a.score); - return { success: true, query: _q, matches: matches.slice(0, _lim), __fallbacks: fallbacks }; - }, [query, limit, refPrefix]).then((res) => { + }); + return { + success: true, query: _q, matches, + ...(matches.length === 0 ? { hint: 'No match. Try fewer/other words, a role filter, browser_snapshot, or browser_text.' } : {}), + __fallbacks: fallbacks, + }; + }, [query, limit, refPrefix, role || null]).then((res) => { if (res && res.__fallbacks) { const map = fallbackByTab.get(tabId) || new Map(); for (const [ref, fbEntry] of Object.entries(res.__fallbacks)) map.set(ref, fbEntry); @@ -411,21 +481,41 @@ export async function handleGetPageText(params) { // Default must match the MCP schema (text.ts: maxLength .default(5000)) — // it drifted 10x here once, so direct-WS callers got 50000 while MCP callers // got 5000 from the same knob. - const { tabId, selector, maxLength = 5000 } = params; + const { tabId, selector, maxLength = 5000, mode = 'all', offset = 0 } = params; await resolveTab(tabId); - const args = selector === undefined ? [null, maxLength] : [selector, maxLength]; - - return safeExec(tabId, (_sel, _max) => { - const root = _sel ? document.querySelector(_sel) : document.body; + const max = Math.min(Math.max(1, Number(maxLength) || 5000), 100_000); + const from = Math.max(0, Number(offset) || 0); + + return execDom(tabId, (_sel, _max, _mode, _from) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const article = _mode === 'article'; + let root = document.body; + if (_sel) { + const hit = D.resolve(null, _sel, null); + if (hit.error === 'INVALID_SELECTOR') return { success: false, error: `Invalid CSS selector: ${_sel}` }; + root = hit.el; + } else if (article) { + root = D.articleRoot(); + } if (!root) return { success: false, error: 'Element not found' }; - let text = root.innerText || root.textContent || ''; - text = text.replace(/\t/g, ' ').replace(/\n\s*\n/g, '\n\n').replace(/ +/g, ' ').trim(); + // Composed text: includes open/closed shadow roots and same-origin frames + // (innerText alone misses web-component content such as caniuse's tables). + let text = D.pageText(root, { article, max: _from + _max + 1000 }); + const total = text.length; + if (_from) text = text.slice(_from); const truncated = text.length > _max; if (truncated) text = text.slice(0, _max) + '...'; - return { success: true, url: location.href, title: document.title, text, length: text.length, truncated }; - }, args); + return { + success: true, url: location.href, title: document.title, text, length: text.length, truncated, + ...(_from ? { offset: _from } : {}), + ...(truncated ? { nextOffset: _from + _max } : {}), + ...(article ? { mode: 'article' } : {}), + ...(total && _from >= total ? { note: `offset ${_from} is past the end (${total} chars)` } : {}), + }; + }, [selector ?? null, max, mode, from]); } /** diff --git a/extension/handlers/interaction-advanced.js b/extension/handlers/interaction-advanced.js index 6fe73ef..61c94fa 100644 --- a/extension/handlers/interaction-advanced.js +++ b/extension/handlers/interaction-advanced.js @@ -3,7 +3,7 @@ * orchestration. Kept separate from the common pointer/keyboard handlers so * each module stays focused and reviewable. */ -import { resolveTab, safeExec } from '../lib/page-exec.js'; +import { resolveTab, safeExec, execDom, getFallback } from '../lib/page-exec.js'; import { withCdp } from '../lib/cdp-session.js'; import { openShield, releaseShield } from '../lib/trusted-input.js'; @@ -64,32 +64,19 @@ export async function handleDrag(params) { let sx = startX, sy = startY, ex = endX, ey = endY; if (sx == null || sy == null || ex == null || ey == null) { - const coords = await safeExec(tabId, (_sRef, _sSel, _eRef, _eSel) => { - function deepQuery(sel) { - const query = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const frame of doc.querySelectorAll('iframe')) { - try { - const child = frame.contentDocument; - if (child) { const el = query(child, depth + 1); if (el) return el; } - } catch {} - } - return null; - }; - return query(document, 0); - } - - function find(ref, selector) { - let el = ref ? deepQuery(`[data-mcp-ref="${ref}"]`) : null; - if (!el && selector) el = deepQuery(selector); + const coords = await execDom(tabId, (_sRef, _sSel, _eRef, _eSel, _sFb, _eFb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + function find(ref, selector, fb) { + if (!ref && !selector) return null; + const el = D.resolve(ref, selector, fb).el; if (!el) return null; el.scrollIntoView({ behavior: 'instant', block: 'center' }); - const rect = el.getBoundingClientRect(); - return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }; + const { x, y } = D.centerOf(el); + return { x, y }; } - return { start: find(_sRef, _sSel), end: find(_eRef, _eSel) }; - }, [startRef, startSelector, endRef, endSelector]); + return { start: find(_sRef, _sSel, _sFb), end: find(_eRef, _eSel, _eFb) }; + }, [startRef, startSelector, endRef, endSelector, getFallback(tabId, startRef), getFallback(tabId, endRef)]); if (coords.start) { sx = coords.start.x; sy = coords.start.y; } if (coords.end) { ex = coords.end.x; ey = coords.end.y; } @@ -127,21 +114,12 @@ export async function handleFillForm(params) { } await resolveTab(tabId); - return safeExec(tabId, (_fields, _submit) => { - function deepQuery(sel) { - const query = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const frame of doc.querySelectorAll('iframe')) { - try { - const child = frame.contentDocument; - if (child) { const el = query(child, depth + 1); if (el) return el; } - } catch {} - } - return null; - }; - return query(document, 0); - } + // Attach each ref's snapshot descriptor so stale refs re-resolve (verified) in the page. + const withFb = fields.map((f) => (f && f.ref ? { ...f, fb: getFallback(tabId, f.ref) } : f)); + + return execDom(tabId, (_fields, _submit) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; const setNativeValue = (target, nextValue) => { const prototype = target instanceof HTMLTextAreaElement @@ -154,9 +132,8 @@ export async function handleFillForm(params) { const results = []; let containingForm = null; for (const field of _fields) { - const { ref, selector, value, clear } = field; - let el = ref ? deepQuery(`[data-mcp-ref="${ref}"]`) : null; - if (!el && selector) el = deepQuery(selector); + const { ref, selector, value, clear, fb } = field; + const el = D.resolve(ref, selector, fb).el; if (!el) { results.push({ selector: selector || ref, success: false, error: 'Not found' }); continue; @@ -164,20 +141,27 @@ export async function handleFillForm(params) { el.focus(); if (el.form && !containingForm) containingForm = el.form; - if (clear !== false) { + const isChoice = el.tagName === 'SELECT' || el.type === 'checkbox' || el.type === 'radio'; + if (clear !== false && !isChoice) { if (el.isContentEditable) el.textContent = ''; else setNativeValue(el, ''); el.dispatchEvent(new Event('input', { bubbles: true })); } if (el.tagName === 'SELECT') { - const option = Array.from(el.options).find((candidate) => candidate.value === String(value)); + // Match the option's value first, then its visible label. + const want = String(value); + const options = Array.from(el.options); + const option = options.find((candidate) => candidate.value === want) + || options.find((candidate) => candidate.textContent.trim() === want.trim()) + || options.find((candidate) => candidate.textContent.trim().toLowerCase() === want.trim().toLowerCase()); if (!option) { results.push({ selector: selector || ref, success: false, error: `Option "${value}" not found` }); continue; } - setNativeValue(el, String(value)); - el.dispatchEvent(new Event('change', { bubbles: true })); + const setter = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value')?.set; + if (setter) setter.call(el, option.value); else el.value = option.value; + el.dispatchEvent(new Event('input', { bubbles: true })); } else if (el.type === 'checkbox' || el.type === 'radio') { const checked = value === true || value === 'true'; if (el.checked !== checked) el.click(); @@ -205,5 +189,5 @@ export async function handleFillForm(params) { return failed === 0 ? { success: true, fields: results } : { success: false, error: `${failed} of ${results.length} fields failed`, fields: results }; - }, [fields, submit]); + }, [withFb, submit]); } diff --git a/extension/handlers/interaction.js b/extension/handlers/interaction.js index cc11fd1..3f81740 100644 --- a/extension/handlers/interaction.js +++ b/extension/handlers/interaction.js @@ -3,9 +3,8 @@ * hover, select, click_text, dialog, drag, fill_form — the write side that * drives the page's event system (synthetic events) or CDP when required. */ -import { resolveTab, requireTarget, safeExec, getFallback } from '../lib/page-exec.js'; +import { resolveTab, requireTarget, execDom, getFallback } from '../lib/page-exec.js'; import { autoReSnapshot } from './inspection.js'; -import { PAGE_FALLBACK_INSTALL } from '../utils/smart-selector.js'; import { trustedSender, locateTarget, releaseShield, cdpClickAt, cdpKeyPress, cdpTypeText, keyDefinition } from '../lib/trusted-input.js'; export { handleDialog, handleDrag, handleFillForm } from './interaction-advanced.js'; @@ -31,10 +30,8 @@ export async function handleClick(params) { const { tabId, ref, selector, button = 'left', doubleClick = false, trusted } = params; await resolveTab(tabId); requireTarget(params); + // Snapshot-time descriptor used by the shared resolver when the ref is stale. const fb = getFallback(tabId, ref); - // Install the fallback page runtime only when a descriptor exists (v2 - // install-once pattern — eval rebuilding is impossible under MV3 CSP). - if (fb) await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); // Trusted path: a real mouse click at the element's centre over CDP, so // focus moves, default actions run and the page sees isTrusted:true. @@ -59,33 +56,19 @@ export async function handleClick(params) { // Zero-size element: no point to hit — fall through to the synthetic path. } - const res = await safeExec(tabId, async (_ref, _sel, _btn, _dbl, _fb) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + const res = await execDom(tabId, async (_ref, _sel, _btn, _dbl, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; - let el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : null; - let via = 'ref'; - if (!el && _sel) { el = deepQuery(_sel); via = 'selector'; } - // Resolver comes from the pre-installed page runtime (no eval). - const resolveFallback = (globalThis.__browserControllerFallbackRuntime || {}).resolveFallback || null; - // Smart-selector fallback (plan task 3): ref broke → try robust selector, - // then text+role+tag scan. The agent doesn't request this; it's automatic. - if (!el && _fb && resolveFallback) { el = resolveFallback(_fb); if (el) via = 'fallback'; } + // ref registry → first visible selector match → verified fallback (lib/page-dom.js). + const found = D.resolve(_ref, _sel, _fb); + if (found.error === 'INVALID_SELECTOR') return { success: false, error: `Invalid CSS selector: ${_sel}` }; + let el = found.el || null; + const via = found.via || 'ref'; if (!el) { // Element is gone (likely virtualized away on scroll). Abort WITHOUT // clicking — the background auto-re-snapshots and embeds fresh refs. - return { success: false, error: 'REF_GONE', _ref }; + return { success: false, error: 'REF_GONE', _ref, url: location.href }; } el.scrollIntoView({ behavior: 'instant', block: 'center' }); @@ -100,7 +83,7 @@ export async function handleClick(params) { if (!visible0) { await new Promise((r) => setTimeout(r, 200)); // re-resolve the element (it may have been re-rendered with a new node) - el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : el; + el = D.resolve(_ref, _sel, _fb).el || el; if (el) el.scrollIntoView({ behavior: 'instant', block: 'center' }); } if (!el) return { success: false, error: 'REF_GONE', _ref }; @@ -146,10 +129,8 @@ export async function handleType(params) { const { tabId, ref, selector, text, clear = false, trusted } = params; await resolveTab(tabId); requireTarget(params); + // Snapshot-time descriptor used by the shared resolver when the ref is stale. const fb = getFallback(tabId, ref); - // Install the fallback page runtime only when a descriptor exists (v2 - // install-once pattern — eval rebuilding is impossible under MV3 CSP). - if (fb) await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); // Trusted path: focus the field, then real key presses over CDP (keydown / // keypress / input / keyup per character). Like a user, this does NOT fire @@ -180,30 +161,18 @@ export async function handleType(params) { await releaseShield(tabId); } - const res = await safeExec(tabId, (_ref, _sel, _text, _clear, _fb) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + const res = await execDom(tabId, (_ref, _sel, _text, _clear, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; - let el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : null; - let via = 'ref'; - if (!el && _sel) { el = deepQuery(_sel); via = 'selector'; } - const resolveFallback = (globalThis.__browserControllerFallbackRuntime || {}).resolveFallback || null; - if (!el && _fb && resolveFallback) { el = resolveFallback(_fb); if (el) via = 'fallback'; } + const found = D.resolve(_ref, _sel, _fb); + if (found.error === 'INVALID_SELECTOR') return { success: false, error: `Invalid CSS selector: ${_sel}` }; + const el = found.el || null; + const via = found.via || 'ref'; if (!el) { // Element gone (virtualized feed) — abort WITHOUT typing; background // auto-re-snapshots and embeds fresh refs for a one-step retry. - return { success: false, error: 'REF_GONE', _ref }; + return { success: false, error: 'REF_GONE', _ref, url: location.href }; } el.focus(); @@ -263,6 +232,7 @@ export async function handlePressKey(params) { const { tabId, ref, selector, trusted } = params; const { key, mods: modifiers } = parseKeyCombo(params.key, params.modifiers || []); await resolveTab(tabId); + const fb = getFallback(tabId, ref); // Trusted path: a real key press, so default actions run (Tab moves focus // and fires blur/focusout, Enter submits, arrows drive autocomplete menus). @@ -270,7 +240,7 @@ export async function handlePressKey(params) { try { keyDefinition(key); } catch { knownKey = false; } const send = knownKey ? await trustedSender(tabId, trusted) : null; if (send) { - const loc = await locateTarget(tabId, { ref, selector, mode: ref || selector ? 'focus' : 'active' }); + const loc = await locateTarget(tabId, { ref, selector, fb, mode: ref || selector ? 'focus' : 'active' }); if (!loc || loc.success === false) { await releaseShield(tabId); if (ref || selector) return { success: false, error: `Element ${ref ? `with ref ${ref}` : `with selector ${selector}`} not found` }; @@ -284,34 +254,18 @@ export async function handlePressKey(params) { return { success: true, key, ...(modifiers.length ? { modifiers } : {}), input: 'cdp', ...(after?.focusedTag ? { focused: after.focusedTag } : {}) }; } - return safeExec(tabId, (_key, _mods, _ref, _sel) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + return execDom(tabId, (_key, _mods, _ref, _sel, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; let target = document.activeElement || document.body; // When the caller names a target, an unresolved ref/selector must FAIL — // silently falling back to activeElement sent Enter to the wrong control // with a success result. (Omitting both is still legitimate: intentional // activeElement targeting.) - if (_ref) { - const el = deepQuery(`[data-mcp-ref="${_ref}"]`); - if (!el) return { success: false, error: `Element with ref ${_ref} not found` }; - el.focus(); - target = el; - } else if (_sel) { - const el = deepQuery(_sel); - if (!el) return { success: false, error: `Element with selector ${_sel} not found` }; + if (_ref || _sel) { + const el = D.resolve(_ref, _sel, _fb).el; + if (!el) return { success: false, error: _ref ? `Element with ref ${_ref} not found` : `Element with selector ${_sel} not found` }; el.focus(); target = el; } @@ -332,17 +286,18 @@ export async function handlePressKey(params) { target.dispatchEvent(new KeyboardEvent('keyup', init)); return { success: true, key: _key }; - }, [key, modifiers, ref, selector]); + }, [key, modifiers, ref, selector, fb]); } export async function handleHover(params) { const { tabId, ref, selector, trusted } = params; await resolveTab(tabId); requireTarget(params); + const fb = getFallback(tabId, ref); const send = await trustedSender(tabId, trusted); if (send) { - const loc = await locateTarget(tabId, { ref, selector }); + const loc = await locateTarget(tabId, { ref, selector, fb }); if (loc?.success && loc.visible) { try { await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: loc.x, y: loc.y }); @@ -355,23 +310,11 @@ export async function handleHover(params) { if (loc && loc.success === false) return { success: false, error: 'Element not found' }; } - return safeExec(tabId, (_ref, _sel) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + return execDom(tabId, (_ref, _sel, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; - let el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : null; - if (!el && _sel) el = deepQuery(_sel); + const el = D.resolve(_ref, _sel, _fb).el; if (!el) return { success: false, error: 'Element not found' }; el.scrollIntoView({ behavior: 'instant', block: 'center' }); @@ -385,7 +328,7 @@ export async function handleHover(params) { el.dispatchEvent(new MouseEvent('mousemove', init)); return { success: true }; - }, [ref, selector]); + }, [ref, selector, fb]); } export async function handleSelect(params) { @@ -395,24 +338,13 @@ export async function handleSelect(params) { if (value === undefined && label === undefined && index === undefined) { throw new Error('One of value, label, or index is required to pick an option.'); } + const fb = getFallback(tabId, ref); - return safeExec(tabId, (_ref, _sel, _val, _lbl, _idx) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + return execDom(tabId, (_ref, _sel, _val, _lbl, _idx, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; - let el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : null; - if (!el && _sel) el = deepQuery(_sel); + const el = D.resolve(_ref, _sel, _fb).el; if (!el) return { success: false, error: 'Element not found' }; if (el.tagName !== 'SELECT') return { success: false, error: 'Not a select element' }; @@ -429,69 +361,98 @@ export async function handleSelect(params) { el.dispatchEvent(new Event('change', { bubbles: true })); el.dispatchEvent(new Event('input', { bubbles: true })); return { success: true, selected: el.value }; - }, [ref, selector, value, label, index]); + }, [ref, selector, value, label, index, fb]); } export async function handleClickByText(params) { - const { tabId, text, index = 0, exact = false } = params; + const { tabId, text, index = 0, exact = false, trusted } = params; await resolveTab(tabId); - - return safeExec(tabId, (_text, _index, _exact) => { - const textLower = _text.toLowerCase(); - const candidates = []; - // Same-origin iframe piercing — walk every frame body, not just the top. - const roots = [document.body]; - (function collectFrames(doc, depth) { - if (depth >= 3) return; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d && d.body) { roots.push(d.body); collectFrames(d, depth + 1); } } catch {} - } - })(document, 0); - let node; - for (const root of roots) { - const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT); - while ((node = walker.nextNode())) { - const s = getComputedStyle(node); - if (s.display === 'none' || s.visibility === 'hidden') continue; - const r = node.getBoundingClientRect(); - if (r.width === 0 || r.height === 0) continue; - - const nodeText = (node.innerText || node.textContent || '').trim(); - const firstLine = nodeText.split('\n')[0].trim(); - const match = _exact - ? firstLine === _text - : firstLine.toLowerCase().includes(textLower); - - if (match) { - candidates.push({ el: node, text: firstLine, depth: getDepth(node) }); + const tempRef = `t${Date.now().toString(36)}${Math.random().toString(36).slice(2, 5)}`; + + // Page side: find the element by accessible name / composed text (shadow + // roots + same-origin frames), climb to the control that owns it, and park + // it in the ref registry so the click itself goes through the normal path. + const found = await execDom(tabId, (_text, _index, _exact, _ref) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const want = D.clean(_text).toLowerCase(); + if (!want) return { success: false, error: 'text is required' }; + const SKIP = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'HEAD', 'HTML', 'BODY', 'META', 'LINK']); + const hits = []; + const seen = new Set(); + const matches = (s) => { + const t = D.clean(s).toLowerCase(); + if (!t) return false; + return _exact ? t === want : t.includes(want); + }; + for (const root of D.allRoots(true)) { + let els = []; + try { els = root.querySelectorAll('*'); } catch {} + for (const el of els) { + if (SKIP.has(el.tagName)) continue; + const own = D.isInteractive(el) ? D.nameOf(el) : D.composedText(el, 200); + // aria-label / title / value also count as the element's text. + if (!matches(own) && !matches(D.attr(el, 'aria-label')) && !matches(D.attr(el, 'title')) + && !(el.tagName === 'INPUT' && matches(el.value))) continue; + // Climb to the control that owns this text (MUI: inside