From 1e3cdeae97f7c7d571e947bbefca77f895bd940f Mon Sep 17 00:00:00 2001 From: noiemany Date: Tue, 29 Sep 2026 17:59:20 +0300 Subject: [PATCH 1/8] feat(input): trusted CDP input for click/type/press_key/hover Synthetic DOM events are isTrusted:false, never move real focus, skip default actions (Tab traversal, Enter submit) and never fire focus/blur while the window is in the background, which broke legacy grids and lookup widgets. The write tools now drive the page over CDP like a user: - lib/cdp-session.js: one debugger session per tab, reused and detached after 30s idle; enables focus emulation and sizes never-shown background tabs (0x0 viewport) to their window. - lib/trusted-input.js: locate in page (iframe offsets, occlusion hint), real mouse/key events, key definitions, per-key typing. - click/type/press_key/hover use it by default (`trusted:false` opts out) and fall back to synthetic events when CDP can't attach. - press_key accepts "ctrl+a" combos; type returns the field value. - Lock shield lets the agent's own trusted input through for a short window; run_action/upload/dialog/drag share the tab session instead of attaching their own (which collided with "already attached"). Co-Authored-By: Claude Opus 5.5 --- extension/handlers/cdp.js | 25 +-- extension/handlers/interaction-advanced.js | 27 ++- extension/handlers/interaction.js | 142 +++++++++++-- extension/lib/cdp-session.js | 118 +++++++++++ extension/lib/overlay.js | 3 + extension/lib/trusted-input.js | 235 +++++++++++++++++++++ mcp-server/src/tools/click.ts | 4 +- mcp-server/src/tools/hover.ts | 1 + mcp-server/src/tools/press-key.ts | 3 +- mcp-server/src/tools/type.ts | 4 +- tests/extension-agent-api.test.ts | 2 +- tests/trusted-input.test.ts | 130 ++++++++++++ 12 files changed, 639 insertions(+), 55 deletions(-) create mode 100644 extension/lib/cdp-session.js create mode 100644 extension/lib/trusted-input.js create mode 100644 tests/trusted-input.test.ts diff --git a/extension/handlers/cdp.js b/extension/handlers/cdp.js index f49b285..cf1a3d7 100644 --- a/extension/handlers/cdp.js +++ b/extension/handlers/cdp.js @@ -5,6 +5,7 @@ */ import { resolveTab, safeExec } from '../lib/page-exec.js'; import { MAX_RESULT_CHARS } from '../lib/state.js'; +import { ensureCdp } from '../lib/cdp-session.js'; export async function handleRunAction(params, _sessionId, _agentName, signal) { const { tabId, code, actionParams = {} } = params; @@ -13,8 +14,8 @@ export async function handleRunAction(params, _sessionId, _agentName, signal) { // run_action stays on CDP (plan decision: CDP-only, can't be scripted — // it bypasses page CSP via the debugger protocol, unlike browser_evaluate). - await chrome.debugger.attach({ tabId: tab.id }, '1.3'); - try { + const send = await ensureCdp(tab.id); + { const paramsJson = JSON.stringify(actionParams); // Dual mode: accept EITHER a {execute:function()} tool wrapper (legacy // skill syntax) OR a plain JS expression/statement (simple usage like @@ -37,8 +38,7 @@ export async function handleRunAction(params, _sessionId, _agentName, signal) { } })()`; - const { result, exceptionDetails } = await chrome.debugger.sendCommand( - { tabId: tab.id }, + const { result, exceptionDetails } = await send( 'Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true }, ); @@ -64,8 +64,6 @@ export async function handleRunAction(params, _sessionId, _agentName, signal) { }; } return { success: true, result: result.value }; - } finally { - try { await chrome.debugger.detach({ tabId: tab.id }); } catch {} } } @@ -101,30 +99,25 @@ export async function handleUploadFile(params) { } // upload_file stays on CDP (DOM.setFileInputFiles is CDP-only). - let attached = false; let uploaded = false; try { - await chrome.debugger.attach({ tabId: tab.id }, '1.3'); - attached = true; - await chrome.debugger.sendCommand({ tabId: tab.id }, 'DOM.enable', {}); - const { root } = await chrome.debugger.sendCommand({ tabId: tab.id }, 'DOM.getDocument', {}); + const send = await ensureCdp(tab.id); + await send('DOM.enable'); + const { root } = await send('DOM.getDocument'); - const { nodeId } = await chrome.debugger.sendCommand({ tabId: tab.id }, 'DOM.querySelector', { + const { nodeId } = await send('DOM.querySelector', { nodeId: root.nodeId, selector: sel, }); if (!nodeId) throw new Error(`File input not found with selector: ${sel}`); - await chrome.debugger.sendCommand({ tabId: tab.id }, 'DOM.setFileInputFiles', { + await send('DOM.setFileInputFiles', { files: filePaths, nodeId, }); uploaded = true; } finally { - if (attached) { - try { await chrome.debugger.detach({ tabId: tab.id }); } catch {} - } // Fire the events React/Vue file inputs listen for after a successful set, // and always remove the short-lived Observation V2 handoff marker. try { diff --git a/extension/handlers/interaction-advanced.js b/extension/handlers/interaction-advanced.js index 48cd14b..6fe73ef 100644 --- a/extension/handlers/interaction-advanced.js +++ b/extension/handlers/interaction-advanced.js @@ -4,6 +4,8 @@ * each module stays focused and reviewable. */ import { resolveTab, safeExec } from '../lib/page-exec.js'; +import { withCdp } from '../lib/cdp-session.js'; +import { openShield, releaseShield } from '../lib/trusted-input.js'; export async function handleDialog(params) { const { tabId, action = 'accept', promptText } = params; @@ -12,17 +14,14 @@ export async function handleDialog(params) { // An ALREADY-OPEN native dialog freezes the page's JS thread — overrides // can't help in that state. CDP handles it out-of-band, so try it first. try { - await chrome.debugger.attach({ tabId: tab.id }, '1.3'); - try { - await chrome.debugger.sendCommand({ tabId: tab.id }, 'Page.enable', {}); - await chrome.debugger.sendCommand({ tabId: tab.id }, 'Page.handleJavaScriptDialog', { + return await withCdp(tab.id, async (send) => { + await send('Page.enable'); + await send('Page.handleJavaScriptDialog', { accept: action === 'accept', promptText: promptText || '', }); return { success: true, handled: 'open-dialog', action }; - } finally { - try { await chrome.debugger.detach({ tabId: tab.id }); } catch {} - } + }); } catch { // No dialog showing (or debugger unavailable) — arm future overrides. } @@ -100,27 +99,25 @@ export async function handleDrag(params) { throw new Error('Could not determine drag coordinates. Provide refs/selectors or explicit x,y coordinates.'); } - await chrome.debugger.attach({ tabId: tab.id }, '1.3'); - try { - await chrome.debugger.sendCommand({ tabId: tab.id }, 'Input.dispatchMouseEvent', { + await openShield(tab.id); + return withCdp(tab.id, async (send) => { + await send('Input.dispatchMouseEvent', { type: 'mousePressed', x: sx, y: sy, button: 'left', clickCount: 1, }); for (let i = 1; i <= steps; i++) { const progress = i / steps; - await chrome.debugger.sendCommand({ tabId: tab.id }, 'Input.dispatchMouseEvent', { + await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: Math.round(sx + (ex - sx) * progress), y: Math.round(sy + (ey - sy) * progress), button: 'left', }); } - await chrome.debugger.sendCommand({ tabId: tab.id }, 'Input.dispatchMouseEvent', { + await send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: ex, y: ey, button: 'left', clickCount: 1, }); return { success: true, from: { x: sx, y: sy }, to: { x: ex, y: ey } }; - } finally { - try { await chrome.debugger.detach({ tabId: tab.id }); } catch {} - } + }).finally(() => releaseShield(tab.id)); } export async function handleFillForm(params) { diff --git a/extension/handlers/interaction.js b/extension/handlers/interaction.js index d466099..af074db 100644 --- a/extension/handlers/interaction.js +++ b/extension/handlers/interaction.js @@ -6,11 +6,24 @@ import { resolveTab, requireTarget, safeExec, getFallback } from '../lib/page-exec.js'; import { autoReSnapshot } from './inspection.js'; import { PAGE_FALLBACK_INSTALL } from '../utils/smart-selector.js'; +import { trustedSender, locateTarget, releaseShield, cdpClickAt, cdpKeyPress, cdpTypeText, keyDefinition } from '../lib/trusted-input.js'; export { handleDialog, handleDrag, handleFillForm } from './interaction-advanced.js'; +/** Shared REF_GONE recovery: re-snapshot and hand fresh refs back (no auto-retry). */ +async function refGone(tabId, res, ref) { + const fresh = await autoReSnapshot(tabId); + return { + success: false, + error: `Element ${res._ref || ref} is gone from the DOM (feed scrolled/virtualized). Fresh refs captured — retry with a new ref.`, + freshRefs: fresh, + }; +} + +const BUTTONS = new Set(['left', 'right', 'middle']); + export async function handleClick(params) { - const { tabId, ref, selector, button = 'left', doubleClick = false } = params; + const { tabId, ref, selector, button = 'left', doubleClick = false, trusted } = params; await resolveTab(tabId); requireTarget(params); const fb = getFallback(tabId, ref); @@ -18,6 +31,29 @@ export async function handleClick(params) { // install-once pattern — eval rebuilding is impossible under MV3 CSP). if (fb) await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); + // Trusted path: a real mouse click at the element's centre over CDP, so + // focus moves, default actions run and the page sees isTrusted:true. + const send = await trustedSender(tabId, trusted); + if (send && BUTTONS.has(button)) { + const loc = await locateTarget(tabId, { ref, selector, fb }); + if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref); + if (loc?.success && loc.visible) { + try { + await cdpClickAt(send, loc.x, loc.y, { button, clickCount: doubleClick ? 2 : 1 }); + } finally { + await releaseShield(tabId); + } + return { + success: true, + input: 'cdp', + ...(loc.via ? { via: loc.via } : {}), + ...(loc.occludedBy ? { warning: `click point is covered by ${loc.occludedBy}` } : {}), + }; + } + await releaseShield(tabId); + // Zero-size element: no point to hit — fall through to the synthetic path. + } + const res = await safeExec(tabId, async (_ref, _sel, _btn, _dbl, _fb) => { // Same-origin iframe piercing (field report: legacy UIs live inside // #mainFrame — top-document lookups missed every element). @@ -97,19 +133,12 @@ export async function handleClick(params) { // Auto-re-snapshot and embed fresh refs so the agent retries in ONE step. // We do NOT auto-retry the click: it's non-idempotent and the element that // re-appears may be a different post after the scroll shifted the feed. - if (res && res.success === false && res.error === 'REF_GONE') { - const fresh = await autoReSnapshot(tabId); - return { - success: false, - error: `Element ${res._ref || ref} is gone from the DOM (feed scrolled/virtualized). Fresh refs captured — retry with a new ref.`, - freshRefs: fresh, - }; - } + if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref); return res; } export async function handleType(params) { - const { tabId, ref, selector, text, clear = false } = params; + const { tabId, ref, selector, text, clear = false, trusted } = params; await resolveTab(tabId); requireTarget(params); const fb = getFallback(tabId, ref); @@ -117,6 +146,35 @@ export async function handleType(params) { // install-once pattern — eval rebuilding is impossible under MV3 CSP). if (fb) await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); + // Trusted path: focus the field, then real key presses over CDP (keydown / + // keypress / input / keyup per character). Like a user, this does NOT fire + // `change` until focus leaves the field — press Tab to commit. + const send = await trustedSender(tabId, trusted); + if (send) { + const loc = await locateTarget(tabId, { ref, selector, fb, mode: clear ? 'clear' : 'focus' }); + if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref); + if (loc?.success && (loc.focused || loc.visible)) { + let after; + try { + // Not focusable by script (custom widget): click it like a user would. + if (!loc.focused) await cdpClickAt(send, loc.x, loc.y); + if (clear && loc.needsSelectAll) await cdpKeyPress(send, 'a', ['ctrl']); + if (clear && loc.hasText && !text) await cdpKeyPress(send, 'Backspace'); + await cdpTypeText(send, text); + } finally { + after = await releaseShield(tabId); + } + return { + success: true, + typed: text, + input: 'cdp', + ...(after?.value != null ? { value: after.value } : {}), + ...(loc.via ? { via: loc.via } : {}), + }; + } + await releaseShield(tabId); + } + const res = await safeExec(tabId, (_ref, _sel, _text, _clear, _fb) => { // Same-origin iframe piercing (field report: legacy UIs live inside // #mainFrame — top-document lookups missed every element). @@ -177,21 +235,50 @@ export async function handleType(params) { // Virtualization recovery (same as click): type target is gone, so // auto-re-snapshot and embed fresh refs. No auto-retry (non-idempotent). - if (res && res.success === false && res.error === 'REF_GONE') { - const fresh = await autoReSnapshot(tabId); - return { - success: false, - error: `Element ${res._ref || ref} is gone from the DOM (feed scrolled/virtualized). Fresh refs captured — retry with a new ref.`, - freshRefs: fresh, - }; - } + if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref); return res; } +/** "ctrl+a" / "Control+Shift+Tab" -> { key: 'a', mods: ['ctrl'] }; plain keys pass through. */ +export function parseKeyCombo(key, modifiers = []) { + const mods = [...modifiers]; + if (typeof key !== 'string' || key.length < 3 || !key.includes('+')) return { key, mods }; + const parts = key.split('+'); + const last = parts.pop() || '+'; + const alias = { control: 'ctrl', ctrl: 'ctrl', alt: 'alt', option: 'alt', shift: 'shift', meta: 'meta', cmd: 'meta', command: 'meta', win: 'meta' }; + for (const part of parts) { + const m = alias[part.trim().toLowerCase()]; + if (!m) return { key, mods: [...modifiers] }; + if (!mods.includes(m)) mods.push(m); + } + return { key: last, mods }; +} + export async function handlePressKey(params) { - const { tabId, key, modifiers = [], ref, selector } = params; + const { tabId, ref, selector, trusted } = params; + const { key, mods: modifiers } = parseKeyCombo(params.key, params.modifiers || []); await resolveTab(tabId); + // Trusted path: a real key press, so default actions run (Tab moves focus + // and fires blur/focusout, Enter submits, arrows drive autocomplete menus). + let knownKey = true; + try { keyDefinition(key); } catch { knownKey = false; } + const send = knownKey ? await trustedSender(tabId, trusted) : null; + if (send) { + const loc = await locateTarget(tabId, { ref, selector, mode: ref || selector ? 'focus' : 'active' }); + if (!loc || loc.success === false) { + await releaseShield(tabId); + if (ref || selector) return { success: false, error: `Element ${ref ? `with ref ${ref}` : `with selector ${selector}`} not found` }; + } + let after; + try { + await cdpKeyPress(send, key, modifiers); + } finally { + after = await releaseShield(tabId); + } + return { success: true, key, ...(modifiers.length ? { modifiers } : {}), input: 'cdp', ...(after?.focusedTag ? { focused: after.focusedTag } : {}) }; + } + return safeExec(tabId, (_key, _mods, _ref, _sel) => { // Same-origin iframe piercing (field report: legacy UIs live inside // #mainFrame — top-document lookups missed every element). @@ -244,10 +331,25 @@ export async function handlePressKey(params) { } export async function handleHover(params) { - const { tabId, ref, selector } = params; + const { tabId, ref, selector, trusted } = params; await resolveTab(tabId); requireTarget(params); + const send = await trustedSender(tabId, trusted); + if (send) { + const loc = await locateTarget(tabId, { ref, selector }); + if (loc?.success && loc.visible) { + try { + await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: loc.x, y: loc.y }); + } finally { + await releaseShield(tabId); + } + return { success: true, input: 'cdp' }; + } + await releaseShield(tabId); + if (loc && loc.success === false) return { success: false, error: 'Element not found' }; + } + return safeExec(tabId, (_ref, _sel) => { // Same-origin iframe piercing (field report: legacy UIs live inside // #mainFrame — top-document lookups missed every element). diff --git a/extension/lib/cdp-session.js b/extension/lib/cdp-session.js new file mode 100644 index 0000000..d46211d --- /dev/null +++ b/extension/lib/cdp-session.js @@ -0,0 +1,118 @@ +/** + * Shared chrome.debugger session per tab. + * + * Trusted input (clicks/keys the page sees as isTrusted:true), background-tab + * screenshots and REPL evaluate all go through CDP. Attaching per call costs a + * round-trip and flashes the "is being debugged" banner on every action, and two + * handlers attaching the same tab at once fail with "Another debugger is already + * attached". So one session is kept per tab and detached after IDLE_MS without use. + * + * Every session also enables focus emulation: the page believes it has focus + * even while its window is in the background, so focus/blur/focusin/focusout + * fire as they would for a real user (legacy grids and lookups depend on them). + */ + +export const IDLE_MS = 30_000; + +/** tabId -> { ready: Promise, timer } */ +const sessions = new Map(); + +function armIdle(tabId) { + const s = sessions.get(tabId); + if (!s) return; + clearTimeout(s.timer); + s.timer = setTimeout(() => { detachCdp(tabId); }, IDLE_MS); +} + +async function attach(tabId) { + const target = { tabId }; + try { + await chrome.debugger.attach(target, '1.3'); + } catch (err) { + // A service-worker restart forgets the map but Chrome may keep our + // attachment: probe it and reuse instead of failing. + if (!/already attached/i.test(String(err?.message || err))) throw err; + await chrome.debugger.sendCommand(target, 'Runtime.evaluate', { expression: '1' }); + } + await chrome.debugger.sendCommand(target, 'Emulation.setFocusEmulationEnabled', { enabled: true }).catch(() => {}); + // Can fail while the page is still loading; locateTarget retries it. + await ensureViewport(tabId).catch(() => {}); +} + +/** + * A tab opened in the background and never shown has a 0x0 viewport: nothing + * is laid out, so every element sits at negative coordinates and mouse input + * or screenshots hit nothing. Give it its window's size for the session + * (emulation overrides are dropped automatically on detach). + */ +export async function ensureViewport(tabId) { + const target = { tabId }; + const { result } = await chrome.debugger.sendCommand(target, 'Runtime.evaluate', { + expression: 'innerWidth * innerHeight', returnByValue: true, + }); + if (result?.value > 0) return; + const tab = await chrome.tabs.get(tabId); + const win = await chrome.windows.get(tab.windowId); + await chrome.debugger.sendCommand(target, 'Emulation.setDeviceMetricsOverride', { + width: Math.max(800, (win.width || 1280) - 16), + height: Math.max(600, (win.height || 900) - 140), // minus tab strip + toolbar + deviceScaleFactor: 0, + mobile: false, + }); +} + +/** Attach (or reuse) the tab's session and return a `send(method, params)` helper. */ +export async function ensureCdp(tabId) { + let s = sessions.get(tabId); + if (!s) { + s = { ready: attach(tabId), timer: null }; + sessions.set(tabId, s); + try { + await s.ready; + } catch (err) { + sessions.delete(tabId); + throw err; + } + } else { + await s.ready; + } + armIdle(tabId); + return (method, params = {}) => chrome.debugger.sendCommand({ tabId }, method, params); +} + +/** Run fn(send) with the tab's session; keeps the session alive for reuse. */ +export async function withCdp(tabId, fn) { + const send = await ensureCdp(tabId); + try { + return await fn(send); + } finally { + armIdle(tabId); + } +} + +export async function detachCdp(tabId) { + const s = sessions.get(tabId); + if (!s) return; + clearTimeout(s.timer); + sessions.delete(tabId); + try { await chrome.debugger.detach({ tabId }); } catch { /* already gone */ } +} + +export function hasCdp(tabId) { + return sessions.has(tabId); +} + +// The user can dismiss the debugger banner, or the tab can close/navigate to a +// protected page: forget the session so the next call re-attaches cleanly. +if (typeof chrome !== 'undefined' && chrome.debugger?.onDetach) { + chrome.debugger.onDetach.addListener((source) => { + const s = sessions.get(source.tabId); + if (s) { clearTimeout(s.timer); sessions.delete(source.tabId); } + }); +} +if (typeof chrome !== 'undefined' && chrome.tabs?.onRemoved) { + chrome.tabs.onRemoved.addListener((tabId) => { + const s = sessions.get(tabId); + if (s) { clearTimeout(s.timer); sessions.delete(tabId); } + }); +} diff --git a/extension/lib/overlay.js b/extension/lib/overlay.js index 43dfc2f..49d51e8 100644 --- a/extension/lib/overlay.js +++ b/extension/lib/overlay.js @@ -44,6 +44,9 @@ export async function showLockShield(tabId, label) { // locked tabs. Real user input is isTrusted===true (DOM invariant, // unforgeable) and gets blocked. (Fix-loop 3: audit C2.) if (e.isTrusted === false) return; + // The agent's own CDP input (trusted-input.js) is isTrusted===true + // too: it opens a short pass-through window around each action. + if (window.__bcAgentInputUntil > Date.now()) return; e.preventDefault(); e.stopImmediatePropagation(); }; diff --git a/extension/lib/trusted-input.js b/extension/lib/trusted-input.js new file mode 100644 index 0000000..bf18bff --- /dev/null +++ b/extension/lib/trusted-input.js @@ -0,0 +1,235 @@ +/** + * Trusted input over CDP (Input.dispatchMouseEvent / Input.dispatchKeyEvent). + * + * Synthetic DOM events (el.dispatchEvent) are isTrusted:false, never move real + * focus, never run default actions (Tab focus traversal, Enter form submit, + * autocomplete menus) and never fire focus/blur while the window is in the + * background. Legacy grids and lookup widgets depend on all of that, so the + * write tools now drive the page the way a user does, like Claude in Chrome. + * The synthetic path stays as the fallback when CDP can't attach. + */ +import { ensureCdp, ensureViewport, hasCdp } from './cdp-session.js'; +import { safeExec } from './page-exec.js'; + +const MOD_BITS = { alt: 1, ctrl: 2, meta: 4, shift: 8 }; + +export function modifierBits(mods = []) { + return mods.reduce((bits, m) => bits | (MOD_BITS[m] || 0), 0); +} + +const NAMED_KEYS = { + Enter: { code: 'Enter', vk: 13, text: '\r' }, + Tab: { code: 'Tab', vk: 9 }, + Escape: { code: 'Escape', vk: 27 }, + Backspace: { code: 'Backspace', vk: 8 }, + Delete: { code: 'Delete', vk: 46 }, + Insert: { code: 'Insert', vk: 45 }, + ArrowUp: { code: 'ArrowUp', vk: 38 }, + ArrowDown: { code: 'ArrowDown', vk: 40 }, + ArrowLeft: { code: 'ArrowLeft', vk: 37 }, + ArrowRight: { code: 'ArrowRight', vk: 39 }, + Home: { code: 'Home', vk: 36 }, + End: { code: 'End', vk: 35 }, + PageUp: { code: 'PageUp', vk: 33 }, + PageDown: { code: 'PageDown', vk: 34 }, + ' ': { code: 'Space', vk: 32, text: ' ' }, + Space: { code: 'Space', vk: 32, text: ' ', key: ' ' }, + Shift: { code: 'ShiftLeft', vk: 16 }, + Control: { code: 'ControlLeft', vk: 17 }, + Alt: { code: 'AltLeft', vk: 18 }, + Meta: { code: 'MetaLeft', vk: 91 }, +}; +for (let i = 1; i <= 12; i++) NAMED_KEYS[`F${i}`] = { code: `F${i}`, vk: 111 + i }; +const KEY_ALIASES = { Esc: 'Escape', Return: 'Enter', Del: 'Delete', Up: 'ArrowUp', Down: 'ArrowDown', Left: 'ArrowLeft', Right: 'ArrowRight' }; + +/** CDP key definition for a key name ("Enter", "a", "7", "ب"). */ +export function keyDefinition(rawKey) { + const name = KEY_ALIASES[rawKey] || rawKey; + const named = NAMED_KEYS[name]; + if (named) return { key: named.key || name, code: named.code, vk: named.vk, text: named.text }; + if ([...name].length !== 1) throw new Error(`Unknown key "${rawKey}"`); + const ch = name; + const upper = ch.toUpperCase(); + if (/^[a-z]$/i.test(ch)) return { key: ch, code: `Key${upper}`, vk: upper.charCodeAt(0), text: ch }; + if (/^[0-9]$/.test(ch)) return { key: ch, code: `Digit${ch}`, vk: ch.charCodeAt(0), text: ch }; + return { key: ch, code: '', vk: 0, text: ch }; +} + +/** One full key press (keyDown[+char] / keyUp). With ctrl/alt/meta no text is produced. */ +export async function cdpKeyPress(send, rawKey, mods = []) { + const def = keyDefinition(rawKey); + const modifiers = modifierBits(mods); + const printable = def.text && !(modifiers & (MOD_BITS.ctrl | MOD_BITS.alt | MOD_BITS.meta)); + const base = { key: def.key, code: def.code, windowsVirtualKeyCode: def.vk, nativeVirtualKeyCode: def.vk, modifiers }; + await send('Input.dispatchKeyEvent', { + type: printable ? 'keyDown' : 'rawKeyDown', + ...base, + ...(printable ? { text: def.text, unmodifiedText: def.text } : {}), + }); + await send('Input.dispatchKeyEvent', { type: 'keyUp', ...base }); +} + +/** Up to this length text is typed key by key (keydown/keypress/input/keyup per char). */ +export const PER_KEY_MAX = 300; + +export async function cdpTypeText(send, text) { + if ([...text].length > PER_KEY_MAX) { + await send('Input.insertText', { text }); + return; + } + for (const ch of text) { + if (ch === '\n') { await cdpKeyPress(send, 'Enter'); continue; } + if (ch === '\t') { await cdpKeyPress(send, 'Tab'); continue; } + await cdpKeyPress(send, ch); + } +} + +export async function cdpClickAt(send, x, y, { button = 'left', clickCount = 1, modifiers = 0 } = {}) { + await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x, y, modifiers }); + for (let n = 1; n <= clickCount; n++) { + await send('Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount: n, modifiers }); + await send('Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount: n, modifiers }); + } +} + +/** + * Page-side: resolve the target (ref → selector → smart fallback, piercing + * same-origin iframes), scroll it into view, optionally focus/select it, and + * return its centre in TOP-level viewport coordinates (what CDP expects). + * Also opens the lock shield for the agent's own trusted input for a few + * seconds, since trusted events are otherwise blocked by it. + * Kept self-contained: it is serialized into the page by chrome.scripting. + */ +function pageLocate(ref, sel, fb, mode) { + function deepQuery(s) { + const q = (doc, depth) => { + try { const el = doc.querySelector(s); if (el) return el; } catch {} + if (depth >= 3) return null; + for (const f of doc.querySelectorAll('iframe')) { + try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} + } + return null; + }; + return q(document, 0); + } + let el = ref ? deepQuery(`[data-mcp-ref="${ref}"]`) : null; + let via = 'ref'; + if (!el && sel) { el = deepQuery(sel); via = 'selector'; } + const resolveFallback = (globalThis.__browserControllerFallbackRuntime || {}).resolveFallback || null; + if (!el && fb && resolveFallback) { el = resolveFallback(fb); if (el) via = 'fallback'; } + if (!el && mode === 'active') { el = document.activeElement; via = 'active'; } + if (!el) return { success: false, error: 'REF_GONE', _ref: ref }; + + // Agent input pass-through for the lock shield (see overlay.js). + window.__bcAgentInputUntil = Date.now() + 8000; + const shield = document.getElementById('__bc-lock-shield'); + if (shield) shield.style.pointerEvents = 'none'; + + if (mode !== 'active') el.scrollIntoView({ behavior: 'instant', block: 'center', inline: 'center' }); + if (mode === 'focus' || mode === 'clear') { + if (typeof el.focus === 'function') el.focus(); + if (mode === 'clear') { + if (el.isContentEditable) { + const r = el.ownerDocument.createRange(); + r.selectNodeContents(el); + const s = el.ownerDocument.defaultView.getSelection(); + s.removeAllRanges(); + s.addRange(r); + } else if (typeof el.select === 'function') { + el.select(); + } + } + } + + const rect = el.getBoundingClientRect(); + let x = rect.left + rect.width / 2; + let y = rect.top + rect.height / 2; + // Add the offsets of every enclosing same-origin iframe. + let win = el.ownerDocument.defaultView; + while (win && win !== window && win.frameElement) { + const fr = win.frameElement.getBoundingClientRect(); + const cs = win.frameElement.ownerDocument.defaultView.getComputedStyle(win.frameElement); + x += fr.left + (parseFloat(cs.borderLeftWidth) || 0) + (parseFloat(cs.paddingLeft) || 0); + y += fr.top + (parseFloat(cs.borderTopWidth) || 0) + (parseFloat(cs.paddingTop) || 0); + win = win.parent; + } + const doc = el.ownerDocument; + const focused = doc.activeElement === el || (el.contains && el.contains(doc.activeElement)); + const hasValue = 'value' in el && !el.isContentEditable && typeof el.value === 'string'; + let fullySelected = false; + try { fullySelected = el.selectionStart === 0 && el.selectionEnd === el.value.length; } catch { /* number/email inputs */ } + // What a real click at (x, y) would hit (top document only). + let occludedBy = null; + if (win === window || !el.ownerDocument.defaultView.frameElement) { + const hit = document.elementFromPoint(x, y); + if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) { + occludedBy = hit.tagName.toLowerCase() + (hit.id ? `#${hit.id}` : '') + + (typeof hit.className === 'string' && hit.className.trim() ? `.${hit.className.trim().split(/\s+/).slice(0, 2).join('.')}` : ''); + } + } + return { + success: true, + x, y, + zeroViewport: window.innerWidth === 0 || window.innerHeight === 0, + visible: rect.width > 0 && rect.height > 0, + focused, + // clear: a value that select() could not select still needs Ctrl+A. + needsSelectAll: hasValue && el.value.length > 0 && !fullySelected, + hasText: hasValue ? el.value.length > 0 : (el.textContent || '').length > 0, + ...(occludedBy ? { occludedBy } : {}), + ...(via !== 'ref' ? { via } : {}), + }; +} + +/** Close the shield pass-through; report the focused field's value for verification. */ +function pageRelease() { + window.__bcAgentInputUntil = 0; + const shield = document.getElementById('__bc-lock-shield'); + if (shield) shield.style.pointerEvents = 'auto'; + let a = document.activeElement; + while (a && a.tagName === 'IFRAME') { + try { a = a.contentDocument.activeElement; } catch { break; } + } + if (!a || a === document.body) return { value: null }; + const value = typeof a.value === 'string' ? a.value : a.isContentEditable ? a.textContent : null; + return { value: value == null ? null : value.slice(0, 500), focusedTag: a.tagName.toLowerCase() + (a.id ? `#${a.id}` : '') }; +} + +export async function locateTarget(tabId, { ref, selector, fb, mode = 'none' }) { + const loc = await safeExec(tabId, pageLocate, [ref, selector, fb, mode]); + // Never-shown background tab: size its viewport, then measure again. + if (loc?.success && loc.zeroViewport && hasCdp(tabId)) { + await ensureViewport(tabId).catch(() => {}); + return safeExec(tabId, pageLocate, [ref, selector, fb, mode]); + } + return loc; +} + +/** Let the agent's own trusted input through the lock shield (for raw-coordinate tools like drag). */ +export async function openShield(tabId) { + try { + await safeExec(tabId, () => { + window.__bcAgentInputUntil = Date.now() + 8000; + const shield = document.getElementById('__bc-lock-shield'); + if (shield) shield.style.pointerEvents = 'none'; + }, []); + } catch { /* protected page: CDP input still works, no shield there */ } +} + +export async function releaseShield(tabId) { + try { return (await safeExec(tabId, pageRelease, [])) || {}; } catch { return {}; /* page navigated away */ } +} + +/** + * Attach the tab's CDP session or return null when CDP is unavailable + * (another debugger owns the tab, policy blocks chrome.debugger…) so callers + * fall back to synthetic events. `trusted:false` forces the synthetic path. + */ +export async function trustedSender(tabId, trusted) { + if (trusted === false) return null; + try { + return await ensureCdp(tabId); + } catch { + return null; + } +} diff --git a/mcp-server/src/tools/click.ts b/mcp-server/src/tools/click.ts index d58cfc5..8e4db7d 100644 --- a/mcp-server/src/tools/click.ts +++ b/mcp-server/src/tools/click.ts @@ -4,13 +4,15 @@ import { requireTabId, forwardHandler } from './types.js'; export const clickTool: ToolDefinition = { name: 'browser_click', - summary: 'Click an element by ref or CSS selector', description: 'Click an element on the page using a ref from snapshot or a CSS selector', + summary: 'Click an element by ref or CSS selector', + description: 'Click an element on the page using a ref from snapshot or a CSS selector. Uses a real mouse click over CDP (isTrusted events, real focus, default actions — works in background windows); falls back to synthetic DOM events if the debugger cannot attach.', inputSchema: z.object({ tabId: requireTabId(), ref: z.string().optional().describe('Element reference from snapshot (e.g. "e12")'), selector: z.string().optional().describe('CSS selector for the element'), button: z.enum(['left', 'right', 'middle']).optional().default('left'), doubleClick: z.boolean().optional().default(false), + trusted: z.boolean().optional().describe('Real (isTrusted) input over CDP — default. false = synthetic DOM events, no debugger banner.'), }).superRefine((params, ctx) => { if (!params.ref && !params.selector) { ctx.addIssue({ code: 'custom', message: 'ref or selector is required', path: ['ref'] }); diff --git a/mcp-server/src/tools/hover.ts b/mcp-server/src/tools/hover.ts index 5950a05..1b5180c 100644 --- a/mcp-server/src/tools/hover.ts +++ b/mcp-server/src/tools/hover.ts @@ -9,6 +9,7 @@ export const hoverTool: ToolDefinition = { tabId: requireTabId(), ref: z.string().optional().describe('Element reference from snapshot'), selector: z.string().optional().describe('CSS selector for the element'), + trusted: z.boolean().optional().describe('Real (isTrusted) mouse move over CDP — default. false = synthetic DOM events, no debugger banner.'), }), timeoutMs: 5_000, handler: forwardHandler('browser_hover'), diff --git a/mcp-server/src/tools/press-key.ts b/mcp-server/src/tools/press-key.ts index 7d6348c..cee74cd 100644 --- a/mcp-server/src/tools/press-key.ts +++ b/mcp-server/src/tools/press-key.ts @@ -5,7 +5,7 @@ import { requireTabId, forwardHandler } from './types.js'; export const pressKeyTool: ToolDefinition = { name: 'browser_press_key', summary: 'Press a keyboard key (Enter, Tab, Escape, etc.)', description: - 'Press a keyboard key or combination (Enter, Escape, Tab, ArrowDown, etc). Supports modifiers like Ctrl+A, Cmd+C.', + 'Press a keyboard key or combination (Enter, Escape, Tab, ArrowDown, etc). Accepts combos as "ctrl+a" or via modifiers. Real key press over CDP: Tab moves focus (fires blur/focusout), Enter submits, arrows drive autocomplete menus.', inputSchema: z.object({ tabId: requireTabId(), key: z.string().describe('Key name (e.g. "Enter", "Escape", "Tab", "ArrowDown", "a")'), @@ -15,6 +15,7 @@ export const pressKeyTool: ToolDefinition = { .describe('Modifier keys to hold'), ref: z.string().optional().describe('Element ref to focus before pressing'), selector: z.string().optional().describe('CSS selector to focus before pressing'), + trusted: z.boolean().optional().describe('Real (isTrusted) input over CDP — default. false = synthetic DOM events, no debugger banner.'), }), timeoutMs: 5_000, handler: forwardHandler('browser_press_key'), diff --git a/mcp-server/src/tools/type.ts b/mcp-server/src/tools/type.ts index 9289c19..7fad852 100644 --- a/mcp-server/src/tools/type.ts +++ b/mcp-server/src/tools/type.ts @@ -4,13 +4,15 @@ import { requireTabId, forwardHandler } from './types.js'; export const typeTool: ToolDefinition = { name: 'browser_type', - summary: 'Type text into an input element', description: 'Type text into an input element', + summary: 'Type text into an input element', + description: 'Focus an input and type text with real key presses over CDP (keydown/keypress/input/keyup per character, like a user). Like a user, `change`/blur fire only when focus leaves — follow with browser_press_key Tab to commit. Returns the field value after typing. Falls back to synthetic events if the debugger cannot attach.', inputSchema: z.object({ tabId: requireTabId(), ref: z.string().optional().describe('Element reference from snapshot'), selector: z.string().optional().describe('CSS selector for the input'), text: z.string().describe('Text to type'), clear: z.boolean().optional().default(false).describe('Clear the field before typing'), + trusted: z.boolean().optional().describe('Real (isTrusted) input over CDP — default. false = synthetic DOM events, no debugger banner.'), }), timeoutMs: 15_000, handler: forwardHandler('browser_type'), diff --git a/tests/extension-agent-api.test.ts b/tests/extension-agent-api.test.ts index 8e225fe..dec7bd2 100644 --- a/tests/extension-agent-api.test.ts +++ b/tests/extension-agent-api.test.ts @@ -196,7 +196,7 @@ describe('observe/act extension handlers', () => { }, 'session-a'); expect(result).toMatchObject({ success: true, ok: true, action: 'upload', files: ['/tmp/resume.pdf'] }); - expect(debuggerCommands).toEqual(['DOM.enable', 'DOM.getDocument', 'DOM.querySelector', 'DOM.setFileInputFiles']); + expect(debuggerCommands.filter((m) => m.startsWith('DOM.'))).toEqual(['DOM.enable', 'DOM.getDocument', 'DOM.querySelector', 'DOM.setFileInputFiles']); expect(result.metrics).toMatchObject({ protocolCalls: 8 }); }); diff --git a/tests/trusted-input.test.ts b/tests/trusted-input.test.ts new file mode 100644 index 0000000..8c20ddf --- /dev/null +++ b/tests/trusted-input.test.ts @@ -0,0 +1,130 @@ +import { describe, it, expect, beforeEach } from 'vitest'; + +// Minimal chrome mock: executeScript answers with the next queued page result, +// debugger records every CDP command. +const cdp: Array<{ method: string; params: Record }> = []; +const pageResults: unknown[] = []; +let attachError: Error | null = null; +let attachCount = 0; + +(globalThis as unknown as { chrome: unknown }).chrome = { + tabs: { + get: async (id: number) => ({ id, url: 'https://example.test', windowId: 1 }), + onRemoved: { addListener: () => {} }, + }, + scripting: { + executeScript: async () => [{ result: pageResults.length ? pageResults.shift() : {} }], + }, + storage: { + session: { get: async () => ({}), set: async () => {} }, + local: { get: async () => ({}), set: async () => {} }, + }, + debugger: { + attach: async () => { attachCount++; if (attachError) throw attachError; }, + detach: async () => {}, + sendCommand: async (_t: unknown, method: string, params: Record = {}) => { + cdp.push({ method, params }); + return {}; + }, + onDetach: { addListener: () => {} }, + }, +}; + +const ti = await import('../extension/lib/trusted-input.js'); +const session = await import('../extension/lib/cdp-session.js'); +const { handleClick, handleType, handlePressKey, parseKeyCombo } = await import('../extension/handlers/interaction.js'); + +const inputs = () => cdp.filter((c) => c.method.startsWith('Input.')); + +describe('trusted input (CDP)', () => { + beforeEach(async () => { + cdp.length = 0; + pageResults.length = 0; + attachError = null; + attachCount = 0; + await session.detachCdp(5); + }); + + it('maps key names to CDP key definitions', () => { + expect(ti.keyDefinition('Enter')).toMatchObject({ key: 'Enter', vk: 13, text: '\r' }); + expect(ti.keyDefinition('Esc')).toMatchObject({ key: 'Escape', vk: 27 }); + expect(ti.keyDefinition('a')).toMatchObject({ code: 'KeyA', vk: 65, text: 'a' }); + expect(ti.keyDefinition('7')).toMatchObject({ code: 'Digit7', vk: 55 }); + expect(ti.keyDefinition('ب')).toMatchObject({ key: 'ب', text: 'ب' }); + expect(ti.keyDefinition('F5')).toMatchObject({ vk: 116 }); + expect(() => ti.keyDefinition('NoSuchKey')).toThrow(/Unknown key/); + }); + + it('parses "ctrl+a" style combos', () => { + expect(parseKeyCombo('ctrl+a')).toEqual({ key: 'a', mods: ['ctrl'] }); + expect(parseKeyCombo('Control+Shift+Tab')).toEqual({ key: 'Tab', mods: ['ctrl', 'shift'] }); + expect(parseKeyCombo('Enter', ['alt'])).toEqual({ key: 'Enter', mods: ['alt'] }); + expect(parseKeyCombo('+')).toEqual({ key: '+', mods: [] }); + }); + + it('a modified key press produces no text (shortcut, not typing)', async () => { + const send = async (method: string, params: Record) => { cdp.push({ method, params }); }; + await ti.cdpKeyPress(send, 'a', ['ctrl']); + expect(cdp[0]).toMatchObject({ params: { type: 'rawKeyDown', modifiers: 2 } }); + expect(cdp[0].params.text).toBeUndefined(); + }); + + it('enables focus emulation once and reuses the session', async () => { + await session.ensureCdp(5); + await session.ensureCdp(5); + expect(attachCount).toBe(1); + expect(cdp.filter((c) => c.method === 'Emulation.setFocusEmulationEnabled')).toHaveLength(1); + }); + + it('click: real mouse events at the located centre', async () => { + pageResults.push({ success: true, x: 100, y: 40, visible: true, focused: false }); + const res = await handleClick({ tabId: 5, selector: '#go' }); + expect(res).toMatchObject({ success: true, input: 'cdp' }); + expect(inputs().map((c) => c.params.type)).toEqual(['mouseMoved', 'mousePressed', 'mouseReleased']); + expect(inputs()[1].params).toMatchObject({ x: 100, y: 40, button: 'left', clickCount: 1 }); + }); + + it('click: double click sends clickCount 1 then 2', async () => { + pageResults.push({ success: true, x: 1, y: 2, visible: true }); + await handleClick({ tabId: 5, selector: '#go', doubleClick: true }); + expect(inputs().filter((c) => c.params.type === 'mousePressed').map((c) => c.params.clickCount)).toEqual([1, 2]); + }); + + it('type: key per character, select-all when clearing an unselectable value', async () => { + pageResults.push({ success: true, x: 1, y: 2, visible: true, focused: true, needsSelectAll: true, hasText: true }); + pageResults.push({ value: 'ab' }); + const res = await handleType({ tabId: 5, selector: '#f', text: 'ab', clear: true }); + expect(res).toMatchObject({ success: true, input: 'cdp', value: 'ab' }); + const downs = inputs().filter((c) => c.params.type !== 'keyUp'); + expect(downs.map((c) => c.params.key)).toEqual(['a', 'a', 'b']); + expect(downs[0].params.modifiers).toBe(2); // Ctrl+A + expect(downs[1].params.text).toBe('a'); + }); + + it('type: clearing to empty presses Backspace', async () => { + pageResults.push({ success: true, x: 1, y: 2, visible: true, focused: true, needsSelectAll: false, hasText: true }); + await handleType({ tabId: 5, selector: '#f', text: '', clear: true }); + expect(inputs().map((c) => c.params.key)).toEqual(['Backspace', 'Backspace']); + }); + + it('press_key: combo goes through CDP', async () => { + pageResults.push({ success: true, x: 0, y: 0, visible: true }); + const res = await handlePressKey({ tabId: 5, key: 'shift+Tab' }); + expect(res).toMatchObject({ success: true, key: 'Tab', modifiers: ['shift'], input: 'cdp' }); + expect(inputs()[0].params).toMatchObject({ key: 'Tab', modifiers: 8 }); + }); + + it('falls back to synthetic events when the debugger cannot attach', async () => { + attachError = new Error('Another debugger is attached'); + pageResults.push({ success: true }); + const res = await handleClick({ tabId: 5, selector: '#go' }); + expect(res).toEqual({ success: true }); + expect(inputs()).toHaveLength(0); + }); + + it('trusted:false forces the synthetic path', async () => { + pageResults.push({ success: true }); + await handleClick({ tabId: 5, selector: '#go', trusted: false }); + expect(attachCount).toBe(0); + }); +}); From be6c7a68ec24a61da3898abfa32ffc88990dfbbc Mon Sep 17 00:00:00 2001 From: noiemany Date: Tue, 29 Sep 2026 18:01:12 +0300 Subject: [PATCH 2/8] feat(batch): browser_batch runs several tool calls in one round-trip Steps run in order in the MCP process, each validated and executed exactly like a standalone call (same schemas, timeouts, error payloads). Stops at the first failing step unless continueOnError; a top-level tabId is applied to steps that don't set one. Nested batches and the meta tool are rejected. Guidance/preamble updated for trusted input. Co-Authored-By: Claude Opus 5.5 --- mcp-server/src/tools/batch.ts | 82 +++++++++++++++++++++++++++++++++++ mcp-server/src/tools/index.ts | 2 + mcp-server/src/tools/meta.ts | 9 ++-- tests/tools/batch.test.ts | 81 ++++++++++++++++++++++++++++++++++ tests/tools/registry.test.ts | 6 ++- 5 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 mcp-server/src/tools/batch.ts create mode 100644 tests/tools/batch.test.ts diff --git a/mcp-server/src/tools/batch.ts b/mcp-server/src/tools/batch.ts new file mode 100644 index 0000000..fbb7cfe --- /dev/null +++ b/mcp-server/src/tools/batch.ts @@ -0,0 +1,82 @@ +import { z } from 'zod'; +import type { ToolDefinition, ToolResult } from './types.js'; +import { optionalTabId } from './types.js'; +import { toolMap } from './index.js'; +import { parseToolParams } from '../register-tools.js'; + +/** Tools that must not run inside a batch (recursion / discovery only). */ +const NOT_BATCHABLE = new Set(['browser_batch', 'browser_tools']); +const MAX_STEPS = 50; + +/** + * Run several browser tool calls in one round-trip (like Claude in Chrome's + * browser_batch). Steps run strictly in order; the batch stops at the first + * failing step so the agent never acts on a page that didn't reach the + * expected state. Each step is validated and executed exactly as if it had + * been called on its own, so per-tool timeouts, retries and error payloads + * are unchanged. + */ +export const batchTool: ToolDefinition = { + name: 'browser_batch', + summary: 'Run several browser actions in one call (stops at first error)', + description: + 'Run a sequence of browser tool calls in ONE call, in order, and get all their results back. Use it to cut round-trips when you already know the next steps, e.g. click a field → type → press Tab → wait → read text. Stops at the first failing step (remaining steps are skipped) unless continueOnError is true. A top-level tabId is applied to every step that does not set its own. Steps cannot be nested batches.', + inputSchema: z.object({ + tabId: optionalTabId().describe('Default tab id for every step that does not set its own tabId'), + actions: z + .array(z.object({ + tool: z.string().describe('Tool name, e.g. "browser_click"'), + params: z.record(z.string(), z.unknown()).optional().describe('That tool\'s arguments'), + })) + .min(1) + .max(MAX_STEPS) + .describe(`Steps to run in order (max ${MAX_STEPS})`), + continueOnError: z.boolean().optional().default(false).describe('Keep going after a failing step'), + }), + // Longest a single MCP call may reasonably take; each step keeps its own + // transport timeout. + timeoutMs: 300_000, + async handler(host, params) { + const { tabId, actions, continueOnError } = params as { + tabId?: number; + actions: Array<{ tool: string; params?: Record }>; + continueOnError: boolean; + }; + const content: ToolResult['content'] = []; + let failed = 0; + let ran = 0; + for (const [i, step] of actions.entries()) { + const label = `[${i + 1}/${actions.length}] ${step.tool}`; + const def = toolMap.get(step.tool); + let result: ToolResult; + if (!def || NOT_BATCHABLE.has(step.tool)) { + result = { content: [{ type: 'text', text: `Error: ${def ? 'cannot be used inside a batch' : 'unknown tool'}` }], isError: true }; + } else { + const stepParams = { ...(step.params || {}) }; + if (tabId !== undefined && stepParams.tabId === undefined && 'tabId' in def.inputSchema.shape) { + stepParams.tabId = tabId; + } + try { + result = await def.handler(host, parseToolParams(def, stepParams)); + } catch (err) { + result = { content: [{ type: 'text', text: `Error: ${err instanceof Error ? err.message : String(err)}` }], isError: true }; + } + } + ran++; + content.push({ type: 'text', text: `${label} ${result.isError ? 'FAILED' : 'ok'}` }); + content.push(...result.content); + if (result.isError) { + failed++; + if (!continueOnError) { + const skipped = actions.length - i - 1; + if (skipped) content.push({ type: 'text', text: `Stopped: ${skipped} remaining step(s) skipped.` }); + break; + } + } + } + return { + content: [{ type: 'text', text: `batch: ${ran - failed}/${actions.length} steps ok` }, ...content], + ...(failed ? { isError: true } : {}), + }; + }, +}; diff --git a/mcp-server/src/tools/index.ts b/mcp-server/src/tools/index.ts index 7e56306..daca021 100644 --- a/mcp-server/src/tools/index.ts +++ b/mcp-server/src/tools/index.ts @@ -24,6 +24,7 @@ import { dragTool } from './drag.js'; import { fillFormTool } from './fill-form.js'; import { observeTool } from './observe.js'; import { actTool } from './act.js'; +import { batchTool } from './batch.js'; export const allTools: ToolDefinition[] = [ navigateTool, @@ -50,6 +51,7 @@ export const allTools: ToolDefinition[] = [ fillFormTool, observeTool, actTool, + batchTool, ]; export const toolMap = new Map( diff --git a/mcp-server/src/tools/meta.ts b/mcp-server/src/tools/meta.ts index 31e9873..4bc663f 100644 --- a/mcp-server/src/tools/meta.ts +++ b/mcp-server/src/tools/meta.ts @@ -44,7 +44,8 @@ export interface MetaToolDeps { */ const TASK_PREAMBLE = 'Task → tool:\n' + - '• Click / type / fill a form → browser_click / browser_type / browser_fill_form (SPA-aware full events, no debugger banner). Prefer these over raw JS — they handle React/Vue controlled inputs and smart-selector fallback.\n' + + '• Click / type / keys → browser_click / browser_type / browser_press_key (real trusted input over CDP, works in background windows; trusted:false = synthetic, no debugger banner). Fill a form → browser_fill_form. Prefer these over raw JS.\n' + + '• Several known steps → browser_batch (one call, stops at the first error).\n' + '• Read visible text → browser_text (cheapest). Page structure / element refs → browser_snapshot. Screenshot → browser_screenshot (cannot be done via JS).\n' + '• Safe Observe → Act loop → browser_observe, then browser_act with its snapshotId/ref. This adds freshness, geometry, allowed-action, and click-occlusion checks.\n' + '• Read/write DOM OR call an internal API (fetch) OR read cookies on a strict-CSP SPA → browser_run_action (runs via CDP, bypasses CSP, returns real values; shows a yellow debugger banner).\n' + @@ -60,9 +61,11 @@ const TASK_PREAMBLE = */ const TOOL_GUIDANCE: Record = { browser_click: - 'Use for ANY click — it dispatches full mouse events and has a smart-selector fallback. Prefer over JS .click().', + 'Use for ANY click — a real (trusted) mouse click over CDP with a smart-selector fallback; works in background windows. Prefer over JS .click().', browser_type: - 'Use for typing into inputs — sets the value with the native setter + input/change events so React/Vue controlled inputs update. Prefer over JS .value= .', + 'Use for typing into inputs — real key presses over CDP, so autocomplete/lookup widgets react like for a user. change/blur fire when focus leaves: follow with browser_press_key Tab. Prefer over JS .value= .', + browser_batch: + 'Use to run several known steps (click → type → Tab → wait → text) in ONE call. Stops at the first failing step. Biggest round-trip saver.', browser_fill_form: 'Use to fill several fields in one call (and optionally submit). Cheaper than repeated browser_type calls.', browser_click_text: diff --git a/tests/tools/batch.test.ts b/tests/tools/batch.test.ts new file mode 100644 index 0000000..0f6ba5d --- /dev/null +++ b/tests/tools/batch.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect } from 'vitest'; +import { toolMap } from '../../mcp-server/src/tools/index.js'; +import type { ToolHost } from '../../mcp-server/src/tools/types.js'; + +const batch = toolMap.get('browser_batch')!; + +function fakeHost(failOn?: string) { + const calls: Array<{ tool: string; params: Record }> = []; + const host: ToolHost = { + async callTool(tool, params) { + calls.push({ tool, params }); + if (tool === failOn) throw Object.assign(new Error('boom'), { result: { success: false, error: 'boom' } }); + return { success: true, tool }; + }, + }; + return { host, calls }; +} + +const run = (host: ToolHost, params: Record) => + batch.handler(host, batch.inputSchema.parse(params) as Record); + +describe('browser_batch', () => { + it('runs steps in order and applies the default tabId', async () => { + const { host, calls } = fakeHost(); + const res = await run(host, { + tabId: 9, + actions: [ + { tool: 'browser_click', params: { selector: '#a' } }, + { tool: 'browser_type', params: { selector: '#a', text: 'x', tabId: 3 } }, + { tool: 'browser_press_key', params: { key: 'Tab' } }, + ], + }); + expect(res.isError).toBeUndefined(); + expect(calls.map((c) => [c.tool, c.params.tabId])).toEqual([ + ['browser_click', 9], ['browser_type', 3], ['browser_press_key', 9], + ]); + expect((res.content[0] as { text: string }).text).toBe('batch: 3/3 steps ok'); + }); + + it('stops at the first failing step', async () => { + const { host, calls } = fakeHost('browser_type'); + const res = await run(host, { + tabId: 1, + actions: [ + { tool: 'browser_click', params: { selector: '#a' } }, + { tool: 'browser_type', params: { selector: '#a', text: 'x' } }, + { tool: 'browser_press_key', params: { key: 'Tab' } }, + ], + }); + expect(res.isError).toBe(true); + expect(calls.map((c) => c.tool)).toEqual(['browser_click', 'browser_type']); + expect(res.content.some((c) => c.type === 'text' && c.text.includes('1 remaining step(s) skipped'))).toBe(true); + }); + + it('continueOnError keeps going', async () => { + const { host, calls } = fakeHost('browser_click'); + const res = await run(host, { + tabId: 1, + continueOnError: true, + actions: [ + { tool: 'browser_click', params: { selector: '#a' } }, + { tool: 'browser_press_key', params: { key: 'Tab' } }, + ], + }); + expect(res.isError).toBe(true); + expect(calls).toHaveLength(2); + expect((res.content[0] as { text: string }).text).toBe('batch: 1/2 steps ok'); + }); + + it('validates each step with its own schema and rejects nesting / unknown tools', async () => { + const { host, calls } = fakeHost(); + const bad = await run(host, { tabId: 1, actions: [{ tool: 'browser_type', params: { selector: '#a' } }] }); + expect(bad.isError).toBe(true); + expect(JSON.stringify(bad.content)).toMatch(/Invalid tool arguments for browser_type/); + const nested = await run(host, { actions: [{ tool: 'browser_batch', params: {} }] }); + expect(JSON.stringify(nested.content)).toMatch(/cannot be used inside a batch/); + const unknown = await run(host, { actions: [{ tool: 'nope' }] }); + expect(JSON.stringify(unknown.content)).toMatch(/unknown tool/); + expect(calls).toHaveLength(0); + }); +}); diff --git a/tests/tools/registry.test.ts b/tests/tools/registry.test.ts index c5f7463..46f2c28 100644 --- a/tests/tools/registry.test.ts +++ b/tests/tools/registry.test.ts @@ -13,8 +13,11 @@ describe('Tool Registry', () => { 'browser_click_text', 'browser_handle_dialog', 'browser_upload_file', 'browser_run_action', 'browser_drag', 'browser_fill_form', - 'browser_observe', 'browser_act', + 'browser_observe', 'browser_act', 'browser_batch', ]; + // Tools that run entirely in the MCP process (they call other tools' handlers) + // and therefore never send a wire name of their own. + const localTools = new Set(['browser_batch']); it(`registers every expected tool (${expectedTools.length})`, () => { expect(allTools.length).toBe(expectedTools.length); @@ -74,6 +77,7 @@ describe('Tool Registry', () => { describe('wire name == .name (no drift)', () => { for (const tool of allTools) { it(`${tool.name} sends its own wire name`, () => { + if (localTools.has(tool.name)) return; const tagged = (tool.handler as { toolName?: string }).toolName; if (tagged !== undefined) { expect(tagged, `${tool.name} factory wire name must equal .name`).toBe(tool.name); From af0aa915a5b5f6daf59f14b968ff18b3cb500e5f Mon Sep 17 00:00:00 2001 From: noiemany Date: Tue, 29 Sep 2026 18:05:32 +0300 Subject: [PATCH 3/8] feat(evaluate): REPL-style evaluate over CDP (top-level await, last value) browser_evaluate now runs Runtime.evaluate in replMode through the tab's shared debugger session: top-level await, statement lists whose last expression is the result, let/const redeclaration across calls, not blocked by page CSP. Objects are serialized page-side (DOM nodes as readable descriptions, cycles/functions/bigint/Map/Set handled) instead of coming back as {}. New `timeout` (default 30s, max 120s) races awaited promises; `mode:"scripting"` keeps the banner-free chrome.scripting path, which is also the fallback when the debugger can't attach. Sessions are no longer idle-detached while a long withCdp() call is running. Co-Authored-By: Claude Opus 5.5 --- extension/handlers/inspection.js | 23 ++++++- extension/lib/cdp-evaluate.js | 109 +++++++++++++++++++++++++++++++ extension/lib/cdp-session.js | 8 ++- mcp-server/src/tools/evaluate.ts | 11 ++-- tests/cdp-evaluate.test.ts | 50 ++++++++++++++ 5 files changed, 192 insertions(+), 9 deletions(-) create mode 100644 extension/lib/cdp-evaluate.js create mode 100644 tests/cdp-evaluate.test.ts diff --git a/extension/handlers/inspection.js b/extension/handlers/inspection.js index cd575f9..3bea899 100644 --- a/extension/handlers/inspection.js +++ b/extension/handlers/inspection.js @@ -6,6 +6,8 @@ import { safeExec, resolveTab, getFallback } from '../lib/page-exec.js'; import { fallbackByTab, lastSnapshotFingerprints, MAX_RESULT_CHARS, persistSessionState } from '../lib/state.js'; import { PAGE_FALLBACK_INSTALL } from '../utils/smart-selector.js'; import { PAGE_LEGACY_REF_INSTALL } from '../utils/legacy-refs.js'; +import { withCdp } from '../lib/cdp-session.js'; +import { cdpEvaluate } from '../lib/cdp-evaluate.js'; export async function handleWait(params, _sessionId, _agentName, signal) { const { tabId, selector, state = 'visible', timeout = 10000, delay } = params; @@ -431,14 +433,29 @@ export async function handleGetPageText(params) { * chrome.debugger, so no yellow "is being debugged" banner. Replaces the old * CDP Runtime.evaluate path. */ -export async function handleEvaluate(params) { - const { tabId, expression } = params; +export async function handleEvaluate(params, _sessionId, _agentName, signal) { + const { tabId, expression, mode = 'cdp', timeout } = params; await resolveTab(tabId); const tab = await chrome.tabs.get(tabId); if (/^(chrome|chrome-extension|devtools|edge|about):/i.test(tab.url || '')) { throw new Error(`Cannot evaluate on protected page (${tab.url}).`); } + // Default: REPL semantics over CDP (top-level await, last expression is the + // result, not blocked by CSP). mode:"scripting" (or no debugger available) + // keeps the banner-free chrome.scripting path below. + if (mode !== 'scripting') { + let attached = false; + try { + return await withCdp(tabId, (send) => { + attached = true; + return cdpEvaluate(send, expression, { timeoutMs: timeout, signal }); + }); + } catch (err) { + if (attached) throw err; // a real evaluate failure, not "no debugger" + } + } + // TWO stacked bugs found live (production stress audit): (1) the old // wrapper `(async () => { ${expression} })()` is a BLOCK body — it evaluates // the expression and DISCARDS it, so every result was undefined even where @@ -467,7 +484,7 @@ export async function handleEvaluate(params) { }); let out = null; - const deadline = Date.now() + 5000; // page-side settle budget (tool budget is 15s) + const deadline = Date.now() + Math.min(timeout ?? 5000, 120_000); // page-side settle budget while (Date.now() < deadline) { const read = await chrome.scripting.executeScript({ target: { tabId }, diff --git a/extension/lib/cdp-evaluate.js b/extension/lib/cdp-evaluate.js new file mode 100644 index 0000000..889c506 --- /dev/null +++ b/extension/lib/cdp-evaluate.js @@ -0,0 +1,109 @@ +/** + * REPL-style evaluate over CDP (Runtime.evaluate replMode), like Claude in + * Chrome's javascript tool: top-level `await`, statement lists whose LAST + * expression is the result, `let`/`const` redeclaration across calls — no + * function wrapper needed. Runs through the debugger, so page CSP does not + * block it. Non-JSON values (DOM nodes, cycles, functions) come back as + * readable descriptions instead of `{}`. + */ +import { MAX_RESULT_CHARS } from './state.js'; + +export const DEFAULT_EVAL_TIMEOUT_MS = 30_000; +export const MAX_EVAL_TIMEOUT_MS = 120_000; + +/** Page-side serializer, run with `this` = the evaluated value. */ +function serializeThis() { + const describe = (n) => { + if (!(n instanceof Node)) return String(n); + if (n.nodeType !== 1) return `#${n.nodeName.toLowerCase()}`; + let s = n.tagName.toLowerCase(); + if (n.id) s += `#${n.id}`; + if (typeof n.className === 'string' && n.className.trim()) s += `.${n.className.trim().split(/\s+/).slice(0, 3).join('.')}`; + const text = (n.innerText || n.value || '').trim().replace(/\s+/g, ' ').slice(0, 80); + return text ? `<${s}> "${text}"` : `<${s}>`; + }; + const seen = new WeakSet(); + const out = JSON.stringify(this, (_k, v) => { + if (typeof v === 'bigint') return `${v}n`; + if (typeof v === 'function') return `[Function ${v.name || 'anonymous'}]`; + if (typeof v === 'symbol') return v.toString(); + if (v === undefined) return null; + if (typeof Node !== 'undefined' && v instanceof Node) return describe(v); + if (v instanceof Map) return Object.fromEntries(v); + if (v instanceof Set) return [...v]; + if (v instanceof Error) return `${v.name}: ${v.message}`; + if (typeof NodeList !== 'undefined' && (v instanceof NodeList || v instanceof HTMLCollection)) return [...v]; + if (v && typeof v === 'object') { + if (seen.has(v)) return '[Circular]'; + seen.add(v); + } + return v; + }); + return out === undefined ? null : out; +} + +function capped(value) { + const raw = typeof value === 'string' ? value : JSON.stringify(value); + if (raw && raw.length > MAX_RESULT_CHARS) { + return { success: true, result: raw.slice(0, MAX_RESULT_CHARS), truncated: true, fullLength: raw.length }; + } + return { success: true, result: value }; +} + +export async function cdpEvaluate(send, expression, { timeoutMs = DEFAULT_EVAL_TIMEOUT_MS, signal } = {}) { + const budget = Math.min(Math.max(1000, timeoutMs), MAX_EVAL_TIMEOUT_MS); + const group = `bc-eval-${Date.now()}`; + let timer; + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve({ timedOut: true }), budget); + }); + const aborted = signal + ? new Promise((resolve) => { + if (signal.aborted) resolve({ aborted: true }); + else signal.addEventListener('abort', () => resolve({ aborted: true }), { once: true }); + }) + : null; + try { + const run = send('Runtime.evaluate', { + expression, + replMode: true, + awaitPromise: true, + returnByValue: false, + userGesture: true, + objectGroup: group, + timeout: budget, // bounds synchronous execution; the race below bounds awaits + }); + const r = await Promise.race([run, timeout, ...(aborted ? [aborted] : [])]); + if (r.aborted) return { success: false, error: 'aborted' }; + if (r.timedOut) { + // Leave the page alone — the pending promise just resolves into nothing. + return { success: false, error: `evaluate did not finish within ${budget}ms (pass a larger timeout, max ${MAX_EVAL_TIMEOUT_MS})` }; + } + if (r.exceptionDetails) { + const d = r.exceptionDetails; + const msg = d.exception?.description || d.exception?.value || d.text || 'evaluate threw'; + return { success: false, error: String(msg).slice(0, 2000) }; + } + const ro = r.result || {}; + if (ro.type === 'undefined') return { success: true, result: null, type: 'undefined' }; + if (ro.unserializableValue !== undefined) return { success: true, result: ro.unserializableValue }; + if ('value' in ro && ro.objectId === undefined) return capped(ro.value); + if (ro.subtype === 'node' || !ro.objectId) return capped(ro.description ?? null); + const ser = await send('Runtime.callFunctionOn', { + objectId: ro.objectId, + functionDeclaration: serializeThis.toString(), + returnByValue: true, + }); + const json = ser?.result?.value; + if (typeof json !== 'string') return capped(ro.description ?? null); + if (json.length > MAX_RESULT_CHARS) return capped(json); + try { + return { success: true, result: JSON.parse(json) }; + } catch { + return capped(json); + } + } finally { + clearTimeout(timer); + send('Runtime.releaseObjectGroup', { objectGroup: group }).catch(() => {}); + } +} diff --git a/extension/lib/cdp-session.js b/extension/lib/cdp-session.js index d46211d..c531a39 100644 --- a/extension/lib/cdp-session.js +++ b/extension/lib/cdp-session.js @@ -21,6 +21,7 @@ function armIdle(tabId) { const s = sessions.get(tabId); if (!s) return; clearTimeout(s.timer); + if (s.busy > 0) return; // a long withCdp() call is still using the session s.timer = setTimeout(() => { detachCdp(tabId); }, IDLE_MS); } @@ -65,7 +66,7 @@ export async function ensureViewport(tabId) { export async function ensureCdp(tabId) { let s = sessions.get(tabId); if (!s) { - s = { ready: attach(tabId), timer: null }; + s = { ready: attach(tabId), timer: null, busy: 0 }; sessions.set(tabId, s); try { await s.ready; @@ -80,12 +81,15 @@ export async function ensureCdp(tabId) { return (method, params = {}) => chrome.debugger.sendCommand({ tabId }, method, params); } -/** Run fn(send) with the tab's session; keeps the session alive for reuse. */ +/** Run fn(send) with the tab's session; no idle detach while fn runs. */ export async function withCdp(tabId, fn) { const send = await ensureCdp(tabId); + const s = sessions.get(tabId); + if (s) { s.busy++; clearTimeout(s.timer); } try { return await fn(send); } finally { + if (s) s.busy--; armIdle(tabId); } } diff --git a/mcp-server/src/tools/evaluate.ts b/mcp-server/src/tools/evaluate.ts index a472471..f9c60ed 100644 --- a/mcp-server/src/tools/evaluate.ts +++ b/mcp-server/src/tools/evaluate.ts @@ -4,13 +4,16 @@ import { requireTabId, forwardHandler } from './types.js'; export const evaluateTool: ToolDefinition = { name: 'browser_evaluate', - summary: 'Run JavaScript in the page MAIN world (CSP-restricted)', + summary: 'Run JavaScript in the page (top-level await, last value returned)', description: - 'Execute JavaScript in a tab and return the result. Use for DOM queries, reading page state, or any operation not covered by other tools. NOTE: runs in the page MAIN world via chrome.scripting and is CSP-restricted — on a strict-CSP SPA it may return null even for valid expressions; in that case use browser_run_action (which runs via CDP and bypasses CSP). Prefer browser_click/browser_type over dispatching events by hand here.', + 'Execute JavaScript in a tab and return the result. Write code as in the DevTools console: top-level `await` works, several statements are fine and the value of the LAST expression is returned (no IIFE / return needed), DOM nodes come back as readable descriptions. Runs over CDP, so page CSP does not block it. mode:"scripting" uses chrome.scripting instead (no debugger banner, CSP-restricted, no top-level await). Prefer browser_click/browser_type over dispatching events by hand here.', inputSchema: z.object({ tabId: requireTabId(), - expression: z.string().describe('JavaScript expression or code to evaluate in the page context'), + expression: z.string().describe('JavaScript to run, e.g. `const r = await fetch("/api"); (await r.json()).items.length`'), + timeout: z.number().int().min(1000).max(120_000).optional().default(30_000).describe('Max ms to wait for the result (awaits included), default 30000'), + mode: z.enum(['cdp', 'scripting']).optional().default('cdp').describe('cdp (default) or scripting (banner-free, CSP-bound)'), }), - timeoutMs: 15_000, + // Must exceed the largest `timeout` so the page-side budget decides. + timeoutMs: 125_000, handler: forwardHandler('browser_evaluate'), }; diff --git a/tests/cdp-evaluate.test.ts b/tests/cdp-evaluate.test.ts new file mode 100644 index 0000000..727268e --- /dev/null +++ b/tests/cdp-evaluate.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect } from 'vitest'; + +const { cdpEvaluate } = await import('../extension/lib/cdp-evaluate.js'); + +type Reply = Record; +function fakeSend(evalReply: Reply | (() => Promise), serialized?: string) { + const calls: Array<{ method: string; params: Reply }> = []; + const send = async (method: string, params: Reply = {}) => { + calls.push({ method, params }); + if (method === 'Runtime.evaluate') return typeof evalReply === 'function' ? evalReply() : evalReply; + if (method === 'Runtime.callFunctionOn') return { result: { value: serialized } }; + return {}; + }; + return { send, calls }; +} + +describe('cdpEvaluate (REPL mode)', () => { + it('uses replMode + awaitPromise and returns primitives directly', async () => { + const { send, calls } = fakeSend({ result: { type: 'number', value: 5 } }); + expect(await cdpEvaluate(send, 'await 2; 5')).toEqual({ success: true, result: 5 }); + expect(calls[0].params).toMatchObject({ replMode: true, awaitPromise: true, userGesture: true }); + expect(calls.at(-1)!.method).toBe('Runtime.releaseObjectGroup'); + }); + + it('serializes objects page-side', async () => { + const { send } = fakeSend({ result: { type: 'object', objectId: 'o1' } }, '{"a":[1,""]}'); + expect(await cdpEvaluate(send, '({a:[1,el]})')).toEqual({ success: true, result: { a: [1, ''] } }); + }); + + it('describes DOM nodes', async () => { + const { send } = fakeSend({ result: { type: 'object', subtype: 'node', objectId: 'n', description: 'input#a' } }); + expect(await cdpEvaluate(send, 'document.activeElement')).toEqual({ success: true, result: 'input#a' }); + }); + + it('reports undefined, NaN and exceptions', async () => { + expect(await cdpEvaluate(fakeSend({ result: { type: 'undefined' } }).send, 'void 0')) + .toMatchObject({ success: true, result: null, type: 'undefined' }); + expect(await cdpEvaluate(fakeSend({ result: { type: 'number', unserializableValue: 'NaN' } }).send, 'NaN')) + .toEqual({ success: true, result: 'NaN' }); + const err = await cdpEvaluate(fakeSend({ result: {}, exceptionDetails: { exception: { description: 'ReferenceError: x is not defined' } } }).send, 'x'); + expect(err).toEqual({ success: false, error: 'ReferenceError: x is not defined' }); + }); + + it('times out on a promise that never settles', async () => { + const { send } = fakeSend(() => new Promise(() => {})); + const res = await cdpEvaluate(send, 'await new Promise(()=>{})', { timeoutMs: 1000 }); + expect(res.success).toBe(false); + expect(String(res.error)).toMatch(/did not finish within 1000ms/); + }); +}); From aa754fbc8799e12645ba8e85984313e29b49ddb2 Mon Sep 17 00:00:00 2001 From: noiemany Date: Tue, 29 Sep 2026 18:05:49 +0300 Subject: [PATCH 4/8] test(router): exclude server-local browser_batch from the dispatch drift guard Co-Authored-By: Claude Opus 5.5 --- tests/extension-router.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/extension-router.test.ts b/tests/extension-router.test.ts index f7577ba..1a973ce 100644 --- a/tests/extension-router.test.ts +++ b/tests/extension-router.test.ts @@ -237,14 +237,17 @@ describe('observe/act concurrency integration', () => { }); describe('dispatch registry ↔ MCP tool registry (drift guard)', () => { + // Server-local tools never reach the extension: browser_batch runs other + // tools' handlers in the MCP process (the meta tool isn't in allTools). + const wireTools = allTools.filter((t) => t.name !== 'browser_batch'); + it('every registered MCP tool has an extension handler', () => { - for (const tool of allTools) { + for (const tool of wireTools) { expect(dispatchedTools, `${tool.name} has no extension handler`).toContain(tool.name); } }); it('dispatches exactly the known tool set (no stray handlers)', () => { - // 22 wire tools; the meta tool (browser_tools) is server-local by design. - expect(dispatchedTools.length).toBe(allTools.length); + expect(dispatchedTools.length).toBe(wireTools.length); }); }); From 17a9e5b82dfa3d5d62e38950e3b3bc9f502ba518 Mon Sep 17 00:00:00 2001 From: noiemany Date: Tue, 29 Sep 2026 18:10:04 +0300 Subject: [PATCH 5/8] feat(screenshot): CDP capture with scale/maxWidth/fullPage, no shield browser_screenshot now uses Page.captureScreenshot through the tab's shared debugger session: - scale / maxWidth / jpeg quality shrink the image (fewer tokens); - fullPage captures the whole scrollable content; - the agent's blue control frame is hidden for every capture (it is shown during every action, not only while locked, and used to end up in the picture); - a background tab doesn't paint (captureScreenshot hangs, also with fromSurface:false), so it is shown for a moment, captured over CDP and the user's tab is switched straight back; - captureVisibleTab remains the fallback, reporting why (cdpFallback). Co-Authored-By: Claude Opus 5.5 --- extension/handlers/tabs.js | 81 +++++++++++++++++++++++++++- mcp-server/src/tools/screenshot.ts | 8 ++- tests/screenshot-cdp.test.ts | 86 ++++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 tests/screenshot-cdp.test.ts diff --git a/extension/handlers/tabs.js b/extension/handlers/tabs.js index 8676752..96f69b3 100644 --- a/extension/handlers/tabs.js +++ b/extension/handlers/tabs.js @@ -15,10 +15,87 @@ import { import { showLockShield, hideLockShield } from '../lib/overlay.js'; import { broadcastStatus } from '../lib/connection.js'; import { lockTabUi, releaseTabUi } from '../lib/lock-ops.js'; +import { withCdp, ensureViewport } from '../lib/cdp-session.js'; + +const CDP_CAPTURE_TIMEOUT_MS = 4000; + +function withTimeout(promise, ms, what) { + let timer; + return Promise.race([ + promise, + new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`${what} timed out`)), ms); }), + ]).finally(() => clearTimeout(timer)); +} + +/** + * CDP capture (Page.captureScreenshot): works on a tab that is NOT the active + * one in its window, so the user's view is never switched, and can downscale + * (`scale`) or cap the width (`maxWidth`) to save image tokens. + */ +async function cdpScreenshot(tabId, { format, quality, scale, maxWidth, fullPage }) { + return withCdp(tabId, async (send) => { + let metrics = await send('Page.getLayoutMetrics'); + if (!(metrics?.cssVisualViewport?.clientWidth > 0)) { + await ensureViewport(tabId); + metrics = await send('Page.getLayoutMetrics'); + } + const vv = metrics.cssVisualViewport; + const content = metrics.cssContentSize || metrics.contentSize; + const width = fullPage ? Math.ceil(content.width) : vv.clientWidth; + const height = fullPage ? Math.min(Math.ceil(content.height), 16_000) : vv.clientHeight; + let s = Math.min(1, Math.max(0.05, scale ?? 1)); + if (maxWidth && width * s > maxWidth) s = maxWidth / width; + const { data } = await withTimeout(send('Page.captureScreenshot', { + format, + ...(format === 'jpeg' ? { quality } : {}), + captureBeyondViewport: !!fullPage, + clip: { x: fullPage ? 0 : vv.pageX, y: fullPage ? 0 : vv.pageY, width, height, scale: s }, + }), CDP_CAPTURE_TIMEOUT_MS, 'Page.captureScreenshot'); + return { data, width: Math.round(width * s), height: Math.round(height * s) }; + }); +} export async function handleScreenshot(params) { - const { tabId, format = 'png', quality = 80 } = params; + const { tabId, format = 'png', quality = 80, scale, maxWidth, fullPage = false } = params; const tab = await resolveTab(tabId); + const protectedPage = /^(chrome|chrome-extension|devtools|edge|about):/i.test(tab.url || ''); + let cdpError = null; + if (!protectedPage) { + // The shield is shown during EVERY agent action (router), not only while + // locked — always take it out of the picture; the router restores it. + const wasLocked = !!tabLocks.owner(tabId); + await hideLockShield(tabId); + const opts = { format, quality, scale, maxWidth, fullPage }; + const done = (shot, via) => ({ success: true, format, via, width: shot.width, height: shot.height, data: shot.data }); + try { + if (tab.active) { + const shot = await cdpScreenshot(tabId, opts); + if (shot?.data) return done(shot, 'cdp'); + } else { + // A background tab produces no compositor frames (Page.captureScreenshot + // just hangs, fromSurface:false too), so show it for a moment, capture + // over CDP (keeps scale/maxWidth/fullPage) and switch straight back. + const fg = await windowCaptureMutex.run(tab.windowId, async () => { + const [previousActive] = await chrome.tabs.query({ active: true, windowId: tab.windowId }); + await chrome.tabs.update(tabId, { active: true }); + try { + await new Promise((r) => setTimeout(r, 150)); + return await cdpScreenshot(tabId, opts); + } finally { + if (previousActive?.id != null && previousActive.id !== tabId) { + await chrome.tabs.update(previousActive.id, { active: true }).catch(() => {}); + } + } + }); + if (fg?.data) return done(fg, 'cdp-activated'); + } + } catch (err) { + // debugger unavailable or the hidden tab would not paint: fall back below + cdpError = String(err?.message || err).slice(0, 200); + } finally { + if (wasLocked) await showLockShield(tabId); + } + } return windowCaptureMutex.run(tab.windowId, async () => { const [previousActive] = await chrome.tabs.query({ active: true, windowId: tab.windowId }); const changedActiveTab = previousActive?.id !== tabId; @@ -34,7 +111,7 @@ export async function handleScreenshot(params) { format, quality: format === 'jpeg' ? quality : undefined, }); - return { success: true, format, data: dataUrl.split(',')[1] }; + return { success: true, format, data: dataUrl.split(',')[1], ...(cdpError ? { cdpFallback: cdpError } : {}) }; } finally { if (wasLocked) await showLockShield(tabId); if (changedActiveTab && previousActive?.id != null) { diff --git a/mcp-server/src/tools/screenshot.ts b/mcp-server/src/tools/screenshot.ts index 1e407ec..ea72ebc 100644 --- a/mcp-server/src/tools/screenshot.ts +++ b/mcp-server/src/tools/screenshot.ts @@ -4,15 +4,19 @@ import { requireTabId, imageResult, jsonError, payloadOf } from './types.js'; export const screenshotTool: ToolDefinition = { name: 'browser_screenshot', - summary: 'Capture a screenshot of a tab', description: 'Capture a screenshot of a tab. Note: Chrome screenshots the tab\'s window, so the tab must be the active one in its window; if it is not, the tool activates it first.', + summary: 'Capture a screenshot of a tab', + description: 'Capture a screenshot of a tab over CDP. Use maxWidth / scale and format:"jpeg" to shrink the image (far fewer tokens); fullPage captures the whole scrollable page. A background tab is shown for a moment and the user\'s tab is switched straight back (Chrome does not paint hidden tabs). The agent\'s blue control frame is never in the picture.', inputSchema: z.object({ tabId: requireTabId(), format: z.enum(['png', 'jpeg']).optional().default('png'), quality: z.number().min(0).max(100).optional().default(80).describe('JPEG quality (ignored for PNG)'), + scale: z.number().min(0.05).max(1).optional().describe('Downscale factor, e.g. 0.5 = half size'), + maxWidth: z.number().int().min(100).max(4000).optional().describe('Cap the image width in pixels (keeps aspect ratio), e.g. 1024'), + fullPage: z.boolean().optional().default(false).describe('Capture the whole scrollable page, not just the viewport'), }), // Read-only: safe to retry. idempotent: true, - timeoutMs: 10_000, + timeoutMs: 15_000, async handler(bridge, params) { let result; try { diff --git a/tests/screenshot-cdp.test.ts b/tests/screenshot-cdp.test.ts new file mode 100644 index 0000000..9f52f23 --- /dev/null +++ b/tests/screenshot-cdp.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect, beforeEach } from 'vitest'; + +const cdp: Array<{ method: string; params: Record }> = []; +const tabs = new Map(); +const updates: Array<[number, unknown]> = []; +let captureHangs = false; + +(globalThis as unknown as { chrome: unknown }).chrome = { + tabs: { + get: async (id: number) => tabs.get(id), + query: async () => [...tabs.values()].filter((t) => t.active), + update: async (id: number, props: { active?: boolean }) => { + updates.push([id, props]); + if (props.active) for (const t of tabs.values()) t.active = t.id === id; + return tabs.get(id); + }, + captureVisibleTab: async () => 'data:image/png;base64,LEGACY', + onRemoved: { addListener: () => {} }, + }, + windows: { get: async () => ({ width: 1200, height: 900 }) }, + scripting: { executeScript: async () => [{ result: null }] }, + storage: { session: { get: async () => ({}), set: async () => {} }, local: { get: async () => ({}), set: async () => {} } }, + action: { setBadgeBackgroundColor: () => {}, setBadgeText: () => {} }, + runtime: { sendMessage: async () => {}, onMessage: { addListener: () => {} } }, + alarms: { create: () => {}, onAlarm: { addListener: () => {} } }, + webRequest: { onCompleted: { addListener: () => {} } }, + debugger: { + attach: async () => {}, + detach: async () => {}, + onDetach: { addListener: () => {} }, + sendCommand: async (_t: { tabId: number }, method: string, params: Record = {}) => { + cdp.push({ method, params }); + if (method === 'Runtime.evaluate') return { result: { value: 1000 } }; + if (method === 'Page.getLayoutMetrics') { + return { cssVisualViewport: { clientWidth: 1000, clientHeight: 600, pageX: 0, pageY: 50 }, cssContentSize: { width: 1000, height: 3000 } }; + } + if (method === 'Page.captureScreenshot') { + const tab = tabs.get(_t.tabId); + if (captureHangs || !tab?.active) return new Promise(() => {}); + return { data: 'CDPDATA' }; + } + return {}; + }, + }, +}; + +const { handleScreenshot } = await import('../extension/handlers/tabs.js'); +const session = await import('../extension/lib/cdp-session.js'); + +describe('browser_screenshot over CDP', () => { + beforeEach(async () => { + cdp.length = 0; + updates.length = 0; + captureHangs = false; + tabs.clear(); + tabs.set(1, { id: 1, url: 'https://a.test', windowId: 7, active: true }); + tabs.set(2, { id: 2, url: 'https://b.test', windowId: 7, active: false }); + await session.detachCdp(1); + await session.detachCdp(2); + }); + + it('captures the viewport with scale / maxWidth', async () => { + const res = await handleScreenshot({ tabId: 1, format: 'jpeg', quality: 60, maxWidth: 500 }); + expect(res).toMatchObject({ success: true, via: 'cdp', width: 500, height: 300, data: 'CDPDATA' }); + const cap = cdp.find((c) => c.method === 'Page.captureScreenshot')!; + expect(cap.params).toMatchObject({ format: 'jpeg', quality: 60, clip: { x: 0, y: 50, width: 1000, height: 600, scale: 0.5 } }); + }); + + it('fullPage clips the whole content', async () => { + await handleScreenshot({ tabId: 1, format: 'png', fullPage: true }); + const cap = cdp.find((c) => c.method === 'Page.captureScreenshot')!; + expect(cap.params).toMatchObject({ captureBeyondViewport: true, clip: { y: 0, height: 3000 } }); + }); + + it('background tab: shows it briefly, captures over CDP, restores the user tab', async () => { + const res = await handleScreenshot({ tabId: 2, format: 'png' }); + expect(res).toMatchObject({ success: true, via: 'cdp-activated', data: 'CDPDATA' }); + expect(updates).toEqual([[2, { active: true }], [1, { active: true }]]); + }); + + it('falls back to captureVisibleTab when CDP capture hangs', async () => { + captureHangs = true; + const res = await handleScreenshot({ tabId: 1, format: 'png' }); + expect(res).toMatchObject({ success: true, data: 'LEGACY', cdpFallback: 'Page.captureScreenshot timed out' }); + }, 10_000); +}); From a72b5bacbb14c6991ea310ec592d2fbdcc71c0e8 Mon Sep 17 00:00:00 2001 From: noiemany Date: Tue, 29 Sep 2026 18:10:44 +0300 Subject: [PATCH 6/8] chore(release): bump version to 2.3.0 and document trusted input, batch, REPL evaluate, CDP screenshots Co-Authored-By: Claude Opus 5.5 --- README.md | 28 ++++++++++++++++------------ extension/manifest.json | 2 +- package-lock.json | 4 ++-- package.json | 2 +- server.json | 4 ++-- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 92cdcc6..4d69a27 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@

CI - v2.2.1 + v2.3.0 License: MIT Node >= 20 TypeScript strict @@ -41,7 +41,9 @@ It already has your browser open right there. It just can't see it. - **Open-dialog rescue.** A native `alert`/`confirm`/`prompt` freezes the page's JS thread — `browser_handle_dialog` dismisses it out-of-band via CDP, no page JS needed, which also un-blocks every other tool on that tab. `browser_tabs close/focus` always work, even on a frozen tab. - **Authenticated local connection.** Token + one-time enrollment secret, so no other local process can silently drive your browser. Everything stays on localhost — no cloud, no telemetry. - **Versioned compatibility handshake.** The daemon and extension advertise an application-protocol version, build version, and capabilities before tool traffic is accepted. Explicitly incompatible protocol majors fail with an actionable error instead of producing unexplained timeouts; the build version is diagnostic and does not by itself make compatible peers fail. -- **No debugger banner.** `browser_evaluate` runs in the page's MAIN world via `chrome.scripting` — no yellow "this tab is being debugged" banner, and real values return across the MV3 world boundary. +- **Real, trusted input.** Clicks, typing and key presses go through the Chrome DevTools Protocol, so the page sees `isTrusted` events, focus really moves, default actions run (Tab moves focus, Enter submits, arrows drive autocomplete menus) and focus/blur fire even while the window is in the background — legacy grids and lookup widgets behave as they do for a person. One debugger session per tab is reused and detached after 30 s idle (the yellow "being debugged" banner shows only while it is attached). Pass `trusted: false` for the old synthetic events with no banner; they are also the automatic fallback when the debugger can't attach. +- **Batches.** `browser_batch` runs a list of tool calls in one round-trip and stops at the first failure — a click → type → Tab → wait → read sequence is one call instead of five. +- **Console-style JavaScript.** `browser_evaluate` accepts code as you'd type it in DevTools: top-level `await`, several statements, the last expression's value is returned, DOM nodes come back as readable descriptions — and page CSP doesn't block it. - **Honest errors.** Every tool failure reaches your agent as a real `isError` result with the full payload — no "success" responses hiding failures mid-workflow. --- @@ -241,7 +243,8 @@ A fixed-height tabbed shell (the body never scrolls, only the lists do): - **Forgot `tabId`?** You'll get a clear error: `tabId is required. Call browser_tabs list first.` - **Protected pages** (`chrome://`, the Web Store, devtools) can't be scripted — you'll get `Cannot access protected page (chrome://...)` instead of a silent hang. - **`browser_navigate`** is the one tool where `tabId` is optional (defaults to the active tab) — but for multi-agent safety, pass it explicitly. **Hash-only changes** (e.g. `/page` → `/page#section`) resolve as soon as the URL is set, without waiting for a `complete` event (SPAs don't reload on hash change, so that event never fires). -- **`browser_evaluate`** runs in the page's MAIN world without the debugger banner and returns JSON-serializable values across the MV3 world boundary. It uses page-side `eval`, so a strict page Content Security Policy can reject it. Use `browser_run_action` when CDP-based execution is required. Both tools are powerful and **non-idempotent**, so they are not auto-retried on timeout. +- **`browser_evaluate`** runs over CDP in REPL mode (top-level `await`, last expression returned, `timeout` up to 120 s). `mode: "scripting"` runs it in the page's MAIN world via `chrome.scripting` instead — no debugger banner, but a strict page CSP can reject it and top-level `await` isn't available. It and `browser_run_action` are powerful and **non-idempotent**, so they are not auto-retried on timeout. +- **`browser_type`** types like a person: `change`/blur fire when focus leaves the field, so follow it with `browser_press_key { key: "Tab" }` to commit a value. It returns the field's value after typing. - **Iframe reach is origin-bound.** Snapshot, find, and interaction handlers can descend into same-origin iframes. Browser same-origin rules prevent those DOM paths from entering cross-origin iframes; use a separately targetable tab or an origin-specific integration for content inside them. - **The control shield is top-frame protection.** Its frame and input interception are installed in the top document. Do not treat it as a security boundary for independently focused or cross-origin child frames; avoid manual input anywhere in a tab while an agent owns it. - **Scrolling virtualized feeds** (Facebook/Instagram/Twitter): `browser_scroll` returns `refsMayBeStale: true` because those sites recycle DOM nodes. Re-snapshot before your next interaction. @@ -252,7 +255,7 @@ A fixed-height tabbed shell (the body never scrolls, only the lists do): ## 🧠 Teach Your Agent -The agent can use all 24 tools out of the box, but it works better when it knows the **tab-first** workflow. From the repo root: +The agent can use all 25 tools out of the box, but it works better when it knows the **tab-first** workflow. From the repo root: ```bash npm run setup:cursor # or: node mcp-server/dist/index.js --setup cursor @@ -281,7 +284,7 @@ See [`agent-config/`](agent-config/) for manual installation or to customize the ## What It Can Do -24 tools. Every page-interaction tool takes a **`tabId`** (the one exception is `browser_navigate`, where it's optional). +25 tools. Every page-interaction tool takes a **`tabId`** (the one exception is `browser_navigate`, where it's optional). **See** @@ -289,7 +292,7 @@ See [`agent-config/`](agent-config/) for manual installation or to customize the |------|-------------| | `browser_observe` | Compact atomic semantic observation with snapshot/document identity, geometry, state, and dynamic allowed actions | | `browser_snapshot` | Accessibility tree with element refs. Compact mode (default) returns only interactive elements. Traverses open shadow DOM + same-origin iframes. | -| `browser_screenshot` | Capture a tab as an image (activates the tab first to capture) | +| `browser_screenshot` | Capture a tab as an image over CDP — `maxWidth` / `scale` / `jpeg` to cut tokens, `fullPage` for the whole page | | `browser_text` | Extract raw text from page or element | | `browser_find` | Query elements by natural language — walks same-origin iframes too | @@ -298,10 +301,11 @@ See [`agent-config/`](agent-config/) for manual installation or to customize the | Tool | What it does | |------|-------------| | `browser_act` | Safely click/type/select/focus/hover/keypress/scroll/upload against a `browser_observe` snapshot | -| `browser_click` | Click by ref or CSS selector — pierces same-origin iframes | +| `browser_click` | Real (trusted) click by ref or CSS selector — pierces same-origin iframes | | `browser_click_text` | Click by visible text. Works through React portals and overlays | -| `browser_type` | Type into inputs and contenteditable fields | -| `browser_press_key` | Key combos (Enter, Escape, Ctrl+A) | +| `browser_type` | Real key presses into inputs and contenteditable fields; returns the resulting value | +| `browser_press_key` | Real key presses and combos (`Enter`, `Tab`, `ctrl+a`) | +| `browser_batch` | Run several tool calls in one round-trip; stops at the first failure | | `browser_scroll` | Scroll pages and virtual containers | | `browser_hover` | Trigger tooltips and dropdowns | | `browser_select` | Pick from native `