From 8004cb4506e9027ee61d8bbe2cfb1347688a2f38 Mon Sep 17 00:00:00 2001 From: Chris Raroque Date: Tue, 28 Apr 2026 23:33:56 -0500 Subject: [PATCH 1/4] feat(browser): full-Chrome fallback integration + parallel sub-agent spawns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a "browser" integration that sub-agents can use when no native Composio toolkit covers the task. Wraps the agent-browser CLI as 10 MCP tools (open, snapshot, click, fill, press, get_text, get_url, wait, screenshot, close) with a single shared Chrome session pinned to a dedicated boop profile dir. Defaults to the user's real Chrome (auto-detected on macOS/Linux, overridable via BOOP_BROWSER_EXECUTABLE) instead of agent-browser's bundled Chrome for Testing — Cloudflare/Reddit fingerprint CfT trivially. Headed/headless is a runtime toggle persisted to settings.browser_headed (UI in the Settings tab). Dispatcher and execution-agent prompts gain a tool-selection priority block making "browser" a fallback only — for gmail/calendar/etc. the agent sticks with the native toolkit. Dispatcher also gets a "fan out" block telling it parallel spawn_agent calls in one turn run concurrently. Debug UI: BrowserSection card in Settings shows install status, an in-card "Show Chrome window" toggle, "Install Chrome for Testing" button (one-time fallback download), and a per-site "Open & sign in" helper that pops a real Chrome window using the boop profile. Co-Authored-By: Claude Opus 4.7 (1M context) --- debug/public/chrome-logo.svg | 1 + debug/src/components/BrowserSection.tsx | 299 ++++++++++++++++++++++++ debug/src/components/SettingsPanel.tsx | 2 + debug/src/lib/branding.tsx | 22 +- package-lock.json | 235 +++++++++++++++++++ package.json | 2 + scripts/browser-login.ts | 41 ++++ server/browser-routes.ts | 98 ++++++++ server/browser/config.ts | 62 +++++ server/browser/tools.ts | 137 +++++++++++ server/execution-agent.ts | 10 + server/index.ts | 2 + server/integrations/browser-loader.ts | 12 + server/integrations/registry.ts | 10 +- server/interaction-agent.ts | 32 ++- server/runtime-config.ts | 33 +++ 16 files changed, 990 insertions(+), 8 deletions(-) create mode 100644 debug/public/chrome-logo.svg create mode 100644 debug/src/components/BrowserSection.tsx create mode 100644 scripts/browser-login.ts create mode 100644 server/browser-routes.ts create mode 100644 server/browser/config.ts create mode 100644 server/browser/tools.ts create mode 100644 server/integrations/browser-loader.ts diff --git a/debug/public/chrome-logo.svg b/debug/public/chrome-logo.svg new file mode 100644 index 00000000..4ff6ab6b --- /dev/null +++ b/debug/public/chrome-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/debug/src/components/BrowserSection.tsx b/debug/src/components/BrowserSection.tsx new file mode 100644 index 00000000..464950c8 --- /dev/null +++ b/debug/src/components/BrowserSection.tsx @@ -0,0 +1,299 @@ +import { useCallback, useEffect, useState } from "react"; +import { useMutation, useQuery } from "convex/react"; +import { api } from "../../../convex/_generated/api.js"; + +interface BrowserStatus { + installed: boolean; + cliVersion: string | null; + chromeVersion: string | null; + raw?: string; +} + +type InstallState = "idle" | "installing" | "done" | "error"; + +const HEADED_KEY = "browser_headed"; + +export function BrowserSection({ isDark }: { isDark: boolean }) { + const [status, setStatus] = useState(null); + const [installState, setInstallState] = useState("idle"); + const [installLog, setInstallLog] = useState(""); + const [loginUrl, setLoginUrl] = useState(""); + const [loginBusy, setLoginBusy] = useState(false); + const [loginMsg, setLoginMsg] = useState<{ tone: "ok" | "err"; text: string } | null>(null); + + const headedRaw = useQuery(api.settings.get, { key: HEADED_KEY }); + const setSetting = useMutation(api.settings.set); + const headedLoading = headedRaw === undefined; + const headedEnabled = headedLoading + ? true + : headedRaw === null + ? true + : headedRaw !== "false"; + const toggleHeaded = useCallback(async () => { + if (headedLoading) return; + await setSetting({ key: HEADED_KEY, value: headedEnabled ? "false" : "true" }); + }, [headedLoading, headedEnabled, setSetting]); + + const refresh = useCallback(async () => { + try { + const r = await fetch("/api/browser/status"); + if (!r.ok) throw new Error(`status ${r.status}`); + const data = (await r.json()) as BrowserStatus; + setStatus(data); + } catch { + setStatus({ installed: false, cliVersion: null, chromeVersion: null }); + } + }, []); + + useEffect(() => { + refresh(); + }, [refresh]); + + const install = useCallback(async () => { + setInstallState("installing"); + setInstallLog("Downloading Chrome for Testing… this can take 30–90 seconds."); + try { + const r = await fetch("/api/browser/install", { method: "POST" }); + const data = await r.json(); + if (data.ok) { + setInstallState("done"); + setInstallLog((data.output ?? "").slice(-1500) || "Installed."); + } else { + setInstallState("error"); + setInstallLog((data.output ?? data.error ?? "Install failed.").slice(-1500)); + } + await refresh(); + } catch (err) { + setInstallState("error"); + setInstallLog(err instanceof Error ? err.message : String(err)); + } + }, [refresh]); + + const startLogin = useCallback(async () => { + const trimmed = loginUrl.trim(); + if (!trimmed) { + setLoginMsg({ tone: "err", text: "Enter a URL first." }); + return; + } + setLoginBusy(true); + setLoginMsg(null); + try { + const r = await fetch("/api/browser/login", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ url: trimmed }), + }); + const data = await r.json(); + if (r.ok && data.ok) { + setLoginMsg({ + tone: "ok", + text: "Chrome opened — sign in there. Cookies persist in the boop profile so future agent runs reuse the login.", + }); + setLoginUrl(""); + } else { + setLoginMsg({ tone: "err", text: data.error ?? "Failed to open Chrome." }); + } + } catch (err) { + setLoginMsg({ + tone: "err", + text: err instanceof Error ? err.message : String(err), + }); + } finally { + setLoginBusy(false); + } + }, [loginUrl]); + + const cardBg = isDark ? "bg-slate-900/40 border-slate-800" : "bg-white border-slate-200"; + const muted = isDark ? "text-slate-400" : "text-slate-500"; + const heading = isDark ? "text-slate-100" : "text-slate-900"; + + const dot = status?.installed + ? "bg-emerald-400" + : installState === "installing" + ? "bg-amber-400" + : "bg-slate-500"; + const statusLabel = status?.installed + ? `Installed${status.chromeVersion ? ` — Chrome ${status.chromeVersion}` : ""}` + : installState === "installing" + ? "Installing…" + : status === null + ? "Checking…" + : "Not installed"; + + return ( +
+
+

+ Full browser use +

+
+ +
+
+
+ 🌐 +
+
+
Browser (full web access)
+

+ Lets sub-agents drive a real Chrome with your saved logins. Use for sites without a + native toolkit (portals, niche SaaS, anything you've signed into via the boop + profile). Native toolkits like Gmail or Slack are still preferred when they cover the + task. +

+ +
+ + {statusLabel} + {status?.cliVersion && ( + (CLI {status.cliVersion}) + )} +
+ +
+
+
+ Show Chrome window when agents browse +
+

+ ON: a real Chrome window pops up while sub-agents browse — visible, but slips + past most bot walls (Cloudflare, Reddit) since it's not headless. OFF: Chrome + runs invisibly. Faster, but easily fingerprinted as a bot. Takes effect within + ~30s. +

+
+ +
+ +
+ + +
+ + {installLog && ( + 
+                {installLog}
+
+ )} + +
+
Log in to a site
+

+ Opens Chrome (boop profile). Sign in by hand once — cookies persist for future + agent runs. +

+
+ setLoginUrl(e.target.value)} + placeholder="https://mail.google.com" + className={`flex-1 text-xs px-3 py-1.5 rounded-md border ${ + isDark + ? "bg-slate-950/40 border-slate-700 text-slate-200 placeholder-slate-600" + : "bg-white border-slate-300 text-slate-800 placeholder-slate-400" + }`} + disabled={loginBusy || !status?.installed} + onKeyDown={(e) => { + if (e.key === "Enter" && !loginBusy && status?.installed) startLogin(); + }} + /> + +
+ {!status?.installed && ( +
+ Install Chrome for Testing first. +
+ )} + {loginMsg && ( +
+ {loginMsg.text} +
+ )} +
+
+
+
+
+ ); +} diff --git a/debug/src/components/SettingsPanel.tsx b/debug/src/components/SettingsPanel.tsx index 93200393..f0ff7f90 100644 --- a/debug/src/components/SettingsPanel.tsx +++ b/debug/src/components/SettingsPanel.tsx @@ -1,6 +1,7 @@ import { useEffect, useState } from "react"; import { useQuery, useMutation } from "convex/react"; import { api } from "../../../convex/_generated/api.js"; +import { BrowserSection } from "./BrowserSection.js"; interface ToggleSetting { kind: "toggle"; @@ -85,6 +86,7 @@ export function SettingsPanel({ isDark }: { isDark: boolean }) { ), )} + ); diff --git a/debug/src/lib/branding.tsx b/debug/src/lib/branding.tsx index ba1dd29b..0cb4ab24 100644 --- a/debug/src/lib/branding.tsx +++ b/debug/src/lib/branding.tsx @@ -7,6 +7,10 @@ type ToolBrand = { displayName: string; domain: string; aliases: string[]; + // Override the favicon-by-domain icon with a local asset URL when the brand's + // canonical favicon doesn't actually depict the product (e.g. "browser" wants + // the Chrome browser pinwheel, not the Chrome Web Store logo). + assetUrl?: string; }; const TOOL_BRANDS: ToolBrand[] = [ @@ -59,6 +63,13 @@ const TOOL_BRANDS: ToolBrand[] = [ { key: "supabase", displayName: "Supabase", domain: "supabase.com", aliases: ["supabase"] }, { key: "granola", displayName: "Granola", domain: "granola.ai", aliases: ["granola", "granola_mcp"] }, { key: "imessage", displayName: "iMessage", domain: "apple.com", aliases: ["imessage", "messages"] }, + { + key: "browser", + displayName: "Browser", + domain: "google.com/chrome", + aliases: ["browser"], + assetUrl: "/chrome-logo.svg", + }, ]; function normalize(value: string): string { @@ -204,8 +215,15 @@ export function IntegrationLogo({ const radius = Math.max(4, Math.round(size * 0.28)); const iconSize = Math.max(12, Math.round(size * 0.72)); - // Prefer an explicit URL (e.g. Composio's branded toolkit logo) over favicon-by-domain. - const imgSrc = !failed && logoUrl ? logoUrl : !failed && brand ? faviconUrl(brand.domain) : null; + // Source priority: + // 1. Brand asset override (a curated local file for brands whose favicon + // doesn't depict the product — e.g. "browser" → Chrome pinwheel, not + // Chrome Web Store logo). + // 2. Explicit URL passed in (Composio's branded toolkit logo). + // 3. Favicon-by-domain fallback. + const imgSrc = !failed + ? (brand?.assetUrl ?? logoUrl ?? (brand ? faviconUrl(brand.domain) : null)) + : null; if (imgSrc) { return ( diff --git a/package-lock.json b/package-lock.json index ceb018d1..3c41194d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,10 +15,12 @@ "@hugeicons/core-free-icons": "^1.0.0", "@hugeicons/react": "^1.0.0", "@modelcontextprotocol/sdk": "^1.0.0", + "agent-browser": "^0.26.0", "convex": "^1.36.1", "cors": "^2.8.5", "croner": "^9.0.0", "dotenv": "^16.4.0", + "execa": "^9.6.1", "express": "^5.0.0", "react-force-graph-2d": "^1.27.0", "ws": "^8.18.0", @@ -1517,6 +1519,24 @@ "win32" ] }, + "node_modules/@sec-ant/readable-stream": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", + "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", + "license": "MIT" + }, + "node_modules/@sindresorhus/merge-streams": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", + "integrity": "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@tailwindcss/node": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.2.2.tgz", @@ -2045,6 +2065,16 @@ "node": ">=12" } }, + "node_modules/agent-browser": { + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/agent-browser/-/agent-browser-0.26.0.tgz", + "integrity": "sha512-pdqSfjwbFSp+qnwlb2g23e9wXveIOfMi19xpPA9xZUbzEAUp6W4YBZj6Ybj8z4M7WkcbGDDYc+oDIHDt9R3EDQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "bin": { + "agent-browser": "bin/agent-browser.js" + } + }, "node_modules/ajv": { "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", @@ -3185,6 +3215,32 @@ "node": ">=18.0.0" } }, + "node_modules/execa": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-9.6.1.tgz", + "integrity": "sha512-9Be3ZoN4LmYR90tUoVu2te2BsbzHfhJyfEiAVfz7N5/zv+jduIfLrV2xdQXOHbaD6KgpGdO9PRPM1Y4Q9QkPkA==", + "license": "MIT", + "dependencies": { + "@sindresorhus/merge-streams": "^4.0.0", + "cross-spawn": "^7.0.6", + "figures": "^6.1.0", + "get-stream": "^9.0.0", + "human-signals": "^8.0.1", + "is-plain-obj": "^4.1.0", + "is-stream": "^4.0.1", + "npm-run-path": "^6.0.0", + "pretty-ms": "^9.2.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^4.0.0", + "yoctocolors": "^2.1.1" + }, + "engines": { + "node": "^18.19.0 || >=20.5.0" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, "node_modules/express": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", @@ -3286,6 +3342,21 @@ } } }, + "node_modules/figures": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-6.1.0.tgz", + "integrity": "sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==", + "license": "MIT", + "dependencies": { + "is-unicode-supported": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/finalhandler": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", @@ -3493,6 +3564,22 @@ "node": ">= 0.4" } }, + "node_modules/get-stream": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-9.0.1.tgz", + "integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==", + "license": "MIT", + "dependencies": { + "@sec-ant/readable-stream": "^0.4.1", + "is-stream": "^4.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-symbol-description": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.1.0.tgz", @@ -3687,6 +3774,15 @@ "url": "https://opencollective.com/express" } }, + "node_modules/human-signals": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-8.0.1.tgz", + "integrity": "sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, "node_modules/iconv-lite": { "version": "0.7.2", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", @@ -3981,6 +4077,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-promise": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", @@ -4035,6 +4143,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", + "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-string": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.1.1.tgz", @@ -4086,6 +4206,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-unicode-supported": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", + "integrity": "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-weakmap": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz", @@ -4876,6 +5008,34 @@ "which": "bin/which" } }, + "node_modules/npm-run-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-6.0.0.tgz", + "integrity": "sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==", + "license": "MIT", + "dependencies": { + "path-key": "^4.0.0", + "unicorn-magic": "^0.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -5002,6 +5162,18 @@ "node": ">=4" } }, + "node_modules/parse-ms": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-4.0.0.tgz", + "integrity": "sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -5166,6 +5338,21 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/pretty-ms": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-9.3.0.tgz", + "integrity": "sha512-gjVS5hOP+M3wMm5nmNOucbIrqudzs9v/57bWRHQWLYklXqoXKrVfYW2W9+glfGsqtPgpiz5WwyEEB+ksXIx3gQ==", + "license": "MIT", + "dependencies": { + "parse-ms": "^4.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -5769,6 +5956,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -5933,6 +6132,18 @@ "node": ">=4" } }, + "node_modules/strip-final-newline": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-4.0.0.tgz", + "integrity": "sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -6169,6 +6380,18 @@ "dev": true, "license": "MIT" }, + "node_modules/unicorn-magic": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.3.0.tgz", + "integrity": "sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -6926,6 +7149,18 @@ "dev": true, "license": "ISC" }, + "node_modules/yoctocolors": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yoctocolors/-/yoctocolors-2.1.2.tgz", + "integrity": "sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/package.json b/package.json index 3fec4d05..111e1f50 100644 --- a/package.json +++ b/package.json @@ -27,10 +27,12 @@ "@hugeicons/core-free-icons": "^1.0.0", "@hugeicons/react": "^1.0.0", "@modelcontextprotocol/sdk": "^1.0.0", + "agent-browser": "^0.26.0", "convex": "^1.36.1", "cors": "^2.8.5", "croner": "^9.0.0", "dotenv": "^16.4.0", + "execa": "^9.6.1", "express": "^5.0.0", "react-force-graph-2d": "^1.27.0", "ws": "^8.18.0", diff --git a/scripts/browser-login.ts b/scripts/browser-login.ts new file mode 100644 index 00000000..79d0e4ef --- /dev/null +++ b/scripts/browser-login.ts @@ -0,0 +1,41 @@ +#!/usr/bin/env tsx +// Manually log in to a site once so the agent can reuse the cookies forever. +// +// Usage: +// npx tsx scripts/browser-login.ts https://mail.google.com +// +// What it does: +// - Launches Chrome under the shared "boop" profile (env: AGENT_BROWSER_PROFILE). +// - Navigates to the URL you pass. +// - You log in by hand. Cookies persist in the profile dir. +// - Future agent runs that use the "browser" integration share the same login. +// +// Don't use this for login flows the agent should automate — those would call +// agent-browser commands themselves. This is for the one-time "I logged into +// my landlord portal in Chrome" handoff. + +import { spawn } from "node:child_process"; +import { browserBaseArgs, CHROME_PATH, PROFILE_DIR } from "../server/browser/config.js"; + +const url = process.argv[2]; +if (!url) { + console.error("Usage: npx tsx scripts/browser-login.ts "); + console.error("Example: npx tsx scripts/browser-login.ts https://mail.google.com"); + process.exit(1); +} + +console.log(`[browser-login] profile=${PROFILE_DIR}`); +console.log(`[browser-login] chrome=${CHROME_PATH ?? "(Chrome for Testing fallback)"}`); +console.log(`[browser-login] url=${url}`); +console.log("[browser-login] Log in by hand in the Chrome window that opens."); +console.log("[browser-login] When you're done, run `npx agent-browser close` (or close Chrome)."); + +// Login script always runs headed — the whole point is for the user to see the +// Chrome window and sign in by hand. Independent from the runtime headed +// toggle, which controls behavior of agent-driven tool calls. +const child = spawn("npx", ["agent-browser", ...browserBaseArgs(), "open", url], { + stdio: "inherit", + env: { ...process.env, AGENT_BROWSER_HEADED: "1" }, +}); + +child.on("exit", (code) => process.exit(code ?? 0)); diff --git a/server/browser-routes.ts b/server/browser-routes.ts new file mode 100644 index 00000000..23e7d88c --- /dev/null +++ b/server/browser-routes.ts @@ -0,0 +1,98 @@ +import express from "express"; +import { execa } from "execa"; +import { browserBaseArgs, getBrowserEnv, PROFILE_DIR } from "./browser/config.js"; + +interface BrowserStatus { + installed: boolean; + cliVersion: string | null; + chromeVersion: string | null; + raw?: string; +} + +async function getStatus(): Promise { + try { + const r = await execa("agent-browser", ["doctor"], { + preferLocal: true, + timeout: 15_000, + reject: false, + }); + const raw = `${r.stdout ?? ""}\n${r.stderr ?? ""}`; + const cliMatch = raw.match(/CLI version\s+([\d.]+)/); + const chromeMatch = raw.match(/Google Chrome for Testing\s+([\d.]+)/); + return { + installed: Boolean(chromeMatch), + cliVersion: cliMatch?.[1] ?? null, + chromeVersion: chromeMatch?.[1] ?? null, + }; + } catch (err) { + return { + installed: false, + cliVersion: null, + chromeVersion: null, + raw: err instanceof Error ? err.message : String(err), + }; + } +} + +export function createBrowserRouter(): express.Router { + const router = express.Router(); + + router.get("/status", async (_req, res) => { + res.json(await getStatus()); + }); + + router.post("/install", async (_req, res) => { + // Chrome for Testing is ~150MB. Bound at 5min — covers slow connections + // without leaving the request hanging forever if something is wedged. + try { + const r = await execa("agent-browser", ["install"], { + preferLocal: true, + timeout: 5 * 60_000, + reject: false, + }); + const after = await getStatus(); + res.json({ + ok: r.exitCode === 0 && after.installed, + exitCode: r.exitCode, + output: `${r.stdout ?? ""}\n${r.stderr ?? ""}`.trim().slice(-4000), + status: after, + }); + } catch (err) { + res + .status(500) + .json({ ok: false, error: err instanceof Error ? err.message : String(err) }); + } + }); + + router.post("/login", async (req, res) => { + const url = typeof req.body?.url === "string" ? req.body.url.trim() : ""; + if (!url) { + res.status(400).json({ error: "url required" }); + return; + } + let parsed: URL; + try { + parsed = new URL(url); + } catch { + res.status(400).json({ error: "invalid url" }); + return; + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + res.status(400).json({ error: "url must be http(s)" }); + return; + } + // Fire and forget — `agent-browser open` returns once navigation completes + // (a few seconds), but the Chrome window stays open for the user to log in. + // We wait briefly to surface any immediate launch errors, then return. + const child = execa("agent-browser", [...browserBaseArgs(), "open", parsed.toString()], { + preferLocal: true, + timeout: 30_000, + reject: false, + env: await getBrowserEnv(), + }); + child.catch((err) => console.error("[browser-login] post-launch error", err)); + res.json({ ok: true, url: parsed.toString(), profile: PROFILE_DIR }); + }); + + return router; +} diff --git a/server/browser/config.ts b/server/browser/config.ts new file mode 100644 index 00000000..23e896fc --- /dev/null +++ b/server/browser/config.ts @@ -0,0 +1,62 @@ +import { existsSync, mkdirSync } from "node:fs"; +import { homedir, platform } from "node:os"; +import { join } from "node:path"; + +// agent-browser's --profile flag accepts either an existing Chrome profile NAME +// or a directory path for a persistent custom profile. We always pass a path so +// we own the dir (real Chrome's profiles are not reusable while Chrome is open). +export const PROFILE_DIR = + process.env.AGENT_BROWSER_PROFILE ?? join(homedir(), ".boop", "agent-browser-profile"); +mkdirSync(PROFILE_DIR, { recursive: true }); + +// Single shared agent-browser session for the whole Boop server. agent-browser +// launches one Chrome per (--session, --profile-dir); since Chrome enforces +// one-process-per-profile-dir via SingletonLock, we can't have multiple +// sessions on the same profile. Pinning to a fixed name means every browser +// tool call across all sub-agents attaches to the same Chrome via the daemon, +// which serializes commands. Parallel spawns that both reach for the browser +// will share tabs — that's a v0 tradeoff, fine for single-user Boop. +export const SESSION = "boop"; + +// Prefer the user's real Chrome over agent-browser's bundled Chrome for Testing — +// many sites (Reddit, Cloudflare-protected) fingerprint CfT and serve a bot wall. +// Real Chrome at our --user-data-dir has no SingletonLock conflict with the +// user's daily Chrome since Chrome locks per profile dir, not per binary. +function detectRealChrome(): string | null { + if (process.env.BOOP_BROWSER_EXECUTABLE) return process.env.BOOP_BROWSER_EXECUTABLE; + const candidates = + platform() === "darwin" + ? [ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser", + ] + : platform() === "linux" + ? ["/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium"] + : []; + return candidates.find((p) => existsSync(p)) ?? null; +} + +export const CHROME_PATH = detectRealChrome(); + +export function browserBaseArgs(): string[] { + const args = ["--profile", PROFILE_DIR, "--session", SESSION]; + if (CHROME_PATH) args.push("--executable-path", CHROME_PATH); + return args; +} + +// agent-browser defaults to --headless=new, which puts "HeadlessChrome" in the +// user-agent and is trivially flagged by Cloudflare/Reddit/etc. — even when +// pointed at the user's real Chrome binary. The CLI flag --headed is silently +// ignored in 0.26.x; the env var AGENT_BROWSER_HEADED=1/0 is what actually +// drives --headless=new on Chrome spawn. The user toggles this from the debug +// UI (settings.browser_headed); we read the live value per call via +// runtime-config and pass it through to every execa call. +import { getBrowserHeaded } from "../runtime-config.js"; + +export async function getBrowserEnv(): Promise> { + const headed = await getBrowserHeaded(); + return { + ...process.env, + AGENT_BROWSER_HEADED: headed ? "1" : "0", + } as Record; +} diff --git a/server/browser/tools.ts b/server/browser/tools.ts new file mode 100644 index 00000000..d1c16e6b --- /dev/null +++ b/server/browser/tools.ts @@ -0,0 +1,137 @@ +import { tool, createSdkMcpServer } from "@anthropic-ai/claude-agent-sdk"; +import { z } from "zod"; +import { execa } from "execa"; +import { browserBaseArgs, CHROME_PATH, getBrowserEnv } from "./config.js"; + +if (CHROME_PATH) { + console.log(`[browser] using real Chrome at ${CHROME_PATH}`); +} else { + console.log("[browser] no real Chrome found — falling back to Chrome for Testing"); +} + +const TIMEOUT_MS = 30_000; + +interface Result { + stdout: string; + stderr: string; + exitCode: number | null; +} + +async function ab(args: string[]): Promise { + try { + const r = await execa("agent-browser", [...browserBaseArgs(), ...args], { + preferLocal: true, + timeout: TIMEOUT_MS, + reject: false, + env: await getBrowserEnv(), + }); + return { + stdout: r.stdout?.toString() ?? "", + stderr: r.stderr?.toString() ?? "", + exitCode: r.exitCode ?? null, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { stdout: "", stderr: message, exitCode: null }; + } +} + +function fmt(r: Result): { content: [{ type: "text"; text: string }] } { + const ok = r.exitCode === 0; + if (ok) { + return { content: [{ type: "text" as const, text: r.stdout || "(no output)" }] }; + } + const hint = + r.stderr.includes("ENOENT") || r.exitCode === null + ? "\n\nIs agent-browser installed? Run `npx agent-browser install` once on this machine." + : ""; + const body = `[browser error] exit=${r.exitCode}\n${r.stderr || r.stdout || "(no output)"}${hint}`; + return { content: [{ type: "text" as const, text: body }] }; +} + +const FALLBACK_DISCLAIMER = + "FALLBACK ONLY. Use a native integration (gmail, calendar, slack, github, notion, linear, etc.) when one covers the task — they're faster, structured, and more reliable. Reach for the browser only for sites/services with no Composio toolkit, or for tasks that genuinely need a real browser (visual layouts, JS-heavy UIs, sites you're already logged into via the boop Chrome profile)."; + +export function createBrowserMcp() { + return createSdkMcpServer({ + name: "browser", + version: "0.1.0", + tools: [ + tool( + "browser_open", + `Launch (or reuse) a Chrome session and navigate to a URL. Uses a dedicated boop Chrome profile so logged-in cookies persist across runs. ${FALLBACK_DISCLAIMER}`, + { + url: z.string().describe("URL to navigate to. Include the scheme (https://...)."), + }, + async (args) => fmt(await ab(["open", args.url])), + ), + tool( + "browser_snapshot", + "Return the page's accessibility tree with @e1, @e2, ... refs you can pass to click/fill/get_text. PRIMARY perception tool — call this instead of screenshot whenever possible (much cheaper in tokens). Returns interactive elements, structure, and visible text.", + {}, + async () => fmt(await ab(["snapshot", "-i", "-c"])), + ), + tool( + "browser_click", + "Click an element by ref (@e2) or CSS selector. Get refs from browser_snapshot first.", + { + selector: z.string().describe("Ref like '@e2' or a CSS selector like '#submit'."), + }, + async (args) => fmt(await ab(["click", args.selector])), + ), + tool( + "browser_fill", + "Clear an input and type text into it. Use a ref (@e3) or CSS selector.", + { + selector: z.string().describe("Ref like '@e3' or a CSS selector."), + text: z.string().describe("Text to type (will replace existing value)."), + }, + async (args) => fmt(await ab(["fill", args.selector, args.text])), + ), + tool( + "browser_press", + "Press a key (Enter, Tab, Escape, or chords like Control+a). Acts on the focused element.", + { + key: z.string().describe("Key name. Examples: 'Enter', 'Tab', 'Escape', 'Control+a'."), + }, + async (args) => fmt(await ab(["press", args.key])), + ), + tool( + "browser_get_text", + "Get the visible text content of an element by ref or CSS selector.", + { + selector: z.string().describe("Ref like '@e1' or a CSS selector."), + }, + async (args) => fmt(await ab(["get", "text", args.selector])), + ), + tool( + "browser_get_url", + "Return the current page URL. Useful after a click/redirect to confirm where you ended up.", + {}, + async () => fmt(await ab(["get", "url"])), + ), + tool( + "browser_wait", + "Wait for an element to appear OR a fixed duration in milliseconds. Use selector form for navigation/load waits, ms form sparingly.", + { + target: z + .string() + .describe("CSS selector to wait for, OR a number of ms (e.g. '1500')."), + }, + async (args) => fmt(await ab(["wait", args.target])), + ), + tool( + "browser_screenshot", + "Take an annotated screenshot (writes a PNG to disk and returns the path). Use ONLY when browser_snapshot isn't enough — visual layout questions, charts, image content. Otherwise prefer snapshot.", + {}, + async () => fmt(await ab(["screenshot", "--annotate"])), + ), + tool( + "browser_close", + "Close the shared browser session. Only call when truly done — other parallel agents may still need it.", + {}, + async () => fmt(await ab(["close"])), + ), + ], + }); +} diff --git a/server/execution-agent.ts b/server/execution-agent.ts index c3e7f301..019b89b9 100644 --- a/server/execution-agent.ts +++ b/server/execution-agent.ts @@ -54,6 +54,15 @@ Research discipline: - Cite real URLs only — NEVER invent sources. If a page failed to load, say so. - Cross-check when it matters: one search is rarely enough for a claim. +Tool selection priority (read this carefully): +1. Native Composio toolkit (gmail, calendar, slack, github, notion, linear, etc.) — ALWAYS first choice when one covers the task. They're structured, fast, and reliable. +2. WebSearch / WebFetch — for public read-only info that doesn't require login. +3. browser_* tools (the "browser" integration) — LAST RESORT. Use ONLY when: + • No Composio toolkit can do the job (e.g. a site that isn't connected), OR + • The task genuinely needs a real logged-in browser (a JS-heavy app, a visual layout question, scraping behind a login that has no API). + If a Gmail task lands in your kit and you have both gmail and browser, USE GMAIL. Do not open Gmail in the browser. Same for any other connected toolkit. + When you do use the browser: call browser_snapshot (cheap, returns refs) before browser_screenshot (expensive). Always browser_close at the end. + MANDATORY: for any task that used WebSearch or WebFetch, end your response with a "Sources:" section listing the ACTUAL URLs you fetched or found. Example: @@ -118,6 +127,7 @@ export async function spawnExecutionAgent(opts: SpawnOptions): Promise { const ok = cancelAgent(req.params.id); diff --git a/server/integrations/browser-loader.ts b/server/integrations/browser-loader.ts new file mode 100644 index 00000000..0b63b91f --- /dev/null +++ b/server/integrations/browser-loader.ts @@ -0,0 +1,12 @@ +import { createBrowserMcp } from "../browser/tools.js"; +import { registerIntegration } from "./registry.js"; + +export function registerBrowserIntegration(): void { + registerIntegration({ + name: "browser", + description: + "Full web browser (real Chrome with your saved logins). Pass this to spawn_agent ONLY when no native Composio toolkit covers the task — for gmail/calendar/slack/github/notion/linear/etc., use those toolkits instead. Best for sites without a native toolkit (portals, niche SaaS, anything you've logged into via the boop profile).", + createServer: async () => createBrowserMcp(), + }); + console.log("[browser] registered (fallback for sites without a native toolkit)"); +} diff --git a/server/integrations/registry.ts b/server/integrations/registry.ts index e7c299b5..215de5e3 100644 --- a/server/integrations/registry.ts +++ b/server/integrations/registry.ts @@ -9,6 +9,7 @@ export interface IntegrationModule { export interface IntegrationContext { conversationId?: string; + agentId?: string; } const registry = new Map(); @@ -28,6 +29,8 @@ export function getIntegration(name: string): IntegrationModule | undefined { export async function loadIntegrations(): Promise { const { registerComposioToolkits } = await import("./composio-loader.js"); await registerComposioToolkits(); + const { registerBrowserIntegration } = await import("./browser-loader.js"); + registerBrowserIntegration(); const loaded = [...registry.keys()]; console.log( `[integrations] loaded: ${loaded.join(", ") || "(none — connect a toolkit from the Debug UI's Connections tab)"}`, @@ -39,15 +42,16 @@ export async function refreshIntegrations(): Promise { await loadIntegrations(); } -export function makeContext(conversationId?: string): IntegrationContext { - return { conversationId }; +export function makeContext(conversationId?: string, agentId?: string): IntegrationContext { + return { conversationId, agentId }; } export async function buildMcpServersForIntegrations( names: string[], conversationId?: string, + agentId?: string, ): Promise> { - const ctx = makeContext(conversationId); + const ctx = makeContext(conversationId, agentId); const out: Record = {}; for (const name of names) { const mod = registry.get(name); diff --git a/server/interaction-agent.ts b/server/interaction-agent.ts index bf5070d2..b11c19d6 100644 --- a/server/interaction-agent.ts +++ b/server/interaction-agent.ts @@ -43,17 +43,31 @@ spawn_agent. No exceptions. Even if you're 99% sure. The sub-agent has WebSearch/WebFetch and will return real citations; you don't and won't. Acknowledgment rule (iMessage UX): -BEFORE every spawn_agent call, you MUST call send_ack first with a short +BEFORE any spawn_agent call(s), you MUST call send_ack first with a short 1-sentence message. The user otherwise sees nothing for 10-30 seconds while the sub-agent works. Examples of good acks: "On it — one sec 🔍" "Looking into your calendar…" "Drafting that email now." "Checking Slack, hold tight." -Order: send_ack → spawn_agent → (wait) → final reply with the result. +Order: send_ack → spawn_agent(s) → (wait) → final reply with the result(s). +ONE ack covers multiple parallel spawns — don't ack each one separately. Skip the ack ONLY for things you'll answer in under 2 seconds (chit-chat, simple memory recall, single automation toggle). +Parallel spawning: +When the user's request decomposes into independent sub-tasks (e.g. "check +my gmail unreads AND summarize today's calendar", or "draft the email and +also find me 3 restaurants nearby"), emit MULTIPLE spawn_agent tool_use +blocks in the SAME assistant turn. They run concurrently and you'll see +all results before your next turn. This is much faster than chaining +sequential spawns. Rules: + - Only fan out for genuinely independent tasks. If task B needs task A's + result, do them sequentially. + - Send ONE send_ack first, then all the spawns in the same turn. + - When relaying, combine the results in one reply — don't make the user + read N separate messages. + Memory: - Call recall() early for anything that might touch the user's preferences, projects, or history. - Call write_memory() aggressively for durable facts. Err on the side of saving. @@ -118,6 +132,18 @@ user once ("what timezone are you in?") and call set_timezone with their answer. Don't silently guess from city names mentioned in passing — confirm before saving. +Choosing integrations for spawn_agent: +- Pick the SPECIFIC native toolkit that matches the task (gmail for email, + calendar for events, slack for slack, etc.). Don't shotgun all of them. +- The "browser" integration is a FALLBACK for sites/services with no native + toolkit. NEVER pass "browser" for a task a native toolkit can do — if the + user asks about Gmail, pass ["gmail"], NOT ["browser"] or ["gmail", "browser"]. + Browser is for tasks like "log into my landlord's tenant portal and grab + this month's invoice" — sites we don't have a Composio toolkit for. The + sub-agent already runs in a logged-in Chrome profile via "browser". +- If you're unsure whether a toolkit exists, prefer the toolkit name and let + the sub-agent fall back if it doesn't have the right tool surface. + Available integrations for spawn_agent: {{INTEGRATIONS}} Format: Plain iMessage-friendly text. Markdown sparingly. Keep replies under ~400 chars when you can.`; @@ -209,7 +235,7 @@ export async function handleUserMessage(opts: HandleOpts): Promise { tools: [ tool( "spawn_agent", - "Spawn a focused sub-agent to do real work using external tools. Returns the agent's final answer. Use for anything requiring lookups, drafting, or actions in the user's integrations.", + "Spawn a focused sub-agent to do real work using external tools. Returns the agent's final answer. Use for anything requiring lookups, drafting, or actions in the user's integrations. Multiple independent spawn_agent calls in one turn run in parallel — fan out when the request has independent sub-tasks instead of chaining serially.", { task: z .string() diff --git a/server/runtime-config.ts b/server/runtime-config.ts index a545356d..7b18a45b 100644 --- a/server/runtime-config.ts +++ b/server/runtime-config.ts @@ -58,3 +58,36 @@ export async function clearRuntimeModel(): Promise { await convex.mutation(api.settings.clear, { key: MODEL_KEY }); cached = null; } + +const BROWSER_HEADED_KEY = "browser_headed"; +const BROWSER_HEADED_TTL_MS = 30 * 1000; +let browserHeadedCache: { at: number; value: boolean } | null = null; + +// Default headed: real visible Chrome window. Headless gets fingerprinted by +// Cloudflare/Reddit/etc., so headed is the safer default. Override via the +// debug UI toggle (writes to settings.browser_headed) or via the env var +// AGENT_BROWSER_HEADED. +const HEADED_DEFAULT = true; + +export async function getBrowserHeaded(): Promise { + if (browserHeadedCache && Date.now() - browserHeadedCache.at < BROWSER_HEADED_TTL_MS) { + return browserHeadedCache.value; + } + let stored: string | null = null; + try { + stored = await convex.query(api.settings.get, { key: BROWSER_HEADED_KEY }); + } catch (err) { + console.warn("[runtime-config] browser_headed:get failed", err); + } + let value = HEADED_DEFAULT; + if (stored === "true") value = true; + else if (stored === "false") value = false; + else if (process.env.AGENT_BROWSER_HEADED === "0") value = false; + else if (process.env.AGENT_BROWSER_HEADED === "1") value = true; + browserHeadedCache = { at: Date.now(), value }; + return value; +} + +export function invalidateBrowserHeadedCache(): void { + browserHeadedCache = null; +} From 5bb02de7c69e07f4ef5a53df3a971ff055a2bb03 Mon Sep 17 00:00:00 2001 From: Chris Raroque Date: Tue, 28 Apr 2026 23:47:11 -0500 Subject: [PATCH 2/4] fix(browser): address Greptile review on PR #31 - Drop browser_close tool (P1). With one shared --session boop daemon, a parallel browser-using sub-agent that finished first would close the session out from under any concurrently running agent. The server now owns the daemon lifecycle; agents can't close it. - Update execution-agent prompt accordingly: no longer instructs the agent to call browser_close at end-of-task. - Replace the misleading "wait briefly to surface launch errors" comment in /browser/login with an accurate description of the fire-and-forget behavior. - Move the runtime-config import in server/browser/config.ts to the top of the file alongside the other imports. Co-Authored-By: Claude Opus 4.7 (1M context) --- server/browser-routes.ts | 7 ++++--- server/browser/config.ts | 3 +-- server/browser/tools.ts | 10 ++++------ server/execution-agent.ts | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/server/browser-routes.ts b/server/browser-routes.ts index 23e7d88c..16768869 100644 --- a/server/browser-routes.ts +++ b/server/browser-routes.ts @@ -81,9 +81,10 @@ export function createBrowserRouter(): express.Router { res.status(400).json({ error: "url must be http(s)" }); return; } - // Fire and forget — `agent-browser open` returns once navigation completes - // (a few seconds), but the Chrome window stays open for the user to log in. - // We wait briefly to surface any immediate launch errors, then return. + // Fire and forget — return 200 as soon as the spawn is in flight. Launch + // failures are logged server-side; the user will see Chrome simply not + // appear and can retry. Blocking the request on Chrome's startup would + // stall the click for 1-3s without much actionable signal. const child = execa("agent-browser", [...browserBaseArgs(), "open", parsed.toString()], { preferLocal: true, timeout: 30_000, diff --git a/server/browser/config.ts b/server/browser/config.ts index 23e896fc..cd235e64 100644 --- a/server/browser/config.ts +++ b/server/browser/config.ts @@ -1,6 +1,7 @@ import { existsSync, mkdirSync } from "node:fs"; import { homedir, platform } from "node:os"; import { join } from "node:path"; +import { getBrowserHeaded } from "../runtime-config.js"; // agent-browser's --profile flag accepts either an existing Chrome profile NAME // or a directory path for a persistent custom profile. We always pass a path so @@ -51,8 +52,6 @@ export function browserBaseArgs(): string[] { // drives --headless=new on Chrome spawn. The user toggles this from the debug // UI (settings.browser_headed); we read the live value per call via // runtime-config and pass it through to every execa call. -import { getBrowserHeaded } from "../runtime-config.js"; - export async function getBrowserEnv(): Promise> { const headed = await getBrowserHeaded(); return { diff --git a/server/browser/tools.ts b/server/browser/tools.ts index d1c16e6b..0c1ea4ab 100644 --- a/server/browser/tools.ts +++ b/server/browser/tools.ts @@ -126,12 +126,10 @@ export function createBrowserMcp() { {}, async () => fmt(await ab(["screenshot", "--annotate"])), ), - tool( - "browser_close", - "Close the shared browser session. Only call when truly done — other parallel agents may still need it.", - {}, - async () => fmt(await ab(["close"])), - ), + // Intentionally no browser_close: the agent-browser daemon is shared across + // every sub-agent (single --session boop). If one agent closed it, parallel + // browser-using agents would see their next call fail. The server owns + // lifecycle; agents just borrow tabs. ], }); } diff --git a/server/execution-agent.ts b/server/execution-agent.ts index 019b89b9..aafcdb87 100644 --- a/server/execution-agent.ts +++ b/server/execution-agent.ts @@ -61,7 +61,7 @@ Tool selection priority (read this carefully): • No Composio toolkit can do the job (e.g. a site that isn't connected), OR • The task genuinely needs a real logged-in browser (a JS-heavy app, a visual layout question, scraping behind a login that has no API). If a Gmail task lands in your kit and you have both gmail and browser, USE GMAIL. Do not open Gmail in the browser. Same for any other connected toolkit. - When you do use the browser: call browser_snapshot (cheap, returns refs) before browser_screenshot (expensive). Always browser_close at the end. + When you do use the browser: call browser_snapshot (cheap, returns refs) before browser_screenshot (expensive). Don't try to close the browser — the server reuses one shared Chrome across agents and manages its lifecycle. MANDATORY: for any task that used WebSearch or WebFetch, end your response with a "Sources:" section listing the ACTUAL URLs you fetched or found. Example: From de103979cc6f026e3db89eef6e2d22a0c0cb9678 Mon Sep 17 00:00:00 2001 From: Chris Raroque Date: Wed, 29 Apr 2026 00:03:40 -0500 Subject: [PATCH 3/4] feat(pause): pause-and-resume for sub-agents that hit a hand-action wall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a pause_for_user MCP tool exposed to execution agents. When a sub-agent hits something that needs the human (login wall, OAuth/2FA, captcha, file pick), it sends a friendly message to the user, persists a continuation, and ends its turn cleanly. The dispatcher stays silent for that turn since the message already went out. On the user's next message, the dispatcher reads the pending continuation and decides: - Reply looks like readiness ("done", "logged in", "ok", or just pushing forward like "what's the balance?") → spawn a fresh sub-agent with the saved resume_task. Browser session/cookies persist across the spawn so the resumed agent picks up where the first left off. - Reply cancels or changes topic → call clear_pending_continuation and proceed normally. New schema: pendingContinuations table (one row per conversation, keyed by conversationId). New convex/pendingContinuations.ts {get,set,clear} mutations. Execution-agent prompt updated to describe the pause flow and the specific trigger conditions (login wall, OAuth, captcha, 2FA, etc.). Dispatcher prompt gets a {{PENDING_CONTINUATION}} block telling it how to handle the next user turn when a continuation is live. Schema enum executionAgents.status gains "paused" so the agents table distinguishes paused agents from completed/failed/cancelled. Co-Authored-By: Claude Opus 4.7 (1M context) --- convex/agents.ts | 3 +- convex/pendingContinuations.ts | 58 ++++++++++++++++++++++ convex/schema.ts | 14 ++++++ server/execution-agent.ts | 29 ++++++++++- server/interaction-agent.ts | 68 ++++++++++++++++++++++++- server/pause-tools.ts | 90 ++++++++++++++++++++++++++++++++++ 6 files changed, 257 insertions(+), 5 deletions(-) create mode 100644 convex/pendingContinuations.ts create mode 100644 server/pause-tools.ts diff --git a/convex/agents.ts b/convex/agents.ts index 57a9de67..6e554492 100644 --- a/convex/agents.ts +++ b/convex/agents.ts @@ -7,6 +7,7 @@ const statusV = v.union( v.literal("completed"), v.literal("failed"), v.literal("cancelled"), + v.literal("paused"), ); export const create = mutation({ @@ -48,7 +49,7 @@ export const update = mutation({ .withIndex("by_agent_id", (q) => q.eq("agentId", agentId)) .unique(); if (!agent) return null; - const completed = patch.status && ["completed", "failed", "cancelled"].includes(patch.status); + const completed = patch.status && ["completed", "failed", "cancelled", "paused"].includes(patch.status); await ctx.db.patch(agent._id, { ...patch, ...(completed ? { completedAt: Date.now() } : {}) }); return agent._id; }, diff --git a/convex/pendingContinuations.ts b/convex/pendingContinuations.ts new file mode 100644 index 00000000..81aff2d7 --- /dev/null +++ b/convex/pendingContinuations.ts @@ -0,0 +1,58 @@ +import { mutation, query } from "./_generated/server.js"; +import { v } from "convex/values"; + +export const get = query({ + args: { conversationId: v.string() }, + handler: async (ctx, args) => { + const row = await ctx.db + .query("pendingContinuations") + .withIndex("by_conversation", (q) => q.eq("conversationId", args.conversationId)) + .unique(); + return row + ? { + resumeTask: row.resumeTask, + integrations: row.integrations, + pausedByAgentId: row.pausedByAgentId, + askedAt: row.askedAt, + } + : null; + }, +}); + +export const set = mutation({ + args: { + conversationId: v.string(), + resumeTask: v.string(), + integrations: v.array(v.string()), + pausedByAgentId: v.optional(v.string()), + }, + handler: async (ctx, args) => { + const existing = await ctx.db + .query("pendingContinuations") + .withIndex("by_conversation", (q) => q.eq("conversationId", args.conversationId)) + .unique(); + const payload = { + conversationId: args.conversationId, + resumeTask: args.resumeTask, + integrations: args.integrations, + pausedByAgentId: args.pausedByAgentId, + askedAt: Date.now(), + }; + if (existing) { + await ctx.db.patch(existing._id, payload); + } else { + await ctx.db.insert("pendingContinuations", payload); + } + }, +}); + +export const clear = mutation({ + args: { conversationId: v.string() }, + handler: async (ctx, args) => { + const existing = await ctx.db + .query("pendingContinuations") + .withIndex("by_conversation", (q) => q.eq("conversationId", args.conversationId)) + .unique(); + if (existing) await ctx.db.delete(existing._id); + }, +}); diff --git a/convex/schema.ts b/convex/schema.ts index 707577e3..fb84831b 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -70,6 +70,7 @@ export default defineSchema({ v.literal("completed"), v.literal("failed"), v.literal("cancelled"), + v.literal("paused"), ), result: v.optional(v.string()), error: v.optional(v.string()), @@ -219,6 +220,19 @@ export default defineSchema({ updatedAt: v.number(), }).index("by_key", ["key"]), + // Per-conversation pause-and-resume slot. A sub-agent that hits a wall + // requiring hand-action (login, OAuth, captcha, file pick) writes here and + // ends its turn; the dispatcher picks this up on the next user message and + // re-spawns with the saved resume task. Only one pending continuation per + // conversation at a time. + pendingContinuations: defineTable({ + conversationId: v.string(), + resumeTask: v.string(), + integrations: v.array(v.string()), + pausedByAgentId: v.optional(v.string()), + askedAt: v.number(), + }).index("by_conversation", ["conversationId"]), + automationRuns: defineTable({ runId: v.string(), automationId: v.string(), diff --git a/server/execution-agent.ts b/server/execution-agent.ts index aafcdb87..bd122fa0 100644 --- a/server/execution-agent.ts +++ b/server/execution-agent.ts @@ -4,6 +4,7 @@ import { convex } from "./convex-client.js"; import { broadcast } from "./broadcast.js"; import { buildMcpServersForIntegrations, listIntegrations } from "./integrations/registry.js"; import { createDraftStagingMcp } from "./draft-tools.js"; +import { createPauseMcp } from "./pause-tools.js"; import { aggregateUsageFromResult, EMPTY_USAGE, type UsageTotals } from "./usage.js"; import { getRuntimeModel } from "./runtime-config.js"; @@ -63,6 +64,13 @@ Tool selection priority (read this carefully): If a Gmail task lands in your kit and you have both gmail and browser, USE GMAIL. Do not open Gmail in the browser. Same for any other connected toolkit. When you do use the browser: call browser_snapshot (cheap, returns refs) before browser_screenshot (expensive). Don't try to close the browser — the server reuses one shared Chrome across agents and manages its lifecycle. +Pause for user (browser flows that need a sign-in): +- If you open a site and hit a login/auth wall, OAuth screen, captcha, 2FA prompt, or any other roadblock that needs the human to do something by hand, do NOT give up and do NOT try to brute-force past it. Call pause_for_user with: + • message: a friendly 1-2 sentence prompt referencing the open Chrome window ("Opened Chase login — sign in via the Chrome window I just popped, then reply when ready.") + • resume_task: a complete, standalone task description for the fresh sub-agent that picks up after the user confirms ("The user has now logged into chase.com. Look up their current checking balance and report it.") +- After calling pause_for_user, RETURN immediately with an empty reply. The dispatcher knows not to relay anything; the user already got your message. Boop re-spawns a fresh agent (with the same browser session — your tabs persist) when they reply. +- ONLY use pause_for_user for genuine hand-action requirements. Don't use it for "I need clarification on the task" — work with what you have or ask in your normal reply. + MANDATORY: for any task that used WebSearch or WebFetch, end your response with a "Sources:" section listing the ACTUAL URLs you fetched or found. Example: @@ -95,7 +103,7 @@ export interface SpawnOptions { export interface SpawnResult { agentId: string; result: string; - status: "completed" | "failed" | "cancelled"; + status: "completed" | "failed" | "cancelled" | "paused"; } export async function spawnExecutionAgent(opts: SpawnOptions): Promise { @@ -132,9 +140,19 @@ export async function spawnExecutionAgent(opts: SpawnOptions): Promise { content: opts.content, }); + const pendingContinuation = await convex.query(api.pendingContinuations.get, { + conversationId: opts.conversationId, + }); + const memoryServer = createMemoryMcp(opts.conversationId); const automationServer = createAutomationMcp(opts.conversationId); const draftDecisionServer = createDraftDecisionMcp(opts.conversationId); @@ -229,6 +249,11 @@ export async function handleUserMessage(opts: HandleOpts): Promise { ], }); + // Set by spawn_agent when a sub-agent paused for user action. The post-loop + // logic uses this to skip the "(no reply)" fallback so the user doesn't + // receive a placeholder message after the sub-agent already sent its own. + let dispatcherSilent = false; + const spawnServer = createSdkMcpServer({ name: "boop-spawn", version: "0.1.0", @@ -252,6 +277,17 @@ export async function handleUserMessage(opts: HandleOpts): Promise { conversationId: opts.conversationId, name: args.name, }); + if (res.status === "paused") { + dispatcherSilent = true; + return { + content: [ + { + type: "text" as const, + text: `[agent ${res.agentId} PAUSED — waiting for user to complete a hand-action]\n\nThe sub-agent already messaged the user with what to do. DO NOT relay anything else for this turn — return an empty assistant message. Boop will re-spawn the agent when the user replies.`, + }, + ], + }; + } return { content: [ { @@ -274,10 +310,32 @@ export async function handleUserMessage(opts: HandleOpts): Promise { .map((m) => `${m.role.toUpperCase()}: ${m.content}`) .join("\n"); + const pendingServer = createSdkMcpServer({ + name: "boop-pending", + version: "0.1.0", + tools: [ + tool( + "clear_pending_continuation", + "Drop any pending continuation set by a paused sub-agent for THIS conversation. Call this when the user changes topic, cancels, or reports the hand-action didn't work — anything that means we shouldn't auto-resume the saved task. No-op when there's nothing pending.", + {}, + async () => { + await convex.mutation(api.pendingContinuations.clear, { + conversationId: opts.conversationId, + }); + return { content: [{ type: "text" as const, text: "Pending continuation cleared." }] }; + }, + ), + ], + }); + + const pendingDescription = pendingContinuation + ? `RESUME_TASK="${pendingContinuation.resumeTask.replace(/"/g, '\\"')}", INTEGRATIONS=[${pendingContinuation.integrations.join(", ")}], asked ${Math.round((Date.now() - pendingContinuation.askedAt) / 1000)}s ago by agent ${pendingContinuation.pausedByAgentId ?? "?"}` + : "(none)"; + const systemPrompt = INTERACTION_SYSTEM.replace( "{{INTEGRATIONS}}", integrations.join(", ") || "(no integrations configured yet)", - ); + ).replace("{{PENDING_CONTINUATION}}", pendingDescription); const prompt = historyBlock ? `Prior turns:\n${historyBlock}\n\nCurrent message:\n${opts.content}` @@ -303,6 +361,7 @@ export async function handleUserMessage(opts: HandleOpts): Promise { "boop-draft-decisions": draftDecisionServer, "boop-ack": ackServer, "boop-self": selfServer, + "boop-pending": pendingServer, }, allowedTools: [ "mcp__boop-memory__write_memory", @@ -315,6 +374,7 @@ export async function handleUserMessage(opts: HandleOpts): Promise { "mcp__boop-draft-decisions__list_drafts", "mcp__boop-draft-decisions__send_draft", "mcp__boop-draft-decisions__reject_draft", + "mcp__boop-pending__clear_pending_continuation", "mcp__boop-ack__send_ack", "mcp__boop-self__get_config", "mcp__boop-self__set_model", @@ -369,7 +429,11 @@ export async function handleUserMessage(opts: HandleOpts): Promise { reply = "Sorry — I hit an error processing that. Try again in a moment."; } - reply = reply.trim() || "(no reply)"; + // When a sub-agent paused for user action it already sent its own message — + // don't fall back to the "(no reply)" placeholder, since that'd send a + // useless string to the user. Returning empty here makes the caller skip + // the iMessage send entirely. + reply = dispatcherSilent ? reply.trim() : reply.trim() || "(no reply)"; if (usage.costUsd > 0 || usage.inputTokens > 0) { log( diff --git a/server/pause-tools.ts b/server/pause-tools.ts new file mode 100644 index 00000000..04f5cbec --- /dev/null +++ b/server/pause-tools.ts @@ -0,0 +1,90 @@ +import { tool, createSdkMcpServer } from "@anthropic-ai/claude-agent-sdk"; +import { z } from "zod"; +import { api } from "../convex/_generated/api.js"; +import { convex } from "./convex-client.js"; +import { broadcast } from "./broadcast.js"; +import { sendImessage } from "./sendblue.js"; + +interface PauseContext { + conversationId: string; + agentId: string; + integrations: string[]; + turnId?: string; + // Set to true by the tool handler when the agent calls pause_for_user. The + // execution-agent loop checks this flag after `query()` finishes so it can + // mark the run as "paused" rather than "completed" and skip relaying a + // duplicate reply to the user (the tool already messaged them directly). + pausedFlag: { paused: boolean }; +} + +export function createPauseMcp(ctx: PauseContext) { + return createSdkMcpServer({ + name: "boop-pause", + version: "0.1.0", + tools: [ + tool( + "pause_for_user", + "Use ONLY when the task can't proceed without a hand-action by the user (login wall, OAuth/2FA, captcha, manual file pick, security challenge). Sends a friendly message to the user, persists a continuation, and ends your turn cleanly. Boop will re-spawn you with the same task plus the user's reply once they confirm. Do NOT use for normal task completion or when you can finish on your own.", + { + message: z + .string() + .describe( + "Short message the user will see. Reference the open Chrome window and tell them what to do (e.g. 'Opened Chase login — sign in via the Chrome window I just popped, then reply when ready.'). 1-2 sentences.", + ), + resume_task: z + .string() + .describe( + "What you'll do AFTER the user confirms. Write it as a complete task description (a fresh sub-agent will receive this verbatim). Include the original goal AND the assumption that the user has now completed the action. Example: 'The user has now logged into chase.com. Look up their current checking balance and report it.'", + ), + }, + async (args) => { + const trimmed = args.message.trim(); + if (!trimmed) { + return { + content: [ + { type: "text" as const, text: "Empty message — provide a real prompt for the user." }, + ], + }; + } + + if (ctx.conversationId.startsWith("sms:")) { + const number = ctx.conversationId.slice(4); + try { + await sendImessage(number, trimmed); + } catch (err) { + console.error("[pause_for_user] sendImessage failed", err); + } + } + await convex.mutation(api.messages.send, { + conversationId: ctx.conversationId, + role: "assistant", + content: trimmed, + turnId: ctx.turnId, + }); + broadcast("assistant_message", { + conversationId: ctx.conversationId, + content: trimmed, + }); + + await convex.mutation(api.pendingContinuations.set, { + conversationId: ctx.conversationId, + resumeTask: args.resume_task, + integrations: ctx.integrations, + pausedByAgentId: ctx.agentId, + }); + + ctx.pausedFlag.paused = true; + + return { + content: [ + { + type: "text" as const, + text: "Paused. Your message was sent to the user and a continuation was saved. END your turn now — return an empty reply. Boop will re-spawn you with the resume task when the user confirms.", + }, + ], + }; + }, + ), + ], + }); +} From 47a21dd33eb83a26e7cab21c8f1466d02c73aa1e Mon Sep 17 00:00:00 2001 From: Chris Raroque Date: Wed, 29 Apr 2026 10:06:29 -0500 Subject: [PATCH 4/4] feat(browser): stealth Chrome + cookie import from daily Chrome; dispatcher anaphora fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Stealth launcher spawns Chrome on a fixed CDP port and patches navigator.webdriver, languages, plugins, chrome.runtime, and Function.prototype.toString via Page.addScriptToEvaluateOnNewDocument before any site script runs. agent-browser attaches via --cdp instead of launching its own Chrome. - Cookie import scans the user's daily Chrome profile (Default, Profile 1, ...) for Google/LinkedIn/X/Reddit/GitHub session cookies, snapshot-reads them safely while Chrome holds the file, copies into boop's profile, and verifies via a CDP probe that distinguishes logged_in / needs_challenge / not_logged_in. - Convex cookieImports table tracks per-service imports with identity + verification state so the debug UI can show "Active as user@example.com · 4m ago". - New debug UI section under Settings → Browser surfaces logged-in sessions per daily profile with one-click Import/Refresh. - Dispatcher: bumped recent-history limit 10 → 30 and added a "Resolving references" block so anaphoric requests like "forward her the flight details" resolve from earlier conversation context (or ask) instead of defaulting to "most recent X". Co-Authored-By: Claude Opus 4.7 (1M context) --- convex/cookieImports.ts | 49 ++ convex/schema.ts | 14 + debug/src/components/BrowserSection.tsx | 3 + debug/src/components/CookieImportSection.tsx | 342 ++++++++++++++ package-lock.json | 423 ++++++++++++++++- package.json | 2 + server/browser-routes.ts | 146 +++++- server/browser/config.ts | 16 +- server/browser/cookies.ts | 351 ++++++++++++++ server/browser/stealth-launcher.ts | 472 +++++++++++++++++++ server/browser/tools.ts | 2 + server/index.ts | 11 + server/interaction-agent.ts | 18 +- 13 files changed, 1840 insertions(+), 9 deletions(-) create mode 100644 convex/cookieImports.ts create mode 100644 debug/src/components/CookieImportSection.tsx create mode 100644 server/browser/cookies.ts create mode 100644 server/browser/stealth-launcher.ts diff --git a/convex/cookieImports.ts b/convex/cookieImports.ts new file mode 100644 index 00000000..0bcd4fe8 --- /dev/null +++ b/convex/cookieImports.ts @@ -0,0 +1,49 @@ +import { mutation, query } from "./_generated/server.js"; +import { v } from "convex/values"; + +export const list = query({ + args: {}, + handler: async (ctx) => { + const rows = await ctx.db.query("cookieImports").collect(); + return rows.map((r) => ({ + service: r.service, + sourceProfile: r.sourceProfile, + identity: r.identity, + cookieCount: r.cookieCount, + lastImportedAt: r.lastImportedAt, + lastVerifiedAt: r.lastVerifiedAt, + verifiedOk: r.verifiedOk, + })); + }, +}); + +export const record = mutation({ + args: { + service: v.string(), + sourceProfile: v.string(), + identity: v.optional(v.string()), + cookieCount: v.number(), + verifiedOk: v.optional(v.boolean()), + }, + handler: async (ctx, args) => { + const now = Date.now(); + const existing = await ctx.db + .query("cookieImports") + .withIndex("by_service", (q) => q.eq("service", args.service)) + .unique(); + const payload = { + service: args.service, + sourceProfile: args.sourceProfile, + identity: args.identity, + cookieCount: args.cookieCount, + lastImportedAt: now, + lastVerifiedAt: args.verifiedOk !== undefined ? now : undefined, + verifiedOk: args.verifiedOk, + }; + if (existing) { + await ctx.db.patch(existing._id, payload); + } else { + await ctx.db.insert("cookieImports", payload); + } + }, +}); diff --git a/convex/schema.ts b/convex/schema.ts index fb84831b..99d88a21 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -233,6 +233,20 @@ export default defineSchema({ askedAt: v.number(), }).index("by_conversation", ["conversationId"]), + // Cookie imports from the user's daily Chrome profile into boop's stealth + // Chrome. One row per (service, profile) — re-importing updates the same + // row. Identity is the Google email / handle we read off the source + // profile so the UI can show "Active as user@example.com". + cookieImports: defineTable({ + service: v.string(), + sourceProfile: v.string(), + identity: v.optional(v.string()), + cookieCount: v.number(), + lastImportedAt: v.number(), + lastVerifiedAt: v.optional(v.number()), + verifiedOk: v.optional(v.boolean()), + }).index("by_service", ["service"]), + automationRuns: defineTable({ runId: v.string(), automationId: v.string(), diff --git a/debug/src/components/BrowserSection.tsx b/debug/src/components/BrowserSection.tsx index 464950c8..ce2e863f 100644 --- a/debug/src/components/BrowserSection.tsx +++ b/debug/src/components/BrowserSection.tsx @@ -1,6 +1,7 @@ import { useCallback, useEffect, useState } from "react"; import { useMutation, useQuery } from "convex/react"; import { api } from "../../../convex/_generated/api.js"; +import { CookieImportSection } from "./CookieImportSection.js"; interface BrowserStatus { installed: boolean; @@ -291,6 +292,8 @@ export function BrowserSection({ isDark }: { isDark: boolean }) { )} + + diff --git a/debug/src/components/CookieImportSection.tsx b/debug/src/components/CookieImportSection.tsx new file mode 100644 index 00000000..5913d34a --- /dev/null +++ b/debug/src/components/CookieImportSection.tsx @@ -0,0 +1,342 @@ +import { useCallback, useEffect, useMemo, useState } from "react"; +import { useQuery } from "convex/react"; +import { api } from "../../../convex/_generated/api.js"; + +interface DailyProfile { + dir: string; + name: string; + userName: string | null; +} + +interface ServiceScan { + service: string; + label: string; + hostsCovered: string[]; + cookieCount: number; + hasSignature: boolean; +} + +interface ImportRecord { + service: string; + sourceProfile: string; + identity?: string; + cookieCount: number; + lastImportedAt: number; + lastVerifiedAt?: number; + verifiedOk?: boolean; +} + +interface ScanResponse { + profile: string; + services: ServiceScan[]; + imports: ImportRecord[]; +} + +const SERVICE_ICON: Record = { + google: "G", + linkedin: "in", + twitter: "X", + reddit: "r", + github: "gh", +}; + +function relTime(epoch: number): string { + const sec = Math.max(1, Math.round((Date.now() - epoch) / 1000)); + if (sec < 60) return `${sec}s ago`; + const min = Math.round(sec / 60); + if (min < 60) return `${min}m ago`; + const hr = Math.round(min / 60); + if (hr < 24) return `${hr}h ago`; + const day = Math.round(hr / 24); + return `${day}d ago`; +} + +interface RowState { + busy?: boolean; + message?: { tone: "ok" | "err"; text: string }; +} + +export function CookieImportSection({ isDark }: { isDark: boolean }) { + const [profiles, setProfiles] = useState([]); + const [profileDir, setProfileDir] = useState(null); + const [scan, setScan] = useState(null); + const [scanning, setScanning] = useState(false); + const [scanError, setScanError] = useState(null); + const [rowState, setRowState] = useState>({}); + + // Imports come straight from Convex so they update live across refreshes. + const imports = useQuery(api.cookieImports.list, {}) ?? []; + const importsByService = useMemo(() => { + const m = new Map(); + for (const r of imports) m.set(r.service, r as ImportRecord); + return m; + }, [imports]); + + const loadProfiles = useCallback(async () => { + try { + const r = await fetch("/api/browser/cookies/profiles"); + if (!r.ok) throw new Error(`status ${r.status}`); + const data = (await r.json()) as { profiles: DailyProfile[] }; + setProfiles(data.profiles); + // Default to a signed-in profile when available. + const preferred = + data.profiles.find((p) => p.userName) ?? data.profiles[0] ?? null; + if (preferred && profileDir === null) setProfileDir(preferred.dir); + } catch (err) { + setScanError(err instanceof Error ? err.message : String(err)); + } + }, [profileDir]); + + const rescan = useCallback( + async (dir: string) => { + setScanning(true); + setScanError(null); + try { + const r = await fetch( + `/api/browser/cookies/scan?profile=${encodeURIComponent(dir)}`, + ); + if (!r.ok) throw new Error(`status ${r.status}`); + const data = (await r.json()) as ScanResponse; + setScan(data.services); + } catch (err) { + setScanError(err instanceof Error ? err.message : String(err)); + } finally { + setScanning(false); + } + }, + [], + ); + + useEffect(() => { + void loadProfiles(); + }, [loadProfiles]); + + useEffect(() => { + if (profileDir) void rescan(profileDir); + }, [profileDir, rescan]); + + const runImport = useCallback( + async (svc: ServiceScan) => { + if (!profileDir) return; + setRowState((s) => ({ ...s, [svc.service]: { busy: true } })); + try { + const r = await fetch("/api/browser/cookies/import", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: profileDir, service: svc.service }), + }); + const data = (await r.json()) as { + ok?: boolean; + imported?: number; + identity?: string; + verified?: { + state: "logged_in" | "needs_challenge" | "not_logged_in"; + finalUrl?: string; + title?: string; + } | null; + error?: string; + runningAgents?: string[]; + }; + if (!r.ok || !data.ok) { + const text = + data.error ?? + (data.runningAgents?.length + ? `Browser is busy with ${data.runningAgents.length} sub-agent(s). Try again in a moment.` + : `Import failed (${r.status}).`); + setRowState((s) => ({ + ...s, + [svc.service]: { busy: false, message: { tone: "err", text } }, + })); + return; + } + const v = data.verified; + let verifyText = ""; + let tone: "ok" | "err" = "ok"; + if (v) { + if (v.state === "logged_in") { + verifyText = " · login verified"; + } else if (v.state === "needs_challenge") { + verifyText = + " · Google recognizes you but wants a one-time device confirmation. Finish it in the open Chrome window — once approved, future runs won't ask again."; + tone = "err"; + } else { + verifyText = " · cookies copied but login didn't carry over"; + tone = "err"; + } + } + setRowState((s) => ({ + ...s, + [svc.service]: { + busy: false, + message: { + tone, + text: `Imported ${data.imported ?? 0} cookies${ + data.identity ? ` for ${data.identity}` : "" + }${verifyText}.`, + }, + }, + })); + } catch (err) { + setRowState((s) => ({ + ...s, + [svc.service]: { + busy: false, + message: { + tone: "err", + text: err instanceof Error ? err.message : String(err), + }, + }, + })); + } + }, + [profileDir], + ); + + const muted = isDark ? "text-slate-400" : "text-slate-500"; + const heading = isDark ? "text-slate-100" : "text-slate-900"; + const rowBg = isDark ? "bg-slate-950/30" : "bg-slate-50"; + const borderTone = isDark ? "border-slate-800" : "border-slate-100"; + + if (profiles.length === 0 && scanError) { + return ( +
+
Logged-in sessions
+

+ Couldn't read daily Chrome profile: {scanError} +

+
+ ); + } + + return ( +
+
+
+
Logged-in sessions
+

+ Lift cookies from your daily Chrome so the boop browser is signed + in to the same accounts. Avoids login walls (Google, X, …) entirely. +

+
+
+ + +
+
+ + {scanError && ( +
+ {scanError} +
+ )} + +
+ {scan === null && !scanError && ( +
Scanning…
+ )} + {scan?.map((svc) => { + const imp = importsByService.get(svc.service); + const state = rowState[svc.service] ?? {}; + const verifyFailed = imp && imp.verifiedOk === false; + const status = svc.hasSignature + ? imp + ? verifyFailed + ? { dot: "bg-amber-400", text: "Cookies expired", tone: muted } + : { dot: "bg-emerald-400", text: "Active", tone: muted } + : { dot: "bg-emerald-400", text: "Logged in", tone: muted } + : svc.cookieCount > 0 + ? { dot: "bg-slate-500", text: "Not signed in", tone: muted } + : { dot: "bg-slate-700", text: "—", tone: muted }; + const canImport = svc.hasSignature && !state.busy; + return ( +
+
+ {SERVICE_ICON[svc.service] ?? "?"} +
+
+
{svc.label}
+
+ + {status.text} + {imp?.identity && ( + · {imp.identity} + )} + {imp && ( + + · {relTime(imp.lastImportedAt)} + + )} +
+ {state.message && ( +
+ {state.message.text} +
+ )} +
+ +
+ ); + })} +
+
+ ); +} diff --git a/package-lock.json b/package-lock.json index 3c41194d..75d03f0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@hugeicons/react": "^1.0.0", "@modelcontextprotocol/sdk": "^1.0.0", "agent-browser": "^0.26.0", + "better-sqlite3": "^12.9.0", "convex": "^1.36.1", "cors": "^2.8.5", "croner": "^9.0.0", @@ -28,6 +29,7 @@ }, "devDependencies": { "@tailwindcss/vite": "^4.0.0", + "@types/better-sqlite3": "^7.6.13", "@types/cors": "^2.8.17", "@types/express": "^5.0.0", "@types/node": "^22.0.0", @@ -1860,6 +1862,16 @@ "@babel/types": "^7.28.2" } }, + "node_modules/@types/better-sqlite3": { + "version": "7.6.13", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz", + "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/body-parser": { "version": "1.19.6", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", @@ -2195,6 +2207,26 @@ "dev": true, "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/baseline-browser-mapping": { "version": "2.10.19", "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.19.tgz", @@ -2208,6 +2240,20 @@ "node": ">=6.0.0" } }, + "node_modules/better-sqlite3": { + "version": "12.9.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.9.0.tgz", + "integrity": "sha512-wqUv4Gm3toFpHDQmaKD4QhZm3g1DjUBI0yzS4UBl6lElUmXFYdTQmmEDpAFa5o8FiFiymURypEnfVHzILKaxqQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + }, + "engines": { + "node": "20.x || 22.x || 23.x || 24.x || 25.x" + } + }, "node_modules/bezier-js": { "version": "6.1.4", "resolved": "https://registry.npmjs.org/bezier-js/-/bezier-js-6.1.4.tgz", @@ -2218,6 +2264,26 @@ "url": "https://github.com/Pomax/bezierjs/blob/master/FUNDING.md" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/body-parser": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", @@ -2287,6 +2353,30 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -2393,6 +2483,12 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -2858,6 +2954,30 @@ } } }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/define-data-property": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", @@ -2907,7 +3027,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, "license": "Apache-2.0", "engines": { "node": ">=8" @@ -2961,6 +3080,15 @@ "node": ">= 0.8" } }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/enhanced-resolve": { "version": "5.20.1", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.20.1.tgz", @@ -3241,6 +3369,15 @@ "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, "node_modules/express": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", @@ -3357,6 +3494,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, "node_modules/finalhandler": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", @@ -3452,6 +3595,12 @@ "node": ">= 0.8" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -3611,6 +3760,12 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, "node_modules/globalthis": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", @@ -3799,6 +3954,26 @@ "url": "https://opencollective.com/express" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/index-array-by": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/index-array-by/-/index-array-by-1.4.2.tgz", @@ -3814,6 +3989,12 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, "node_modules/internal-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.1.0.tgz", @@ -4757,6 +4938,18 @@ "url": "https://opencollective.com/express" } }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", @@ -4770,6 +4963,21 @@ "node": "*" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -4795,6 +5003,12 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, "node_modules/negotiator": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", @@ -4811,6 +5025,18 @@ "dev": true, "license": "MIT" }, + "node_modules/node-abi": { + "version": "3.89.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz", + "integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/node-releases": { "version": "2.0.37", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.37.tgz", @@ -5323,6 +5549,33 @@ "url": "https://opencollective.com/preact" } }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/prettier": { "version": "3.8.3", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.3.tgz", @@ -5391,6 +5644,16 @@ "node": ">= 0.10" } }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/pusher-js": { "version": "8.5.0", "resolved": "https://registry.npmjs.org/pusher-js/-/pusher-js-8.5.0.tgz", @@ -5439,6 +5702,21 @@ "node": ">= 0.10" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, "node_modules/react": { "version": "19.2.5", "resolved": "https://registry.npmjs.org/react/-/react-19.2.5.tgz", @@ -5524,6 +5802,20 @@ "node": ">=4" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz", @@ -5690,6 +5982,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/safe-push-apply": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/safe-push-apply/-/safe-push-apply-1.0.0.tgz", @@ -5968,6 +6280,51 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -6044,6 +6401,15 @@ "node": ">= 0.4" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string.prototype.padend": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/string.prototype.padend/-/string.prototype.padend-3.1.6.tgz", @@ -6144,6 +6510,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -6190,6 +6565,34 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tar-fs": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tinycolor2": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz", @@ -6242,6 +6645,18 @@ "fsevents": "~2.3.3" } }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/tweetnacl": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz", @@ -6432,6 +6847,12 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, "node_modules/validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", diff --git a/package.json b/package.json index 111e1f50..8f58a5e9 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "@hugeicons/react": "^1.0.0", "@modelcontextprotocol/sdk": "^1.0.0", "agent-browser": "^0.26.0", + "better-sqlite3": "^12.9.0", "convex": "^1.36.1", "cors": "^2.8.5", "croner": "^9.0.0", @@ -40,6 +41,7 @@ }, "devDependencies": { "@tailwindcss/vite": "^4.0.0", + "@types/better-sqlite3": "^7.6.13", "@types/cors": "^2.8.17", "@types/express": "^5.0.0", "@types/node": "^22.0.0", diff --git a/server/browser-routes.ts b/server/browser-routes.ts index 16768869..d659ad11 100644 --- a/server/browser-routes.ts +++ b/server/browser-routes.ts @@ -1,6 +1,20 @@ import express from "express"; import { execa } from "execa"; import { browserBaseArgs, getBrowserEnv, PROFILE_DIR } from "./browser/config.js"; +import { + ensureStealthChrome, + stopStealthChromeAndWait, +} from "./browser/stealth-launcher.js"; +import { + importCookiesForService, + listDailyProfiles, + scanProfile, + SERVICES, + verifyService, +} from "./browser/cookies.js"; +import { runningAgentIds } from "./execution-agent.js"; +import { convex } from "./convex-client.js"; +import { api } from "../convex/_generated/api.js"; interface BrowserStatus { installed: boolean; @@ -64,6 +78,122 @@ export function createBrowserRouter(): express.Router { } }); + router.get("/cookies/profiles", async (_req, res) => { + try { + const profiles = listDailyProfiles().map((p) => ({ + dir: p.dir, + name: p.name, + userName: p.userName, + })); + res.json({ profiles }); + } catch (err) { + res.status(500).json({ + error: err instanceof Error ? err.message : String(err), + }); + } + }); + + router.get("/cookies/scan", async (req, res) => { + const profile = typeof req.query.profile === "string" ? req.query.profile : ""; + if (!profile) { + res.status(400).json({ error: "profile query param required" }); + return; + } + try { + const services = scanProfile(profile); + const imports = await convex.query(api.cookieImports.list, {}); + res.json({ profile, services, imports }); + } catch (err) { + res.status(500).json({ + error: err instanceof Error ? err.message : String(err), + }); + } + }); + + router.post("/cookies/import", async (req, res) => { + const profile = typeof req.body?.profile === "string" ? req.body.profile.trim() : ""; + const service = typeof req.body?.service === "string" ? req.body.service.trim() : ""; + const verify = req.body?.verify !== false; // default on + if (!profile || !service) { + res.status(400).json({ error: "profile and service required" }); + return; + } + if (!SERVICES.some((s) => s.id === service)) { + res.status(400).json({ error: `unknown service: ${service}` }); + return; + } + + // Restarting Chrome would kill any tab a sub-agent is currently using. + // Refuse rather than wedge an in-flight task. + if (runningAgentIds().length > 0) { + res.status(409).json({ + error: "browser is in use by a sub-agent — try again when it finishes", + runningAgents: runningAgentIds(), + }); + return; + } + + try { + // Make sure boop Chrome was at least bootstrapped so its profile dir + // and Cookies DB schema exist, THEN stop it for the SQLite write. + await ensureStealthChrome(); + await stopStealthChromeAndWait(); + + const result = importCookiesForService(profile, service); + + // Re-boot Chrome with the new cookies in place. + await ensureStealthChrome(); + + let verified: { + state: "logged_in" | "needs_challenge" | "not_logged_in"; + finalUrl?: string; + title?: string; + } | null = null; + if (verify) { + try { + const v = await verifyService(service); + verified = { state: v.state, finalUrl: v.finalUrl, title: v.title }; + } catch (err) { + console.warn("[cookies] verify failed:", err); + } + } + + // Read the user_name off the source profile so the UI can show + // "Active as user@example.com" without a separate scan call. + const sp = listDailyProfiles().find((p) => p.dir === profile); + const identity = sp?.userName ?? undefined; + + await convex.mutation(api.cookieImports.record, { + service, + sourceProfile: profile, + identity, + cookieCount: result.imported, + // Only treat the green "Active" path as verifiedOk; both + // not_logged_in and needs_challenge are explicit "do something" + // states the UI should distinguish. + verifiedOk: verified ? verified.state === "logged_in" : undefined, + }); + + res.json({ + ok: true, + imported: result.imported, + identity, + verified, + }); + } catch (err) { + // Best-effort restart even on error so we don't leave Chrome stopped. + try { + await ensureStealthChrome(); + } catch { + /* ignore — surface the original error */ + } + res.status(500).json({ + ok: false, + error: err instanceof Error ? err.message : String(err), + }); + } + }); + router.post("/login", async (req, res) => { const url = typeof req.body?.url === "string" ? req.body.url.trim() : ""; if (!url) { @@ -81,10 +211,18 @@ export function createBrowserRouter(): express.Router { res.status(400).json({ error: "url must be http(s)" }); return; } - // Fire and forget — return 200 as soon as the spawn is in flight. Launch - // failures are logged server-side; the user will see Chrome simply not - // appear and can retry. Blocking the request on Chrome's startup would - // stall the click for 1-3s without much actionable signal. + // Make sure stealth Chrome is up + connected, then open the URL in a tab. + // Stealth bootstrap is awaited so a 200 response means Chrome is actually + // open on the user's screen (not just queued). + try { + await ensureStealthChrome(); + } catch (err) { + res.status(500).json({ + ok: false, + error: err instanceof Error ? err.message : String(err), + }); + return; + } const child = execa("agent-browser", [...browserBaseArgs(), "open", parsed.toString()], { preferLocal: true, timeout: 30_000, diff --git a/server/browser/config.ts b/server/browser/config.ts index cd235e64..14c30c4e 100644 --- a/server/browser/config.ts +++ b/server/browser/config.ts @@ -39,10 +39,20 @@ function detectRealChrome(): string | null { export const CHROME_PATH = detectRealChrome(); +// We launch Chrome ourselves (see server/browser/stealth-launcher.ts) and +// patch navigator.webdriver via CDP before any page loads — that's the +// signal Google checks. `--disable-blink-features=AutomationControlled` +// alone was patched out in Chrome 122+; doing nothing about webdriver +// means Google's "browser may not be secure" wall fires on every sign-in. +// +// `--cdp ` is the per-command flag that points agent-browser at our +// already-running Chrome. Without it, the daemon launches its own bundled +// Chrome for Testing in a temp profile and ignores ours entirely (which +// silently breaks the patch — your patches go to the wrong browser). +export const STEALTH_CDP_PORT = 9222; + export function browserBaseArgs(): string[] { - const args = ["--profile", PROFILE_DIR, "--session", SESSION]; - if (CHROME_PATH) args.push("--executable-path", CHROME_PATH); - return args; + return ["--session", SESSION, "--cdp", String(STEALTH_CDP_PORT)]; } // agent-browser defaults to --headless=new, which puts "HeadlessChrome" in the diff --git a/server/browser/cookies.ts b/server/browser/cookies.ts new file mode 100644 index 00000000..7efd8214 --- /dev/null +++ b/server/browser/cookies.ts @@ -0,0 +1,351 @@ +import { + copyFileSync, + existsSync, + readFileSync, + unlinkSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import Database from "better-sqlite3"; +import { PROFILE_DIR } from "./config.js"; +import { evalOnNewTab } from "./stealth-launcher.js"; + +// Daily Chrome on macOS keeps profiles under here. Each subdir is one profile; +// 'Default' is the first profile, 'Profile 1', 'Profile 2', ... are the rest. +const DAILY_CHROME_DIR = join( + homedir(), + "Library", + "Application Support", + "Google", + "Chrome", +); + +// Each service describes (a) which cookie host_keys belong to it, +// (b) cookie names that signal an active login, (c) a URL we can visit +// post-import to confirm the cookie actually authenticates us. +export interface ServiceDef { + id: string; + label: string; + hostPatterns: string[]; // matched as `host_key LIKE '%' + p` + signatures: string[]; // any one of these cookie names ⇒ logged in + verifyUrl: string; + // Verification: after navigating to verifyUrl, we must end up on this + // host. If the final URL's host doesn't match, we got bounced (almost + // always to the service's sign-in / device-confirm flow). + expectHost: string; + // Hosts that, when present in the final URL, indicate a device- + // confirmation / 2FA challenge — cookies authenticated us but the site + // wants additional verification before letting us in. UI surfaces this + // distinctly from "not signed in at all." + challengePathContains?: string[]; +} + +export const SERVICES: ServiceDef[] = [ + { + id: "google", + label: "Google", + hostPatterns: [ + ".google.com", + ".youtube.com", + ".googleapis.com", + ".gstatic.com", + ".googleusercontent.com", + ], + signatures: ["SID", "HSID", "SSID", "SAPISID", "__Secure-1PSID"], + verifyUrl: "https://myaccount.google.com/", + expectHost: "myaccount.google.com", + challengePathContains: [ + "/signin/v2/challenge", + "/v3/signin/confirmidentifier", + "/v3/signin/challenge", + ], + }, + { + id: "linkedin", + label: "LinkedIn", + hostPatterns: [".linkedin.com"], + signatures: ["li_at"], + verifyUrl: "https://www.linkedin.com/feed/", + expectHost: "www.linkedin.com", + challengePathContains: ["/checkpoint/", "/uas/"], + }, + { + id: "twitter", + label: "X (Twitter)", + hostPatterns: [".x.com", ".twitter.com"], + signatures: ["auth_token"], + verifyUrl: "https://x.com/home", + expectHost: "x.com", + challengePathContains: ["/i/flow/login", "/account/access"], + }, + { + id: "reddit", + label: "Reddit", + hostPatterns: [".reddit.com"], + signatures: ["reddit_session", "token_v2"], + verifyUrl: "https://www.reddit.com/", + expectHost: "www.reddit.com", + challengePathContains: ["/login"], + }, + { + id: "github", + label: "GitHub", + hostPatterns: [".github.com"], + signatures: ["user_session"], + verifyUrl: "https://github.com/notifications", + expectHost: "github.com", + challengePathContains: ["/login", "/sessions/two-factor"], + }, +]; + +export interface DailyProfile { + dir: string; // 'Default', 'Profile 1', etc. + name: string; + userName: string | null; + cookiesPath: string; +} + +interface ChromeProfileInfo { + name?: string; + user_name?: string; +} + +function cookiesPathFor(profileDir: string): string | null { + // Chrome 96+ stores cookies under Network/, older under the profile root. + const network = join(profileDir, "Network", "Cookies"); + if (existsSync(network)) return network; + const legacy = join(profileDir, "Cookies"); + if (existsSync(legacy)) return legacy; + return null; +} + +export function listDailyProfiles(): DailyProfile[] { + if (!existsSync(DAILY_CHROME_DIR)) return []; + let infoCache: Record = {}; + try { + const localState = JSON.parse( + readFileSync(join(DAILY_CHROME_DIR, "Local State"), "utf-8"), + ) as { profile?: { info_cache?: Record } }; + infoCache = localState.profile?.info_cache ?? {}; + } catch { + /* fall through to filesystem scan */ + } + const out: DailyProfile[] = []; + for (const dir of Object.keys(infoCache)) { + const profileDir = join(DAILY_CHROME_DIR, dir); + const cp = cookiesPathFor(profileDir); + if (!cp) continue; + out.push({ + dir, + name: infoCache[dir]?.name ?? dir, + userName: infoCache[dir]?.user_name?.trim() || null, + cookiesPath: cp, + }); + } + return out; +} + +// Open a Cookies SQLite file safely while Chrome may still hold it. Strategy: +// snapshot the .db plus its WAL/SHM sidecars to a temp location, then open +// the temp copy read-only. This avoids any lock contention with running +// Chrome and guarantees a consistent view. +function snapshotCookieDb(srcPath: string): string { + const tmp = join("/tmp", `boop-cookies-${process.pid}-${Date.now()}.db`); + copyFileSync(srcPath, tmp); + if (existsSync(srcPath + "-wal")) copyFileSync(srcPath + "-wal", tmp + "-wal"); + if (existsSync(srcPath + "-shm")) copyFileSync(srcPath + "-shm", tmp + "-shm"); + return tmp; +} + +function cleanupSnapshot(tmp: string): void { + for (const suffix of ["", "-wal", "-shm"]) { + try { + unlinkSync(tmp + suffix); + } catch { + /* ignore */ + } + } +} + +export interface ServiceScan { + service: string; + label: string; + hostsCovered: string[]; + cookieCount: number; + hasSignature: boolean; +} + +export function scanProfile(profileDir: string): ServiceScan[] { + const cp = cookiesPathFor(join(DAILY_CHROME_DIR, profileDir)); + if (!cp) return []; + const tmp = snapshotCookieDb(cp); + try { + const db = new Database(tmp, { readonly: true, fileMustExist: true }); + try { + const out: ServiceScan[] = []; + for (const svc of SERVICES) { + const where = svc.hostPatterns.map(() => "host_key LIKE ?").join(" OR "); + const args = svc.hostPatterns.map((p) => "%" + p); + const rows = db + .prepare(`SELECT host_key, name FROM cookies WHERE ${where}`) + .all(...args) as Array<{ host_key: string; name: string }>; + const hasSignature = svc.signatures.some((sig) => + rows.some((r) => r.name === sig), + ); + out.push({ + service: svc.id, + label: svc.label, + hostsCovered: [...new Set(rows.map((r) => r.host_key))].sort(), + cookieCount: rows.length, + hasSignature, + }); + } + return out; + } finally { + db.close(); + } + } finally { + cleanupSnapshot(tmp); + } +} + +interface PragmaRow { + cid: number; + name: string; + type: string; + notnull: number; + dflt_value: string | null; + pk: number; +} + +function tableColumns(db: Database.Database, table: string): string[] { + const rows = db.pragma(`table_info(${table})`) as PragmaRow[]; + return rows.map((r) => r.name); +} + +export interface ImportResult { + imported: number; + skipped: number; +} + +// Copy cookies from `sourceProfile` (Default, Profile 1, ...) into the boop +// profile's Cookies DB for a single service. Caller MUST have stopped boop's +// stealth Chrome first — otherwise the dest DB is held with a write lock and +// we'll error out (or worse, silently fail). +export function importCookiesForService( + sourceProfile: string, + serviceId: string, +): ImportResult { + const svc = SERVICES.find((s) => s.id === serviceId); + if (!svc) throw new Error(`unknown service: ${serviceId}`); + + const srcCookies = cookiesPathFor(join(DAILY_CHROME_DIR, sourceProfile)); + if (!srcCookies) { + throw new Error(`No Cookies DB found for source profile "${sourceProfile}"`); + } + const destDir = join(PROFILE_DIR, "Default"); + // The boop profile's Default/Network/ dir may not exist if Chrome has never + // been launched with this profile. Caller is expected to have done at + // least one ensureStealthChrome cycle to bootstrap the dir + schema. + const destCookies = cookiesPathFor(destDir); + if (!destCookies) { + throw new Error( + `Boop Cookies DB not found at ${destDir}. Open the browser at least once to initialize it.`, + ); + } + + const tmpSrc = snapshotCookieDb(srcCookies); + let srcDb: Database.Database | null = null; + let dstDb: Database.Database | null = null; + try { + srcDb = new Database(tmpSrc, { readonly: true, fileMustExist: true }); + dstDb = new Database(destCookies, { fileMustExist: true }); + + const srcCols = tableColumns(srcDb, "cookies"); + const dstCols = tableColumns(dstDb, "cookies"); + const cols = srcCols.filter((c) => dstCols.includes(c)); + if (!cols.includes("host_key") || !cols.includes("name")) { + throw new Error("Cookies table missing host_key/name columns; schema mismatch."); + } + + const where = svc.hostPatterns.map(() => "host_key LIKE ?").join(" OR "); + const args = svc.hostPatterns.map((p) => "%" + p); + const select = srcDb.prepare( + `SELECT ${cols.join(", ")} FROM cookies WHERE ${where}`, + ); + const rows = select.all(...args) as Array>; + + const placeholders = cols.map(() => "?").join(", "); + const insert = dstDb.prepare( + `INSERT OR REPLACE INTO cookies (${cols.join(", ")}) VALUES (${placeholders})`, + ); + const tx = dstDb.transaction((records: Array>) => { + let n = 0; + for (const r of records) { + insert.run(...cols.map((c) => r[c] ?? null)); + n++; + } + return n; + }); + const imported = tx(rows); + return { imported, skipped: 0 }; + } finally { + try { + srcDb?.close(); + } catch { + /* ignore */ + } + try { + dstDb?.close(); + } catch { + /* ignore */ + } + cleanupSnapshot(tmpSrc); + } +} + +export type VerifyState = "logged_in" | "needs_challenge" | "not_logged_in"; + +export interface VerifyResult { + state: VerifyState; + finalUrl: string; + title: string; +} + +// Open verifyUrl in a fresh tab and decide what kind of state we're in. +// We staying on `expectHost` ⇒ logged_in. Bounced to a known challenge +// path (Google's /v3/signin/confirmidentifier, X's flow/login, etc.) ⇒ +// needs_challenge — cookies authenticated us but the site wants a one- +// time human action. Bounced anywhere else ⇒ not_logged_in (cookies +// didn't carry over). +export async function verifyService(serviceId: string): Promise { + const svc = SERVICES.find((s) => s.id === serviceId); + if (!svc) throw new Error(`unknown service: ${serviceId}`); + const probe = (await evalOnNewTab( + svc.verifyUrl, + "JSON.stringify({ url: location.href, title: document.title })", + )) as string | undefined; + let finalUrl = ""; + let title = ""; + try { + const parsed = JSON.parse(probe ?? "{}") as { url?: string; title?: string }; + finalUrl = parsed.url ?? ""; + title = parsed.title ?? ""; + } catch { + /* leave empty */ + } + let host = ""; + try { + host = new URL(finalUrl).host; + } catch { + /* malformed url */ + } + let state: VerifyState = "logged_in"; + if (host && host !== svc.expectHost) { + const challenged = (svc.challengePathContains ?? []).some((p) => + finalUrl.includes(p), + ); + state = challenged ? "needs_challenge" : "not_logged_in"; + } + return { state, finalUrl, title }; +} diff --git a/server/browser/stealth-launcher.ts b/server/browser/stealth-launcher.ts new file mode 100644 index 00000000..daea99c8 --- /dev/null +++ b/server/browser/stealth-launcher.ts @@ -0,0 +1,472 @@ +// Stealth Chrome launcher for the browser integration. +// +// Why this exists: agent-browser, as of 0.26.x, launches Chrome with +// `--remote-debugging-port=0`. That flag flips `navigator.webdriver = true` +// on every page, which is the single signal Google's "this browser may not +// be secure" sign-in wall keys on. Just adding `--disable-blink-features= +// AutomationControlled` to the launch args was patched out in Chrome 122+ +// — verified empirically: the flag lands but `navigator.webdriver` stays +// `true`. Stealth libraries (Patchright, puppeteer-extra-plugin-stealth) +// solve this by patching the JS runtime via CDP's +// `Page.addScriptToEvaluateOnNewDocument` BEFORE site scripts run. We do +// the same here, then have agent-browser attach via `connect 9222` instead +// of launching its own Chrome. +// +// The script we inject runs once per page, before any site code, and rewrites +// the navigator object so site scripts see a "normal" non-automated browser. +// Chrome's internal automation state is unchanged — only the JS-visible view. + +import { spawn, type ChildProcess } from "node:child_process"; +import { setTimeout as sleep } from "node:timers/promises"; +import WebSocket from "ws"; +import { CHROME_PATH, PROFILE_DIR, STEALTH_CDP_PORT } from "./config.js"; + +const STEALTH_PORT = STEALTH_CDP_PORT; + +// Patches site-visible navigator/window properties. Most important is +// navigator.webdriver — Google's "browser may not be secure" wall keys on +// it. In Chrome 122+, the property lives on Navigator.prototype as a +// non-configurable getter; just doing `Object.defineProperty(navigator, +// 'webdriver', ...)` on the instance silently no-ops because the prototype +// getter wins. The technique that actually works (used by puppeteer-extra- +// plugin-stealth and Patchright) is to delete the property off the +// prototype OR redefine it on the prototype with configurable:true. +const STEALTH_SCRIPT = String.raw` +(() => { + try { window.__boopStealth = (window.__boopStealth || 0) + 1; } catch (e) {} + + // Capture the original Function.prototype.toString FIRST so we can mask + // our own patches. Sites use \`navigator.webdriver.toString()\` and + // descriptor.get.toString() to detect tampering — real Chrome returns + // 'function get webdriver() { [native code] }', a custom getter returns + // its own source, which is the giveaway. + const _origToString = Function.prototype.toString; + const _origToStringText = _origToString.call(_origToString); + const _fakedFns = new WeakMap(); + + // 1. navigator.webdriver. Two layers: + // a) try to delete from prototype (works on older Chrome) + // b) define as a VALUE on the instance — no getter for sites to + // introspect, navigator.webdriver === false straight up. + try { delete Object.getPrototypeOf(navigator).webdriver; } catch (e) {} + try { + Object.defineProperty(navigator, 'webdriver', { + value: false, + writable: false, + configurable: true, + enumerable: true, + }); + } catch (e) { try { window.__boopStealthWebdriverErr = String(e); } catch {} } + + // Helper: build a getter via object-literal so .name === 'get ' + // and the function inherits the real "getter" form. We then proxy its + // toString below. + const makeGetter = (prop, returnValue) => { + const desc = Object.getOwnPropertyDescriptor( + { get [prop]() { return returnValue; } }, + prop, + ); + return desc.get; + }; + + // 2. Languages on the prototype. We record the getter in _fakedFns so + // Function.prototype.toString returns '[native code]' for it. + try { + const langGetter = makeGetter('languages', ['en-US', 'en']); + _fakedFns.set(langGetter, 'function get languages() { [native code] }'); + Object.defineProperty(Navigator.prototype, 'languages', { + get: langGetter, + configurable: true, + }); + } catch (e) {} + + // 3. Plugins — empty list is a strong automation tell. Three PDF entries + // is what stock desktop Chrome ships with. + try { + const mkPlugin = (name) => Object.freeze({ + name, + filename: 'internal-pdf-viewer', + description: 'Portable Document Format', + length: 1, + }); + const plugins = Object.freeze([ + mkPlugin('PDF Viewer'), + mkPlugin('Chrome PDF Viewer'), + mkPlugin('Chromium PDF Viewer'), + ]); + const pluginsGetter = makeGetter('plugins', plugins); + _fakedFns.set(pluginsGetter, 'function get plugins() { [native code] }'); + Object.defineProperty(Navigator.prototype, 'plugins', { + get: pluginsGetter, + configurable: true, + }); + } catch (e) {} + + // 4. window.chrome — populated object, not bare. Real Chrome has runtime, + // loadTimes, csi, app properties. + try { + if (!window.chrome) window.chrome = {}; + if (!window.chrome.runtime) { + window.chrome.runtime = { + OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' }, + OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' }, + PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' }, + PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' }, + connect: function connect() {}, + sendMessage: function sendMessage() {}, + }; + _fakedFns.set(window.chrome.runtime.connect, 'function connect() { [native code] }'); + _fakedFns.set(window.chrome.runtime.sendMessage, 'function sendMessage() { [native code] }'); + } + } catch (e) {} + + // 5. permissions.query for notifications — real Chrome returns 'prompt' + // on unconfigured origins; automation often returns 'denied'. + try { + const orig = window.navigator.permissions && window.navigator.permissions.query + ? window.navigator.permissions.query.bind(window.navigator.permissions) + : null; + if (orig) { + const fakeQuery = function query(parameters) { + if (parameters && parameters.name === 'notifications') { + return Promise.resolve({ state: Notification.permission, name: 'notifications', onchange: null }); + } + return orig(parameters); + }; + _fakedFns.set(fakeQuery, 'function query() { [native code] }'); + window.navigator.permissions.query = fakeQuery; + } + } catch (e) {} + + // 6. CRITICAL: patch Function.prototype.toString last. Sites detect + // stealth by calling fakeFn.toString() and checking for source code + // instead of '[native code]'. Our Proxy returns the recorded native + // string for any function we faked, and the real native string for + // everything else (including .toString itself). + try { + const proxiedToString = new Proxy(_origToString, { + apply(target, thisArg, argsList) { + if (thisArg === proxiedToString) return _origToStringText; + if (_fakedFns.has(thisArg)) return _fakedFns.get(thisArg); + return Reflect.apply(target, thisArg, argsList); + }, + }); + Function.prototype.toString = proxiedToString; + } catch (e) {} +})(); +`; + +let chromeProcess: ChildProcess | null = null; +let browserWs: WebSocket | null = null; +let bootPromise: Promise | null = null; +let nextId = 1; +let exitWaiters: Array<() => void> = []; + +interface CdpMessage { + id?: number; + method?: string; + params?: Record; + sessionId?: string; + result?: Record; + error?: { code: number; message: string }; +} + +function send( + ws: WebSocket, + method: string, + params: Record = {}, + sessionId?: string, +): Promise> { + return new Promise((resolve, reject) => { + const id = nextId++; + const message: CdpMessage = sessionId + ? { id, method, params, sessionId } + : { id, method, params }; + const handler = (raw: WebSocket.RawData) => { + let parsed: CdpMessage; + try { + parsed = JSON.parse(raw.toString()) as CdpMessage; + } catch { + return; + } + if (parsed.id !== id) return; + ws.off("message", handler); + if (parsed.error) reject(new Error(`${method}: ${parsed.error.message}`)); + else resolve(parsed.result ?? {}); + }; + ws.on("message", handler); + try { + ws.send(JSON.stringify(message)); + } catch (err) { + ws.off("message", handler); + reject(err as Error); + } + }); +} + +async function spawnChrome(): Promise { + if (chromeProcess) return; + const binary = + CHROME_PATH ?? "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"; + const args = [ + `--remote-debugging-port=${STEALTH_PORT}`, + `--user-data-dir=${PROFILE_DIR}`, + "--no-first-run", + "--no-default-browser-check", + "--disable-blink-features=AutomationControlled", + "--disable-features=AutomationControlled", + ]; + console.log(`[stealth] launching ${binary} with port ${STEALTH_PORT}`); + chromeProcess = spawn(binary, args, { stdio: "ignore", detached: false }); + chromeProcess.on("exit", (code) => { + console.log(`[stealth] Chrome exited (code=${code})`); + chromeProcess = null; + if (browserWs) { + try { + browserWs.close(); + } catch { + /* ignore */ + } + browserWs = null; + } + bootPromise = null; + const waiters = exitWaiters; + exitWaiters = []; + for (const w of waiters) w(); + }); +} + +async function waitForCdpEndpoint(): Promise { + for (let i = 0; i < 60; i++) { + try { + const res = await fetch(`http://127.0.0.1:${STEALTH_PORT}/json/version`); + if (res.ok) { + const json = (await res.json()) as { webSocketDebuggerUrl?: string }; + if (json.webSocketDebuggerUrl) return json.webSocketDebuggerUrl; + } + } catch { + /* retry */ + } + await sleep(250); + } + throw new Error( + `[stealth] Chrome didn't expose CDP on :${STEALTH_PORT} within 15s`, + ); +} + +async function attachWebSocket(url: string): Promise { + const ws = new WebSocket(url); + await new Promise((resolve, reject) => { + ws.once("open", () => resolve()); + ws.once("error", reject); + }); + return ws; +} + +async function patchPageTarget(ws: WebSocket, sessionId: string): Promise { + await send(ws, "Page.enable", {}, sessionId); + await send( + ws, + "Page.addScriptToEvaluateOnNewDocument", + { source: STEALTH_SCRIPT }, + sessionId, + ); + // Re-eval the patches in the CURRENT document too, in case the target was + // already on a real page when we attached (about:blank counts but a re-nav + // is on the way). addScriptToEvaluateOnNewDocument only fires on FUTURE + // documents, so without this the very first page in the tab keeps the + // pre-patch values. + await send( + ws, + "Runtime.evaluate", + { expression: STEALTH_SCRIPT, awaitPromise: false }, + sessionId, + ).catch(() => { + /* about:blank etc. may reject, ignore */ + }); +} + +async function setupAutoAttachAndPatch(ws: WebSocket): Promise { + // CRITICAL: attach the events listener BEFORE calling Target.setAutoAttach. + // setAutoAttach immediately fires Target.attachedToTarget for every + // existing target. If we register the listener after, those events were + // already on the wire and the patch never lands on the about:blank tab — + // which becomes the first page that gets navigated by agent-browser. + ws.on("message", async (raw) => { + let parsed: CdpMessage; + try { + parsed = JSON.parse(raw.toString()) as CdpMessage; + } catch { + return; + } + if (parsed.method !== "Target.attachedToTarget") return; + const params = parsed.params as + | { sessionId: string; targetInfo?: { type?: string } } + | undefined; + if (!params?.sessionId) return; + const targetType = params.targetInfo?.type ?? ""; + const isPagey = targetType === "page" || targetType === "iframe"; + try { + if (isPagey) await patchPageTarget(ws, params.sessionId); + } catch (err) { + console.warn( + `[stealth] failed to patch target ${targetType}:`, + err instanceof Error ? err.message : err, + ); + } finally { + try { + await send(ws, "Runtime.runIfWaitingForDebugger", {}, params.sessionId); + } catch { + /* target may already be running */ + } + } + }); + + // Now enable autoAttach. waitForDebuggerOnStart pauses NEW targets so the + // patch lands before the first script runs. Existing targets attach + // immediately and aren't paused — we handle those via the explicit + // Runtime.evaluate fallback in patchPageTarget. + await send(ws, "Target.setAutoAttach", { + autoAttach: true, + waitForDebuggerOnStart: true, + flatten: true, + }); +} + +export async function ensureStealthChrome(): Promise { + if (browserWs) return; + if (bootPromise) return bootPromise; + bootPromise = (async () => { + await spawnChrome(); + const wsUrl = await waitForCdpEndpoint(); + browserWs = await attachWebSocket(wsUrl); + await setupAutoAttachAndPatch(browserWs); + console.log( + `[stealth] ready — Chrome on :${STEALTH_PORT}, navigator.webdriver patched on every new document. Pass --cdp ${STEALTH_PORT} to agent-browser to drive it.`, + ); + })(); + try { + await bootPromise; + } catch (err) { + bootPromise = null; + throw err; + } +} + +export function stopStealthChrome(): void { + if (browserWs) { + try { + browserWs.close(); + } catch { + /* ignore */ + } + browserWs = null; + } + if (chromeProcess) { + try { + chromeProcess.kill("SIGTERM"); + } catch { + /* ignore */ + } + chromeProcess = null; + } + bootPromise = null; +} + +// Like stopStealthChrome, but resolves only after the Chrome process has +// fully exited. Use before any operation that needs exclusive write access +// to the profile dir (e.g. cookie SQLite writes) — otherwise Chrome's open +// file handle keeps the WAL locked. +export async function stopStealthChromeAndWait(timeoutMs = 8000): Promise { + const proc = chromeProcess; + if (!proc) { + stopStealthChrome(); + return; + } + const done = new Promise((resolve) => { + exitWaiters.push(resolve); + }); + stopStealthChrome(); + await Promise.race([ + done, + new Promise((resolve) => setTimeout(resolve, timeoutMs)), + ]); +} + +export function stealthRunning(): boolean { + return !!chromeProcess && !!browserWs; +} + +// Open a new tab, navigate to `url`, wait for load, evaluate JS, close. +// Returns whatever `Runtime.evaluate` does on `expression`. Used for +// post-import login verification — we need a page session distinct from +// any tabs the user / sub-agents may have open. +export async function evalOnNewTab( + url: string, + expression: string, + navTimeoutMs = 12_000, +): Promise { + if (!browserWs) throw new Error("stealth Chrome not running"); + const ws = browserWs; + + // Create a new target (tab) via the browser-level session. + const created = (await send(ws, "Target.createTarget", { url: "about:blank" })) as { + targetId?: string; + }; + const targetId = created.targetId; + if (!targetId) throw new Error("Target.createTarget returned no targetId"); + + // Attach a flat page session so we can talk to it directly. + const attached = (await send(ws, "Target.attachToTarget", { + targetId, + flatten: true, + })) as { sessionId?: string }; + const sessionId = attached.sessionId; + if (!sessionId) throw new Error("Target.attachToTarget returned no sessionId"); + + try { + await send(ws, "Page.enable", {}, sessionId); + + // Navigate and wait for load via Page.frameStoppedLoading. + const nav = new Promise((resolve, reject) => { + const timer = setTimeout(() => { + ws.off("message", listener); + reject(new Error("navigation timeout")); + }, navTimeoutMs); + const listener = (raw: WebSocket.RawData) => { + let parsed: CdpMessage; + try { + parsed = JSON.parse(raw.toString()) as CdpMessage; + } catch { + return; + } + if (parsed.sessionId !== sessionId) return; + if (parsed.method === "Page.frameStoppedLoading") { + clearTimeout(timer); + ws.off("message", listener); + resolve(); + } + }; + ws.on("message", listener); + }); + + await send(ws, "Page.navigate", { url }, sessionId); + await nav.catch(() => { + /* fall through — we still try to read the page */ + }); + + const result = (await send( + ws, + "Runtime.evaluate", + { expression, returnByValue: true }, + sessionId, + )) as { result?: { value?: unknown } }; + return result.result?.value; + } finally { + try { + await send(ws, "Target.closeTarget", { targetId }); + } catch { + /* ignore */ + } + } +} diff --git a/server/browser/tools.ts b/server/browser/tools.ts index 0c1ea4ab..0d4353fd 100644 --- a/server/browser/tools.ts +++ b/server/browser/tools.ts @@ -2,6 +2,7 @@ import { tool, createSdkMcpServer } from "@anthropic-ai/claude-agent-sdk"; import { z } from "zod"; import { execa } from "execa"; import { browserBaseArgs, CHROME_PATH, getBrowserEnv } from "./config.js"; +import { ensureStealthChrome } from "./stealth-launcher.js"; if (CHROME_PATH) { console.log(`[browser] using real Chrome at ${CHROME_PATH}`); @@ -19,6 +20,7 @@ interface Result { async function ab(args: string[]): Promise { try { + await ensureStealthChrome(); const r = await execa("agent-browser", [...browserBaseArgs(), ...args], { preferLocal: true, timeout: TIMEOUT_MS, diff --git a/server/index.ts b/server/index.ts index 833b0960..461248a6 100644 --- a/server/index.ts +++ b/server/index.ts @@ -14,6 +14,7 @@ import { startConsolidationLoop } from "./consolidation.js"; import { cancelAgent, retryAgent } from "./execution-agent.js"; import { createComposioRouter } from "./composio-routes.js"; import { createBrowserRouter } from "./browser-routes.js"; +import { stopStealthChrome } from "./browser/stealth-launcher.js"; import { ensureProactiveWatcher } from "./proactive-email.js"; async function main() { @@ -109,6 +110,16 @@ async function main() { console.log(` sendblue POST http://localhost:${port}/sendblue/webhook`); console.log(` websocket WS ws://localhost:${port}/ws`); }); + + // Make sure the Chrome we own dies when the server does. tsx watch sends + // SIGTERM on file changes; without this Chrome leaks across reloads and + // the next stealth-bootstrap fights its own zombie for the profile lock. + for (const sig of ["SIGTERM", "SIGINT", "SIGHUP"] as const) { + process.on(sig, () => { + stopStealthChrome(); + process.exit(0); + }); + } } main().catch((err) => { diff --git a/server/interaction-agent.ts b/server/interaction-agent.ts index e4318bd6..09b138af 100644 --- a/server/interaction-agent.ts +++ b/server/interaction-agent.ts @@ -68,6 +68,22 @@ sequential spawns. Rules: - When relaying, combine the results in one reply — don't make the user read N separate messages. +Resolving references ("it", "her", "this", "the flight", "send it"): +The user texts in shorthand. Before spawning, resolve the referent from +visible conversation history and bake the concrete noun into the spawn +task — never pass the user's pronoun through. "Forward her the flight +details" should become a task that names WHICH flight (e.g. "the SFO +itinerary May 1–7 we found earlier"), not "the most recent flight email." +"Most recent X" is NOT a safe default for ambiguous references. +- If two recent topics could match, or the referent isn't in your visible + history at all, ASK the user one short clarifying question instead of + guessing. +- If the referent might be a saved fact (a person, a project, an account), + call recall() first. +- Topic hops (the user wandered to YouTube/Twitter/etc.) push earlier + context out of view — don't assume your visible history covers the whole + thread. When in doubt, ask. + Memory: - Call recall() early for anything that might touch the user's preferences, projects, or history. - Call write_memory() aggressively for durable facts. Err on the side of saving. @@ -303,7 +319,7 @@ export async function handleUserMessage(opts: HandleOpts): Promise { const history = await convex.query(api.messages.recent, { conversationId: opts.conversationId, - limit: 10, + limit: 30, }); const historyBlock = history .slice(0, -1)