From 7a105fee5b45dfc754d2270a3d996b5ffe6014cf Mon Sep 17 00:00:00 2001 From: fettpl <38704082+fettpl@users.noreply.github.com> Date: Sat, 18 Jul 2026 22:13:30 +0200 Subject: [PATCH 1/4] fix(agent): pin embedded browser network requests --- frontend/.depcheckrc.json | 3 +- package.json | 1 + services/agent-runtime/package.json | 3 +- services/agent-runtime/scripts/test.mjs | 34 ++ .../browser-host-concurrency.test.ts | 497 ++++++++++++++++++ .../src/browser-host/browser-host.ts | 403 ++++++++++---- .../browser-network-policy.test.ts | 349 ++++++++++++ ...rowser-request-classes.integration.test.ts | 266 ++++++++++ .../chrome-manager-revocation.test.ts | 215 ++++++++ .../agent-runtime/src/browser-host/chrome.ts | 338 ++++++++---- .../src/browser-host/hosted-page.ts | 63 ++- .../src/browser-host/network-policy.ts | 152 ++++++ .../src/browser-host/pinning-proxy.ts | 311 +++++++++++ .../reader-network-policy.test.ts | 115 ++++ .../agent-runtime/src/browser-host/reader.ts | 114 ++-- .../src/http/browser-handlers.ts | 32 +- shared/agent/sanitize-embedded-browser-url.ts | 279 ++++++---- 17 files changed, 2791 insertions(+), 384 deletions(-) create mode 100644 services/agent-runtime/scripts/test.mjs create mode 100644 services/agent-runtime/src/browser-host/browser-host-concurrency.test.ts create mode 100644 services/agent-runtime/src/browser-host/browser-network-policy.test.ts create mode 100644 services/agent-runtime/src/browser-host/browser-request-classes.integration.test.ts create mode 100644 services/agent-runtime/src/browser-host/chrome-manager-revocation.test.ts create mode 100644 services/agent-runtime/src/browser-host/network-policy.ts create mode 100644 services/agent-runtime/src/browser-host/pinning-proxy.ts create mode 100644 services/agent-runtime/src/browser-host/reader-network-policy.test.ts diff --git a/frontend/.depcheckrc.json b/frontend/.depcheckrc.json index e86bac966..1a4e62383 100644 --- a/frontend/.depcheckrc.json +++ b/frontend/.depcheckrc.json @@ -17,7 +17,8 @@ "lint-staged", "concurrently", "electron-builder", - "madge" + "madge", + "playwright" ], "ignore-patterns": ["*.d.ts", ".next/**", "dist-desktop/**", "desktop/dist/**"], "specials": ["next", "webpack", "babel"], diff --git a/package.json b/package.json index 31e097af3..89a053e51 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "release:check-commits": "node scripts/check-conventional-commits.mjs --range origin/main..HEAD", "release:stage-desktop": "node scripts/stage-desktop-release.mjs", "setup:git-hooks": "node scripts/setup-git-hooks.mjs", + "test:integration": "npm --prefix services/agent-runtime test", "check": "npm run check:contracts && npm run check:structure && npm run check:frontend && npm run check:controller", "check:contracts": "node scripts/validate-shared-contracts.mjs", "check:structure": "node scripts/validate-barrel-dir-siblings.mjs", diff --git a/services/agent-runtime/package.json b/services/agent-runtime/package.json index 7e432c10e..140dc14cf 100644 --- a/services/agent-runtime/package.json +++ b/services/agent-runtime/package.json @@ -12,7 +12,8 @@ "bundle": "bun build src/server.ts --target=node --outfile=dist/standalone.mjs", "build": "tsc -p tsconfig.build.json && node scripts/postbuild.mjs", "dev": "bun --watch src/server.ts", - "start": "node dist/server.js" + "start": "node dist/server.js", + "test": "node scripts/test.mjs" }, "dependencies": { "@earendil-works/pi-ai": "0.80.8", diff --git a/services/agent-runtime/scripts/test.mjs b/services/agent-runtime/scripts/test.mjs new file mode 100644 index 000000000..f586a88d4 --- /dev/null +++ b/services/agent-runtime/scripts/test.mjs @@ -0,0 +1,34 @@ +import { readdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { spawnSync } from "node:child_process"; + +const packageDirectory = dirname(import.meta.dirname); +const compiledDirectory = join( + packageDirectory, + "dist", + "services", + "agent-runtime", + "src", +); +const filters = process.argv.slice(2); + +function testFiles(directory) { + return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const path = join(directory, entry.name); + if (entry.isDirectory()) return testFiles(path); + return entry.isFile() && entry.name.endsWith(".test.js") ? [path] : []; + }); +} + +function run(executable, arguments_) { + const result = spawnSync(executable, arguments_, { cwd: packageDirectory, stdio: "inherit" }); + if (result.error) throw result.error; + if (result.status !== 0) process.exit(result.status ?? 1); +} + +run(process.platform === "win32" ? "npm.cmd" : "npm", ["run", "build", "--silent"]); +const files = testFiles(compiledDirectory).filter( + (file) => filters.length === 0 || filters.some((filter) => file.includes(filter)), +); +if (files.length === 0) throw new Error("No matching agent runtime tests"); +run(process.execPath, ["--test", ...files]); diff --git a/services/agent-runtime/src/browser-host/browser-host-concurrency.test.ts b/services/agent-runtime/src/browser-host/browser-host-concurrency.test.ts new file mode 100644 index 000000000..938464a86 --- /dev/null +++ b/services/agent-runtime/src/browser-host/browser-host-concurrency.test.ts @@ -0,0 +1,497 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + BrowserHost, + type BrowserHostManager, + type BrowserHostOptions, + type BrowserPage, + type ConsoleEntry, + type PageState, + type ScreencastFrame, +} from "./browser-host"; +import type { ManagedChromeTarget } from "./chrome"; +import type { SnapshotElement } from "./dom-scripts"; +import { createBrowserNetworkPolicy } from "./network-policy"; + +class Deferred { + private readonly state = Promise.withResolvers(); + readonly promise = this.state.promise; + + resolve(value: T): void { + this.state.resolve(value); + } +} + +type Barrier = { + release: Deferred; + started: Deferred; +}; + +function barrier(): Barrier { + return { release: new Deferred(), started: new Deferred() }; +} + +const pageState = (url: string): PageState => ({ + url, + title: url, + canGoBack: false, + canGoForward: false, + loading: false, +}); + +class FakePage implements BrowserPage { + closed = false; + latestFrame: ScreencastFrame | null; + subscriptions = 0; + unsubscriptions = 0; + private readonly loadSubscribers = new Set<() => void>(); + private readonly stateSubscribers = new Set<(state: PageState) => void>(); + private state = pageState("about:blank"); + private closing: Promise | null = null; + + constructor( + readonly id: string, + readonly contextId: string, + readonly mode: ManagedChromeTarget["mode"], + private readonly navigationBarrier: Barrier | null, + private readonly frameBarrier: Barrier | null, + private readonly releaseContext: () => Promise, + ) { + this.latestFrame = { data: `frame-${id}`, metadata: {} }; + } + + close(): Promise { + if (this.closing) return this.closing; + this.closed = true; + this.closing = this.releaseContext(); + return this.closing; + } + + async call( + method: string, + params: Record = {}, + ): Promise> { + if (method === "Page.navigate") { + this.navigationBarrier?.started.resolve(); + await this.navigationBarrier?.release.promise; + const url = params["url"]; + if (typeof url !== "string") throw new Error("Missing navigation URL"); + this.state = pageState(url); + for (const subscriber of this.loadSubscribers) subscriber(); + for (const subscriber of this.stateSubscribers) subscriber(this.state); + } + return {}; + } + + async readState(): Promise { + if (this.closed) throw new Error("Target closed"); + return this.state; + } + + subscribeLoad(subscriber: () => void): () => void { + this.loadSubscribers.add(subscriber); + return () => this.loadSubscribers.delete(subscriber); + } + + subscribeFrames(_subscriber: (frame: ScreencastFrame) => void): { + ready: Promise; + unsubscribe: () => void; + } { + this.subscriptions += 1; + this.frameBarrier?.started.resolve(); + let active = true; + return { + ready: this.frameBarrier?.release.promise ?? Promise.resolve(), + unsubscribe: () => { + if (!active) return; + active = false; + this.unsubscriptions += 1; + }, + }; + } + + subscribeState(subscriber: (state: PageState) => void): () => void { + this.stateSubscribers.add(subscriber); + return () => this.stateSubscribers.delete(subscriber); + } + + async invokeScript(_script: string, _args: unknown[]): Promise { + throw new Error("Not implemented"); + } + + setRefMap(_elements: SnapshotElement[]): void {} + + resolveRef(_ref: string): string | null { + return null; + } + + drainConsole(_limit: number): ConsoleEntry[] { + return []; + } +} + +class FakeManager implements BrowserHostManager { + readonly contexts = new Set(); + readonly modes: ManagedChromeTarget["mode"][] = []; + readonly pages: FakePage[] = []; + stops = 0; + private readonly createBarriers: Barrier[] = []; + private readonly disposalPlans: { barrier?: Barrier; error?: Error }[] = []; + private readonly navigationBarriers: Barrier[] = []; + private readonly frameBarriers: Barrier[] = []; + private serial = 0; + private readonly policy = createBrowserNetworkPolicy(); + + isAvailable(): boolean { + return true; + } + + blockNextCreate(): Barrier { + const next = barrier(); + this.createBarriers.push(next); + return next; + } + + blockNextNavigation(): Barrier { + const next = barrier(); + this.navigationBarriers.push(next); + return next; + } + + blockNextDispose(): Barrier { + const next = barrier(); + this.disposalPlans.push({ barrier: next }); + return next; + } + + rejectNextDispose(error: Error): void { + this.disposalPlans.push({ error }); + } + + blockNextFrame(): Barrier { + const next = barrier(); + this.frameBarriers.push(next); + return next; + } + + async createTarget(mode: ManagedChromeTarget["mode"]): Promise { + const blocked = this.createBarriers.shift(); + blocked?.started.resolve(); + await blocked?.release.promise; + const serial = ++this.serial; + const contextId = `context-${serial}`; + const targetId = `target-${serial}`; + this.contexts.add(contextId); + this.modes.push(mode); + return { contextId, mode, policy: this.policy, targetId }; + } + + async ensure(): Promise<{ port: number }> { + throw new Error("Unexpected real Chromium attach"); + } + + async disposeContext(contextId: string): Promise { + const plan = this.disposalPlans.shift(); + plan?.barrier?.started.resolve(); + await plan?.barrier?.release.promise; + if (plan?.error) throw plan.error; + this.contexts.delete(contextId); + } + + attach(managed: ManagedChromeTarget): BrowserPage { + const page = new FakePage( + managed.targetId, + managed.contextId, + managed.mode, + this.navigationBarriers.shift() ?? null, + this.frameBarriers.shift() ?? null, + () => this.disposeContext(managed.contextId), + ); + this.pages.push(page); + return page; + } + + async stop(): Promise { + this.stops += 1; + this.contexts.clear(); + } +} + +type ScheduledTask = { callback: () => void; cancelled: boolean }; + +class FakeScheduler { + readonly tasks: ScheduledTask[] = []; + + schedule = (callback: () => void): (() => void) => { + const task = { callback, cancelled: false }; + this.tasks.push(task); + return () => { + task.cancelled = true; + }; + }; +} + +function testHost( + manager: FakeManager, + schedulePollIdle?: BrowserHostOptions["schedulePollIdle"], +): BrowserHost { + return new BrowserHost(manager, { + attachPage: async (managed) => manager.attach(managed), + schedulePollIdle, + }); +} + +function activePages(manager: FakeManager): FakePage[] { + return manager.pages.filter((page) => !page.closed); +} + +async function expectConcurrentStartup(first: "navigate" | "poll"): Promise { + const manager = new FakeManager(); + const host = testHost(manager); + const create = manager.blockNextCreate(); + const fixture = "http://localhost:4173/visible"; + const firstRequest = first === "poll" ? host.pollFrame() : host.navigate(fixture); + await create.started.promise; + const secondRequest = first === "poll" ? host.navigate(fixture) : host.pollFrame(); + create.release.resolve(); + const results = await Promise.allSettled([firstRequest, secondRequest]); + assert.deepEqual(results.map((result) => result.status), ["fulfilled", "fulfilled"]); + assert.equal((await host.getUrl()).url, fixture); + assert.equal(activePages(manager).length, 1); + assert.equal(manager.contexts.size, 1); + await host.stop(); +} + +test("first frame and navigation share one active page when frame starts first", async () => { + await expectConcurrentStartup("poll"); +}); + +test("first frame and navigation share one active page when navigation starts first", async () => { + await expectConcurrentStartup("navigate"); +}); + +test("poll ownership follows the active page across network modes", async () => { + const manager = new FakeManager(); + const host = testHost(manager); + await host.navigate("https://public.test/page"); + const publicPage = manager.pages[0]; + assert.ok(publicPage); + await host.pollFrame(); + await host.navigate("http://localhost:4173/page"); + const loopbackPage = manager.pages[1]; + assert.ok(loopbackPage); + const polled = await host.pollFrame(); + assert.equal(publicPage.subscriptions, 1); + assert.equal(publicPage.unsubscriptions, 1); + assert.equal(loopbackPage.subscriptions, 1); + assert.equal(polled.state.url, "http://localhost:4173/page"); + assert.equal(polled.frame?.data, loopbackPage.latestFrame?.data); + await host.stop(); +}); + +test("a mode switch rebinds polling while the previous screencast is seeding", async () => { + const manager = new FakeManager(); + const frame = manager.blockNextFrame(); + const host = testHost(manager); + await host.navigate("https://public.test/page"); + const publicPage = manager.pages[0]; + assert.ok(publicPage); + const poll = host.pollFrame(); + await frame.started.promise; + const navigation = host.navigate("http://localhost:4173/page"); + await navigation; + frame.release.resolve(); + const polled = await poll; + const loopbackPage = manager.pages[1]; + assert.ok(loopbackPage); + assert.equal(polled.state.url, "http://localhost:4173/page"); + assert.equal(publicPage.unsubscriptions, 1); + assert.equal(loopbackPage.subscriptions, 1); + assert.equal(activePages(manager).length, 1); + assert.equal(manager.contexts.size, 1); + await host.stop(); +}); + +test("concurrent frame polls wait for one seeded screencast", async () => { + const manager = new FakeManager(); + const host = testHost(manager); + const frame = manager.blockNextFrame(); + await host.navigate("https://public.test/page"); + const first = host.pollFrame(); + await frame.started.promise; + let secondSettled = false; + const second = host.pollFrame().finally(() => { + secondSettled = true; + }); + await Promise.resolve(); + assert.equal(secondSettled, false); + frame.release.resolve(); + await Promise.all([first, second]); + assert.equal(manager.pages[0]?.subscriptions, 1); + await host.stop(); +}); + +test("a stale idle callback cannot unsubscribe the current page", async () => { + const manager = new FakeManager(); + const scheduler = new FakeScheduler(); + const host = testHost(manager, scheduler.schedule); + await host.navigate("https://public.test/page"); + await host.pollFrame(); + const stale = scheduler.tasks[0]; + assert.ok(stale); + await host.navigate("http://localhost:4173/page"); + await host.pollFrame(); + const current = manager.pages[1]; + assert.ok(current); + assert.equal(stale.cancelled, true); + stale.callback(); + assert.equal(current.subscriptions, 1); + assert.equal(current.unsubscriptions, 0); + await host.pollFrame(); + assert.equal(current.subscriptions, 1); + await host.stop(); +}); + +test("cross-mode navigation preserves request order", async () => { + const manager = new FakeManager(); + const host = testHost(manager); + const firstNavigation = manager.blockNextNavigation(); + const first = host.navigate("https://public.test/first"); + await firstNavigation.started.promise; + const second = host.navigate("http://localhost:4173/second"); + await Promise.resolve(); + assert.deepEqual(manager.modes, ["public"]); + firstNavigation.release.resolve(); + assert.deepEqual(await Promise.all([first, second]), [ + { title: "https://public.test/first", url: "https://public.test/first" }, + { title: "http://localhost:4173/second", url: "http://localhost:4173/second" }, + ]); + assert.equal((await host.getUrl()).url, "http://localhost:4173/second"); + assert.deepEqual(manager.modes, ["public", "loopback"]); + assert.equal(activePages(manager).length, 1); + assert.equal(manager.contexts.size, 1); + await host.stop(); +}); + +test("navigation recovery cannot create a target after stop", async () => { + const manager = new FakeManager(); + const blocked = manager.blockNextNavigation(); + const host = testHost(manager); + const navigation = host.navigate("https://public.test/page"); + await blocked.started.promise; + await host.stop(); + blocked.release.resolve(); + await assert.rejects(navigation, /Browser host stopped/); + assert.deepEqual(manager.modes, ["public"]); + assert.equal(activePages(manager).length, 0); + assert.equal(manager.contexts.size, 0); +}); + +test("initial target creation cannot attach after stop", async () => { + const manager = new FakeManager(); + const blocked = manager.blockNextCreate(); + const host = testHost(manager); + const navigation = host.navigate("https://public.test/page"); + await blocked.started.promise; + const stopping = host.stop(); + blocked.release.resolve(); + await assert.rejects(navigation, /Browser host stopped/); + await stopping; + assert.deepEqual(manager.modes, ["public"]); + assert.equal(manager.pages.length, 0); + assert.equal(manager.contexts.size, 0); +}); + +test("initial target attach cannot publish after stop", async () => { + const manager = new FakeManager(); + const blocked = barrier(); + const host = new BrowserHost(manager, { + attachPage: async (managed) => { + blocked.started.resolve(); + await blocked.release.promise; + return manager.attach(managed); + }, + }); + const navigation = host.navigate("https://public.test/page"); + await blocked.started.promise; + const stopping = host.stop(); + blocked.release.resolve(); + await assert.rejects(navigation, /Browser host stopped/); + await stopping; + assert.equal(activePages(manager).length, 0); + assert.equal(manager.contexts.size, 0); +}); + +test("loopback replacement waits for confirmed context revocation", async () => { + const manager = new FakeManager(); + const host = testHost(manager); + await host.navigate("http://localhost:4173/private"); + const loopbackPage = manager.pages[0]; + assert.ok(loopbackPage); + const disposal = manager.blockNextDispose(); + let replacementSettled = false; + const replacement = host.navigate("https://public.test/page").finally(() => { + replacementSettled = true; + }); + await disposal.started.promise; + await Promise.resolve(); + assert.equal(replacementSettled, false); + assert.deepEqual(manager.modes, ["loopback"]); + assert.equal(manager.contexts.has(loopbackPage.contextId), true); + disposal.release.resolve(); + assert.deepEqual(await replacement, { + title: "https://public.test/page", + url: "https://public.test/page", + }); + assert.deepEqual(manager.modes, ["loopback", "public"]); + assert.equal(manager.contexts.has(loopbackPage.contextId), false); + assert.equal(manager.contexts.size, 1); + await host.stop(); +}); + +test("failed loopback revocation stops Chromium before replacement", async () => { + const manager = new FakeManager(); + const host = testHost(manager); + await host.navigate("http://localhost:4173/private"); + const expected = new Error("context revocation failed"); + manager.rejectNextDispose(expected); + await assert.rejects(host.navigate("https://public.test/page"), (error) => error === expected); + assert.deepEqual(manager.modes, ["loopback"]); + assert.equal(manager.contexts.size, 0); + assert.equal(manager.stops, 1); + assert.deepEqual(await host.navigate("https://public.test/recovered"), { + title: "https://public.test/recovered", + url: "https://public.test/recovered", + }); + assert.deepEqual(manager.modes, ["loopback", "public"]); + assert.equal(manager.contexts.size, 1); + await host.stop(); +}); + +test("transition failures preserve their original error and release the context", async () => { + const manager = new FakeManager(); + const expected = new Error("attach failed"); + const host = new BrowserHost(manager, { + attachPage: async () => { + throw expected; + }, + }); + await assert.rejects(host.page(), (error) => error === expected); + assert.equal(manager.contexts.size, 0); + await host.stop(); +}); + +test("attach failures survive rejected cleanup after fail-closed stop", async () => { + const manager = new FakeManager(); + const expected = new Error("attach failed"); + manager.rejectNextDispose(new Error("cleanup failed")); + const host = new BrowserHost(manager, { + attachPage: async () => { + throw expected; + }, + }); + await assert.rejects(host.page(), (error) => error === expected); + assert.equal(manager.contexts.size, 0); + assert.equal(manager.stops, 1); + await host.stop(); +}); diff --git a/services/agent-runtime/src/browser-host/browser-host.ts b/services/agent-runtime/src/browser-host/browser-host.ts index 68f45c90a..43992b740 100644 --- a/services/agent-runtime/src/browser-host/browser-host.ts +++ b/services/agent-runtime/src/browser-host/browser-host.ts @@ -6,9 +6,17 @@ // // Server-only: imported from API routes, never from client components. -import { getGlobalSingleton } from "../instances"; +import { Effect, Fiber, Schema, Semaphore } from "effect"; +import { + browserNavigation, + type BrowserNetworkMode, +} from "../../../../shared/agent/sanitize-embedded-browser-url"; import { delay } from "../async"; -import { chromeManager } from "./chrome"; +import { getGlobalSingleton } from "../instances"; +import { + chromeManager, + type ManagedChromeTarget, +} from "./chrome"; import { CLICK_SCRIPT, FILL_SCRIPT, SNAPSHOT_SCRIPT, type SnapshotResult } from "./dom-scripts"; import { HostedPage, @@ -26,6 +34,14 @@ const TEXT_CAP_BYTES = 500 * 1024; const HTML_CAP_BYTES = 1024 * 1024; const LOAD_EVENT_TIMEOUT_MS = 8_000; const SNAPSHOT_LIMIT = 200; +const CdpTargetSchema = Schema.Struct({ + id: Schema.String, + title: Schema.String, + type: Schema.String, + url: Schema.String, + webSocketDebuggerUrl: Schema.optional(Schema.String), +}); +const CdpTargetsSchema = Schema.Array(CdpTargetSchema); // Stop compositing screencast frames once the panel stops polling for this long. const POLL_IDLE_MS = 2_000; @@ -33,32 +49,21 @@ const POLL_IDLE_MS = 2_000; // Discover the headless Chromium's page targets over its HTTP control endpoint. // A freshly launched browser can briefly report zero pages, so we poll a few // times before giving up. -async function fetchTargets(port: number): Promise { - for (let attempt = 0; attempt < 10; attempt += 1) { +async function fetchTarget(port: number, targetId: string): Promise { + for (let attempt = 0; attempt < 20; attempt += 1) { const response = await fetch(`http://127.0.0.1:${port}/json`, { cache: "no-store" }); if (!response.ok) throw new Error(`Chromium /json returned HTTP ${response.status}`); - const targets = (await response.json()) as CdpTarget[]; - const pages = Array.isArray(targets) ? targets.filter((target) => target.type === "page") : []; - if (pages.length > 0 || attempt === 9) return pages; + const targets = Schema.decodeUnknownSync(CdpTargetsSchema)(await response.json()); + const target = targets.find( + (candidate) => + candidate.id === targetId && + candidate.type === "page" && + candidate.webSocketDebuggerUrl?.includes("/devtools/page/"), + ); + if (target) return target; await delay(100); } - return []; -} - -async function createBlankPage(port: number): Promise { - // Newer Chromium requires PUT for /json/new (GET is blocked). The created - // target must be a page with its own page-level WebSocket — never the - // browser endpoint, which rejects Page.* with "Not attached to an active page". - const response = await fetch(`http://127.0.0.1:${port}/json/new?about:blank`, { - method: "PUT", - cache: "no-store", - }); - if (!response.ok) throw new Error(`Chromium /json/new returned HTTP ${response.status}`); - const created = (await response.json()) as CdpTarget; - if (created.type !== "page" || !created.webSocketDebuggerUrl) { - throw new Error("Chromium did not return a navigable page"); - } - return created; + throw new Error(`Chromium target did not become available: ${targetId}`); } function normalizeUrl(value: string): string { @@ -69,107 +74,269 @@ function capString(value: string, maxBytes: number): string { return value.length > maxBytes ? value.slice(0, maxBytes) : value; } +function targetUnavailable(error: unknown): boolean { + const message = String(error); + return ["Not attached", "Inspected target navigated or closed", "Target closed"].some((value) => + message.includes(value), + ); +} + +export type BrowserPage = Pick< + HostedPage, + | "call" + | "close" + | "closed" + | "contextId" + | "drainConsole" + | "id" + | "invokeScript" + | "latestFrame" + | "mode" + | "readState" + | "resolveRef" + | "setRefMap" + | "subscribeFrames" + | "subscribeLoad" + | "subscribeState" +>; + +export type BrowserHostManager = { + createTarget: (mode: BrowserNetworkMode) => Promise; + disposeContext: (contextId: string) => Promise; + ensure: () => Promise<{ port: number }>; + isAvailable: () => boolean; + stop: () => Promise; +}; + +type AttachPage = (managed: ManagedChromeTarget, timeoutMs: number) => Promise; +type SchedulePollIdle = (callback: () => void, delayMs: number) => () => void; + +export type BrowserHostOptions = { + attachPage?: AttachPage; + schedulePollIdle?: SchedulePollIdle; +}; + +type PollOwner = { + pageId: string; + unsubscribe: () => void; +}; + +function scheduledPollIdle(callback: () => void, delayMs: number): () => void { + const fiber = Effect.runFork(Effect.sleep(delayMs).pipe(Effect.andThen(Effect.sync(callback)))); + return () => void Effect.runPromise(Fiber.interrupt(fiber)); +} + +async function attachHostedPage( + manager: BrowserHostManager, + managed: ManagedChromeTarget, + timeoutMs: number, +): Promise { + const process = await manager.ensure(); + const target = await fetchTarget(process.port, managed.targetId); + return HostedPage.attach(target, { + contextId: managed.contextId, + mode: managed.mode, + policy: managed.policy, + release: () => manager.disposeContext(managed.contextId), + timeoutMs, + }); +} + // Top-level manager: owns the active page id, a per-page cache with // reconnect-when-closed, and the exported tool surface. -class BrowserHost { - private pages = new Map(); +export class BrowserHost { + private pages = new Map(); private activeId: string | null = null; private timeoutMs = 10_000; - - isAvailable(): boolean { - return chromeManager.isAvailable(); + private readonly transitionLock = Semaphore.makeUnsafe(1); + private readonly navigationLock = Semaphore.makeUnsafe(1); + private readonly pollLock = Semaphore.makeUnsafe(1); + private readonly manager: BrowserHostManager; + private readonly attachPage: AttachPage; + private readonly schedulePollIdle: SchedulePollIdle; + private pollOwner: PollOwner | null = null; + private pollIdleCancel: (() => void) | null = null; + private pollGeneration = 0; + private stopped = false; + + constructor( + manager: BrowserHostManager = chromeManager, + { attachPage, schedulePollIdle = scheduledPollIdle }: BrowserHostOptions = {}, + ) { + this.manager = manager; + this.attachPage = + attachPage ?? ((managed, timeoutMs) => attachHostedPage(manager, managed, timeoutMs)); + this.schedulePollIdle = schedulePollIdle; } - private async port(): Promise { - const proc = await chromeManager.ensure(); - return proc.port; + isAvailable(): boolean { + return this.manager.isAvailable(); } // Resolve a hosted page, reconnecting if the cached client closed. With no // pageId, picks the active page, then the first target, creating one if none. - async page(pageId?: string): Promise { - const port = await this.port(); + async page(pageId?: string): Promise { + return this.withPermit(this.transitionLock, () => this.pageUnlocked(pageId)); + } + + private async pageUnlocked( + pageId?: string, + mode?: BrowserNetworkMode, + ): Promise { + this.assertRunning(); const targetId = pageId ?? this.activeId; const cached = targetId ? this.pages.get(targetId) : undefined; - if (cached && !cached.closed) { + if (cached && !cached.closed && (!mode || cached.mode === mode)) { this.activeId = cached.id; return cached; } - if (cached) this.pages.delete(cached.id); - const target = await this.resolveTarget(port, targetId); - const hosted = await HostedPage.attach(target, this.timeoutMs); - this.pages.set(hosted.id, hosted); - this.activeId = hosted.id; - return hosted; - } - - private async resolveTarget(port: number, targetId: string | null): Promise { - const targets = await fetchTargets(port); - const navigable = targets.filter((target) => - target.webSocketDebuggerUrl?.includes("/devtools/page/"), - ); - const match = targetId ? navigable.find((target) => target.id === targetId) : navigable[0]; - const target = match ?? navigable[0]; - if (target) return target; - return createBlankPage(port); + if (cached) await this.closePage(cached); + return this.freshPageUnlocked(mode ?? "public"); } - async ensurePage(): Promise { + private pageForMode(mode: BrowserNetworkMode, pageId?: string): Promise { + return this.withPermit(this.transitionLock, () => this.pageUnlocked(pageId, mode)); + } + + async ensurePage(): Promise { return this.page(); } async navigate(url: string, pageId?: string): Promise<{ url: string; title: string }> { + const navigation = browserNavigation(normalizeUrl(url)); + if (!navigation) throw new Error("Browser network policy blocked navigation URL"); + return this.withPermit(this.navigationLock, () => + this.navigateSerialized(navigation.mode, navigation.url, pageId), + ); + } + + private async navigateSerialized( + mode: BrowserNetworkMode, + url: string, + pageId?: string, + ): Promise<{ url: string; title: string }> { + const page = await this.pageForMode(mode, pageId); try { - return await this.navigateOnce(await this.page(pageId), url); + return await this.navigateOnce(page, url); } catch (error) { // A target discovered at launch can briefly report "Not attached to an // active page"; recover by opening a fresh tab and retrying once. - if (!String(error).includes("Not attached")) throw error; - return this.navigateOnce(await this.freshPage(), url); + if (!targetUnavailable(error)) throw error; + const replacement = await this.withPermit(this.transitionLock, () => + this.freshPageUnlocked(mode), + ); + return this.navigateOnce(replacement, url); } } private async navigateOnce( - page: HostedPage, + page: BrowserPage, url: string, ): Promise<{ url: string; title: string }> { - const loaded = this.waitForLoad(page); - await page.call("Page.navigate", { url: normalizeUrl(url) }); - await loaded; - const state = await page.readState(); - return { url: state.url, title: state.title }; + const loading = this.waitForLoad(page); + try { + const result = (await page.call("Page.navigate", { url })) as { errorText?: unknown }; + if (typeof result.errorText === "string" && result.errorText) { + throw new Error(`Browser navigation failed: ${result.errorText}`); + } + await loading.done; + const state = await page.readState(); + return { url: state.url, title: state.title }; + } finally { + loading.cancel(); + } } - private async freshPage(): Promise { - const port = await this.port(); + private async freshPageUnlocked(mode: BrowserNetworkMode): Promise { // freshPage is a recovery path: the current active page is unusable ("Not // attached"). Close it so its CDP WebSocket doesn't leak — a new attach // below replaces it. const previous = this.activeId ? this.pages.get(this.activeId) : undefined; - if (previous) { - this.pages.delete(previous.id); - previous.close(); + if (previous) await this.closePage(previous); + this.assertRunning(); + const managed = await this.manager.createTarget(mode); + if (this.stopped) { + await this.discardContext(managed.contextId); + this.assertRunning(); + } + return this.attachTarget(managed); + } + + private async attachTarget(managed: ManagedChromeTarget): Promise { + let hosted: BrowserPage | null = null; + try { + hosted = await this.attachPage(managed, this.timeoutMs); + this.assertRunning(); + this.pages.set(hosted.id, hosted); + this.activeId = hosted.id; + return hosted; + } catch (error) { + if (hosted) await this.discardPage(hosted); + else await this.discardContext(managed.contextId); + throw error; + } + } + + private assertRunning(): void { + if (this.stopped) throw new Error("Browser host stopped"); + } + + private async discardPage(page: BrowserPage): Promise { + try { + await page.close(); + } catch { + await this.stopManager(); + } + } + + private async closePage(page: BrowserPage): Promise { + this.clearPollOwner(page.id); + try { + await page.close(); + } catch (error) { + await this.stopManager(); + throw error; + } finally { + this.pages.delete(page.id); + if (this.activeId === page.id) this.activeId = null; } - const target = await createBlankPage(port); - const hosted = await HostedPage.attach(target, this.timeoutMs); - this.pages.set(hosted.id, hosted); - this.activeId = hosted.id; - return hosted; } - private waitForLoad(page: HostedPage): Promise { - return new Promise((resolve) => { + private async discardContext(contextId: string): Promise { + try { + await this.manager.disposeContext(contextId); + } catch { + await this.stopManager(); + } + } + + private async stopManager(): Promise { + await this.manager.stop().catch(() => undefined); + } + + private withPermit(semaphore: Semaphore.Semaphore, task: () => Promise): Promise { + return Effect.runPromise( + semaphore.withPermit(Effect.tryPromise({ try: task, catch: (error) => error })), + ); + } + + private waitForLoad(page: BrowserPage): { cancel: () => void; done: Promise } { + let cancel = () => undefined; + const done = new Promise((resolve) => { + const off = page.subscribeLoad(() => { + cancel(); + resolve(); + }); const timer = setTimeout(() => { - dispose(); + cancel(); resolve(); }, LOAD_EVENT_TIMEOUT_MS); - const off = page.subscribeLoad(() => { + cancel = () => { clearTimeout(timer); - dispose(); - resolve(); - }); - const dispose = () => off(); + off(); + }; }); + return { cancel: () => cancel(), done }; } async getUrl(pageId?: string): Promise<{ url: string; title: string }> { @@ -241,7 +408,7 @@ class BrowserHost { return page.invokeScript<{ found: boolean }>(FILL_SCRIPT, [selector, args.value]); } - private resolveSelector(page: HostedPage, args: { selector?: string; ref?: string }): string { + private resolveSelector(page: BrowserPage, args: { selector?: string; ref?: string }): string { if (args.selector) return args.selector; if (!args.ref) throw new Error("selector or ref required"); const selector = page.resolveRef(args.ref); @@ -333,25 +500,52 @@ class BrowserHost { // A poll keeps the screencast running via a self-renewing frame subscription // that auto-stops once polling lapses (POLL_IDLE_MS) — Chrome stops // compositing screencast frames when nobody is watching. - private pollUnsubscribe: (() => void) | null = null; - private pollIdleTimer: ReturnType | null = null; async pollFrame(pageId?: string): Promise<{ frame: ScreencastFrame | null; state: PageState }> { - const page = await this.page(pageId); - if (!this.pollUnsubscribe) { - // A no-op subscriber is enough to make the page start Page.startScreencast; - // we read latestFrame rather than receiving pushes. Await the screencast - // seed so this first poll already carries a frame instead of null. - const { unsubscribe, ready } = page.subscribeFrames(() => undefined); - this.pollUnsubscribe = unsubscribe; - await ready; + return this.withPermit(this.pollLock, () => this.pollFrameSerialized(pageId)); + } + + private async pollFrameSerialized( + pageId?: string, + ): Promise<{ frame: ScreencastFrame | null; state: PageState }> { + for (let attempt = 0; attempt < 2; attempt += 1) { + const page = await this.page(pageId); + try { + await this.bindPollOwner(page); + this.renewPollIdle(page.id); + return { frame: page.latestFrame, state: await page.readState() }; + } catch (error) { + if (pageId || !page.closed || attempt === 1) throw error; + } } - if (this.pollIdleTimer) clearTimeout(this.pollIdleTimer); - this.pollIdleTimer = setTimeout(() => { - this.pollUnsubscribe?.(); - this.pollUnsubscribe = null; - this.pollIdleTimer = null; + throw new Error("Browser page unavailable"); + } + + private async bindPollOwner(page: BrowserPage): Promise { + if (this.pollOwner?.pageId === page.id) return; + this.clearPollOwner(); + const { unsubscribe, ready } = page.subscribeFrames(() => undefined); + this.pollOwner = { pageId: page.id, unsubscribe }; + await ready; + } + + private renewPollIdle(pageId: string): void { + this.pollIdleCancel?.(); + const generation = ++this.pollGeneration; + this.pollIdleCancel = this.schedulePollIdle(() => { + if (generation !== this.pollGeneration || this.pollOwner?.pageId !== pageId) return; + this.pollOwner.unsubscribe(); + this.pollOwner = null; + this.pollIdleCancel = null; }, POLL_IDLE_MS); - return { frame: page.latestFrame, state: await page.readState() }; + } + + private clearPollOwner(pageId?: string): void { + if (pageId && this.pollOwner?.pageId !== pageId) return; + this.pollGeneration += 1; + this.pollIdleCancel?.(); + this.pollIdleCancel = null; + this.pollOwner?.unsubscribe(); + this.pollOwner = null; } async dispatchMouse(args: MouseInput, pageId?: string): Promise { @@ -369,11 +563,16 @@ class BrowserHost { }); } - stop(): void { - for (const page of this.pages.values()) page.close(); - this.pages.clear(); - this.activeId = null; - chromeManager.stop(); + async stop(): Promise { + this.stopped = true; + await this.withPermit(this.transitionLock, async () => { + this.clearPollOwner(); + const pages = [...this.pages.values()]; + this.pages.clear(); + this.activeId = null; + await Promise.allSettled(pages.map((page) => page.close())); + await this.manager.stop(); + }); } } diff --git a/services/agent-runtime/src/browser-host/browser-network-policy.test.ts b/services/agent-runtime/src/browser-host/browser-network-policy.test.ts new file mode 100644 index 000000000..c0227b67b --- /dev/null +++ b/services/agent-runtime/src/browser-host/browser-network-policy.test.ts @@ -0,0 +1,349 @@ +import assert from "node:assert/strict"; +import { createServer, request as httpRequest } from "node:http"; +import { + connect as netConnect, + createServer as createSocketServer, + type Server, +} from "node:net"; +import test from "node:test"; +import { + browserNavigation, + classifyBrowserAddress, + type BrowserAddressClass, +} from "../../../../shared/agent/sanitize-embedded-browser-url"; +import { chromeArguments } from "./chrome"; +import { + createBrowserNetworkPolicy, + type BrowserHostResolver, +} from "./network-policy"; +import { createPinningProxy, type PinnedDial } from "./pinning-proxy"; + +const addressCases: ReadonlyArray = [ + ["8.8.8.8", "public"], + ["93.184.216.34", "public"], + ["127.0.0.1", "loopback"], + ["127.255.255.255", "loopback"], + ["0.0.0.0", "blocked"], + ["10.0.0.1", "blocked"], + ["100.64.0.1", "blocked"], + ["100.127.255.255", "blocked"], + ["169.254.169.254", "blocked"], + ["172.31.255.255", "blocked"], + ["192.0.0.1", "blocked"], + ["192.0.2.1", "blocked"], + ["192.31.196.1", "blocked"], + ["192.52.193.1", "blocked"], + ["192.88.99.1", "blocked"], + ["192.168.1.1", "blocked"], + ["192.175.48.1", "blocked"], + ["198.18.0.1", "blocked"], + ["198.19.255.255", "blocked"], + ["198.51.100.1", "blocked"], + ["203.0.113.1", "blocked"], + ["224.0.0.1", "blocked"], + ["240.0.0.1", "blocked"], + ["255.255.255.255", "blocked"], + ["2001:4860:4860::8888", "public"], + ["2606:4700:4700::1111", "public"], + ["::1", "loopback"], + ["::", "blocked"], + ["64:ff9b::808:808", "blocked"], + ["64:ff9b:1::1", "blocked"], + ["100::1", "blocked"], + ["2001::1", "blocked"], + ["2001:db8::1", "blocked"], + ["2002::1", "blocked"], + ["2620:4f:8000::1", "blocked"], + ["3fff::1", "blocked"], + ["5f00::1", "blocked"], + ["fc00::1", "blocked"], + ["fd00::1", "blocked"], + ["fe80::1", "blocked"], + ["fec0::1", "blocked"], + ["ff02::1", "blocked"], + ["::ffff:8.8.8.8", "public"], + ["::ffff:808:808", "public"], + ["::ffff:10.0.0.1", "blocked"], + ["::ffff:a00:1", "blocked"], + ["0:0:0:0:0:ffff:127.0.0.1", "loopback"], + ["::ffff:7f00:1", "loopback"], + ["fe80::1%lo0", "blocked"], + ["not-an-address", "blocked"], +]; + +test("classifies complete browser address policy ranges", () => { + for (const [address, expected] of addressCases) { + assert.equal(classifyBrowserAddress(address), expected, address); + } +}); + +test("derives public and explicit loopback navigation modes", () => { + assert.deepEqual(browserNavigation("https://example.com/path"), { + mode: "public", + url: "https://example.com/path", + }); + for (const value of [ + "http://localhost:3000", + "http://localhost.:3000", + "http://app.localhost:3000", + "http://app.localhost.:3000", + "http://127.2.3.4:3000", + "http://[::1]:3000", + "http://[::ffff:7f00:1]:3000", + ]) { + assert.equal(browserNavigation(value)?.mode, "loopback", value); + } + for (const value of [ + "http://host.local", + "http://10.0.0.1", + "http://[fe80::1]", + "file:///tmp/private", + "ftp://example.com/file", + "https://user:password@example.com", + ]) { + assert.equal(browserNavigation(value), null, value); + } +}); + +test("Chromium launch policy removes implicit bypasses and non-proxied transports", () => { + const proxy = "http://127.0.0.1:4567"; + const args = chromeArguments("/tmp/browser-profile", proxy); + assert.ok(args.includes(`--proxy-server=${proxy}`)); + assert.ok(args.includes("--proxy-bypass-list=<-loopback>")); + assert.ok(args.includes("--disable-quic")); + assert.ok(args.includes("--force-webrtc-ip-handling-policy=disable_non_proxied_udp")); + assert.equal(args.some((argument) => argument.includes("direct://")), false); +}); + +function resolver(entries: Record>): BrowserHostResolver { + return async (hostname) => entries[hostname] ?? []; +} + +test("fails closed on zero, malformed, blocked, and mixed DNS answers", async () => { + const policy = createBrowserNetworkPolicy({ + resolver: resolver({ + blocked: [{ address: "10.0.0.1", family: 4 }], + malformed: [{ address: "8.8.8.8", family: 6 }], + mixed: [ + { address: "8.8.8.8", family: 4 }, + { address: "127.0.0.1", family: 4 }, + ], + }), + }); + for (const hostname of ["empty", "blocked", "malformed", "mixed"]) { + await assert.rejects(policy.resolve(`http://${hostname}/`, "public"), /blocked|resolved|address/i); + } + const timedOut = createBrowserNetworkPolicy({ + resolver: () => new Promise(() => undefined), + timeoutMs: 1, + }); + await assert.rejects(timedOut.resolve("https://timeout.test", "public"), /timed out/u); +}); + +test("allows one DNS class compatible with the top-level mode and pins its first answer", async () => { + const policy = createBrowserNetworkPolicy({ + resolver: resolver({ + loopback: [ + { address: "127.0.0.1", family: 4 }, + { address: "::1", family: 6 }, + ], + public: [ + { address: "8.8.8.8", family: 4 }, + { address: "2001:4860:4860::8888", family: 6 }, + ], + }), + }); + assert.equal((await policy.resolve("https://public/", "public")).address.address, "8.8.8.8"); + assert.equal((await policy.resolve("http://loopback/", "loopback")).address.address, "127.0.0.1"); + assert.equal((await policy.resolve("https://public/", "loopback")).address.address, "8.8.8.8"); + await assert.rejects(policy.resolve("http://loopback/", "public"), /blocked/i); +}); + +function listen(server: Server): Promise { + return new Promise((resolveListen, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + reject(new Error("Missing test server address")); + return; + } + resolveListen(address.port); + }); + }); +} + +function close(server: Server): Promise { + return new Promise((resolveClose, reject) => + server.close((error) => (error ? reject(error) : resolveClose())), + ); +} + +function proxyGet(port: number, target: string): Promise<{ body: string; status: number }> { + return new Promise((resolveRequest, reject) => { + const outgoing = httpRequest( + { + host: "127.0.0.1", + method: "GET", + path: target, + port, + headers: { host: new URL(target).host }, + }, + (response) => { + const chunks: Buffer[] = []; + response.on("data", (chunk: Buffer) => chunks.push(chunk)); + response.on("end", () => + resolveRequest({ + body: Buffer.concat(chunks).toString("utf8"), + status: response.statusCode ?? 0, + }), + ); + }, + ); + outgoing.once("error", reject); + outgoing.end(); + }); +} + +function proxyExchange(port: number, message: string): Promise { + return new Promise((resolveExchange, reject) => { + const socket = netConnect({ host: "127.0.0.1", port }); + const chunks: Buffer[] = []; + socket.setTimeout(2_000, () => socket.destroy(new Error("Proxy exchange timed out"))); + socket.once("connect", () => socket.write(message)); + socket.on("data", (chunk: Buffer) => chunks.push(chunk)); + socket.once("end", () => resolveExchange(Buffer.concat(chunks).toString("utf8"))); + socket.once("error", reject); + }); +} + +test("blocks a denied destination before opening any target connection", async () => { + let accepted = 0; + const blocked = createServer((_request, response) => response.end("unsafe")); + blocked.on("connection", () => { + accepted += 1; + }); + const blockedPort = await listen(blocked); + const dial: PinnedDial = ({ port }) => { + throw new Error(`unexpected dial to ${port}`); + }; + const proxy = await createPinningProxy({ + dial, + mode: "public", + policy: createBrowserNetworkPolicy({ + resolver: resolver({ blocked: [{ address: "127.0.0.1", family: 4 }] }), + }), + }); + try { + const result = await proxyGet(proxy.port, `http://blocked:${blockedPort}/private`); + assert.equal(result.status, 403); + assert.equal(accepted, 0); + } finally { + await proxy.close(); + await close(blocked); + } +}); + +test("pins allowed HTTP to one address while preserving the original Host", async () => { + let host = ""; + const destinations: string[] = []; + const origin = createServer((request, response) => { + host = request.headers.host ?? ""; + response.end("pinned"); + }); + const originPort = await listen(origin); + const proxy = await createPinningProxy({ + dial: (destination) => { + destinations.push(`${destination.address.address}|${destination.hostname}`); + return netConnect({ host: "127.0.0.1", port: originPort }); + }, + mode: "loopback", + policy: createBrowserNetworkPolicy({ + resolver: resolver({ origin: [{ address: "8.8.8.8", family: 4 }] }), + }), + }); + try { + const result = await proxyGet(proxy.port, `http://origin:${originPort}/resource`); + assert.deepEqual(result, { body: "pinned", status: 200 }); + assert.equal(host, `origin:${originPort}`); + assert.deepEqual(destinations, ["8.8.8.8|origin"]); + } finally { + await proxy.close(); + await close(origin); + } +}); + +test("blocks CONNECT and WebSocket upgrade before dialing a target", async () => { + let dials = 0; + const proxy = await createPinningProxy({ + dial: () => { + dials += 1; + throw new Error("unexpected target dial"); + }, + mode: "public", + policy: createBrowserNetworkPolicy({ + resolver: resolver({ blocked: [{ address: "10.0.0.1", family: 4 }] }), + }), + }); + try { + const connectResponse = await proxyExchange( + proxy.port, + "CONNECT blocked:443 HTTP/1.1\r\nHost: blocked:443\r\n\r\n", + ); + const upgradeResponse = await proxyExchange( + proxy.port, + "GET http://blocked/socket HTTP/1.1\r\nHost: blocked\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", + ); + assert.match(connectResponse, /^HTTP\/1\.1 403/u); + assert.match(upgradeResponse, /^HTTP\/1\.1 403/u); + assert.equal(dials, 0); + } finally { + await proxy.close(); + } +}); + +test("CONNECT tunnels bytes only to the policy-selected address", async () => { + const origin = createSocketServer((socket) => socket.pipe(socket)); + const originPort = await listen(origin); + const destinations: string[] = []; + const proxy = await createPinningProxy({ + dial: (destination) => { + destinations.push( + `${destination.address.address}|${destination.hostname}|${destination.port}`, + ); + return netConnect({ host: "127.0.0.1", port: originPort }); + }, + mode: "public", + policy: createBrowserNetworkPolicy({ + resolver: resolver({ secure: [{ address: "8.8.8.8", family: 4 }] }), + }), + }); + try { + const tunneled = await new Promise((resolveTunnel, reject) => { + const socket = netConnect({ host: "127.0.0.1", port: proxy.port }); + let output = ""; + let connected = false; + socket.setTimeout(2_000, () => socket.destroy(new Error("CONNECT tunnel timed out"))); + socket.once("connect", () => + socket.write("CONNECT secure:443 HTTP/1.1\r\nHost: secure:443\r\n\r\n"), + ); + socket.on("data", (chunk: Buffer) => { + output += chunk.toString("utf8"); + if (!connected && output.includes("\r\n\r\n")) { + connected = true; + socket.write("pinned"); + } + if (connected && output.endsWith("pinned")) { + socket.end(); + resolveTunnel(output); + } + }); + socket.once("error", reject); + }); + assert.match(tunneled, /^HTTP\/1\.1 200 Connection Established/u); + assert.deepEqual(destinations, ["8.8.8.8|secure|443"]); + } finally { + await proxy.close(); + await close(origin); + } +}); diff --git a/services/agent-runtime/src/browser-host/browser-request-classes.integration.test.ts b/services/agent-runtime/src/browser-host/browser-request-classes.integration.test.ts new file mode 100644 index 000000000..0350d4f33 --- /dev/null +++ b/services/agent-runtime/src/browser-host/browser-request-classes.integration.test.ts @@ -0,0 +1,266 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import { createServer, type Server } from "node:http"; +import { connect as netConnect } from "node:net"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { Schema } from "effect"; +import { BrowserHost } from "./browser-host"; +import { + ChromeManager, + launchChrome, + type ChromeLaunchOptions, +} from "./chrome"; +import { + createBrowserNetworkPolicy, + type BrowserNetworkPolicy, + type PinnedBrowserDestination, +} from "./network-policy"; +import { createPinningProxy } from "./pinning-proxy"; + +const PUBLIC_ADDRESS = "8.8.8.8"; +const BrowserContextsSchema = Schema.Struct({ browserContextIds: Schema.Array(Schema.String) }); + +function listen(server: Server): Promise { + return new Promise((resolveListen, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + reject(new Error("Missing integration server address")); + return; + } + resolveListen(address.port); + }); + }); +} + +function close(server: Server): Promise { + return new Promise((resolveClose, reject) => { + if (!server.listening) { + resolveClose(); + return; + } + server.closeAllConnections(); + server.close((error) => (error ? reject(error) : resolveClose())); + }); +} + +function matrixPage(blockedBase: string, loopbackBase: string): string { + return ``; +} + +function serviceWorkerPage(): string { + return ``; +} + +function fixtureServer(blockedBase: string, loopbackBase: string, hosts: string[]): Server { + return createServer((request, response) => { + hosts.push(request.headers.host ?? ""); + if (request.url === "/redirect") { + response.writeHead(302, { location: `${blockedBase}/redirect-target` }); + response.end(); + return; + } + if (request.url === "/download-redirect") { + response.writeHead(302, { location: `${blockedBase}/download` }); + response.end(); + return; + } + if (request.url === "/sw.js") { + response.setHeader("content-type", "text/javascript"); + response.end( + `self.addEventListener("install", (event) => event.waitUntil(fetch(${JSON.stringify( + `http://blocked.test:${new URL(blockedBase).port}/service-worker`, + )}).catch(() => undefined)));`, + ); + return; + } + response.setHeader("content-type", "text/html"); + response.end( + request.url === "/service-worker" + ? serviceWorkerPage() + : matrixPage(blockedBase, loopbackBase), + ); + }); +} + +function instrumentPolicy( + policy: BrowserNetworkPolicy, + attempts: string[], +): BrowserNetworkPolicy { + return { + allows: policy.allows, + resolve: async (raw, mode) => { + attempts.push(raw); + return policy.resolve(raw, mode); + }, + }; +} + +function expectedPath(attempts: string[], pathname: string): boolean { + return attempts.some((raw) => { + try { + return new URL(raw).pathname === pathname; + } catch { + return false; + } + }); +} + +function expectedWebSocket(attempts: string[], port: number): boolean { + return attempts.some((raw) => { + try { + const url = new URL(raw); + return url.protocol === "https:" && url.hostname === "blocked.test" && url.port === String(port); + } catch { + return false; + } + }); +} + +function waitForExit(manager: ChromeManager): Promise { + const child = manager.current()?.child; + if (!child || child.exitCode !== null) return Promise.resolve(); + return new Promise((resolveExit, reject) => { + const timer = setTimeout(() => reject(new Error("Chromium did not exit after stop")), 5_000); + child.once("close", () => { + clearTimeout(timer); + resolveExit(); + }); + }); +} + +async function browserContextIds(manager: ChromeManager): Promise { + const process = manager.current(); + assert.ok(process); + return Schema.decodeUnknownSync(BrowserContextsSchema)( + await process.browser.call("Target.getBrowserContexts"), + ).browserContextIds; +} + +test("Chromium blocks every browser request class before the denied socket accepts", { timeout: 40_000 }, async () => { + let blockedConnections = 0; + const blocked = createServer((_request, response) => response.end("unsafe")); + blocked.on("connection", () => { + blockedConnections += 1; + }); + const blockedPort = await listen(blocked); + const blockedBase = `http://blocked.test:${blockedPort}`; + const loopbackBase = `http://127.0.0.1:${blockedPort}`; + const hosts: string[] = []; + const fixture = fixtureServer(blockedBase, loopbackBase, hosts); + const fixturePort = await listen(fixture); + const attempts: string[] = []; + const basePolicy = createBrowserNetworkPolicy({ + resolver: async (hostname) => { + if (hostname === "page.test") return [{ address: PUBLIC_ADDRESS, family: 4 }]; + if (hostname === "localhost") return [{ address: "127.0.0.1", family: 4 }]; + if (hostname === "blocked.test") return [{ address: "10.0.0.1", family: 4 }]; + return []; + }, + }); + const policy = instrumentPolicy(basePolicy, attempts); + const dials: PinnedBrowserDestination[] = []; + const pinnedProxies = await Promise.all( + (["public", "loopback"] as const).map((mode) => + createPinningProxy({ + dial: (destination) => { + dials.push(destination); + return netConnect({ host: "127.0.0.1", port: destination.port }); + }, + mode, + policy, + }), + ), + ); + const activeProxies = { loopback: pinnedProxies[1], public: pinnedProxies[0] }; + const profile = await mkdtemp(path.join(os.tmpdir(), "local-studio-browser-policy-")); + const launcher = (binary: string, options: ChromeLaunchOptions = {}) => + launchChrome(binary, { ...options, dataDir: profile, proxies: activeProxies }); + const manager = new ChromeManager(policy, launcher); + const host = new BrowserHost(manager); + try { + assert.equal(manager.isAvailable(), true); + await host.navigate(`http://page.test:${fixturePort}/matrix`); + const publicContexts = await browserContextIds(manager); + assert.equal(publicContexts.length, 1); + await host.evaluate( + "new Promise((resolve, reject) => { const started = Date.now(); const poll = () => { if (window.workerStarted) resolve(true); else if (window.workerError || Date.now() - started > 5000) reject(new Error(window.workerError || 'worker timed out')); else setTimeout(poll, 50); }; poll(); })", + ); + await host.evaluate("window.startDownload()"); + await new Promise((resolve) => setTimeout(resolve, 500)); + await host.navigate(`http://page.test:${fixturePort}/redirect`).catch(() => undefined); + await host.navigate(`http://localhost:${fixturePort}/service-worker`); + const loopbackContexts = await browserContextIds(manager); + assert.equal(loopbackContexts.length, 1); + assert.notEqual(loopbackContexts[0], publicContexts[0]); + await host.evaluate( + "new Promise((resolve, reject) => { const started = Date.now(); const poll = () => { if (window.serviceWorkerReady) resolve(true); else if (window.serviceWorkerError || Date.now() - started > 5000) reject(new Error(window.serviceWorkerError || 'service worker timed out')); else setTimeout(poll, 50); }; poll(); })", + ); + await host.evaluate("new Promise((resolve) => setTimeout(resolve, 500))"); + + for (const pathname of [ + "/image", + "/script", + "/frame", + "/fetch", + "/direct-loopback", + "/xhr", + "/events", + "/worker-fetch", + "/download", + "/redirect-target", + "/service-worker", + ]) { + assert.equal( + expectedPath(attempts, pathname), + true, + `${pathname}: ${attempts.join(", ")}`, + ); + } + assert.equal(expectedWebSocket(attempts, blockedPort), true, attempts.join(", ")); + assert.equal(blockedConnections, 0); + assert.ok( + dials.some( + (destination) => + destination.hostname === "page.test" && destination.address.address === PUBLIC_ADDRESS, + ), + ); + assert.ok( + dials.some( + (destination) => + destination.hostname === "localhost" && destination.address.address === "127.0.0.1", + ), + ); + assert.ok(hosts.includes(`page.test:${fixturePort}`)); + assert.ok(hosts.includes(`localhost:${fixturePort}`)); + } finally { + const exited = waitForExit(manager); + await host.stop(); + await exited; + await Promise.allSettled([activeProxies.public.close(), activeProxies.loopback.close()]); + await Promise.allSettled([close(fixture), close(blocked)]); + await rm(profile, { force: true, recursive: true }); + } +}); diff --git a/services/agent-runtime/src/browser-host/chrome-manager-revocation.test.ts b/services/agent-runtime/src/browser-host/chrome-manager-revocation.test.ts new file mode 100644 index 000000000..4f4ce2224 --- /dev/null +++ b/services/agent-runtime/src/browser-host/chrome-manager-revocation.test.ts @@ -0,0 +1,215 @@ +import assert from "node:assert/strict"; +import { spawn, type ChildProcess } from "node:child_process"; +import test from "node:test"; +import type { BrowserNetworkMode } from "../../../../shared/agent/sanitize-embedded-browser-url"; +import { + ChromeManager, + type ChromeBrowser, + type ChromeLaunchOptions, + type ChromeProcess, +} from "./chrome"; +import { createBrowserNetworkPolicy } from "./network-policy"; +import type { PinningProxy } from "./pinning-proxy"; + +type Disposal = { + error?: Error; + release: ReturnType>; + started: ReturnType>; +}; + +function disposal(error?: Error): Disposal { + return { + error, + release: Promise.withResolvers(), + started: Promise.withResolvers(), + }; +} + +class FakeBrowser implements ChromeBrowser { + readonly contexts = new Set(); + closed = false; + private nextDisposal: Disposal | null = null; + private serial = 0; + + blockNextDisposal(): Disposal { + const next = disposal(); + this.nextDisposal = next; + return next; + } + + rejectNextDisposal(error: Error): void { + const next = disposal(error); + next.release.resolve(); + this.nextDisposal = next; + } + + async call( + method: string, + params: Record = {}, + ): Promise> { + if (method === "Target.createBrowserContext") { + const browserContextId = `context-${++this.serial}`; + this.contexts.add(browserContextId); + return { browserContextId }; + } + if (method === "Target.createTarget") return { targetId: `target-${this.serial}` }; + if (method === "Target.getBrowserContexts") { + return { browserContextIds: [...this.contexts] }; + } + if (method === "Target.disposeBrowserContext") { + const contextId = params["browserContextId"]; + if (typeof contextId !== "string") throw new Error("Missing browser context ID"); + const current = this.nextDisposal; + this.nextDisposal = null; + current?.started.resolve(); + await current?.release.promise; + if (current?.error) throw current.error; + this.contexts.delete(contextId); + } + return {}; + } + + close(): void { + this.closed = true; + } +} + +type FixtureProcess = { + browser: FakeBrowser; + process: ChromeProcess; + proxyCloses: Record; + terminate: ChildProcess["kill"]; +}; + +function proxy(mode: BrowserNetworkMode, closes: Record): PinningProxy { + return { + close: async () => { + closes[mode] += 1; + }, + mode, + port: 1, + url: "http://127.0.0.1:1", + }; +} + +class ChromeFixture { + readonly processes: FixtureProcess[] = []; + private readonly policy = createBrowserNetworkPolicy(); + + launch = async ( + _binary: string, + _options: ChromeLaunchOptions = {}, + ): Promise => { + const browser = new FakeBrowser(); + const child = spawn(process.execPath, ["-e", "setInterval(() => undefined, 1000)"], { + stdio: "ignore", + }); + const proxyCloses = { loopback: 0, public: 0 }; + const launched: ChromeProcess = { + browser, + child, + contexts: new Set(), + policy: this.policy, + port: 1, + proxies: { + loopback: proxy("loopback", proxyCloses), + public: proxy("public", proxyCloses), + }, + wsEndpoint: "ws://127.0.0.1:1/devtools/browser/test", + }; + this.processes.push({ + browser, + process: launched, + proxyCloses, + terminate: child.kill.bind(child), + }); + return launched; + }; +} + +function waitForExit(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(); + return new Promise((resolveExit) => child.once("exit", () => resolveExit())); +} + +async function reapFixture(fixture: ChromeFixture): Promise { + await Promise.all( + fixture.processes.map(async ({ process: launched, terminate }) => { + if (launched.child.exitCode === null && launched.child.signalCode === null) { + terminate("SIGKILL"); + } + await waitForExit(launched.child); + }), + ); +} + +function testManager(fixture: ChromeFixture, terminationTimeoutMs = 1_000): ChromeManager { + return new ChromeManager( + createBrowserNetworkPolicy(), + fixture.launch, + () => process.execPath, + terminationTimeoutMs, + ); +} + +test("context tracking survives until Chromium confirms revocation", async (context) => { + const fixture = new ChromeFixture(); + context.after(() => reapFixture(fixture)); + const manager = testManager(fixture); + const target = await manager.createTarget("loopback"); + const active = fixture.processes[0]; + assert.ok(active); + const pending = active.browser.blockNextDisposal(); + const revocation = manager.disposeContext(target.contextId); + await pending.started.promise; + assert.equal(active.process.contexts.has(target.contextId), true); + assert.equal(active.browser.contexts.has(target.contextId), true); + pending.release.resolve(); + await revocation; + assert.equal(active.process.contexts.has(target.contextId), false); + assert.equal(active.browser.contexts.has(target.contextId), false); + await manager.stop(); +}); + +test("rejected revocation reaps Chromium before a clean restart", async (context) => { + const fixture = new ChromeFixture(); + context.after(() => reapFixture(fixture)); + const manager = testManager(fixture); + const target = await manager.createTarget("loopback"); + const active = fixture.processes[0]; + assert.ok(active); + const expected = new Error("revocation rejected"); + active.browser.rejectNextDisposal(expected); + await assert.rejects(manager.disposeContext(target.contextId), (error) => error === expected); + assert.equal(manager.current(), null); + assert.equal(active.process.contexts.size, 0); + assert.equal(active.browser.closed, true); + assert.notEqual(active.process.child.signalCode, null); + assert.deepEqual(active.proxyCloses, { loopback: 1, public: 1 }); + await manager.createTarget("public"); + assert.equal(fixture.processes.length, 2); + await manager.stop(); +}); + +test("unconfirmed termination poisons the manager and blocks restart", async (context) => { + const fixture = new ChromeFixture(); + context.after(() => reapFixture(fixture)); + const manager = testManager(fixture, 20); + const target = await manager.createTarget("loopback"); + const active = fixture.processes[0]; + assert.ok(active); + active.browser.rejectNextDisposal(new Error("revocation rejected")); + active.process.child.kill = () => false; + let terminationError: unknown; + await assert.rejects(manager.disposeContext(target.contextId), (error) => { + terminationError = error; + return String(error).includes("Timed out confirming Chromium termination"); + }); + assert.equal(active.process.contexts.has(target.contextId), true); + assert.equal(active.browser.contexts.has(target.contextId), true); + assert.equal(fixture.processes.length, 1); + await assert.rejects(manager.ensure(), (error) => error === terminationError); + assert.equal(fixture.processes.length, 1); + active.terminate("SIGKILL"); + await waitForExit(active.process.child); +}); diff --git a/services/agent-runtime/src/browser-host/chrome.ts b/services/agent-runtime/src/browser-host/chrome.ts index c7c4b76b8..7e1a2ba96 100644 --- a/services/agent-runtime/src/browser-host/chrome.ts +++ b/services/agent-runtime/src/browser-host/chrome.ts @@ -1,26 +1,64 @@ -// Locate and launch a headless Chromium for the server-side CDP browser host. -// -// CDP client and snapshot approach adapted from Ghostex (MIT, maddada). -// -// The frontend's pi agent drives a real headless Chromium over raw CDP instead -// of the old renderer-bridge embedded webview. This module owns binary -// discovery + process lifecycle; the actual protocol work lives in cdp.ts and -// browser-host.ts. Server-only: never import from client components. - import { execFileSync, spawn, type ChildProcess } from "node:child_process"; import { existsSync } from "node:fs"; import os from "node:os"; import path from "node:path"; +import { Effect, Schema } from "effect"; +import type { BrowserNetworkMode } from "../../../../shared/agent/sanitize-embedded-browser-url"; import { getGlobalSingleton } from "../instances"; +import { CdpClient } from "./cdp"; +import { browserNetworkPolicy, type BrowserNetworkPolicy } from "./network-policy"; +import { + createBrowserPinningProxies, + type PinningProxy, +} from "./pinning-proxy"; const CHROME_LAUNCH_TIMEOUT_MS = 15_000; +const CHROME_TERMINATION_TIMEOUT_MS = 5_000; +const PROXY_BYPASS_LIST = "<-loopback>"; +const BrowserContextResultSchema = Schema.Struct({ browserContextId: Schema.String }); +const TargetResultSchema = Schema.Struct({ targetId: Schema.String }); + +type BrowserPinningProxies = Record; + +export type ChromeLaunchOptions = { + dataDir?: string; + policy?: BrowserNetworkPolicy; + proxies?: BrowserPinningProxies; +}; + +export type ChromeProcess = { + browser: ChromeBrowser; + child: ChildProcess; + contexts: Set; + policy: BrowserNetworkPolicy; + port: number; + proxies: BrowserPinningProxies; + wsEndpoint: string; +}; + +export type ChromeBrowser = Pick; + +export type ManagedChromeTarget = { + contextId: string; + mode: BrowserNetworkMode; + policy: BrowserNetworkPolicy; + targetId: string; +}; -// Where Chromium keeps its profile. Stable so we reuse one profile dir and the -// smoke/cleanup steps can target it via `pkill -f local-studio-browser-profile`. function chromeDataDir(): string { return path.join(os.tmpdir(), "local-studio-browser-profile"); } +function resolveOnPath(binary: string): string | null { + if (binary.includes("/")) return existsSync(binary) ? binary : null; + try { + const resolved = execFileSync("which", [binary], { encoding: "utf8" }).trim(); + return resolved && existsSync(resolved) ? resolved : null; + } catch { + return null; + } +} + function platformChromeCandidates(): string[] { if (process.platform === "darwin") { return [ @@ -64,35 +102,35 @@ function platformChromeCandidates(): string[] { .filter((value): value is string => Boolean(value)); } -function resolveOnPath(binary: string): string | null { - if (binary.includes("/")) return existsSync(binary) ? binary : null; - try { - const resolved = execFileSync("which", [binary], { encoding: "utf8" }).trim(); - return resolved && existsSync(resolved) ? resolved : null; - } catch { - return null; - } -} - -// Discovery order: explicit env override first, then platform defaults. export function findChromeBinary(): string | null { const override = process.env.LOCAL_STUDIO_CHROME_PATH?.trim(); if (override) return existsSync(override) ? override : null; - for (const candidate of platformChromeCandidates()) { - if (existsSync(candidate)) return candidate; - } - return null; + return platformChromeCandidates().find(existsSync) ?? null; } -export type ChromeProcess = { - child: ChildProcess; - wsEndpoint: string; - port: number; -}; +export function chromeArguments(dataDir: string, publicProxyUrl: string): string[] { + return [ + "--headless=new", + "--remote-debugging-port=0", + "--no-first-run", + "--no-default-browser-check", + "--disable-dev-shm-usage", + "--disable-background-networking", + "--disable-component-update", + "--disable-default-apps", + "--disable-quic", + "--disable-sync", + "--no-pings", + "--force-webrtc-ip-handling-policy=disable_non_proxied_udp", + `--proxy-server=${publicProxyUrl}`, + `--proxy-bypass-list=${PROXY_BYPASS_LIST}`, + "--window-size=1280,800", + `--user-data-dir=${dataDir}`, + ]; +} function parseDevToolsEndpoint(line: string): string | null { - const match = line.match(/DevTools listening on (ws:\/\/\S+)/); - return match ? match[1] : null; + return line.match(/DevTools listening on (ws:\/\/\S+)/u)?.[1] ?? null; } function portFromWsEndpoint(endpoint: string): number { @@ -103,84 +141,124 @@ function portFromWsEndpoint(endpoint: string): number { } } -// Launch headless Chromium with remote debugging on an ephemeral port and parse -// the `DevTools listening on ws://...` line from stderr to learn the endpoint. -export function launchChrome(binary: string): Promise { - const dataDir = chromeDataDir(); - const child = spawn( - binary, - [ - "--headless=new", - "--remote-debugging-port=0", - "--no-first-run", - "--no-default-browser-check", - "--disable-dev-shm-usage", - "--window-size=1280,800", - `--user-data-dir=${dataDir}`, - ], - { stdio: ["ignore", "ignore", "pipe"] }, - ); - - return new Promise((resolve, reject) => { - let settled = false; - let stderrBuffer = ""; - - const finish = (error: Error | null, endpoint?: string) => { - if (settled) return; - settled = true; +function devToolsEndpoint(child: ChildProcess): Promise { + return new Promise((resolveEndpoint, reject) => { + let buffer = ""; + const cleanup = () => { clearTimeout(timer); child.stderr?.off("data", onStderr); - if (error) { - child.kill("SIGKILL"); - reject(error); - return; - } - const wsEndpoint = endpoint as string; - resolve({ child, wsEndpoint, port: portFromWsEndpoint(wsEndpoint) }); + child.off("error", onError); + child.off("exit", onExit); + }; + const succeed = (endpoint: string) => { + cleanup(); + resolveEndpoint(endpoint); + }; + const fail = (error: Error) => { + cleanup(); + reject(error); }; - const onStderr = (chunk: Buffer | string) => { - stderrBuffer += String(chunk); - const endpoint = parseDevToolsEndpoint(stderrBuffer); - if (endpoint) finish(null, endpoint); + buffer += String(chunk); + const endpoint = parseDevToolsEndpoint(buffer); + if (endpoint) succeed(endpoint); }; + const onError = (error: Error) => fail(error); + const onExit = (code: number | null) => + fail(new Error(`Chromium exited before ready (code ${code ?? "null"})`)); + const timer = setTimeout( + () => fail(new Error("Timed out waiting for Chromium DevTools endpoint")), + CHROME_LAUNCH_TIMEOUT_MS, + ); + child.stderr?.on("data", onStderr); + child.once("error", onError); + child.once("exit", onExit); + }); +} - const timer = setTimeout(() => { - finish(new Error("Timed out waiting for Chromium DevTools endpoint")); - }, CHROME_LAUNCH_TIMEOUT_MS); +function childExit(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(); + return new Promise((resolveExit) => child.once("exit", () => resolveExit())); +} - child.stderr?.on("data", onStderr); - child.once("error", (error) => finish(error)); - child.once("exit", (code) => { - if (!settled) finish(new Error(`Chromium exited before ready (code ${code ?? "null"})`)); - }); +async function terminateChild(child: ChildProcess, timeoutMs: number): Promise { + const exited = childExit(child); + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await Effect.runPromise( + Effect.tryPromise({ try: () => exited, catch: (error) => error }).pipe( + Effect.timeoutOrElse({ + duration: timeoutMs, + orElse: () => Effect.fail(new Error("Timed out confirming Chromium termination")), + }), + ), + ); +} + +async function closeProxies(proxies: BrowserPinningProxies): Promise { + await Promise.allSettled([proxies.public.close(), proxies.loopback.close()]); +} + +export async function launchChrome( + binary: string, + { + dataDir = chromeDataDir(), + policy = browserNetworkPolicy, + proxies, + }: ChromeLaunchOptions = {}, +): Promise { + const activeProxies = proxies ?? (await createBrowserPinningProxies(policy)); + const child = spawn(binary, chromeArguments(dataDir, activeProxies.public.url), { + stdio: ["ignore", "ignore", "pipe"], }); + try { + const wsEndpoint = await devToolsEndpoint(child); + const browser = await CdpClient.connect(wsEndpoint); + return { + browser, + child, + contexts: new Set(), + policy, + port: portFromWsEndpoint(wsEndpoint), + proxies: activeProxies, + wsEndpoint, + }; + } catch (error) { + child.kill("SIGKILL"); + await closeProxies(activeProxies); + throw error; + } } -// Singleton process manager. Lazy-launches on first use, detects process exit -// and clears state so the next caller relaunches, and exposes stop()/isAvailable(). -class ChromeManager { +export class ChromeManager { private process: ChromeProcess | null = null; private launching: Promise | null = null; + private stopping: Promise | null = null; + private readonly releases = new WeakMap>(); + + constructor( + private readonly policy: BrowserNetworkPolicy = browserNetworkPolicy, + private readonly launcher: typeof launchChrome = launchChrome, + private readonly resolveBinary: () => string | null = findChromeBinary, + private readonly terminationTimeoutMs = CHROME_TERMINATION_TIMEOUT_MS, + ) {} isAvailable(): boolean { - return findChromeBinary() !== null; + return this.resolveBinary() !== null; } async ensure(): Promise { + if (this.stopping) await this.stopping; if (this.process) return this.process; if (this.launching) return this.launching; - const binary = findChromeBinary(); + const binary = this.resolveBinary(); if (!binary) { throw new Error("Browser unavailable: no Chromium found — set LOCAL_STUDIO_CHROME_PATH"); } - this.launching = launchChrome(binary) - .then((proc) => { - this.process = proc; - proc.child.once("exit", () => { - if (this.process === proc) this.process = null; - }); - return proc; + this.launching = this.launcher(binary, { policy: this.policy }) + .then((process) => { + this.process = process; + process.child.once("exit", () => void this.release(process, false)); + return process; }) .finally(() => { this.launching = null; @@ -188,28 +266,84 @@ class ChromeManager { return this.launching; } + async createTarget(mode: BrowserNetworkMode): Promise { + const process = await this.ensure(); + const context = Schema.decodeUnknownSync(BrowserContextResultSchema)( + await process.browser.call("Target.createBrowserContext", { + proxyBypassList: PROXY_BYPASS_LIST, + proxyServer: process.proxies[mode].url, + }), + ); + process.contexts.add(context.browserContextId); + try { + const target = Schema.decodeUnknownSync(TargetResultSchema)( + await process.browser.call("Target.createTarget", { + browserContextId: context.browserContextId, + url: "about:blank", + }), + ); + return { + contextId: context.browserContextId, + mode, + policy: process.policy, + targetId: target.targetId, + }; + } catch (error) { + await this.disposeContext(context.browserContextId).catch(() => undefined); + throw error; + } + } + + async disposeContext(contextId: string): Promise { + const process = this.process; + if (!process || !process.contexts.has(contextId)) return; + try { + await process.browser.call("Target.disposeBrowserContext", { browserContextId: contextId }); + process.contexts.delete(contextId); + } catch (error) { + await this.stopProcess(process); + throw error; + } + } + current(): ChromeProcess | null { return this.process; } - stop(): void { - const proc = this.process; - this.process = null; - if (proc) proc.child.kill("SIGKILL"); + async stop(): Promise { + if (this.stopping) return this.stopping; + if (this.process) return this.stopProcess(this.process); + if (!this.launching) return; + await this.launching.catch(() => undefined); + if (this.process) await this.stopProcess(this.process); + } + + private async stopProcess(process: ChromeProcess): Promise { + const stopping = this.release(process, true); + this.stopping = stopping; + await stopping; + if (this.stopping === stopping) this.stopping = null; + } + + private release(process: ChromeProcess, kill: boolean): Promise { + const existing = this.releases.get(process); + if (existing) return existing; + if (this.process === process) this.process = null; + process.browser.close(); + const released = Promise.all([ + kill ? terminateChild(process.child, this.terminationTimeoutMs) : Promise.resolve(), + closeProxies(process.proxies), + ]).then(() => { + process.contexts.clear(); + }); + this.releases.set(process, released); + return released; } } export const chromeManager = getGlobalSingleton("chromeManager", () => new ChromeManager()); -// Kill the spawned Chromium when the server process exits, so a normal -// shutdown / restart doesn't orphan a headless browser holding the fixed -// profile-dir lock. Registered once (guarded through the package's global -// singleton registry), synchronous (safe in "exit"). Next's graceful -// SIGTERM/SIGINT handling calls process.exit, which fires this; a raw SIGKILL -// can't be intercepted by anything. getGlobalSingleton("chromeExitHook", () => { - if (typeof process !== "undefined") { - process.on("exit", () => chromeManager.stop()); - } + if (typeof process !== "undefined") process.on("exit", () => void chromeManager.stop()); return true; }); diff --git a/services/agent-runtime/src/browser-host/hosted-page.ts b/services/agent-runtime/src/browser-host/hosted-page.ts index 97bf238d0..a6b81ce77 100644 --- a/services/agent-runtime/src/browser-host/hosted-page.ts +++ b/services/agent-runtime/src/browser-host/hosted-page.ts @@ -2,6 +2,8 @@ import { CdpClient, type CdpEvent } from "./cdp"; import type { SnapshotElement } from "./dom-scripts"; +import type { BrowserNetworkMode } from "../../../../shared/agent/sanitize-embedded-browser-url"; +import type { BrowserNetworkPolicy } from "./network-policy"; const CONSOLE_RING_SIZE = 1000; @@ -32,6 +34,14 @@ export type CdpTarget = { webSocketDebuggerUrl?: string; }; +export type HostedPageConfig = { + contextId: string; + mode: BrowserNetworkMode; + policy: BrowserNetworkPolicy; + release: () => Promise; + timeoutMs: number; +}; + function remoteObjectText(value: unknown): string { const object = value as { value?: unknown; description?: string; type?: string } | undefined; if (!object) return ""; @@ -43,7 +53,12 @@ function remoteObjectText(value: unknown): string { export class HostedPage { readonly id: string; + readonly contextId: string; + readonly mode: BrowserNetworkMode; private client: CdpClient; + private policy: BrowserNetworkPolicy; + private release: () => Promise; + private closing: Promise | null = null; private console: ConsoleEntry[] = []; private refMap = new Map(); private captureEnabled = false; @@ -56,28 +71,47 @@ export class HostedPage { private screencastReady: Promise | null = null; latestFrame: ScreencastFrame | null = null; - private constructor(id: string, client: CdpClient) { + private constructor(id: string, client: CdpClient, config: HostedPageConfig) { this.id = id; + this.contextId = config.contextId; + this.mode = config.mode; this.client = client; + this.policy = config.policy; + this.release = config.release; } - static async attach(target: CdpTarget, timeoutMs: number): Promise { - const client = await CdpClient.connect(target.webSocketDebuggerUrl as string, timeoutMs); - const page = new HostedPage(target.id, client); - await page.enableCapture(); - return page; + static async attach(target: CdpTarget, config: HostedPageConfig): Promise { + if (!target.webSocketDebuggerUrl) throw new Error("Chromium target has no debugger endpoint"); + const client = await CdpClient.connect(target.webSocketDebuggerUrl, config.timeoutMs); + const page = new HostedPage(target.id, client, config); + try { + await page.enableCapture(); + return page; + } catch (error) { + await page.close().catch(() => undefined); + throw error; + } } get closed(): boolean { return this.client.closed; } - close(): void { + close(): Promise { this.client.close(); + this.closing ??= this.release(); + return this.closing; } private async enableCapture(): Promise { if (this.captureEnabled) return; + this.client.on("Fetch.requestPaused", (event) => void this.authorizeRequest(event)); + await this.client.call("Fetch.enable", { + patterns: ["http://*", "https://*", "ws://*", "wss://*"].map((urlPattern) => ({ + requestStage: "Request", + urlPattern, + })), + }); await this.client.call("Runtime.enable"); await this.client.call("Log.enable"); await this.client.call("Page.enable"); @@ -87,6 +121,21 @@ export class HostedPage { this.captureEnabled = true; } + private async authorizeRequest(event: CdpEvent): Promise { + const requestId = event.params?.requestId; + const request = event.params?.request as { url?: unknown } | undefined; + if (typeof requestId !== "string") return; + try { + if (typeof request?.url !== "string") throw new Error("Missing browser request URL"); + await this.policy.resolve(request.url, this.mode); + await this.client.call("Fetch.continueRequest", { requestId }); + } catch { + await this.client + .call("Fetch.failRequest", { errorReason: "BlockedByClient", requestId }) + .catch(() => undefined); + } + } + private pushConsole(entry: ConsoleEntry): void { this.console.push(entry); if (this.console.length > CONSOLE_RING_SIZE) { diff --git a/services/agent-runtime/src/browser-host/network-policy.ts b/services/agent-runtime/src/browser-host/network-policy.ts new file mode 100644 index 000000000..1b1a3c34f --- /dev/null +++ b/services/agent-runtime/src/browser-host/network-policy.ts @@ -0,0 +1,152 @@ +import { lookup } from "node:dns/promises"; +import { Effect, Schema } from "effect"; +import { + browserAddressFamily, + classifyBrowserAddress, + sanitizeBrowserNetworkUrl, + type BrowserAddressClass, + type BrowserNetworkMode, +} from "../../../../shared/agent/sanitize-embedded-browser-url"; + +const RESOLUTION_TIMEOUT_MS = 5_000; +const ResolvedAddressSchema = Schema.Struct({ + address: Schema.String, + family: Schema.Union([Schema.Literal(4), Schema.Literal(6)]), +}); +const ResolvedAddressesSchema = Schema.Array(ResolvedAddressSchema); + +export type ResolvedBrowserAddress = { address: string; family: 4 | 6 }; +export type BrowserHostResolver = ( + hostname: string, +) => Promise>; +export type PinnedBrowserDestination = { + address: ResolvedBrowserAddress; + addressClass: Exclude; + hostname: string; + mode: BrowserNetworkMode; + port: number; + url: URL; +}; +export type BrowserNetworkPolicy = { + allows: (raw: string, mode: BrowserNetworkMode) => boolean; + resolve: (raw: string, mode: BrowserNetworkMode) => Promise; +}; + +export class BrowserNetworkPolicyError extends Error { + override name = "BrowserNetworkPolicyError"; +} + +function normalizedHostname(hostname: string): string { + const unwrapped = hostname.replace(/^\[|\]$/gu, "").toLowerCase(); + return unwrapped.endsWith(".") ? unwrapped.slice(0, -1) : unwrapped; +} + +function destinationPort(url: URL): number { + if (url.port) return Number(url.port); + return url.protocol === "https:" || url.protocol === "wss:" ? 443 : 80; +} + +async function systemResolver(hostname: string): Promise> { + const results = await lookup(hostname, { all: true, verbatim: true }); + return results.map(({ address, family }) => ({ address, family: family === 6 ? 6 : 4 })); +} + +function allowedAddressClass( + addressClass: BrowserAddressClass, + mode: BrowserNetworkMode, +): addressClass is Exclude { + return addressClass === "public" || (addressClass === "loopback" && mode === "loopback"); +} + +function literalDestination(hostname: string): ResolvedBrowserAddress | null { + const family = browserAddressFamily(hostname); + return family ? { address: hostname, family } : null; +} + +function validatedAnswers(input: unknown): ReadonlyArray { + try { + const addresses = Schema.decodeUnknownSync(ResolvedAddressesSchema)(input); + for (const address of addresses) { + if (browserAddressFamily(address.address) !== address.family) { + throw new BrowserNetworkPolicyError("Resolved host returned an invalid address family"); + } + } + return addresses; + } catch (error) { + if (error instanceof BrowserNetworkPolicyError) throw error; + throw new BrowserNetworkPolicyError(`Resolved host returned invalid addresses: ${String(error)}`); + } +} + +async function resolvedAnswers( + hostname: string, + resolver: BrowserHostResolver, + timeoutMs: number, +): Promise> { + const resolution = Effect.tryPromise({ + try: () => resolver(hostname), + catch: (error) => + new BrowserNetworkPolicyError(`Host resolution failed for ${hostname}: ${String(error)}`), + }).pipe( + Effect.timeoutOrElse({ + duration: timeoutMs, + orElse: () => + Effect.fail(new BrowserNetworkPolicyError(`Host resolution timed out for ${hostname}`)), + }), + ); + return validatedAnswers(await Effect.runPromise(resolution)); +} + +async function pinnedAddress( + hostname: string, + mode: BrowserNetworkMode, + resolver: BrowserHostResolver, + timeoutMs: number, +): Promise<{ address: ResolvedBrowserAddress; addressClass: Exclude }> { + const literal = literalDestination(hostname); + const addresses = literal ? [literal] : await resolvedAnswers(hostname, resolver, timeoutMs); + if (addresses.length === 0) { + throw new BrowserNetworkPolicyError(`Host resolved to no addresses: ${hostname}`); + } + const classes = new Set(addresses.map(({ address }) => classifyBrowserAddress(address))); + if (classes.size !== 1) { + throw new BrowserNetworkPolicyError(`Host resolved to mixed address classes: ${hostname}`); + } + const addressClass = classes.values().next().value; + const address = addresses[0]; + if (!addressClass || !address) { + throw new BrowserNetworkPolicyError(`Host resolved to no usable addresses: ${hostname}`); + } + if (!allowedAddressClass(addressClass, mode)) { + throw new BrowserNetworkPolicyError(`Browser network policy blocked destination: ${hostname}`); + } + return { address, addressClass }; +} + +export function createBrowserNetworkPolicy({ + resolver = systemResolver, + timeoutMs = RESOLUTION_TIMEOUT_MS, +}: { + resolver?: BrowserHostResolver; + timeoutMs?: number; +} = {}): BrowserNetworkPolicy { + const allows = (raw: string, mode: BrowserNetworkMode) => + sanitizeBrowserNetworkUrl(raw, mode) !== null; + const resolve = async (raw: string, mode: BrowserNetworkMode) => { + const safe = sanitizeBrowserNetworkUrl(raw, mode); + if (!safe) throw new BrowserNetworkPolicyError("Browser network policy blocked URL"); + const url = new URL(safe); + const hostname = normalizedHostname(url.hostname); + const destination = await pinnedAddress(hostname, mode, resolver, timeoutMs); + return { + ...destination, + hostname, + mode, + port: destinationPort(url), + url, + }; + }; + return { allows, resolve }; +} + +export const browserNetworkPolicy = createBrowserNetworkPolicy(); diff --git a/services/agent-runtime/src/browser-host/pinning-proxy.ts b/services/agent-runtime/src/browser-host/pinning-proxy.ts new file mode 100644 index 000000000..a3b737ad7 --- /dev/null +++ b/services/agent-runtime/src/browser-host/pinning-proxy.ts @@ -0,0 +1,311 @@ +import { + Agent, + createServer, + request as httpRequest, + type IncomingHttpHeaders, + type IncomingMessage, + type Server, + type ServerResponse, +} from "node:http"; +import { connect as netConnect, type Socket } from "node:net"; +import type { Duplex } from "node:stream"; +import { Effect } from "effect"; +import type { BrowserNetworkMode } from "../../../../shared/agent/sanitize-embedded-browser-url"; +import { + BrowserNetworkPolicyError, + type BrowserNetworkPolicy, + type PinnedBrowserDestination, +} from "./network-policy"; + +const PROXY_HOST = "127.0.0.1"; +const CONNECT_TIMEOUT_MS = 10_000; + +export type PinnedDial = (destination: PinnedBrowserDestination) => Socket; +export type PinningProxy = { + close: () => Promise; + mode: BrowserNetworkMode; + port: number; + url: string; +}; + +function defaultDial(destination: PinnedBrowserDestination): Socket { + return netConnect({ + family: destination.address.family, + host: destination.address.address, + port: destination.port, + }); +} + +function statusFor(error: unknown): number { + return error instanceof BrowserNetworkPolicyError ? 403 : 502; +} + +function responseMessage(status: number): string { + return status === 403 ? "Browser network policy blocked destination" : "Pinned proxy failure"; +} + +function failHttp(response: ServerResponse, error: unknown): void { + if (response.headersSent) { + response.destroy(); + return; + } + const status = statusFor(error); + response.writeHead(status, { connection: "close", "content-type": "text/plain" }); + response.end(responseMessage(status)); +} + +function failSocket(socket: Duplex, error: unknown): void { + if (!socket.destroyed) { + const status = statusFor(error); + const message = responseMessage(status); + socket.end( + `HTTP/1.1 ${status} ${status === 403 ? "Forbidden" : "Bad Gateway"}\r\nConnection: close\r\nContent-Length: ${Buffer.byteLength(message)}\r\n\r\n${message}`, + ); + } +} + +function withoutHopHeaders(headers: IncomingHttpHeaders): IncomingHttpHeaders { + const output: IncomingHttpHeaders = { ...headers }; + for (const name of [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "transfer-encoding", + "upgrade", + ]) { + delete output[name]; + } + return output; +} + +function requestHeaders(headers: IncomingHttpHeaders, host: string): IncomingHttpHeaders { + return { ...withoutHopHeaders(headers), host }; +} + +function absoluteRequestUrl(request: IncomingMessage): URL { + const url = new URL(request.url ?? ""); + if (url.protocol !== "http:" || url.username || url.password) { + throw new BrowserNetworkPolicyError("Pinned proxy rejected request URL"); + } + return url; +} + +function authorityUrl(authority: string): URL { + const url = new URL(`https://${authority}`); + if (url.username || url.password || url.pathname !== "/" || url.search || url.hash) { + throw new BrowserNetworkPolicyError("Pinned proxy rejected CONNECT authority"); + } + return url; +} + +function rememberSocket(socket: Socket, sockets: Set): Socket { + sockets.add(socket); + socket.on("error", () => undefined); + socket.once("close", () => sockets.delete(socket)); + return socket; +} + +function pinnedSocket(socket: Socket, sockets: Set): Socket { + rememberSocket(socket, sockets); + socket.setTimeout(CONNECT_TIMEOUT_MS, () => socket.destroy(new Error("Pinned connection timed out"))); + socket.once("connect", () => socket.setTimeout(0)); + return socket; +} + +function responseHeaders(headers: IncomingHttpHeaders): IncomingHttpHeaders { + return withoutHopHeaders(headers); +} + +function pinnedAgent( + destination: PinnedBrowserDestination, + dial: PinnedDial, + sockets: Set, +): Agent { + const agent = new Agent({ keepAlive: false }); + agent.createConnection = () => pinnedSocket(dial(destination), sockets); + return agent; +} + +async function forwardHttp( + request: IncomingMessage, + response: ServerResponse, + mode: BrowserNetworkMode, + policy: BrowserNetworkPolicy, + dial: PinnedDial, + sockets: Set, +): Promise { + const url = absoluteRequestUrl(request); + const destination = await policy.resolve(url.toString(), mode); + const outgoing = httpRequest( + { + agent: pinnedAgent(destination, dial, sockets), + family: destination.address.family, + headers: requestHeaders(request.headers, url.host), + hostname: destination.address.address, + method: request.method, + path: `${url.pathname}${url.search}`, + port: destination.port, + protocol: "http:", + }, + (origin) => { + response.writeHead(origin.statusCode ?? 502, responseHeaders(origin.headers)); + origin.pipe(response); + }, + ); + outgoing.once("error", (error) => failHttp(response, error)); + request.once("aborted", () => outgoing.destroy()); + request.pipe(outgoing); +} + +function serializedHeaders(headers: IncomingHttpHeaders, host: string): string { + const output = requestHeaders(headers, host); + output.connection = "Upgrade"; + output.upgrade = headers.upgrade ?? "websocket"; + return Object.entries(output) + .flatMap(([name, value]) => + Array.isArray(value) ? value.map((entry) => `${name}: ${entry}`) : [`${name}: ${value ?? ""}`], + ) + .join("\r\n"); +} + +async function forwardUpgrade( + request: IncomingMessage, + client: Duplex, + head: Buffer, + mode: BrowserNetworkMode, + policy: BrowserNetworkPolicy, + dial: PinnedDial, + sockets: Set, +): Promise { + const url = absoluteRequestUrl(request); + const websocketUrl = `ws://${url.host}${url.pathname}${url.search}`; + const destination = await policy.resolve(websocketUrl, mode); + const upstream = pinnedSocket(dial(destination), sockets); + let connected = false; + upstream.once("error", (error) => + connected ? client.destroy() : failSocket(client, error), + ); + upstream.once("connect", () => { + connected = true; + upstream.write( + `${request.method ?? "GET"} ${url.pathname}${url.search} HTTP/${request.httpVersion}\r\n${serializedHeaders( + request.headers, + url.host, + )}\r\n\r\n`, + ); + if (head.length) upstream.write(head); + client.pipe(upstream).pipe(client); + }); +} + +async function forwardConnect( + request: IncomingMessage, + client: Duplex, + head: Buffer, + mode: BrowserNetworkMode, + policy: BrowserNetworkPolicy, + dial: PinnedDial, + sockets: Set, +): Promise { + const url = authorityUrl(request.url ?? ""); + const destination = await policy.resolve(url.toString(), mode); + const upstream = pinnedSocket(dial(destination), sockets); + let connected = false; + upstream.once("error", (error) => + connected ? client.destroy() : failSocket(client, error), + ); + upstream.once("connect", () => { + connected = true; + client.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + if (head.length) upstream.write(head); + client.pipe(upstream).pipe(client); + }); +} + +function listen(server: Server): Promise { + const effect = Effect.tryPromise({ + try: () => + new Promise((resolveListen, reject) => { + server.once("error", reject); + server.listen(0, PROXY_HOST, () => { + const address = server.address(); + if (!address || typeof address === "string") { + reject(new Error("Pinned proxy did not bind a TCP port")); + return; + } + resolveListen(address.port); + }); + }), + catch: (error) => new Error(`Pinned proxy failed to listen: ${String(error)}`), + }); + return Effect.runPromise(effect); +} + +function closeServer(server: Server, sockets: Set): Promise { + for (const socket of sockets) socket.destroy(); + const effect = Effect.tryPromise({ + try: () => + new Promise((resolveClose, reject) => { + if (!server.listening) { + resolveClose(); + return; + } + server.close((error) => (error ? reject(error) : resolveClose())); + }), + catch: (error) => new Error(`Pinned proxy failed to close: ${String(error)}`), + }); + return Effect.runPromise(effect); +} + +export async function createPinningProxy({ + dial = defaultDial, + mode, + policy, +}: { + dial?: PinnedDial; + mode: BrowserNetworkMode; + policy: BrowserNetworkPolicy; +}): Promise { + const sockets = new Set(); + const server = createServer((request, response) => { + void forwardHttp(request, response, mode, policy, dial, sockets).catch((error) => + failHttp(response, error), + ); + }); + server.on("connection", (socket) => rememberSocket(socket, sockets)); + server.on("connect", (request, client, head) => { + void forwardConnect(request, client, head, mode, policy, dial, sockets).catch((error) => + failSocket(client, error), + ); + }); + server.on("upgrade", (request, client, head) => { + void forwardUpgrade(request, client, head, mode, policy, dial, sockets).catch((error) => + failSocket(client, error), + ); + }); + const port = await listen(server); + let closing: Promise | null = null; + return { + close: () => (closing ??= closeServer(server, sockets)), + mode, + port, + url: `http://${PROXY_HOST}:${port}`, + }; +} + +export async function createBrowserPinningProxies( + policy: BrowserNetworkPolicy, +): Promise> { + const publicProxy = await createPinningProxy({ mode: "public", policy }); + try { + const loopbackProxy = await createPinningProxy({ mode: "loopback", policy }); + return { loopback: loopbackProxy, public: publicProxy }; + } catch (error) { + await publicProxy.close(); + throw error; + } +} diff --git a/services/agent-runtime/src/browser-host/reader-network-policy.test.ts b/services/agent-runtime/src/browser-host/reader-network-policy.test.ts new file mode 100644 index 000000000..1655f07a8 --- /dev/null +++ b/services/agent-runtime/src/browser-host/reader-network-policy.test.ts @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import test from "node:test"; +import { createBrowserNetworkPolicy } from "./network-policy"; +import { fetchReadable, type ReaderTransport } from "./reader"; + +function response(url: string, options: { location?: string; status?: number } = {}) { + const status = options.status ?? 200; + return { + body: "# Ready", + contentType: "text/markdown", + ok: status >= 200 && status < 300, + status, + url, + ...(options.location ? { location: options.location } : {}), + }; +} + +test("reader keeps one explicit mode and pins every redirect before transport", async () => { + const resolutions: string[] = []; + const policy = createBrowserNetworkPolicy({ + resolver: async (hostname) => { + resolutions.push(hostname); + return [{ address: "8.8.8.8", family: 4 }]; + }, + }); + const destinations: string[] = []; + const transport: ReaderTransport = async (destination) => { + destinations.push(`${destination.address.address}|${destination.url.hostname}`); + return destination.url.hostname === "first.test" + ? response(destination.url.toString(), { + location: "https://second.test/final", + status: 302, + }) + : response(destination.url.toString()); + }; + + const result = await fetchReadable("https://first.test/start", "public", { + policy, + transport, + }); + + assert.deepEqual(resolutions, ["first.test", "second.test"]); + assert.deepEqual(destinations, ["8.8.8.8|first.test", "8.8.8.8|second.test"]); + assert.equal(result.url, "https://second.test/final"); +}); + +test("reader rejects a blocked redirect before the transport opens a connection", async () => { + const policy = createBrowserNetworkPolicy({ + resolver: async (hostname) => [ + { address: hostname === "blocked.test" ? "10.0.0.1" : "8.8.8.8", family: 4 }, + ], + }); + let requests = 0; + const transport: ReaderTransport = async (destination) => { + requests += 1; + return response(destination.url.toString(), { + location: "http://blocked.test/secret", + status: 302, + }); + }; + + await assert.rejects( + fetchReadable("https://allowed.test", "public", { policy, transport }), + /blocked destination/u, + ); + assert.equal(requests, 1); +}); + +test("reader allows loopback only when the caller selected loopback mode", async () => { + const policy = createBrowserNetworkPolicy(); + const transport: ReaderTransport = async (destination) => response(destination.url.toString()); + + await assert.rejects( + fetchReadable("http://127.0.0.1:4321", "public", { policy, transport }), + /url rejected/u, + ); + const result = await fetchReadable("http://127.0.0.1:4321", "loopback", { + policy, + transport, + }); + assert.equal(result.url, "http://127.0.0.1:4321/"); +}); + +test("reader transport pins the selected address and preserves the original Host", async () => { + let host = ""; + const server = createServer((request, response) => { + host = request.headers.host ?? ""; + response.setHeader("content-type", "text/plain"); + response.end("reader pinned"); + }); + const port = await new Promise((resolveListen, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + reject(new Error("Missing reader fixture address")); + return; + } + resolveListen(address.port); + }); + }); + try { + const policy = createBrowserNetworkPolicy({ + resolver: async () => [{ address: "127.0.0.1", family: 4 }], + }); + const result = await fetchReadable(`http://localhost:${port}/`, "loopback", { policy }); + assert.equal(result.text, "reader pinned"); + assert.equal(host, `localhost:${port}`); + } finally { + await new Promise((resolveClose, reject) => + server.close((error) => (error ? reject(error) : resolveClose())), + ); + } +}); diff --git a/services/agent-runtime/src/browser-host/reader.ts b/services/agent-runtime/src/browser-host/reader.ts index aa434d6a8..3739cc2ab 100644 --- a/services/agent-runtime/src/browser-host/reader.ts +++ b/services/agent-runtime/src/browser-host/reader.ts @@ -7,10 +7,15 @@ // route; it is shared so the embedded [verb] path can fall back without an HTTP // self-call. -import { lookup } from "node:dns/promises"; import { request as httpRequest, type RequestOptions } from "node:http"; import { request as httpsRequest } from "node:https"; -import { sanitizePublicBrowserUrl } from "../../../../shared/agent/sanitize-embedded-browser-url"; +import type { LookupFunction } from "node:net"; +import type { BrowserNetworkMode } from "../../../../shared/agent/sanitize-embedded-browser-url"; +import { + browserNetworkPolicy, + type BrowserNetworkPolicy, + type PinnedBrowserDestination, +} from "./network-policy"; const MAX_BYTES = 512 * 1024; const FETCH_TIMEOUT_MS = 12_000; @@ -27,11 +32,7 @@ export type ReaderResult = { contentType: string; }; -type ResolvedHostAddress = { address: string; family: 4 | 6 }; -type ResolvedHostInput = string | ResolvedHostAddress; -type ReaderHostResolver = (hostname: string) => Promise; - -type BoundedResponse = { +export type ReaderResponse = { status: number; ok: boolean; url: string; @@ -40,23 +41,16 @@ type BoundedResponse = { location?: string; }; -declare global { - // Test-only hooks for simulating DNS answers / responses without real network. - var __LOCAL_STUDIO_BROWSER_READER_HOST_RESOLVER_FOR_TEST: ReaderHostResolver | undefined; - var __LOCAL_STUDIO_BROWSER_READER_REQUEST_FOR_TEST: - | ((url: string, address: ResolvedHostAddress) => Promise) - | undefined; -} +export type ReaderTransport = (destination: PinnedBrowserDestination) => Promise; +export type ReaderDependencies = { + policy?: BrowserNetworkPolicy; + transport?: ReaderTransport; +}; -async function resolveReaderHost(hostname: string): Promise { - const testResolver = globalThis.__LOCAL_STUDIO_BROWSER_READER_HOST_RESOLVER_FOR_TEST; - if (testResolver) return (await testResolver(hostname)).map(normalizeResolvedAddress); - const results = await lookup(hostname, { all: true, verbatim: true }); - return results.map((result) => ({ - address: result.address, - family: result.family === 6 ? 6 : 4, - })); -} +type ReaderRuntime = { + policy: BrowserNetworkPolicy; + transport: ReaderTransport; +}; function decodeEntities(value: string): string { return value @@ -127,16 +121,19 @@ function cleanMarkdown(markdown: string): string { .trim(); } -async function fetchBoundedUrl(url: string, redirects = 0): Promise { - const addresses = await publicResolvedAddresses(url); - const response = await requestBoundedUrl(url, addresses[0]); +async function fetchBoundedUrl( + url: string, + mode: BrowserNetworkMode, + runtime: ReaderRuntime, + redirects = 0, +): Promise { + const destination = await runtime.policy.resolve(url, mode); + const response = await runtime.transport(destination); if (isRedirectStatus(response.status)) { if (redirects >= MAX_REDIRECTS) throw new Error("Too many redirects"); if (!response.location) throw new Error("Redirect missing Location header"); const nextUrl = new URL(response.location, url).toString(); - const safeRedirect = sanitizePublicBrowserUrl(nextUrl); - if (!safeRedirect) throw new Error("Redirect rejected (must stay public http/https)"); - return fetchBoundedUrl(safeRedirect, redirects + 1); + return fetchBoundedUrl(nextUrl, mode, runtime, redirects + 1); } return response; } @@ -145,43 +142,16 @@ function isRedirectStatus(status: number): boolean { return status >= 300 && status < 400; } -async function publicResolvedAddresses(raw: string): Promise { - const url = new URL(raw); - const addresses = await resolveReaderHost(url.hostname); - if (!addresses.length) throw new Error("Host resolved to no addresses"); - for (const address of addresses) { - if (!sanitizePublicBrowserUrl(`${url.protocol}//${hostForAddress(address.address)}/`)) { - throw new Error("Resolved host rejected (must stay public http/https)"); - } - } - return addresses; -} - -function hostForAddress(address: string): string { - return address.includes(":") ? `[${address}]` : address; -} - -function normalizeResolvedAddress(input: ResolvedHostInput): ResolvedHostAddress { - if (typeof input !== "string") return input; - return { address: input, family: input.includes(":") ? 6 : 4 }; -} - -function requestBoundedUrl(url: string, address: ResolvedHostAddress): Promise { - const testRequest = globalThis.__LOCAL_STUDIO_BROWSER_READER_REQUEST_FOR_TEST; - if (testRequest) return testRequest(url, address); - const parsed = new URL(url); +function requestBoundedUrl(destination: PinnedBrowserDestination): Promise { + const parsed = destination.url; + const address = destination.address; const request = parsed.protocol === "https:" ? httpsRequest : httpRequest; const options: RequestOptions = { headers: { Accept: ACCEPT, "User-Agent": USER_AGENT }, - lookup: (( - _hostname: string, - lookupOptions: unknown, - callback: (...args: unknown[]) => void, - ) => { - const wantsAll = Boolean((lookupOptions as { all?: boolean } | undefined)?.all); - if (wantsAll) callback(null, [address]); + lookup: ((_hostname, lookupOptions, callback) => { + if (lookupOptions.all) callback(null, [address]); else callback(null, address.address, address.family); - }) as RequestOptions["lookup"], + }) satisfies LookupFunction, }; return new Promise((resolve, reject) => { @@ -212,7 +182,7 @@ function requestBoundedUrl(url: string, address: ResolvedHostAddress): Promise= 200 && status < 300, - url, + url: destination.url.toString(), contentType, body, ...(location ? { location } : {}), @@ -242,7 +212,7 @@ function concatBytes(chunks: Uint8Array[], total: number): Uint8Array { return output; } -function renderReadable(response: BoundedResponse, fallbackUrl: string): ReaderResult { +function renderReadable(response: ReaderResponse, fallbackUrl: string): ReaderResult { const contentType = response.contentType; const finalUrl = response.url || fallbackUrl; if (contentType.startsWith("text/html") || contentType.includes("xhtml")) { @@ -273,10 +243,18 @@ function renderReadable(response: BoundedResponse, fallbackUrl: string): ReaderR // Fetch a public URL and return reading-mode text. Throws on rejected/invalid // URLs or upstream failures; callers map errors to their own response shape. -export async function fetchReadable(rawUrl: string): Promise { - const safe = sanitizePublicBrowserUrl(rawUrl); - if (!safe) throw new Error("url rejected (must be public http/https)"); - const response = await fetchBoundedUrl(safe); +export async function fetchReadable( + rawUrl: string, + mode: BrowserNetworkMode = "public", + dependencies: ReaderDependencies = {}, +): Promise { + const runtime = { + policy: dependencies.policy ?? browserNetworkPolicy, + transport: dependencies.transport ?? requestBoundedUrl, + }; + if (!runtime.policy.allows(rawUrl, mode)) throw new Error("url rejected by browser network policy"); + const safe = new URL(rawUrl.trim()).toString(); + const response = await fetchBoundedUrl(safe, mode, runtime); if (!response.ok) throw new Error(`Upstream returned HTTP ${response.status}`); return renderReadable(response, safe); } diff --git a/services/agent-runtime/src/http/browser-handlers.ts b/services/agent-runtime/src/http/browser-handlers.ts index 40b93bd9a..db21ef166 100644 --- a/services/agent-runtime/src/http/browser-handlers.ts +++ b/services/agent-runtime/src/http/browser-handlers.ts @@ -1,6 +1,9 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import { sanitizeBrowserPaneUrl } from "../../../../shared/agent/sanitize-embedded-browser-url"; +import { + browserNavigation, + type BrowserNavigation, +} from "../../../../shared/agent/sanitize-embedded-browser-url"; import { browserHost, type KeyInput, type MouseInput } from "../browser-host/browser-host"; import { fetchReadable } from "../browser-host/reader"; @@ -29,7 +32,7 @@ const ALLOWED_VERBS = new Set([ const UNAVAILABLE_ERROR = "Browser unavailable: no Chromium found — set LOCAL_STUDIO_CHROME_PATH"; -let lastFallbackUrl = ""; +let lastFallback: BrowserNavigation | null = null; type VerbResult = { ok: boolean; data?: unknown; error?: string }; @@ -115,9 +118,9 @@ async function runHostVerb(verb: string, payload: Record): Prom async function navigateVerb(payload: Record): Promise { // Pane rules: public web plus loopback (previewing local dev servers is the // pane's main job); other private ranges stay blocked. - const url = sanitizeBrowserPaneUrl(String(payload.url ?? "")); - if (!url) return { ok: false, error: "valid public or localhost http(s) url required" }; - const result = await browserHost.navigate(url); + const navigation = browserNavigation(String(payload.url ?? "")); + if (!navigation) return { ok: false, error: "valid public or localhost http(s) url required" }; + const result = await browserHost.navigate(navigation.url); return { ok: true, data: result }; } @@ -148,20 +151,21 @@ function requireSelector(payload: Record): string { // previewable even when there's no headless Chromium to drive a full surface. async function fallbackVerb(verb: string, payload: Record): Promise { if (verb === "navigate") { - const url = sanitizeBrowserPaneUrl(String(payload.url ?? "")); - if (!url) return { ok: false, error: "valid public or localhost http(s) url required" }; - const reader = await fetchReadable(url); - lastFallbackUrl = reader.url; + const navigation = browserNavigation(String(payload.url ?? "")); + if (!navigation) return { ok: false, error: "valid public or localhost http(s) url required" }; + const reader = await fetchReadable(navigation.url, navigation.mode); + lastFallback = { mode: navigation.mode, url: reader.url }; return { ok: true, data: { url: reader.url, title: reader.title, readingMode: true } }; } if (verb === "get-url") { - return { ok: true, data: { url: lastFallbackUrl, title: "" } }; + return { ok: true, data: { url: lastFallback?.url ?? "", title: "" } }; } if (verb === "get-text" || verb === "get-html") { - const url = sanitizeBrowserPaneUrl(String(payload.url ?? "")) || lastFallbackUrl; - if (!url) return { ok: false, error: UNAVAILABLE_ERROR }; - const reader = await fetchReadable(url); - lastFallbackUrl = reader.url; + const requested = browserNavigation(String(payload.url ?? "")); + const navigation = requested ?? lastFallback; + if (!navigation) return { ok: false, error: UNAVAILABLE_ERROR }; + const reader = await fetchReadable(navigation.url, navigation.mode); + lastFallback = { mode: navigation.mode, url: reader.url }; return verb === "get-text" ? { ok: true, data: { text: reader.text, readingMode: true } } : { ok: true, data: { html: reader.markdown ?? reader.text, readingMode: true } }; diff --git a/shared/agent/sanitize-embedded-browser-url.ts b/shared/agent/sanitize-embedded-browser-url.ts index de2d39a4b..4fc4c71f6 100644 --- a/shared/agent/sanitize-embedded-browser-url.ts +++ b/shared/agent/sanitize-embedded-browser-url.ts @@ -1,123 +1,224 @@ -/** - * Normalize and allow-list URLs for the Computer embedded browser. - * Public URLs align loosely with controller browser_open_url rules - * (no loopback / private nets). Local file URLs are intentionally separate so - * agent/browser-tool and server-side fetch paths cannot accidentally read disk. - */ -function parseUrl(raw: string): URL | null { +export type BrowserNetworkMode = "public" | "loopback"; +export type BrowserAddressClass = "public" | "loopback" | "blocked"; +export type BrowserNavigation = { mode: BrowserNetworkMode; url: string }; + +type Ipv4 = readonly [number, number, number, number]; +type Ipv6 = readonly [number, number, number, number, number, number, number, number]; + +const BLOCKED_IPV4_RANGES: ReadonlyArray = [ + [[0, 0, 0, 0], 8], + [[10, 0, 0, 0], 8], + [[100, 64, 0, 0], 10], + [[169, 254, 0, 0], 16], + [[172, 16, 0, 0], 12], + [[192, 0, 0, 0], 24], + [[192, 0, 2, 0], 24], + [[192, 31, 196, 0], 24], + [[192, 52, 193, 0], 24], + [[192, 88, 99, 0], 24], + [[192, 168, 0, 0], 16], + [[192, 175, 48, 0], 24], + [[198, 18, 0, 0], 15], + [[198, 51, 100, 0], 24], + [[203, 0, 113, 0], 24], + [[224, 0, 0, 0], 4], + [[240, 0, 0, 0], 4], +]; + +const BLOCKED_IPV6_RANGES: ReadonlyArray = [ + [[0x2001, 0, 0, 0, 0, 0, 0, 0], 23], + [[0x2001, 0x0db8, 0, 0, 0, 0, 0, 0], 32], + [[0x2002, 0, 0, 0, 0, 0, 0, 0], 16], + [[0x2620, 0x004f, 0x8000, 0, 0, 0, 0, 0], 48], + [[0x3fff, 0, 0, 0, 0, 0, 0, 0], 20], +]; + +function parseUrl(raw: string, protocols: ReadonlySet): URL | null { const trimmed = raw.trim(); if (!trimmed) return null; try { - return new URL(trimmed); + const url = new URL(trimmed); + if (!protocols.has(url.protocol) || url.username || url.password) return null; + return url; } catch { return null; } } -function isLocalHostname(host: string): boolean { - return host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local"); +function ipv4Octets(input: string): Ipv4 | null { + const match = input.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/u); + if (!match) return null; + const [first, second, third, fourth] = match.slice(1).map(Number); + if ( + first === undefined || + second === undefined || + third === undefined || + fourth === undefined || + [first, second, third, fourth].some( + (value) => !Number.isInteger(value) || value < 0 || value > 255, + ) + ) { + return null; + } + return [first, second, third, fourth]; } -function ipv4Octets(host: string): [number, number, number, number] | null { - const match = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/); - if (!match) return null; - const octets = match.slice(1).map(Number) as [number, number, number, number]; - return octets.every((octet) => octet >= 0 && octet <= 255) ? octets : null; -} - -function isPrivateIpv4([a, b]: [number, number, number, number]): boolean { - return ( - a === 0 || - a === 10 || - a === 127 || - (a === 100 && b >= 64 && b <= 127) || - (a === 169 && b === 254) || - (a === 172 && b >= 16 && b <= 31) || - (a === 192 && b === 168) || - (a === 198 && (b === 18 || b === 19)) || - a >= 224 - ); +function ipv4Value(address: Ipv4): number { + return (((address[0] * 256 + address[1]) * 256 + address[2]) * 256 + address[3]) >>> 0; } -function isBlockedIpv4(host: string): boolean { - const octets = ipv4Octets(host); - return !octets || isPrivateIpv4(octets); +function ipv4InRange(address: Ipv4, range: Ipv4, prefix: number): boolean { + const shift = 32 - prefix; + return (ipv4Value(address) >>> shift) === (ipv4Value(range) >>> shift); } -function isPrivateIpv6(host: string): boolean { - const normalized = host.replace(/^\[|\]$/g, "").toLowerCase(); - const mappedIpv4 = ipv4FromMappedIpv6(normalized); - if (mappedIpv4) return isPrivateIpv4(mappedIpv4); - return ( - normalized === "::" || - normalized === "::1" || - normalized.startsWith("fc") || - normalized.startsWith("fd") || - /^fe[89ab]/.test(normalized) - ); +function hextet(input: string): number | null { + if (!/^[0-9a-f]{1,4}$/iu.test(input)) return null; + return Number.parseInt(input, 16); } -function ipv4FromMappedIpv6(host: string): [number, number, number, number] | null { - const tail = host.startsWith("::ffff:") - ? host.slice("::ffff:".length) - : host.startsWith("0:0:0:0:0:ffff:") - ? host.slice("0:0:0:0:0:ffff:".length) - : ""; - if (!tail) return null; - const dotted = ipv4Octets(tail); - if (dotted) return dotted; - const parts = tail.split(":"); - if (parts.length !== 2) return null; - const words = parts.map((part) => Number.parseInt(part, 16)); - if (words.some((word) => !Number.isInteger(word) || word < 0 || word > 0xffff)) { +function expandedIpv6Input(input: string): string | null { + const value = input.replace(/^\[|\]$/gu, "").toLowerCase(); + if (!value || value.includes("%")) return null; + const lastColon = value.lastIndexOf(":"); + if (!value.includes(".") || lastColon < 0) return value; + const ipv4 = ipv4Octets(value.slice(lastColon + 1)); + if (!ipv4) return null; + const high = ((ipv4[0] << 8) | ipv4[1]).toString(16); + const low = ((ipv4[2] << 8) | ipv4[3]).toString(16); + return `${value.slice(0, lastColon)}:${high}:${low}`; +} + +function ipv6Words(input: string): Ipv6 | null { + const expanded = expandedIpv6Input(input); + if (!expanded || !expanded.includes(":")) return null; + const halves = expanded.split("::"); + if (halves.length > 2) return null; + const left = halves[0] ? halves[0].split(":") : []; + const right = halves[1] ? halves[1].split(":") : []; + if ([...left, ...right].some((part) => hextet(part) === null)) return null; + const missing = 8 - left.length - right.length; + if ((halves.length === 1 && missing !== 0) || (halves.length === 2 && missing < 1)) return null; + const words = [...left, ...Array.from({ length: missing }, () => "0"), ...right].map((part) => + Number.parseInt(part, 16), + ); + const [first, second, third, fourth, fifth, sixth, seventh, eighth, extra] = words; + if ( + first === undefined || + second === undefined || + third === undefined || + fourth === undefined || + fifth === undefined || + sixth === undefined || + seventh === undefined || + eighth === undefined || + extra !== undefined + ) { return null; } - const [high, low] = words as [number, number]; - return [high >> 8, high & 0xff, low >> 8, low & 0xff]; + return [first, second, third, fourth, fifth, sixth, seventh, eighth]; } -function isBlockedPublicHost(host: string): boolean { - if (isLocalHostname(host)) return true; - if (/^\d{1,3}(\.\d{1,3}){3}$/.test(host)) return isBlockedIpv4(host); - return host.includes(":") && isPrivateIpv6(host); +function ipv6InRange(address: Ipv6, range: Ipv6, prefix: number): boolean { + const completeWords = Math.floor(prefix / 16); + for (let index = 0; index < completeWords; index += 1) { + if (address[index] !== range[index]) return false; + } + const remaining = prefix % 16; + if (remaining === 0) return true; + const mask = (0xffff << (16 - remaining)) & 0xffff; + const addressWord = address[completeWords]; + const rangeWord = range[completeWords]; + return addressWord !== undefined && rangeWord !== undefined + ? (addressWord & mask) === (rangeWord & mask) + : false; } -export function sanitizePublicBrowserUrl(raw: string): string | null { - const url = parseUrl(raw); +function mappedIpv4(address: Ipv6): Ipv4 | null { + if (address.slice(0, 5).some((word) => word !== 0) || address[5] !== 0xffff) return null; + return [address[6] >> 8, address[6] & 0xff, address[7] >> 8, address[7] & 0xff]; +} + +function classifyIpv4(address: Ipv4): BrowserAddressClass { + if (address[0] === 127) return "loopback"; + return BLOCKED_IPV4_RANGES.some(([range, prefix]) => ipv4InRange(address, range, prefix)) + ? "blocked" + : "public"; +} + +function classifyIpv6(address: Ipv6): BrowserAddressClass { + const mapped = mappedIpv4(address); + if (mapped) return classifyIpv4(mapped); + if (address.slice(0, 7).every((word) => word === 0) && address[7] === 1) return "loopback"; + if (!ipv6InRange(address, [0x2000, 0, 0, 0, 0, 0, 0, 0], 3)) return "blocked"; + return BLOCKED_IPV6_RANGES.some(([range, prefix]) => ipv6InRange(address, range, prefix)) + ? "blocked" + : "public"; +} + +export function classifyBrowserAddress(input: string): BrowserAddressClass { + const host = input.replace(/^\[|\]$/gu, ""); + const ipv4 = ipv4Octets(host); + if (ipv4) return classifyIpv4(ipv4); + const ipv6 = ipv6Words(host); + return ipv6 ? classifyIpv6(ipv6) : "blocked"; +} + +export function browserAddressFamily(input: string): 4 | 6 | null { + const host = input.replace(/^\[|\]$/gu, ""); + if (ipv4Octets(host)) return 4; + return ipv6Words(host) ? 6 : null; +} + +function localHostname(host: string): boolean { + return host === "localhost" || host.endsWith(".localhost"); +} + +function blockedHostname(host: string): boolean { + return host.endsWith(".local"); +} + +function literalAddressClass(host: string): BrowserAddressClass | null { + return browserAddressFamily(host) ? classifyBrowserAddress(host) : null; +} + +function urlMode(url: URL): BrowserNetworkMode | null { + const rawHost = url.hostname.toLowerCase(); + const host = rawHost.endsWith(".") ? rawHost.slice(0, -1) : rawHost; + if (localHostname(host)) return "loopback"; + if (blockedHostname(host)) return null; + const addressClass = literalAddressClass(host); + if (addressClass === "loopback") return "loopback"; + return addressClass === "blocked" ? null : "public"; +} + +export function browserNavigation(raw: string): BrowserNavigation | null { + const url = parseUrl(raw, new Set(["http:", "https:"])); if (!url) return null; - if (url.protocol !== "http:" && url.protocol !== "https:") return null; + const mode = urlMode(url); + return mode ? { mode, url: url.toString() } : null; +} - const host = url.hostname.toLowerCase(); - return isBlockedPublicHost(host) ? null : url.toString(); +export function sanitizeBrowserNetworkUrl(raw: string, mode: BrowserNetworkMode): string | null { + const url = parseUrl(raw, new Set(["http:", "https:", "ws:", "wss:"])); + if (!url) return null; + const destinationMode = urlMode(url); + if (!destinationMode || (mode === "public" && destinationMode === "loopback")) return null; + return url.toString(); } -function isLoopbackHost(host: string): boolean { - if (host === "localhost" || host.endsWith(".localhost")) return true; - const octets = ipv4Octets(host); - if (octets) return octets[0] === 127; - const normalized = host.replace(/^\[|\]$/g, "").toLowerCase(); - return normalized === "::1"; +export function sanitizePublicBrowserUrl(raw: string): string | null { + const navigation = browserNavigation(raw); + return navigation?.mode === "public" ? navigation.url : null; } -/** - * The browser pane's navigate rules: public URLs plus loopback — the pane - * exists to preview the dev servers the agent is running on this machine. - * Other private ranges stay blocked (the agent drives this browser; don't - * hand it the LAN). - */ export function sanitizeBrowserPaneUrl(raw: string): string | null { - const url = parseUrl(raw); - if (!url) return null; - if (url.protocol !== "http:" && url.protocol !== "https:") return null; - const host = url.hostname.toLowerCase(); - if (isLoopbackHost(host)) return url.toString(); - return isBlockedPublicHost(host) ? null : url.toString(); + return browserNavigation(raw)?.url ?? null; } export function sanitizeLocalFileUrl(raw: string): string | null { - const url = parseUrl(raw); - if (!url || url.protocol !== "file:") return null; + const url = parseUrl(raw, new Set(["file:"])); + if (!url) return null; const host = url.hostname.toLowerCase(); - if (host && host !== "localhost") return null; - return url.toString(); + return host && host !== "localhost" ? null : url.toString(); } From 19859e01c03fdf961eac82eeaff0ec01da099392 Mon Sep 17 00:00:00 2001 From: fettpl <38704082+fettpl@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:11:44 +0200 Subject: [PATCH 2/4] fix(agent): prevent browser publication during shutdown --- .../browser-host-concurrency.test.ts | 47 +++++++++++++++---- .../src/browser-host/browser-host.ts | 1 + 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/services/agent-runtime/src/browser-host/browser-host-concurrency.test.ts b/services/agent-runtime/src/browser-host/browser-host-concurrency.test.ts index 7a73d7420..8b81536d1 100644 --- a/services/agent-runtime/src/browser-host/browser-host-concurrency.test.ts +++ b/services/agent-runtime/src/browser-host/browser-host-concurrency.test.ts @@ -132,12 +132,17 @@ class FakePage implements BrowserPage { class FakeContext implements BrowserContextSurface { readonly rawPages: RawPage[] = []; - constructor(private readonly createRawPage: () => RawPage) {} + constructor( + private readonly createRawPage: () => RawPage, + private readonly pageCreationBarrier: Barrier | null, + ) {} - newPage(): Promise { + async newPage(): Promise { + this.pageCreationBarrier?.started.resolve(); + await this.pageCreationBarrier?.release.promise; const page = this.createRawPage(); this.rawPages.push(page); - return Promise.resolve(page); + return page; } pages(): RawPage[] { @@ -187,6 +192,7 @@ class FakeManager implements BrowserHostManager { private stopped = false; private readonly ensureBarriers: Barrier[] = []; private readonly navigationBarriers: Barrier[] = []; + private readonly pageCreationBarriers: Barrier[] = []; blockNextEnsure(): Barrier { const next = barrier(); @@ -200,6 +206,12 @@ class FakeManager implements BrowserHostManager { return next; } + blockNextPageCreation(): Barrier { + const next = barrier(); + this.pageCreationBarriers.push(next); + return next; + } + async ensure(mode: BrowserNetworkMode): Promise { const pending = this.ensureBarriers.shift(); pending?.started.resolve(); @@ -207,12 +219,15 @@ class FakeManager implements BrowserHostManager { if (this.stopped) throw new Error("Browser manager stopped"); if (this.active?.mode === mode && !this.active.closed()) return this.active; await this.active?.close(); - const context = new FakeContext(() => ({ - closed: false, - id: `page-${++this.pageSerial}`, - navigationBarrier: this.navigationBarriers.shift() ?? null, - state: state("about:blank"), - })); + const context = new FakeContext( + () => ({ + closed: false, + id: `page-${++this.pageSerial}`, + navigationBarrier: this.navigationBarriers.shift() ?? null, + state: state("about:blank"), + }), + this.pageCreationBarriers.shift() ?? null, + ); const session = new FakeSession(context, ++this.generation, mode); this.active = session; this.sessions.push(session); @@ -312,6 +327,20 @@ test("stop during context creation prevents page publication", async () => { assert.equal(activeRawPages(manager).length, 0); }); +test("stop during page creation prevents page publication", async () => { + const manager = new FakeManager(); + const host = hostFor(manager); + const pending = manager.blockNextPageCreation(); + const navigation = host.navigate("https://public.test/page"); + await pending.started.promise; + const stopping = host.stop(); + pending.release.resolve(); + await assert.rejects(navigation, /Browser host stopped/u); + await stopping; + assert.equal(manager.sessions[0]?.context.rawPages.length, 1); + assert.equal(activeRawPages(manager).length, 0); +}); + test("stop is terminal and idempotent", async () => { const manager = new FakeManager(); const host = hostFor(manager); diff --git a/services/agent-runtime/src/browser-host/browser-host.ts b/services/agent-runtime/src/browser-host/browser-host.ts index 382d09d79..0dbb65c1f 100644 --- a/services/agent-runtime/src/browser-host/browser-host.ts +++ b/services/agent-runtime/src/browser-host/browser-host.ts @@ -118,6 +118,7 @@ export class BrowserHost { .find((candidate) => Array.from(this.pages.values()).every((hosted) => !hosted.matches(candidate)), ) ?? (await session.context.newPage()); + this.assertRunning(); const hosted = this.attachPage(rawPage); this.pages.set(hosted.id, hosted); this.activeId = hosted.id; From a9457a041b9f50bd5da98f3c466ec2f904a8db4c Mon Sep 17 00:00:00 2001 From: fettpl <38704082+fettpl@users.noreply.github.com> Date: Sun, 19 Jul 2026 00:09:54 +0200 Subject: [PATCH 3/4] fix(agent): unify visible and tool browser surface --- frontend/desktop/logic/security.ts | 29 -- frontend/desktop/logic/window-manager.ts | 1 - frontend/desktop/main.ts | 2 - frontend/package.json | 4 +- .../scripts/agent-browser-live-store.test.ts | 257 +++++++++++++++ .../agent-browser-tools-regressions.test.ts | 232 +++++++++++++ .../src/features/agent/tools/browser-url.ts | 90 +++-- .../agent/ui/agent-browser-effects.ts | 88 +---- .../features/agent/ui/agent-browser-input.ts | 58 ++++ .../agent/ui/agent-browser-live-store.ts | 307 ++++++++++++++++++ .../agent/ui/agent-browser-location.ts | 14 + .../features/agent/ui/agent-browser-panel.tsx | 46 +-- .../agent/ui/agent-browser-screencast.tsx | 169 ++-------- .../src/features/agent/ui/agent-browser.tsx | 120 ++----- .../features/agent/ui/assistant-markdown.tsx | 21 +- .../features/agent/ui/computer-tab-panel.tsx | 17 +- .../agent/ui/filesystem-panel-effects.ts | 44 ++- .../src/features/agent/workspace-file-link.ts | 111 +++++++ package.json | 1 + ...rowser-request-classes.integration.test.ts | 85 ++++- 20 files changed, 1212 insertions(+), 484 deletions(-) create mode 100644 frontend/scripts/agent-browser-live-store.test.ts create mode 100644 frontend/scripts/agent-browser-tools-regressions.test.ts create mode 100644 frontend/src/features/agent/ui/agent-browser-input.ts create mode 100644 frontend/src/features/agent/ui/agent-browser-live-store.ts create mode 100644 frontend/src/features/agent/ui/agent-browser-location.ts create mode 100644 frontend/src/features/agent/workspace-file-link.ts diff --git a/frontend/desktop/logic/security.ts b/frontend/desktop/logic/security.ts index 4603372cc..b35bfe1dd 100644 --- a/frontend/desktop/logic/security.ts +++ b/frontend/desktop/logic/security.ts @@ -86,35 +86,6 @@ export function hardenWebContents(window: electron.BrowserWindow, appOrigin: str }); } -export function registerNavigationPolicy(appOrigin: string): void { - electron.app.on("web-contents-created", (_, contents: electron.WebContents) => { - contents.on("will-attach-webview", (_event, webPreferences, _params) => { - delete webPreferences.preload; - webPreferences.nodeIntegration = false; - webPreferences.contextIsolation = true; - webPreferences.sandbox = true; - }); - - contents.on("will-navigate", (event) => { - // Guest WebContents (the embedded browser webview plus cross-origin - // iframes / OOPIFs) must be able to perform their own navigations. - // Keep the app shell origin-locked, but do not turn the Computer browser - // into a single-load preview. - if ( - contents.getType() === "webview" || - electron.BrowserWindow.fromWebContents(contents) == null - ) { - return; - } - const targetUrl = event.url; - const targetOrigin = safeOrigin(targetUrl); - if (!targetOrigin || targetOrigin !== appOrigin) { - event.preventDefault(); - } - }); - }); -} - function safeOrigin(input: string | undefined): string | null { if (!input) return null; try { diff --git a/frontend/desktop/logic/window-manager.ts b/frontend/desktop/logic/window-manager.ts index 23c895280..d2bd2bbed 100644 --- a/frontend/desktop/logic/window-manager.ts +++ b/frontend/desktop/logic/window-manager.ts @@ -28,7 +28,6 @@ export function createMainWindow(appUrl: string): BrowserWindow { contextIsolation: true, nodeIntegration: false, sandbox: true, - webviewTag: true, webSecurity: true, devTools: !process.env.LOCAL_STUDIO_DESKTOP_DISABLE_DEVTOOLS, allowRunningInsecureContent: false, diff --git a/frontend/desktop/main.ts b/frontend/desktop/main.ts index 8fa952494..4bfd3dd72 100644 --- a/frontend/desktop/main.ts +++ b/frontend/desktop/main.ts @@ -8,7 +8,6 @@ import { writeJsonAtomic } from "./helpers/fs-json"; import { log } from "./helpers/logger"; import { isHttpUrl } from "./helpers/url"; import { createMainWindow } from "./logic/window-manager"; -import { registerNavigationPolicy } from "./logic/security"; import { startFrontendServer, stopFrontendServer, type ServerHandle } from "./logic/app-server"; import { checkForUpdates, getUpdateState, initializeAutoUpdates } from "./logic/update-manager"; import { addProject, listProjectsWithMeta, removeProject } from "./logic/projects-store"; @@ -71,7 +70,6 @@ async function processMemorySummary(): Promise { async function bootstrap(): Promise { if (!frontendServer) { frontendServer = await startFrontendServer({ onExit: handleFrontendServerExit }); - registerNavigationPolicy(new URL(frontendServer.runtime.url).origin); startFrontendHealthMonitor(); } if (!mainWindow) { diff --git a/frontend/package.json b/frontend/package.json index 8f191762e..f72ef0485 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -11,6 +11,8 @@ "dev": "concurrently -k -n NEXT,AGENT -c cyan,green \"next dev\" \"npm --prefix ../services/agent-runtime run dev\"", "build": "npm --prefix ../services/agent-runtime run bundle && next build && node scripts/complete-standalone-build.mjs && node scripts/assert-standalone-build.mjs", "start": "node scripts/start-standalone.mjs", + "test": "bun test scripts", + "test:regression": "bun test", "perf:audit": "node scripts/perf-audit.mjs", "perf:browser": "node scripts/browser-perf-audit.mjs", "lint": "eslint", @@ -39,7 +41,7 @@ "typecheck:desktop": "tsc -p desktop/tsconfig.json", "check:cycles": "madge --extensions ts,tsx --circular src", "check:static": "npm run lint && npm run typecheck && npm run typecheck:desktop && npm run check:cycles && npm run check:ui-structure", - "check:quality": "node scripts/validate-package-json.mjs && npm run check:static && npm run check:cleanup && npm run build", + "check:quality": "node scripts/validate-package-json.mjs && npm run check:static && npm run check:cleanup && npm run test && npm run build", "precommit": "lint-staged --config .lintstagedrc.json && npm run typecheck" }, "dependencies": { diff --git a/frontend/scripts/agent-browser-live-store.test.ts b/frontend/scripts/agent-browser-live-store.test.ts new file mode 100644 index 000000000..2820ff81c --- /dev/null +++ b/frontend/scripts/agent-browser-live-store.test.ts @@ -0,0 +1,257 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { Effect } from "effect"; +import { + createBrowserLiveStore, + type BrowserLiveTransport, +} from "../src/features/agent/ui/agent-browser-live-store"; + +const A = "http://page.test/a"; +const B = "http://page.test/b"; +const C = "http://page.test/c"; + +function frame(url: string) { + return { + status: 200, + body: { + ok: true, + data: { + frame: Buffer.from(url).toString("base64"), + url, + title: url === A ? "A" : url === B ? "B" : "", + canGoBack: false, + canGoForward: false, + }, + }, + }; +} + +async function waitFor(predicate: () => boolean): Promise { + await Effect.runPromise( + Effect.gen(function* () { + for (let attempt = 0; attempt < 500; attempt += 1) { + if (predicate()) return; + yield* Effect.sleep(2); + } + return yield* Effect.fail(new Error("Timed out waiting for browser store state")); + }), + ); +} + +function harness(initialUrl: string) { + let hostUrl = initialUrl; + const navigations: string[] = []; + const transport: BrowserLiveTransport = { + frame: async () => frame(hostUrl), + navigate: async (url) => { + navigations.push(url); + hostUrl = url; + return { status: 200, body: { ok: true, data: { url } } }; + }, + }; + return { + host: (url: string) => { + hostUrl = url; + }, + navigations, + store: createBrowserLiveStore({ pollIntervalMs: 2, transport }), + }; +} + +test("agent navigation leaves the initial start page through observed host state", async () => { + const { host, navigations, store } = harness("about:blank"); + const unsubscribe = store.subscribeState(() => undefined); + try { + await waitFor(() => store.getStateSnapshot().hydrated); + host(B); + await waitFor(() => store.getStateSnapshot().location?.url === B); + assert.equal(store.getStateSnapshot().state?.url, B); + assert.deepEqual(navigations, []); + } finally { + unsubscribe(); + } +}); + +test("remount hydrates agent navigation without replaying the cached visible URL", async () => { + const { host, navigations, store } = harness(A); + const unsubscribeA = store.subscribeState(() => undefined); + await waitFor(() => store.getStateSnapshot().location?.url === A); + unsubscribeA(); + assert.equal(store.getStateSnapshot().hydrated, false); + host(B); + const unsubscribeB = store.subscribeState(() => undefined); + try { + await waitFor(() => store.getStateSnapshot().location?.url === B); + assert.equal(store.getStateSnapshot().state?.url, B); + assert.deepEqual(navigations, []); + } finally { + unsubscribeB(); + } +}); + +test("remount keeps location blocked until explicit navigation settles", async () => { + let hostUrl = "about:blank"; + const request = Promise.withResolvers<{ status: number; body: unknown }>(); + const store = createBrowserLiveStore({ + pollIntervalMs: 2, + transport: { + frame: async () => frame(hostUrl), + navigate: async (url) => { + const response = await request.promise; + hostUrl = url; + return response; + }, + }, + }); + const unsubscribeA = store.subscribeState(() => undefined); + await waitFor(() => store.getStateSnapshot().hydrated); + const navigation = store.navigate(A); + unsubscribeA(); + const unsubscribeB = store.subscribeState(() => undefined); + try { + await waitFor(() => store.getStateSnapshot().hydrated); + assert.equal(store.getStateSnapshot().state?.url, "about:blank"); + assert.equal(store.getStateSnapshot().location, null); + request.resolve({ status: 200, body: { ok: true, data: { url: A } } }); + await navigation; + await waitFor(() => store.getStateSnapshot().location?.url === A); + assert.equal(store.getStateSnapshot().state?.url, A); + } finally { + unsubscribeB(); + } +}); + +test("navigation settles against the host redirect destination", async () => { + let hostUrl = "about:blank"; + const store = createBrowserLiveStore({ + pollIntervalMs: 2, + transport: { + frame: async () => frame(hostUrl), + navigate: async () => { + hostUrl = C; + return { status: 200, body: { ok: true, data: { url: C } } }; + }, + }, + }); + const unsubscribe = store.subscribeState(() => undefined); + try { + await waitFor(() => store.getStateSnapshot().hydrated); + await store.navigate(A); + await waitFor(() => store.getStateSnapshot().location?.url === C); + assert.equal(store.getStateSnapshot().state?.url, C); + } finally { + unsubscribe(); + } +}); + +test("explicit navigation dispatches in issue order and finishes at the latest target", async () => { + let hostUrl = "about:blank"; + const dispatches: string[] = []; + const requests = new Map>(); + const store = createBrowserLiveStore({ + pollIntervalMs: 2, + transport: { + frame: async () => frame(hostUrl), + navigate: async (url) => { + dispatches.push(url); + const request = Promise.withResolvers<{ status: number; body: unknown }>(); + requests.set(url, request); + const response = await request.promise; + hostUrl = url; + return response; + }, + }, + }); + const unsubscribe = store.subscribeState(() => undefined); + try { + await waitFor(() => store.getStateSnapshot().hydrated); + const navigateA = store.navigate(A); + const navigateB = store.navigate(B); + await waitFor(() => requests.has(A)); + assert.deepEqual(dispatches, [A]); + assert.equal(requests.has(B), false); + requests.get(A)?.resolve({ status: 200, body: { ok: true, data: { url: A } } }); + await navigateA; + await waitFor(() => requests.has(B)); + await waitFor(() => store.getStateSnapshot().state?.url === A); + assert.notEqual(store.getStateSnapshot().location?.url, A); + requests.get(B)?.resolve({ status: 200, body: { ok: true, data: { url: B } } }); + await navigateB; + await waitFor(() => store.getStateSnapshot().location?.url === B); + assert.deepEqual(dispatches, [A, B]); + assert.equal(hostUrl, B); + assert.equal(store.getStateSnapshot().state?.url, B); + } finally { + unsubscribe(); + } +}); + +test("first poll after navigation settlement accepts an immediate redirect", async () => { + let hostUrl = "about:blank"; + const frames: PromiseWithResolvers<{ status: number; body: unknown }>[] = []; + const store = createBrowserLiveStore({ + pollIntervalMs: 2, + transport: { + frame: () => { + const request = Promise.withResolvers<{ status: number; body: unknown }>(); + frames.push(request); + return request.promise; + }, + navigate: async (url) => { + hostUrl = url; + return { status: 200, body: { ok: true, data: { url } } }; + }, + }, + }); + const unsubscribe = store.subscribeState(() => undefined); + try { + await waitFor(() => frames.length === 1); + frames[0]?.resolve(frame(hostUrl)); + await waitFor(() => store.getStateSnapshot().hydrated); + await waitFor(() => frames.length === 2); + await store.navigate(A); + hostUrl = B; + frames[1]?.resolve(frame(hostUrl)); + await waitFor(() => frames.length === 3); + assert.notEqual(store.getStateSnapshot().location?.url, B); + frames[2]?.resolve(frame(hostUrl)); + await waitFor(() => store.getStateSnapshot().location?.url === B); + assert.equal(store.getStateSnapshot().state?.url, B); + } finally { + unsubscribe(); + } +}); + +test("failed navigation releases location only to a post-settlement poll", async () => { + let hostUrl = "about:blank"; + const frames: PromiseWithResolvers<{ status: number; body: unknown }>[] = []; + const store = createBrowserLiveStore({ + pollIntervalMs: 2, + transport: { + frame: () => { + const request = Promise.withResolvers<{ status: number; body: unknown }>(); + frames.push(request); + return request.promise; + }, + navigate: async () => ({ status: 500, body: { ok: false, error: "Navigation failed" } }), + }, + }); + const unsubscribe = store.subscribeState(() => undefined); + try { + await waitFor(() => frames.length === 1); + frames[0]?.resolve(frame(hostUrl)); + await waitFor(() => store.getStateSnapshot().hydrated); + await waitFor(() => frames.length === 2); + await store.navigate(A); + assert.equal(store.getStateSnapshot().navigationError, "Navigation failed"); + hostUrl = C; + frames[1]?.resolve(frame(hostUrl)); + await waitFor(() => frames.length === 3); + assert.notEqual(store.getStateSnapshot().location?.url, C); + frames[2]?.resolve(frame(hostUrl)); + await waitFor(() => store.getStateSnapshot().location?.url === C); + assert.equal(store.getStateSnapshot().state?.url, C); + } finally { + unsubscribe(); + } +}); diff --git a/frontend/scripts/agent-browser-tools-regressions.test.ts b/frontend/scripts/agent-browser-tools-regressions.test.ts new file mode 100644 index 000000000..eb826daad --- /dev/null +++ b/frontend/scripts/agent-browser-tools-regressions.test.ts @@ -0,0 +1,232 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { readFileSnippet } from "../src/features/agent/fs-store"; +import { + normalizeBrowserInput, + resolveBrowserInput, +} from "../src/features/agent/tools/browser-url"; +import { + browserKeyInputs, + browserMouseButton, + browserViewportPoint, +} from "../src/features/agent/ui/agent-browser-input"; +import { browserLocationUpdate } from "../src/features/agent/ui/agent-browser-location"; +import { workspaceFilePath } from "../src/features/agent/workspace-file-link"; + +const frontendRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const repositoryRoot = resolve(frontendRoot, ".."); + +function frontendSource(path: string): string { + return readFileSync(resolve(frontendRoot, path), "utf8"); +} + +test("desktop and web render the same CDP browser surface", () => { + const browser = frontendSource("src/features/agent/ui/agent-browser.tsx"); + const panel = frontendSource("src/features/agent/ui/agent-browser-panel.tsx"); + const tabs = frontendSource("src/features/agent/ui/computer-tab-panel.tsx"); + assert.match(browser, / { + const windowManager = frontendSource("desktop/logic/window-manager.ts"); + const security = frontendSource("desktop/logic/security.ts"); + assert.doesNotMatch(windowManager, new RegExp(`${"webview"}Tag\\s*:\\s*true`, "u")); + assert.equal(security.includes(`will-attach-${"webview"}`), false); + assert.equal(security.includes(`getType() === "${"webview"}"`), false); +}); + +test("visible navigation, state, history, viewport, and input use browser-host routes", () => { + const browser = frontendSource("src/features/agent/ui/agent-browser.tsx"); + const panel = frontendSource("src/features/agent/ui/agent-browser-panel.tsx"); + const surface = frontendSource("src/features/agent/ui/agent-browser-screencast.tsx"); + const liveStore = frontendSource("src/features/agent/ui/agent-browser-live-store.ts"); + const extension = frontendSource("desktop/resources/pi-extensions/browser.ts"); + for (const verb of ["back", "forward", "reload"]) { + assert.match(browser, new RegExp(`postLiveVerb\\(\"${verb}\"\\)`, "u")); + } + for (const route of ["frame", "navigate"]) { + assert.equal(liveStore.includes(`/api/agent/browser/${route}`), true, route); + } + for (const route of ["input", "viewport"]) { + assert.equal(surface.includes(`postBrowser("${route}"`), true, route); + } + assert.equal(extension.includes("/api/agent/browser/${verb}"), true); + assert.equal(panel.includes('fetch("/api/agent/browser/navigate"'), false); + assert.equal(surface.includes("/api/agent/browser/navigate"), false); + assert.equal(panel.includes("navigateBrowserHost(result.url)"), true); +}); + +test("browser navigation rejects files and protected destinations", () => { + const cwd = "/workspace/project"; + for (const value of [ + "file:///workspace/project/private.txt", + "/workspace/project/private.txt", + "./private.txt", + "../private.txt", + "http://10.0.0.1/private", + "https://user:password@example.com/private", + ]) { + assert.equal(normalizeBrowserInput(value, cwd), "", value); + } + assert.equal( + normalizeBrowserInput("http://localhost:3000/page", cwd), + "http://localhost:3000/page", + ); + assert.equal(normalizeBrowserInput("https://example.com/page", cwd), "https://example.com/page"); + assert.deepEqual(resolveBrowserInput("src/app.ts", cwd), { + kind: "file", + path: "src/app.ts", + }); + assert.deepEqual(resolveBrowserInput("file:///workspace/project/src/app.ts", cwd), { + kind: "file", + path: "src/app.ts", + }); + assert.equal(resolveBrowserInput("/workspace/private.txt", cwd).kind, "unsupported"); + assert.equal(resolveBrowserInput("ftp://example.com/private", cwd).kind, "unsupported"); +}); + +test("browser location emits each observed host change once", () => { + assert.deepEqual(browserLocationUpdate("", "https://page-a.test/"), { + emittedUrl: "https://page-a.test/", + location: "https://page-a.test/", + }); + assert.deepEqual(browserLocationUpdate("https://page-a.test/", "https://page-b.test/"), { + emittedUrl: "https://page-b.test/", + location: "https://page-b.test/", + }); + assert.deepEqual(browserLocationUpdate("https://page-b.test/", "https://page-b.test/"), { + emittedUrl: "https://page-b.test/", + location: null, + }); +}); + +test("pointer and keyboard input preserve viewport and shortcut behavior", () => { + assert.deepEqual( + browserViewportPoint( + { height: 400, left: 100, top: 50, width: 800 }, + { height: 800, width: 1600 }, + { clientX: 500, clientY: 250 }, + ), + { x: 800, y: 400 }, + ); + assert.deepEqual( + browserViewportPoint(null, { height: 800, width: 1600 }, { clientX: 500, clientY: 250 }), + { x: 0, y: 0 }, + ); + assert.equal(browserMouseButton(0), "left"); + assert.equal(browserMouseButton(1), "middle"); + assert.equal(browserMouseButton(2), "right"); + assert.deepEqual( + browserKeyInputs("down", { + altKey: false, + code: "KeyA", + ctrlKey: false, + key: "a", + metaKey: false, + }), + [ + { code: "KeyA", key: "a", kind: "key", type: "down" }, + { code: "KeyA", key: "a", kind: "key", text: "a", type: "char" }, + ], + ); + assert.deepEqual( + browserKeyInputs("down", { + altKey: false, + code: "KeyK", + ctrlKey: false, + key: "k", + metaKey: true, + }), + [], + ); + assert.deepEqual( + browserKeyInputs("down", { + altKey: false, + code: "Enter", + ctrlKey: false, + key: "Enter", + metaKey: false, + }), + [ + { code: "Enter", key: "Enter", kind: "key", type: "down" }, + { code: "Enter", key: "Enter", kind: "key", text: "\r", type: "char" }, + ], + ); +}); + +test("workspace file intent is lexical, bounded, and platform-neutral", () => { + for (const [value, cwd, expected] of [ + ["src/app.ts", "/workspace/project", "src/app.ts"], + ["/workspace/project/src/app.ts", "/workspace/project", "src/app.ts"], + ["file:///workspace/project/src/app.ts", "/workspace/project", "src/app.ts"], + ["C:\\workspace\\project\\src\\app.ts", "C:\\workspace\\project", "src/app.ts"], + ]) { + assert.equal(workspaceFilePath(value, cwd), expected, value); + } + for (const value of [ + "../secret.txt", + "/workspace/secret.txt", + "file:///workspace/secret.txt", + "file:///workspace/project/%2e%2e/secret.txt", + "file:///workspace/project/%ZZ/secret.txt", + "file:///workspace/project/%00secret.txt", + ]) { + assert.equal(workspaceFilePath(value, "/workspace/project"), null, value); + } +}); + +test("assistant file links use the Files action", () => { + const markdown = frontendSource("src/features/agent/ui/assistant-markdown.tsx"); + assert.equal(markdown.includes("requestFileOpen"), true); + assert.equal(markdown.includes("Local paths resolve to a file:// URL"), false); +}); + +test("the Files API retains the workspace root boundary", () => { + const store = readFileSync( + resolve(repositoryRoot, "frontend/src/features/agent/fs-store.ts"), + "utf8", + ); + const route = readFileSync( + resolve(repositoryRoot, "frontend/src/app/api/agent/fs/file/route.ts"), + "utf8", + ); + assert.match(store, /ensureInside\(root, path\.resolve\(root, relPath\)\)/u); + assert.match(store, /Path escapes project root/u); + assert.match(route, /readFileSnippet\(cwd, relPath\)/u); + assert.doesNotMatch(route, /file:\/\//u); +}); + +test("the Files API rejects a real path traversal before reading", async () => { + const root = await mkdtemp(join(tmpdir(), "local-studio-file-boundary-")); + const workspace = join(root, "workspace"); + await mkdir(workspace); + await writeFile(join(workspace, "inside.txt"), "inside", "utf8"); + await writeFile(join(root, "outside.txt"), "outside", "utf8"); + try { + assert.deepEqual(await readFileSnippet(workspace, "inside.txt"), { + content: "inside", + size: 6, + truncated: false, + }); + await assert.rejects(readFileSnippet(workspace, "../outside.txt"), /escapes project root/u); + await assert.rejects(readFileSnippet("/", "etc/passwd"), /allowed workspace root/u); + } finally { + await rm(root, { force: true, recursive: true }); + } +}); diff --git a/frontend/src/features/agent/tools/browser-url.ts b/frontend/src/features/agent/tools/browser-url.ts index 373de581b..aaa10610f 100644 --- a/frontend/src/features/agent/tools/browser-url.ts +++ b/frontend/src/features/agent/tools/browser-url.ts @@ -1,63 +1,47 @@ -// Browser URL normalization for the embedded browser tool. Handles file://, -// relative paths under the project cwd, http(s), localhost, and a search-engine -// fallback for free-text input. Keep the fallback away from Google because the -// embedded WebKit view can get trapped on Google bot-protection refresh loops. - -import { sanitizeLocalFileUrl } from "@/features/agent/sanitize-embedded-browser-url"; +import { sanitizeBrowserPaneUrl } from "@/features/agent/sanitize-embedded-browser-url"; import { DEFAULT_BROWSER_URL } from "@/features/agent/tools/persistence"; +import { workspaceFilePath } from "@/features/agent/workspace-file-link"; + +export type BrowserInputResolution = + | { kind: "file"; path: string } + | { kind: "navigate"; url: string } + | { kind: "unsupported"; message: string }; -function encodeFilePath(pathValue: string): string { - const normalized = pathValue.replace(/\\/g, "/"); - const withLeadingSlash = normalized.startsWith("/") ? normalized : `/${normalized}`; - return `file://${withLeadingSlash.split("/").map(encodeURIComponent).join("/")}`; +const FILE_INPUT = /^(?:file:\/\/|~\/|\.{1,2}[\\/]|\/|[A-Za-z]:[\\/])/iu; +const URI_SCHEME = /^[A-Za-z][A-Za-z0-9+.-]*:/u; +const UNSUPPORTED_FILE = "Local files can only be opened from the active workspace."; +const UNSUPPORTED_URL = "Only public or localhost HTTP(S) URLs are supported."; + +function navigation(url: string): BrowserInputResolution { + const accepted = sanitizeBrowserPaneUrl(url); + return accepted + ? { kind: "navigate", url: accepted } + : { kind: "unsupported", message: UNSUPPORTED_URL }; } -function resolveRelativeFilePath(cwd: string, value: string): string { - const segments = `${cwd.replace(/\/+$/, "")}/${value}`.split("/"); - const resolved: string[] = []; - for (const segment of segments) { - if (!segment || segment === ".") continue; - if (segment === "..") { - resolved.pop(); - continue; - } - resolved.push(segment); - } - return `/${resolved.join("/")}`; +function fileResolution(value: string, cwd: string): BrowserInputResolution { + const path = workspaceFilePath(value, cwd); + return path ? { kind: "file", path } : { kind: "unsupported", message: UNSUPPORTED_FILE }; } -function expandHomeFilePath(cwd: string, value: string): string | null { - const homeMatch = cwd.match(/^(\/Users\/[^/]+|\/home\/[^/]+)(?:\/|$)/); - if (!homeMatch) return null; - return `${homeMatch[1]}${value.slice(1)}`; +export function resolveBrowserInput(raw: string, cwd: string): BrowserInputResolution { + const value = raw.trim(); + if (!value) return { kind: "navigate", url: DEFAULT_BROWSER_URL }; + if (FILE_INPUT.test(value)) return fileResolution(value, cwd); + if (/^https?:\/\//iu.test(value)) return navigation(value); + if (URI_SCHEME.test(value)) return { kind: "unsupported", message: UNSUPPORTED_URL }; + if (/^(localhost|127\.0\.0\.1|\[::1\])(:\d+)?([/?#].*)?$/iu.test(value)) { + return navigation(`http://${value}`); + } + if (/^[\w.-]+:\d+([/?#].*)?$/u.test(value)) return navigation(`http://${value}`); + if (/^[\w-]+(\.[\w-]+)+([/:?#].*)?$/u.test(value)) { + return navigation(`https://${value}`); + } + if (value.includes("/") || value.includes("\\")) return fileResolution(value, cwd); + return navigation(`https://duckduckgo.com/?q=${encodeURIComponent(value)}`); } export function normalizeBrowserInput(raw: string, cwd: string): string { - const value = raw.trim(); - if (!value) return DEFAULT_BROWSER_URL; - if (/^file:\/\//i.test(value)) { - return sanitizeLocalFileUrl(value) ?? ""; - } - if (value.startsWith("~/") && cwd) { - const expanded = expandHomeFilePath(cwd, value); - if (expanded) return encodeFilePath(expanded); - } - if (value.startsWith("/")) return encodeFilePath(value); - if ((value.startsWith("./") || value.startsWith("../")) && cwd) { - return encodeFilePath(resolveRelativeFilePath(cwd, value)); - } - if (/^https?:\/\//i.test(value)) return value; - if (/^(localhost|127\.0\.0\.1|\[::1\])(:\d+)?([/?#].*)?$/i.test(value)) { - return `http://${value}`; - } - if (/^[\w.-]+:\d+([/?#].*)?$/.test(value)) { - return `http://${value}`; - } - if (/^[\w-]+(\.[\w-]+)+([/:?#].*)?$/.test(value)) { - return `https://${value}`; - } - if (value.includes("/") && cwd) { - return encodeFilePath(resolveRelativeFilePath(cwd, value)); - } - return `https://duckduckgo.com/?q=${encodeURIComponent(value)}`; + const result = resolveBrowserInput(raw, cwd); + return result.kind === "navigate" ? result.url : ""; } diff --git a/frontend/src/features/agent/ui/agent-browser-effects.ts b/frontend/src/features/agent/ui/agent-browser-effects.ts index 5be8be693..929b04a79 100644 --- a/frontend/src/features/agent/ui/agent-browser-effects.ts +++ b/frontend/src/features/agent/ui/agent-browser-effects.ts @@ -1,5 +1,4 @@ -import { useRef, type Dispatch, type RefObject, type SetStateAction } from "react"; -import type { BrowserPaneState } from "@/features/agent/ui/agent-browser-screencast"; +import type { Dispatch, SetStateAction } from "react"; import { useMountSubscription } from "@/hooks/use-mount-subscription"; export type LocalhostSite = { @@ -52,107 +51,22 @@ export function useLocalhostSitesEffects({ }, [enabled, onErrorChange, onLoadingChange, onSitesChange]); } -type BrowserWebview = HTMLElement & { - executeJavaScript: (script: string, userGesture?: boolean) => Promise; - getURL: () => string; - loadURL: (url: string) => Promise; - getTitle?: () => string; - canGoBack?: () => boolean; - canGoForward?: () => boolean; -}; - type UseAgentBrowserEffectsParams = { url: string; readingMode: boolean; - isElectron: boolean; - webviewRef: RefObject; fetchReadable: (target: string) => Promise; - onLocationChange?: (value: string) => void; - onNavState?: (state: BrowserPaneState) => void; enabled?: boolean; }; -export const shouldLoadBrowserUrl = (desired: string, current: string, observed: string): boolean => - Boolean(desired && desired !== observed && desired !== current); - -export const shouldSyncBrowserLocation = ( - desired: string, - observed: string, - current: string, -): boolean => desired === observed || current === desired; - export function useAgentBrowserEffects({ url, readingMode, - isElectron, - webviewRef, fetchReadable, - onLocationChange, - onNavState, enabled = true, }: UseAgentBrowserEffectsParams): void { - const observedUrl = useRef(url); - useMountSubscription(() => { if (enabled && url && readingMode) { void fetchReadable(url); } }, [enabled, fetchReadable, readingMode, url]); - - useMountSubscription(() => { - if (!enabled || !isElectron || readingMode) return; - const webview = webviewRef.current; - if (!webview) return; - const navigate = () => { - if (typeof webview.getURL !== "function" || typeof webview.loadURL !== "function") return; - try { - const current = webview.getURL(); - if (shouldLoadBrowserUrl(url, current, observedUrl.current)) { - void webview - .loadURL(url) - .then(() => { - const loaded = webview.getURL(); - observedUrl.current = loaded; - if (loaded) onLocationChange?.(loaded); - }) - .catch(() => undefined); - } - } catch { - return; - } - }; - navigate(); - webview.addEventListener("dom-ready", navigate as EventListener); - return () => webview.removeEventListener("dom-ready", navigate as EventListener); - }, [enabled, isElectron, onLocationChange, readingMode, url, webviewRef]); - - useMountSubscription(() => { - if (!enabled || !isElectron || readingMode) return; - const webview = webviewRef.current; - if (!webview) return; - const sync = () => { - try { - const current = webview.getURL(); - if (!shouldSyncBrowserLocation(url, observedUrl.current, current)) return; - observedUrl.current = current; - if (current) onLocationChange?.(current); - onNavState?.({ - url: current || url, - title: typeof webview.getTitle === "function" ? webview.getTitle() : "", - canGoBack: typeof webview.canGoBack === "function" ? webview.canGoBack() : false, - canGoForward: typeof webview.canGoForward === "function" ? webview.canGoForward() : false, - }); - } catch { - // Ignore transient webview state while navigating. - } - }; - webview.addEventListener("did-navigate", sync as EventListener); - webview.addEventListener("did-navigate-in-page", sync as EventListener); - webview.addEventListener("did-stop-loading", sync as EventListener); - return () => { - webview.removeEventListener("did-navigate", sync as EventListener); - webview.removeEventListener("did-navigate-in-page", sync as EventListener); - webview.removeEventListener("did-stop-loading", sync as EventListener); - }; - }, [enabled, isElectron, onLocationChange, onNavState, readingMode, url, webviewRef]); } diff --git a/frontend/src/features/agent/ui/agent-browser-input.ts b/frontend/src/features/agent/ui/agent-browser-input.ts new file mode 100644 index 000000000..55d9daa15 --- /dev/null +++ b/frontend/src/features/agent/ui/agent-browser-input.ts @@ -0,0 +1,58 @@ +type BrowserPoint = { x: number; y: number }; + +type BrowserBounds = { + height: number; + left: number; + top: number; + width: number; +}; + +type BrowserKeyEvent = { + altKey: boolean; + code: string; + ctrlKey: boolean; + key: string; + metaKey: boolean; +}; + +export type BrowserKeyInput = { + code: string; + key: string; + kind: "key"; + text?: string; + type: "char" | "down" | "up"; +}; + +export function browserViewportPoint( + bounds: BrowserBounds | null, + viewport: { height: number; width: number }, + point: { clientX: number; clientY: number }, +): BrowserPoint { + if (!bounds || bounds.width === 0 || bounds.height === 0) return { x: 0, y: 0 }; + return { + x: Math.round(((point.clientX - bounds.left) / bounds.width) * viewport.width), + y: Math.round(((point.clientY - bounds.top) / bounds.height) * viewport.height), + }; +} + +export function browserMouseButton(button: number): "left" | "middle" | "right" { + return button === 1 ? "middle" : button === 2 ? "right" : "left"; +} + +export function browserKeyInputs(type: "down" | "up", event: BrowserKeyEvent): BrowserKeyInput[] { + if (event.metaKey) return []; + const inputs: BrowserKeyInput[] = [{ kind: "key", type, key: event.key, code: event.code }]; + if (type === "down" && event.key.length === 1 && !event.ctrlKey && !event.altKey) { + inputs.push({ + kind: "key", + type: "char", + key: event.key, + code: event.code, + text: event.key, + }); + } + if (type === "down" && event.key === "Enter") { + inputs.push({ kind: "key", type: "char", key: "Enter", code: "Enter", text: "\r" }); + } + return inputs; +} diff --git a/frontend/src/features/agent/ui/agent-browser-live-store.ts b/frontend/src/features/agent/ui/agent-browser-live-store.ts new file mode 100644 index 000000000..211dc0431 --- /dev/null +++ b/frontend/src/features/agent/ui/agent-browser-live-store.ts @@ -0,0 +1,307 @@ +"use client"; + +import { useSyncExternalStore } from "react"; +import { Effect, Fiber, Schedule, Schema, Semaphore } from "effect"; +import { browserLocationUpdate } from "@/features/agent/ui/agent-browser-location"; + +export type BrowserPaneState = { + url: string; + title: string; + canGoBack: boolean; + canGoForward: boolean; +}; + +export type BrowserLiveStateSnapshot = { + hydrated: boolean; + state: BrowserPaneState | null; + location: { revision: number; url: string } | null; + navigationError: string | null; + unavailable: string | null; +}; + +export type BrowserLiveFrameSnapshot = { + frame: string | null; +}; + +type BrowserTransportResponse = { + status: number; + body: unknown; +}; + +export type BrowserLiveTransport = { + frame: () => Promise; + navigate: (url: string) => Promise; +}; + +export type BrowserLiveStore = { + getFrameSnapshot: () => BrowserLiveFrameSnapshot; + getStateSnapshot: () => BrowserLiveStateSnapshot; + navigate: (url: string) => Promise; + subscribeFrame: (listener: () => void) => () => void; + subscribeState: (listener: () => void) => () => void; +}; + +type BrowserLiveStoreOptions = { + pollIntervalMs?: number; + transport?: BrowserLiveTransport; +}; + +const BrowserPaneStateSchema = Schema.Struct({ + url: Schema.String, + title: Schema.String, + canGoBack: Schema.Boolean, + canGoForward: Schema.Boolean, +}); +const BrowserFrameResponseSchema = Schema.Struct({ + ok: Schema.Boolean, + error: Schema.optional(Schema.String), + data: Schema.optional( + Schema.Struct({ + frame: Schema.NullOr(Schema.String), + ...BrowserPaneStateSchema.fields, + }), + ), +}); +const BrowserActionResponseSchema = Schema.Struct({ + ok: Schema.Boolean, + error: Schema.optional(Schema.String), + data: Schema.optional(Schema.Struct({ url: Schema.optional(Schema.String) })), +}); +const EMPTY_STATE: BrowserLiveStateSnapshot = { + hydrated: false, + state: null, + location: null, + navigationError: null, + unavailable: null, +}; +const EMPTY_FRAME: BrowserLiveFrameSnapshot = { frame: null }; +const POLL_INTERVAL_MS = 110; + +async function request(path: string, init?: RequestInit): Promise { + const response = await fetch(path, init); + return { status: response.status, body: await response.json() }; +} + +const defaultTransport: BrowserLiveTransport = { + frame: () => request("/api/agent/browser/frame", { cache: "no-store" }), + navigate: (url) => + request("/api/agent/browser/navigate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ url }), + }), +}; + +function samePaneState(left: BrowserPaneState | null, right: BrowserPaneState): boolean { + return ( + left?.url === right.url && + left.title === right.title && + left.canGoBack === right.canGoBack && + left.canGoForward === right.canGoForward + ); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : "Browser command failed"; +} + +export function createBrowserLiveStore({ + pollIntervalMs = POLL_INTERVAL_MS, + transport = defaultTransport, +}: BrowserLiveStoreOptions = {}): BrowserLiveStore { + let stateSnapshot = EMPTY_STATE; + let frameSnapshot = EMPTY_FRAME; + let pollFiber: Fiber.Fiber | null = null; + let generation = 0; + let navigationSequence = 0; + let settledNavigationSequence = 0; + let pollSequence = 0; + let locationPollBarrier: number | null = null; + let locationRevision = 0; + let emittedUrl = ""; + const navigationLock = Semaphore.makeUnsafe(1); + const stateListeners = new Set<() => void>(); + const frameListeners = new Set<() => void>(); + + const emitState = (next: BrowserLiveStateSnapshot) => { + if (next === stateSnapshot) return; + stateSnapshot = next; + for (const listener of stateListeners) listener(); + }; + + const emitFrame = (frame: string) => { + if (frameSnapshot.frame === frame) return; + frameSnapshot = { frame }; + for (const listener of frameListeners) listener(); + }; + + const locationIsAuthoritative = (pollRequestSequence: number) => { + if (settledNavigationSequence !== navigationSequence) return false; + if (locationPollBarrier !== null && pollRequestSequence <= locationPollBarrier) return false; + locationPollBarrier = null; + return true; + }; + + const recordFrame = ( + state: BrowserPaneState, + frame: string | null, + pollRequestSequence: number, + ) => { + let nextLocation = stateSnapshot.location; + if (locationIsAuthoritative(pollRequestSequence)) { + const location = browserLocationUpdate(emittedUrl, state.url); + emittedUrl = location.emittedUrl; + if (location.location) { + nextLocation = { revision: (locationRevision += 1), url: location.location }; + } + } + const nextState = samePaneState(stateSnapshot.state, state) ? stateSnapshot.state : state; + if ( + !stateSnapshot.hydrated || + nextState !== stateSnapshot.state || + nextLocation !== stateSnapshot.location || + stateSnapshot.unavailable + ) { + emitState({ + ...stateSnapshot, + hydrated: true, + state: nextState, + location: nextLocation, + unavailable: null, + }); + } + if (frame) emitFrame(frame); + }; + + const pollOnce = async (activeGeneration: number) => { + const pollRequestSequence = (pollSequence += 1); + const response = await transport.frame(); + if (activeGeneration !== generation) return; + const payload = Schema.decodeUnknownSync(BrowserFrameResponseSchema)(response.body); + if (response.status === 503) { + const unavailable = payload.error ?? "Browser unavailable"; + if ( + !stateSnapshot.hydrated || + stateSnapshot.unavailable !== unavailable || + stateSnapshot.state + ) { + emitState({ + ...stateSnapshot, + hydrated: true, + state: null, + unavailable, + }); + } + return; + } + if (response.status < 200 || response.status >= 300 || !payload.ok || !payload.data) { + throw new Error(payload.error ?? `Browser frame failed with HTTP ${response.status}`); + } + recordFrame(payload.data, payload.data.frame, pollRequestSequence); + }; + + const settleNavigation = (navigationRequestSequence: number) => { + if (navigationRequestSequence !== navigationSequence) return false; + settledNavigationSequence = navigationRequestSequence; + locationPollBarrier = pollSequence; + return true; + }; + + const stop = () => { + generation += 1; + const fiber = pollFiber; + pollFiber = null; + if (fiber) void Effect.runPromise(Fiber.interrupt(fiber)); + stateSnapshot = EMPTY_STATE; + frameSnapshot = EMPTY_FRAME; + locationRevision = 0; + emittedUrl = ""; + locationPollBarrier = null; + }; + + const start = () => { + if (pollFiber || stateListeners.size + frameListeners.size === 0) return; + const activeGeneration = (generation += 1); + pollFiber = Effect.runFork( + Effect.tryPromise({ + try: () => pollOnce(activeGeneration), + catch: (error) => error, + }).pipe( + Effect.catch(() => Effect.void), + Effect.repeat(Schedule.spaced(pollIntervalMs)), + Effect.asVoid, + ), + ) as Fiber.Fiber; + }; + + const unsubscribe = (listeners: Set<() => void>, listener: () => void) => { + listeners.delete(listener); + if (stateListeners.size + frameListeners.size === 0) stop(); + }; + + const subscribe = (listeners: Set<() => void>, listener: () => void) => { + listeners.add(listener); + start(); + return () => unsubscribe(listeners, listener); + }; + + const navigate = (url: string) => { + const target = url.trim(); + if (!target) return Promise.resolve(); + const navigationRequestSequence = (navigationSequence += 1); + if (stateSnapshot.navigationError) { + emitState({ ...stateSnapshot, navigationError: null }); + } + const program = Effect.gen(function* () { + const response = yield* Effect.tryPromise({ + try: () => transport.navigate(target), + catch: (error) => error, + }); + const payload = yield* Schema.decodeUnknownEffect(BrowserActionResponseSchema)(response.body); + if (response.status < 200 || response.status >= 300 || !payload.ok) { + return yield* Effect.fail( + new Error(payload.error ?? `Browser navigation failed with HTTP ${response.status}`), + ); + } + settleNavigation(navigationRequestSequence); + }).pipe( + Effect.catch((error) => + Effect.sync(() => { + if (!settleNavigation(navigationRequestSequence)) return; + emitState({ ...stateSnapshot, navigationError: errorMessage(error) }); + }), + ), + ); + return Effect.runPromise(navigationLock.withPermit(program)); + }; + + return { + getFrameSnapshot: () => frameSnapshot, + getStateSnapshot: () => stateSnapshot, + navigate, + subscribeFrame: (listener) => subscribe(frameListeners, listener), + subscribeState: (listener) => subscribe(stateListeners, listener), + }; +} + +const browserLiveStore = createBrowserLiveStore(); + +export function navigateBrowserHost(url: string): Promise { + return browserLiveStore.navigate(url); +} + +export function useBrowserLiveFrame(): BrowserLiveFrameSnapshot { + return useSyncExternalStore( + browserLiveStore.subscribeFrame, + browserLiveStore.getFrameSnapshot, + () => EMPTY_FRAME, + ); +} + +export function useBrowserLiveState(): BrowserLiveStateSnapshot { + return useSyncExternalStore( + browserLiveStore.subscribeState, + browserLiveStore.getStateSnapshot, + () => EMPTY_STATE, + ); +} diff --git a/frontend/src/features/agent/ui/agent-browser-location.ts b/frontend/src/features/agent/ui/agent-browser-location.ts new file mode 100644 index 000000000..1524b7d33 --- /dev/null +++ b/frontend/src/features/agent/ui/agent-browser-location.ts @@ -0,0 +1,14 @@ +export type BrowserLocationUpdate = { + emittedUrl: string; + location: string | null; +}; + +export function browserLocationUpdate( + emittedUrl: string, + observedUrl: string, +): BrowserLocationUpdate { + if (!observedUrl || emittedUrl === observedUrl) { + return { emittedUrl, location: null }; + } + return { emittedUrl: observedUrl, location: observedUrl }; +} diff --git a/frontend/src/features/agent/ui/agent-browser-panel.tsx b/frontend/src/features/agent/ui/agent-browser-panel.tsx index e11eda097..df7e52f73 100644 --- a/frontend/src/features/agent/ui/agent-browser-panel.tsx +++ b/frontend/src/features/agent/ui/agent-browser-panel.tsx @@ -24,11 +24,8 @@ import { usePersistentTerminalOwners, type TerminalOwnersSnapshot, } from "@/features/agent/ui/use-persistent-terminal-owners"; -import { normalizeBrowserInput } from "@/features/agent/tools/browser-url"; -import { - sanitizeBrowserPaneUrl, - sanitizeLocalFileUrl, -} from "@/features/agent/sanitize-embedded-browser-url"; +import { resolveBrowserInput } from "@/features/agent/tools/browser-url"; +import { navigateBrowserHost } from "@/features/agent/ui/agent-browser-live-store"; import { useTools } from "@/features/agent/tools/context"; import type { ComputerTab } from "@/features/agent/tools/types"; import type { GitSummary, Project } from "@/features/agent/projects/types"; @@ -103,10 +100,6 @@ function closePersistedTerminalOwner(ownerKey: string) { if (owner) void terminalBridge()?.closeOwner?.(owner.mountKey); } -function acceptedBrowserUrl(url: string): string | null { - return /^file:\/\//i.test(url) ? sanitizeLocalFileUrl(url) : sanitizeBrowserPaneUrl(url); -} - export function AgentBrowserPanel({ handles, activeProject, @@ -122,10 +115,10 @@ export function AgentBrowserPanel({ const [sideChatSeed, setSideChatSeed] = useState(() => createSideChatSession(null, null, ""), ); + const [browserNavigationError, setBrowserNavigationError] = useState(null); const sideChatSession = sessions.find((session) => session.id === sideChatSeed.id) ?? sideChatSeed; const { registerComputerAside, startComputerResize } = handles; - const isElectron = typeof navigator !== "undefined" && /electron/i.test(navigator.userAgent); const terminalOwner = useMemo( () => terminalOwnerFor(activeProject, focusedSession), [activeProject, focusedSession], @@ -172,17 +165,23 @@ export function AgentBrowserPanel({ [selectTerminalOwner, visibleTerminalState.owners], ); const navigateBrowser = (value: string) => { - const next = normalizeBrowserInput(value, focusedSession?.cwd ?? activeProject?.path ?? ""); - if (!next) return; - const accepted = acceptedBrowserUrl(next); - if (!accepted) return; - tools.setBrowserUrl(accepted, accepted); - if (/^file:\/\//i.test(accepted)) return; - void fetch("/api/agent/browser/navigate", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ url: accepted }), - }).catch(() => undefined); + const result = resolveBrowserInput(value, focusedSession?.cwd ?? activeProject?.path ?? ""); + if (result.kind === "unsupported") { + setBrowserNavigationError(result.message); + return false; + } + setBrowserNavigationError(null); + if (result.kind === "file") { + tools.requestFileOpen(result.path); + return true; + } + tools.setBrowserUrl(result.url, result.url); + void navigateBrowserHost(result.url); + return true; + }; + const syncBrowserLocation = (value: string) => { + setBrowserNavigationError(null); + tools.setBrowserUrl(value, value); }; const openSideChat = useCallback( (draft?: SideChatDraft) => { @@ -286,10 +285,11 @@ export function AgentBrowserPanel({ gitSummary={gitSummary} models={models} modelsLoading={modelsLoading} - isElectron={isElectron} + browserNavigationError={browserNavigationError} onCloseSideChat={closeSideChat} onCompactSession={handles.compactFocusedSession} onNavigateBrowser={navigateBrowser} + onBrowserLocationChange={syncBrowserLocation} onOpenSideChat={openSideChat} onOpenTerminal={openTerminalForFocusedSession} onRenameSideChat={renameSideChat} @@ -348,7 +348,7 @@ const TAB_OPTIONS: Array<{ { tab: "browser", label: "Browser", - description: "Web, localhost, and file previews", + description: "Web and localhost previews", icon: Globe2, }, { tab: "diff", label: "Git", description: "Diffs, branch, commit, and push", icon: GitBranch }, diff --git a/frontend/src/features/agent/ui/agent-browser-screencast.tsx b/frontend/src/features/agent/ui/agent-browser-screencast.tsx index a5146935d..cd624c2fe 100644 --- a/frontend/src/features/agent/ui/agent-browser-screencast.tsx +++ b/frontend/src/features/agent/ui/agent-browser-screencast.tsx @@ -1,19 +1,6 @@ "use client"; import { effectTimeout, type EffectTimer } from "@/lib/effect-timers"; - -/** - * Live surface for the agent browser pane: renders the server-side headless - * Chromium (features/agent/browser-host) as a CDP screencast and forwards - * pointer/keyboard/wheel input back to it. The user and the agent are looking - * at — and driving — the same browser. - * - * Transport: polls /api/agent/browser/frame (~10fps) for the latest JPEG + - * nav state — Next's standalone server buffers locally-built SSE streams, and - * polling also survives a buffering proxy / Cloudflare for remote deploys. - * Input POSTs to /api/agent/browser/input, viewport sync to .../viewport. - */ - import { useRef, useState, @@ -22,33 +9,19 @@ import { type WheelEvent as ReactWheelEvent, } from "react"; import { useMountSubscription } from "@/hooks/use-mount-subscription"; - -export type BrowserPaneState = { - url: string; - title: string; - canGoBack: boolean; - canGoForward: boolean; -}; - -type FramePayload = { - ok: boolean; - error?: string; - data?: { frame: string | null } & BrowserPaneState; -}; +import { + browserKeyInputs, + browserMouseButton, + browserViewportPoint, +} from "@/features/agent/ui/agent-browser-input"; +import { useBrowserLiveFrame } from "@/features/agent/ui/agent-browser-live-store"; type Props = { - /** Desired URL from the address bar; navigated server-side when it diverges. */ - url: string; - onState: (state: BrowserPaneState) => void; - /** Called once when the host reports no Chromium — the pane should fall back to reading mode. */ - onUnavailable: (error: string) => void; - /** Frame polling pauses entirely while the surface is hidden. */ - visible?: boolean; + navigationError: string | null; }; const VIEWPORT_MIN = { width: 320, height: 240 }; const VIEWPORT_MAX = { width: 1920, height: 1200 }; -const POLL_INTERVAL_MS = 110; // ~9fps const MOVE_THROTTLE_MS = 33; function postBrowser(path: string, body: unknown): void { @@ -59,96 +32,11 @@ function postBrowser(path: string, body: unknown): void { }).catch(() => undefined); } -export function ScreencastSurface({ url, onState, onUnavailable, visible = true }: Props) { +export function ScreencastSurface({ navigationError }: Props) { const [container, setContainer] = useState(null); - const [frameSrc, setFrameSrc] = useState(null); - const [navError, setNavError] = useState(null); - const serverUrlRef = useRef(""); + const { frame } = useBrowserLiveFrame(); const viewportRef = useRef({ width: 1280, height: 800 }); const lastMoveAtRef = useRef(0); - const onStateRef = useRef(onState); - const onUnavailableRef = useRef(onUnavailable); - - // Mirror the latest callbacks into refs in the commit phase (never during - // render), so the long-lived poll loop always calls the current handlers - // without restarting. - useMountSubscription(() => { - onStateRef.current = onState; - onUnavailableRef.current = onUnavailable; - }, [onState, onUnavailable]); - - // ── Frame poll loop: sequential (no overlap), backs off on transient error, - // surfaces 503 once as unavailable. Pauses while the pane is hidden (panel - // collapsed) and idles at 1s while the document itself is hidden, so a - // background browser tab doesn't burn ~9 fetches+JPEG decodes per second. ── - useMountSubscription(() => { - if (!visible) return; - let disposed = false; - let timer: EffectTimer | null = null; - - const tick = async () => { - if (disposed) return; - if (typeof document !== "undefined" && document.visibilityState === "hidden") { - timer = effectTimeout(() => void tick(), 1_000); - return; - } - try { - const response = await fetch("/api/agent/browser/frame", { cache: "no-store" }); - if (response.status === 503) { - const payload = (await response.json().catch(() => null)) as FramePayload | null; - onUnavailableRef.current(payload?.error || "Browser unavailable"); - return; // stop polling; pane switches to reading mode - } - const payload = (await response.json()) as FramePayload; - if (!disposed && payload.ok && payload.data) { - if (payload.data.frame) setFrameSrc(`data:image/jpeg;base64,${payload.data.frame}`); - serverUrlRef.current = payload.data.url; - onStateRef.current({ - url: payload.data.url, - title: payload.data.title, - canGoBack: payload.data.canGoBack, - canGoForward: payload.data.canGoForward, - }); - } - } catch { - // transient — keep polling - } - if (!disposed) timer = effectTimeout(() => void tick(), POLL_INTERVAL_MS); - }; - - void tick(); - return () => { - disposed = true; - if (timer) timer.cancel(); - }; - }, [visible]); - - // ── Address-bar navigation: navigate server-side when the desired URL - // diverges from what the host last reported ──────────────────────────── - useMountSubscription(() => { - const target = url.trim(); - if (!target || target === serverUrlRef.current) return; - let cancelled = false; - void fetch("/api/agent/browser/navigate", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ url: target }), - }) - .then(async (response) => { - const payload = (await response.json()) as { ok: boolean; error?: string }; - if (cancelled) return; - setNavError(payload.ok ? null : (payload.error ?? "Navigation failed")); - }) - .catch((error) => { - if (!cancelled) { - setNavError(error instanceof Error ? error.message : "Navigation failed"); - } - }); - return () => { - cancelled = true; - }; - }, [url]); - // ── Viewport sync: match the headless viewport to the pane size ──────── useMountSubscription(() => { if (!container) return; @@ -180,16 +68,9 @@ export function ScreencastSurface({ url, onState, onUnavailable, visible = true // ── Input forwarding ──────────────────────────────────────────────────── const toViewport = (event: { clientX: number; clientY: number }) => { const rect = container?.getBoundingClientRect(); - if (!rect || rect.width === 0 || rect.height === 0) return { x: 0, y: 0 }; - return { - x: Math.round(((event.clientX - rect.left) / rect.width) * viewportRef.current.width), - y: Math.round(((event.clientY - rect.top) / rect.height) * viewportRef.current.height), - }; + return browserViewportPoint(rect ?? null, viewportRef.current, event); }; - const buttonName = (button: number) => - button === 1 ? "middle" : button === 2 ? "right" : "left"; - const handlePointerDown = (event: ReactPointerEvent) => { container?.focus(); event.currentTarget.setPointerCapture(event.pointerId); @@ -199,7 +80,7 @@ export function ScreencastSurface({ url, onState, onUnavailable, visible = true type: "down", x, y, - button: buttonName(event.button), + button: browserMouseButton(event.button), clickCount: Math.max(1, event.detail), }); }; @@ -211,7 +92,7 @@ export function ScreencastSurface({ url, onState, onUnavailable, visible = true type: "up", x, y, - button: buttonName(event.button), + button: browserMouseButton(event.button), clickCount: Math.max(1, event.detail), }); }; @@ -230,22 +111,10 @@ export function ScreencastSurface({ url, onState, onUnavailable, visible = true }; const handleKey = (type: "down" | "up") => (event: ReactKeyboardEvent) => { - // Leave app-level shortcuts (⌘K etc.) alone; forward everything else. - if (event.metaKey) return; + const inputs = browserKeyInputs(type, event); + if (inputs.length === 0) return; event.preventDefault(); - postBrowser("input", { kind: "key", type, key: event.key, code: event.code }); - if (type === "down" && event.key.length === 1 && !event.ctrlKey && !event.altKey) { - postBrowser("input", { - kind: "key", - type: "char", - key: event.key, - code: event.code, - text: event.key, - }); - } - if (type === "down" && event.key === "Enter") { - postBrowser("input", { kind: "key", type: "char", key: "Enter", code: "Enter", text: "\r" }); - } + for (const input of inputs) postBrowser("input", input); }; return ( @@ -263,9 +132,9 @@ export function ScreencastSurface({ url, onState, onUnavailable, visible = true onKeyUp={handleKey("up")} onContextMenu={(event) => event.preventDefault()} > - {frameSrc ? ( + {frame ? ( )} - {navError ? ( + {navigationError ? (
- {navError} + {navigationError}
) : null} diff --git a/frontend/src/features/agent/ui/agent-browser.tsx b/frontend/src/features/agent/ui/agent-browser.tsx index 734c2464e..93a72820f 100644 --- a/frontend/src/features/agent/ui/agent-browser.tsx +++ b/frontend/src/features/agent/ui/agent-browser.tsx @@ -1,25 +1,11 @@ "use client"; -/** - * Embedded browser pane for the agent surface. - * - * Two surfaces, switched by a toggle on the toolbar: - * - * 1. Live mode (default in Electron) — renders the page through ``. - * Auto-detects "blank" (empty body / failed navigation) and falls back to - * Reading mode without user intervention. - * 2. Reading mode (default in dev) — pulls the page through - * `/api/agent/browser/fetch`, strips scripts/styles, and renders clean - * text with markdown links. Always works because we're not relying on the - * upstream's CSP/X-Frame-Options. - */ import { useCallback, useRef, useState, type FormEvent } from "react"; import { ArrowLeftIcon, ArrowRightIcon, CloseIcon, ReloadIcon } from "@/ui/icons"; +import { Alert } from "@/ui"; import { DEFAULT_BROWSER_URL } from "@/features/agent/tools/persistence"; -import { - ScreencastSurface, - type BrowserPaneState, -} from "@/features/agent/ui/agent-browser-screencast"; +import { ScreencastSurface } from "@/features/agent/ui/agent-browser-screencast"; +import { useBrowserLiveState } from "@/features/agent/ui/agent-browser-live-store"; import { useAgentBrowserEffects, useLocalhostSitesEffects, @@ -27,33 +13,16 @@ import { } from "@/features/agent/ui/agent-browser-effects"; import { LocalhostStartPage } from "@/features/agent/ui/agent-browser-start-page"; import { ReadingView, type ReadablePage } from "@/features/agent/ui/agent-browser-reading-view"; - -type WebviewElement = HTMLElement & { - goBack: () => void; - goForward: () => void; - reload: () => void; - canGoBack: () => boolean; - canGoForward: () => boolean; - src: string; - loadURL: (url: string) => Promise; - getURL: () => string; - getTitle: () => string; - executeJavaScript: (script: string, userGesture?: boolean) => Promise; - capturePage: () => Promise<{ toDataURL: () => string }>; - addEventListener: HTMLElement["addEventListener"]; - removeEventListener: HTMLElement["removeEventListener"]; -}; +import { useMountSubscription } from "@/hooks/use-mount-subscription"; type Props = { url: string; inputValue: string; onInputChange: (value: string) => void; - onNavigate: (value: string) => void; + onNavigate: (value: string) => boolean; onLocationChange: (value: string) => void; onClose: () => void; - isElectron: boolean; - /** Screencast polling pauses while the hosting panel is hidden. */ - visible?: boolean; + navigationError: string | null; }; export function AgentBrowser({ @@ -63,16 +32,10 @@ export function AgentBrowser({ onNavigate, onLocationChange, onClose, - isElectron, - visible = true, + navigationError, }: Props) { - const webviewRef = useRef(null); - const [initialWebviewUrl] = useState(url); - // Live mode is the server-side screencast; it is the default everywhere and - // falls back to reading mode only when the host has no Chromium. const [readingMode, setReadingMode] = useState(false); const [liveUnavailable, setLiveUnavailable] = useState(null); - const [navState, setNavState] = useState(null); const [readable, setReadable] = useState(null); const [readingError, setReadingError] = useState(null); const [readingLoading, setReadingLoading] = useState(false); @@ -82,6 +45,9 @@ export function AgentBrowser({ const [localSites, setLocalSites] = useState([]); const [localSitesLoading, setLocalSitesLoading] = useState(false); const [localSitesError, setLocalSitesError] = useState(null); + const live = useBrowserLiveState(); + const onLocationChangeRef = useRef(onLocationChange); + const navState = live.state; const showStartPage = !hasOpenedUrl && url === DEFAULT_BROWSER_URL; const addressValue = showStartPage && inputValue === DEFAULT_BROWSER_URL ? "" : inputValue; @@ -108,11 +74,7 @@ export function AgentBrowser({ useAgentBrowserEffects({ url, readingMode, - isElectron, - webviewRef, fetchReadable, - onLocationChange, - onNavState: setNavState, enabled: !showStartPage, }); useLocalhostSitesEffects({ @@ -121,24 +83,26 @@ export function AgentBrowser({ onSitesChange: setLocalSites, onErrorChange: setLocalSitesError, }); + useMountSubscription(() => { + onLocationChangeRef.current = onLocationChange; + }, [onLocationChange]); + useMountSubscription(() => { + if (live.location) onLocationChangeRef.current(live.location.url); + }, [live.location?.revision]); + useMountSubscription(() => { + setLiveUnavailable(live.unavailable); + if (live.unavailable) setReadingMode(true); + }, [live.unavailable]); const postLiveVerb = useCallback((verb: "back" | "forward" | "reload") => { void fetch(`/api/agent/browser/${verb}`, { method: "POST" }).catch(() => undefined); }, []); const handleBack = () => { if (readingMode) return; - if (isElectron) { - webviewRef.current?.goBack(); - return; - } postLiveVerb("back"); }; const handleForward = () => { if (readingMode) return; - if (isElectron) { - webviewRef.current?.goForward(); - return; - } postLiveVerb("forward"); }; const handleReload = () => { @@ -162,17 +126,12 @@ export function AgentBrowser({ void fetchReadable(url); return; } - if (isElectron) { - webviewRef.current?.reload(); - return; - } postLiveVerb("reload"); }; const navigateFromBrowser = (value: string) => { const clean = value.trim(); if (!clean) return; - setHasOpenedUrl(true); - onNavigate(clean); + if (onNavigate(clean)) setHasOpenedUrl(true); }; const handleSubmit = (event: FormEvent) => { event.preventDefault(); @@ -256,6 +215,11 @@ export function AgentBrowser({ + {navigationError ? ( + + {navigationError} + + ) : null} {liveUnavailable ? (
{liveUnavailable}. Set LOCAL_STUDIO_CHROME_PATH to a Chromium-based browser binary to @@ -281,38 +245,8 @@ export function AgentBrowser({ loading={readingLoading} onLinkClick={onNavigate} /> - ) : isElectron ? ( - // Desktop: a real embedded Chromium webview. Loads file://, localhost, - // and the public web directly — the same surface the agent drives. - (() => { - type AnyTag = "webview"; - const Tag = "webview" as AnyTag; - return ( - { - webviewRef.current = node; - }} - src={initialWebviewUrl} - // @ts-expect-error — Electron-specific attribute. - allowpopups="true" - className="size-full" - style={{ width: "100%", height: "100%", display: "flex" }} - /> - ); - })() ) : ( - { - setNavState(state); - if (state.url && state.url !== url) onLocationChange(state.url); - }} - onUnavailable={(error) => { - setLiveUnavailable(error); - setReadingMode(true); - }} - /> + )}
diff --git a/frontend/src/features/agent/ui/assistant-markdown.tsx b/frontend/src/features/agent/ui/assistant-markdown.tsx index 3d0c85d0f..7e3d7fae7 100644 --- a/frontend/src/features/agent/ui/assistant-markdown.tsx +++ b/frontend/src/features/agent/ui/assistant-markdown.tsx @@ -20,6 +20,7 @@ import { peekHighlightFenced, } from "@/features/agent/highlight-cache"; import { normalizeBrowserInput } from "@/features/agent/tools/browser-url"; +import { navigateBrowserHost } from "@/features/agent/ui/agent-browser-live-store"; import { useToolsActions } from "@/features/agent/tools/context"; import { CopyablePathChip } from "@/features/agent/ui/copyable-path-chip"; @@ -247,25 +248,19 @@ function normalizeLooseMarkdownEmphasis(text: string): string { } type ToolHandlers = { + requestFileOpen: (path: string) => void; setComputerOpen: (open: boolean) => void; setComputerTab: (tab: "browser" | "files" | "status" | "canvas") => void; setBrowserUrl: (url: string, input?: string) => void; }; function buildComponentsWithAppLinks(tools: ToolHandlers): Components { - // Open a referenced file or URL in the in-app sidepanel browser. Local paths - // resolve to a file:// URL the browser renders directly; the chip's copy - // button stays independent (it only copies the raw path/link). - const openInBrowser = (raw: string) => { + const openFile = (raw: string) => { const cleaned = raw .trim() .replace(/^`+|`+$/g, "") .replace(/:\d+(?::\d+)?$/, ""); - const next = normalizeBrowserInput(cleaned, ""); - if (!next) return; - tools.setComputerOpen(true); - tools.setComputerTab("browser"); - tools.setBrowserUrl(next, next); + if (cleaned) tools.requestFileOpen(cleaned); }; return { ...components, @@ -281,7 +276,7 @@ function buildComponentsWithAppLinks(tools: ToolHandlers): Components { const value = nodeToPlainText(children).trim(); if (isFileReference(value)) { return ( - + {children} ); @@ -292,7 +287,7 @@ function buildComponentsWithAppLinks(tools: ToolHandlers): Components { const fileHref = typeof href === "string" && isFileReference(href); if (fileHref) { return ( - + {children} ); @@ -311,6 +306,7 @@ function buildComponentsWithAppLinks(tools: ToolHandlers): Components { tools.setComputerOpen(true); tools.setComputerTab("browser"); tools.setBrowserUrl(next, next); + void navigateBrowserHost(next); }} className="chat-ref-chip" title={href} @@ -333,11 +329,12 @@ function AssistantMarkdownInner({ text }: { text: string }) { const componentsWithAppLinks = useMemo( () => buildComponentsWithAppLinks({ + requestFileOpen: tools.requestFileOpen, setComputerOpen: tools.setComputerOpen, setComputerTab: tools.setComputerTab, setBrowserUrl: tools.setBrowserUrl, }), - [tools.setComputerOpen, tools.setComputerTab, tools.setBrowserUrl], + [tools.requestFileOpen, tools.setComputerOpen, tools.setComputerTab, tools.setBrowserUrl], ); return (
diff --git a/frontend/src/features/agent/ui/computer-tab-panel.tsx b/frontend/src/features/agent/ui/computer-tab-panel.tsx index 35ded5e7e..22a45f626 100644 --- a/frontend/src/features/agent/ui/computer-tab-panel.tsx +++ b/frontend/src/features/agent/ui/computer-tab-panel.tsx @@ -69,10 +69,11 @@ type ComputerTabPanelProps = { gitSummary?: GitSummary | null; models: AgentModel[]; modelsLoading: boolean; - isElectron: boolean; + browserNavigationError: string | null; onCloseSideChat: () => void; onCompactSession?: () => Promise; - onNavigateBrowser: (value: string) => void; + onBrowserLocationChange: (value: string) => void; + onNavigateBrowser: (value: string) => boolean; onOpenSideChat: (draft?: SideChatDraft) => void; onOpenTerminal: () => void; onRenameSideChat: (tabId: string, title: string) => void; @@ -197,17 +198,21 @@ function SideChatTab({ ); } -function BrowserTab({ isElectron, onNavigateBrowser, tools }: ComputerTabPanelProps) { +function BrowserTab({ + browserNavigationError, + onBrowserLocationChange, + onNavigateBrowser, + tools, +}: ComputerTabPanelProps) { return ( tools.setBrowserUrl(next, next)} + onLocationChange={onBrowserLocationChange} onClose={() => tools.setComputerOpen(false)} - isElectron={isElectron} - visible={tools.computer.open} + navigationError={browserNavigationError} /> ); } diff --git a/frontend/src/features/agent/ui/filesystem-panel-effects.ts b/frontend/src/features/agent/ui/filesystem-panel-effects.ts index 654cdc3c6..9653f3cee 100644 --- a/frontend/src/features/agent/ui/filesystem-panel-effects.ts +++ b/frontend/src/features/agent/ui/filesystem-panel-effects.ts @@ -2,6 +2,7 @@ import { useRef, type Dispatch, type MutableRefObject, type SetStateAction } fro import type { FileOpenRequest } from "@/features/agent/tools/types"; import type { FileComment, FsEntry } from "@/features/agent/filesystem-types"; import { useMountSubscription } from "@/hooks/use-mount-subscription"; +import { workspaceFilePath } from "@/features/agent/workspace-file-link"; type UseFilesystemPanelEffectsParams = { cwd: string | null; @@ -119,11 +120,20 @@ export function useFilesystemPanelEffects({ return; } handledFileOpenRequest.current = fileOpenRequest.id; - const rel = relativePathForRequest(fileOpenRequest.path, cwd); - if (!rel) return; + const rel = cwd ? workspaceFilePath(fileOpenRequest.path, cwd) : null; + if (!rel) { + setOpenFile(null); + setSaveError( + cwd + ? "Only files inside the active workspace can be opened." + : "Select a project to open local files.", + ); + return; + } + setSaveError(null); setOpenFile(rel); if (cwd) setLastOpenFileByProject(cwd, rel); - }, [cwd, fileOpenRequest, setLastOpenFileByProject, setOpenFile]); + }, [cwd, fileOpenRequest, setLastOpenFileByProject, setOpenFile, setSaveError]); useMountSubscription(() => { if (!cwd || !openFile) { @@ -131,7 +141,7 @@ export function useFilesystemPanelEffects({ setDraftContent(""); setFileTruncated(false); setFileSize(0); - setSaveError(null); + if (!cwd) setSaveError(null); setComments([]); return; } @@ -158,17 +168,21 @@ export function useFilesystemPanelEffects({ }; const commentsBody = (await commentsResponse.json()) as { comments?: FileComment[] }; if (cancelled) return; + if (!fileResponse.ok || fileBody.error) { + throw new Error(fileBody.error || `File read failed with HTTP ${fileResponse.status}`); + } const nextContent = fileBody.content ?? ""; setFileContent(nextContent); setDraftContent(nextContent); setFileTruncated(fileBody.truncated ?? false); setFileSize(fileBody.size ?? 0); setComments(commentsBody.comments ?? []); - } catch { + } catch (error) { if (!cancelled) { setFileContent(""); setDraftContent(""); setComments([]); + setSaveError(error instanceof Error ? error.message : "File read failed."); } } finally { if (!cancelled) setLoadingFile(false); @@ -189,23 +203,3 @@ export function useFilesystemPanelEffects({ setSaveError, ]); } - -function relativePathForRequest(path: string, cwd: string | null): string | null { - let raw = path.trim(); - if (!raw) return null; - if (/^file:\/\//i.test(raw)) { - try { - raw = decodeURIComponent(new URL(raw).pathname); - } catch { - return null; - } - } - raw = raw.replace(/^`|`$/g, "").replace(/:\d+(?::\d+)?$/, ""); - if (!raw || raw.includes("\0")) return null; - if (cwd && raw.startsWith(`${cwd.replace(/\/+$/, "")}/`)) { - return raw.slice(cwd.replace(/\/+$/, "").length + 1); - } - if (raw.startsWith("./")) return raw.slice(2); - if (!raw.startsWith("/") && !raw.startsWith("../")) return raw; - return null; -} diff --git a/frontend/src/features/agent/workspace-file-link.ts b/frontend/src/features/agent/workspace-file-link.ts new file mode 100644 index 000000000..87cec8c84 --- /dev/null +++ b/frontend/src/features/agent/workspace-file-link.ts @@ -0,0 +1,111 @@ +const WINDOWS_ABSOLUTE_PATH = /^[A-Za-z]:\//u; +const LINE_SUFFIX = /:\d+(?::\d+)?$/u; + +type AbsolutePath = { + caseInsensitive: boolean; + root: string; + segments: string[]; +}; + +function cleanReference(raw: string): string { + return raw + .trim() + .replace(/^`+|`+$/gu, "") + .replace(LINE_SUFFIX, ""); +} + +function slashPath(value: string): string { + return value.replaceAll("\\", "/"); +} + +function absolutePath(value: string): AbsolutePath | null { + const path = slashPath(value); + if (WINDOWS_ABSOLUTE_PATH.test(path)) { + return { + caseInsensitive: true, + root: path.slice(0, 2).toLowerCase(), + segments: normalizedSegments(path.slice(3).split("/")), + }; + } + if (!path.startsWith("/")) return null; + return { + caseInsensitive: false, + root: "/", + segments: normalizedSegments(path.slice(1).split("/")), + }; +} + +function normalizedSegments(values: string[], initial: string[] = []): string[] { + const segments = [...initial]; + for (const value of values) { + if (!value || value === ".") continue; + if (value === "..") { + segments.pop(); + continue; + } + segments.push(value); + } + return segments; +} + +function decodedFileUrl(value: string): string | null { + try { + const url = new URL(value); + if ( + url.protocol !== "file:" || + (url.hostname && url.hostname.toLowerCase() !== "localhost") || + url.search || + url.hash + ) { + return null; + } + const path = decodeURIComponent(url.pathname); + if (path.includes("\0")) return null; + return /^\/[A-Za-z]:\//u.test(path) ? path.slice(1) : path; + } catch { + return null; + } +} + +function homeDirectory(cwd: string): string | null { + const path = slashPath(cwd); + return ( + path.match(/^\/(?:Users|home)\/[^/]+/u)?.[0] ?? + path.match(/^[A-Za-z]:\/Users\/[^/]+/iu)?.[0] ?? + null + ); +} + +function inputPath(raw: string, cwd: string): string | null { + const value = cleanReference(raw); + if (!value || value.includes("\0")) return null; + if (/^file:\/\//iu.test(value)) return decodedFileUrl(value); + if (!value.startsWith("~/")) return slashPath(value); + const home = homeDirectory(cwd); + return home ? `${home}/${value.slice(2)}` : null; +} + +function sameSegment(left: string, right: string, caseInsensitive: boolean): boolean { + return caseInsensitive ? left.toLowerCase() === right.toLowerCase() : left === right; +} + +export function workspaceFilePath(raw: string, cwd: string): string | null { + const root = absolutePath(cwd); + const path = inputPath(raw, cwd); + if (!root || !path) return null; + const absolute = absolutePath(path); + const target = absolute ?? { + ...root, + segments: normalizedSegments(slashPath(path).split("/"), root.segments), + }; + if (target.root !== root.root || target.caseInsensitive !== root.caseInsensitive) return null; + if ( + root.segments.some( + (segment, index) => !sameSegment(segment, target.segments[index] ?? "", root.caseInsensitive), + ) + ) { + return null; + } + const relative = target.segments.slice(root.segments.length).join("/"); + return relative || null; +} diff --git a/package.json b/package.json index 58e9f8961..b93f374d7 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "release:check-commits": "node scripts/check-conventional-commits.mjs --range origin/main..HEAD", "release:stage-desktop": "node scripts/stage-desktop-release.mjs", "setup:git-hooks": "node scripts/setup-git-hooks.mjs", + "test:frontend:regression": "npm --prefix frontend run test:regression --", "test:integration": "npm --prefix services/agent-runtime test", "check": "npm run check:contracts && npm run check:structure && npm run check:frontend && npm run check:controller", "check:contracts": "node scripts/validate-shared-contracts.mjs", diff --git a/services/agent-runtime/src/browser-host/browser-request-classes.integration.test.ts b/services/agent-runtime/src/browser-host/browser-request-classes.integration.test.ts index 60f7443f4..a92d14c59 100644 --- a/services/agent-runtime/src/browser-host/browser-request-classes.integration.test.ts +++ b/services/agent-runtime/src/browser-host/browser-request-classes.integration.test.ts @@ -5,7 +5,8 @@ import { connect as netConnect } from "node:net"; import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { BrowserHost } from "./browser-host"; +import { Schema } from "effect"; +import { BrowserHost, type ScreencastFrame } from "./browser-host"; import { HostedPage } from "./hosted-page"; import { createBrowserNetworkPolicy, @@ -20,6 +21,12 @@ import { } from "./playwright"; const PUBLIC_ADDRESS = "8.8.8.8"; +const FrameColorSchema = Schema.Union([ + Schema.Literal("red"), + Schema.Literal("green"), + Schema.Literal("other"), +]); +type FrameColor = typeof FrameColorSchema.Type; function listen(server: Server): Promise { return new Promise((resolveListen, reject) => { @@ -104,7 +111,54 @@ function fixtureServer(blockedBase: string, loopbackBase: string, hosts: string[ }); } -function instrumentPolicy(policy: BrowserNetworkPolicy, attempts: string[]): BrowserNetworkPolicy { +function screencastServer(): Server { + return createServer((request, response) => { + const agentPage = request.url === "/b"; + const marker = agentPage ? "AGENT_B_SENTINEL" : "VISIBLE_A_SENTINEL"; + const background = agentPage ? "#00ff00" : "#ff0000"; + response.setHeader("content-type", "text/html"); + response.end( + `${marker}

${marker}

`, + ); + }); +} + +async function classifyFrameColor( + host: BrowserHost, + frame: ScreencastFrame, +): Promise { + const source = JSON.stringify(`data:image/jpeg;base64,${frame.data}`); + const result = await host.evaluate(`new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => { + const canvas = document.createElement("canvas"); + canvas.width = image.naturalWidth; + canvas.height = image.naturalHeight; + const context = canvas.getContext("2d"); + if (!context) { reject(new Error("Missing canvas context")); return; } + context.drawImage(image, 0, 0); + const positions = [0.25, 0.5, 0.75]; + const pixels = positions.flatMap((x) => positions.map((y) => + context.getImageData(Math.floor(canvas.width * x), Math.floor(canvas.height * y), 1, 1).data + )); + const channels = pixels.reduce( + (totals, pixel) => totals.map((total, index) => total + pixel[index]), + [0, 0, 0] + ).map((total) => total / pixels.length); + if (channels[0] > 200 && channels[1] < 70 && channels[2] < 70) { resolve("red"); return; } + if (channels[1] > 200 && channels[0] < 70 && channels[2] < 70) { resolve("green"); return; } + resolve("other"); + }; + image.onerror = () => reject(new Error("Frame decode failed")); + image.src = ${source}; + })`); + return Schema.decodeUnknownSync(FrameColorSchema)(result); +} + +function instrumentPolicy( + policy: BrowserNetworkPolicy, + attempts: string[], +): BrowserNetworkPolicy { return { allows: policy.allows, resolve: async (raw, mode) => { @@ -139,6 +193,33 @@ function attemptedWebSocket(attempts: string[], port: number): boolean { }); } +test("static navigation refreshes the visible screencast frame", { timeout: 20_000 }, async () => { + const fixture = screencastServer(); + const fixturePort = await listen(fixture); + const profile = await mkdtemp(path.join(os.tmpdir(), "local-studio-browser-frame-")); + const policy = createBrowserNetworkPolicy(); + const manager = new PlaywrightManager({ + launch: createPlaywrightSessionLauncher((mode) => path.join(profile, mode)), + policy, + resolveBinary: findBrowserBinary, + }); + const host = new BrowserHost(manager, { attachPage: HostedPage.attach }); + try { + await host.navigate(`http://127.0.0.1:${fixturePort}/a`); + const first = (await host.pollFrame()).frame; + assert.ok(first); + assert.equal(await classifyFrameColor(host, first), "red"); + await host.navigate(`http://127.0.0.1:${fixturePort}/b`); + const second = (await host.pollFrame()).frame; + assert.ok(second); + assert.equal(await classifyFrameColor(host, second), "green"); + assert.equal(await host.getText(), "AGENT_B_SENTINEL"); + } finally { + await host.stop().catch(() => undefined); + await Promise.allSettled([close(fixture), rm(profile, { force: true, recursive: true })]); + } +}); + test( "Playwright blocks redirects and every browser request class before denied sockets accept", { timeout: 40_000 }, From 0937de14dfd0ae7bdd61ddde42fc0d1ce60229c3 Mon Sep 17 00:00:00 2001 From: fettpl <38704082+fettpl@users.noreply.github.com> Date: Sun, 19 Jul 2026 11:08:12 +0200 Subject: [PATCH 4/4] fix: isolate browser state by agent session --- .../resources/pi-extensions/browser.ts | 68 ++- .../pi-extensions/sitegeist-browser.ts | 98 ++-- .../scripts/agent-browser-live-store.test.ts | 119 ++++- .../agent-browser-tools-regressions.test.ts | 4 +- .../agent/ui/agent-browser-effects.ts | 22 +- .../agent/ui/agent-browser-live-store.ts | 100 +++- .../features/agent/ui/agent-browser-panel.tsx | 13 +- .../agent/ui/agent-browser-screencast.tsx | 91 +++- .../src/features/agent/ui/agent-browser.tsx | 40 +- .../features/agent/ui/computer-tab-panel.tsx | 2 + .../browser-host-concurrency.test.ts | 316 +++++++++--- .../src/browser-host/browser-host.ts | 478 ++++++++++++++++-- .../browser-network-policy.test.ts | 6 +- ...rowser-request-classes.integration.test.ts | 135 +++-- .../src/browser-host/browser-session.ts | 31 ++ .../playwright-manager-revocation.test.ts | 17 + .../src/browser-host/playwright.ts | 115 +++-- .../src/browser-runtime-session.test.ts | 50 ++ .../src/browser-session-contract.ts | 29 ++ .../src/http/browser-handlers.ts | 322 +++++++++--- .../src/http/browser-session-handlers.test.ts | 310 ++++++++++++ .../http/browser-standalone-session.test.ts | 131 +++++ .../agent-runtime/src/pi-runtime-helpers.ts | 58 ++- services/agent-runtime/src/pi-runtime.ts | 152 +++--- services/agent-runtime/src/server.ts | 4 +- 25 files changed, 2255 insertions(+), 456 deletions(-) create mode 100644 services/agent-runtime/src/browser-host/browser-session.ts create mode 100644 services/agent-runtime/src/browser-runtime-session.test.ts create mode 100644 services/agent-runtime/src/browser-session-contract.ts create mode 100644 services/agent-runtime/src/http/browser-session-handlers.test.ts create mode 100644 services/agent-runtime/src/http/browser-standalone-session.test.ts diff --git a/frontend/desktop/resources/pi-extensions/browser.ts b/frontend/desktop/resources/pi-extensions/browser.ts index ba7276f3b..5ced9ab77 100644 --- a/frontend/desktop/resources/pi-extensions/browser.ts +++ b/frontend/desktop/resources/pi-extensions/browser.ts @@ -1,4 +1,5 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { Schema } from "effect"; import { Type } from "typebox"; type ToolResult = { @@ -6,19 +7,39 @@ type ToolResult = { details: Record; }; -const FRONTEND_BASE = process.env.LOCAL_STUDIO_FRONTEND_BASE ?? "http://127.0.0.1:3000"; -const BROWSER_SESSION_ID = process.env.LOCAL_STUDIO_BROWSER_SESSION_ID ?? ""; const DEFAULT_BROWSER_TOOL_TIMEOUT_MS = 60_000; +const BrowserActionResponseSchema = Schema.Struct({ + ok: Schema.Boolean, + data: Schema.optional(Schema.Unknown), + error: Schema.optional(Schema.String), +}); function readTimeoutMs(name: string, fallback: number): number { const value = Number(process.env[name]); return Number.isFinite(value) && value > 0 ? Math.trunc(value) : fallback; } -const BROWSER_TOOL_TIMEOUT_MS = readTimeoutMs( - "LOCAL_STUDIO_BROWSER_TOOL_TIMEOUT_MS", - DEFAULT_BROWSER_TOOL_TIMEOUT_MS, -); +type BrowserExtensionConfig = { + frontendBase: string; + sessionHeader: string; + sessionId: string; + timeoutMs: number; +}; + +function browserExtensionConfig(): BrowserExtensionConfig { + const sessionHeader = process.env.LOCAL_STUDIO_BROWSER_SESSION_HEADER ?? ""; + const sessionId = process.env.LOCAL_STUDIO_BROWSER_SESSION_ID ?? ""; + if (!sessionHeader || !sessionId) throw new Error("Browser session configuration is missing"); + return { + frontendBase: process.env.LOCAL_STUDIO_FRONTEND_BASE ?? "http://127.0.0.1:3000", + sessionHeader, + sessionId, + timeoutMs: readTimeoutMs( + "LOCAL_STUDIO_BROWSER_TOOL_TIMEOUT_MS", + DEFAULT_BROWSER_TOOL_TIMEOUT_MS, + ), + }; +} function failedToolResult( verb: string, @@ -33,21 +54,20 @@ function failedToolResult( } async function callBrowserAction( + config: BrowserExtensionConfig, verb: string, payload: Record, signal: AbortSignal, ): Promise { const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), BROWSER_TOOL_TIMEOUT_MS); + const timeout = setTimeout(() => controller.abort(), config.timeoutMs); const abort = () => controller.abort(); signal.addEventListener("abort", abort, { once: true }); if (signal.aborted) controller.abort(); - const response = await fetch(`${FRONTEND_BASE}/api/agent/browser/${verb}`, { + const response = await fetch(`${config.frontendBase}/api/agent/browser/${verb}`, { method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify( - BROWSER_SESSION_ID ? { ...payload, sessionId: BROWSER_SESSION_ID } : payload, - ), + headers: { "Content-Type": "application/json", [config.sessionHeader]: config.sessionId }, + body: JSON.stringify(payload), signal: controller.signal, }).finally(() => { clearTimeout(timeout); @@ -57,7 +77,7 @@ async function callBrowserAction( const errBody = await response.text().catch(() => ""); throw new Error(`HTTP ${response.status} ${errBody}`); } - const result = (await response.json()) as { ok: boolean; data?: unknown; error?: string }; + const result = Schema.decodeUnknownSync(BrowserActionResponseSchema)(await response.json()); if (!result.ok) throw new Error(result.error || "browser bridge returned ok=false"); const text = typeof result.data === "string" ? result.data : JSON.stringify(result.data, null, 2); return { @@ -67,18 +87,22 @@ async function callBrowserAction( } async function safeBrowserAction( + config: BrowserExtensionConfig, verb: string, payload: Record, signal: AbortSignal, ): Promise { try { - return await callBrowserAction(verb, payload, signal); + return await callBrowserAction(config, verb, payload, signal); } catch (error) { return failedToolResult(verb, payload, error); } } export default function registerBrowserExtension(pi: ExtensionAPI) { + const config = browserExtensionConfig(); + const run = (verb: string, payload: Record, signal: AbortSignal) => + safeBrowserAction(config, verb, payload, signal); pi.registerTool({ name: "browser_navigate", label: "Browser: Navigate", @@ -88,7 +112,7 @@ export default function registerBrowserExtension(pi: ExtensionAPI) { url: Type.String({ description: "Absolute http(s) URL to load" }), }), async execute(_id, params, signal) { - return safeBrowserAction("navigate", { url: params.url }, signal); + return run("navigate", { url: params.url }, signal); }, }); @@ -98,7 +122,7 @@ export default function registerBrowserExtension(pi: ExtensionAPI) { description: "Return the current URL of the embedded browser.", parameters: Type.Object({}), async execute(_id, _params, signal) { - return safeBrowserAction("get-url", {}, signal); + return run("get-url", {}, signal); }, }); @@ -109,7 +133,7 @@ export default function registerBrowserExtension(pi: ExtensionAPI) { "Return the visible text of the current page (innerText of ). Use after navigating to read page contents.", parameters: Type.Object({}), async execute(_id, _params, signal) { - return safeBrowserAction("get-text", {}, signal); + return run("get-text", {}, signal); }, }); @@ -120,7 +144,7 @@ export default function registerBrowserExtension(pi: ExtensionAPI) { "Return the rendered HTML of the current page. Useful when text alone isn't enough.", parameters: Type.Object({}), async execute(_id, _params, signal) { - return safeBrowserAction("get-html", {}, signal); + return run("get-html", {}, signal); }, }); @@ -130,7 +154,7 @@ export default function registerBrowserExtension(pi: ExtensionAPI) { description: "Capture a PNG screenshot of the current page; returns a base64 data URI.", parameters: Type.Object({}), async execute(_id, _params, signal) { - return safeBrowserAction("screenshot", {}, signal); + return run("screenshot", {}, signal); }, }); @@ -142,7 +166,7 @@ export default function registerBrowserExtension(pi: ExtensionAPI) { selector: Type.String({ description: "CSS selector for the element to click" }), }), async execute(_id, params, signal) { - return safeBrowserAction("click", { selector: params.selector }, signal); + return run("click", { selector: params.selector }, signal); }, }); @@ -154,7 +178,7 @@ export default function registerBrowserExtension(pi: ExtensionAPI) { deltaY: Type.Number({ description: "Pixels to scroll vertically" }), }), async execute(_id, params, signal) { - return safeBrowserAction("scroll", { deltaY: params.deltaY }, signal); + return run("scroll", { deltaY: params.deltaY }, signal); }, }); @@ -168,7 +192,7 @@ export default function registerBrowserExtension(pi: ExtensionAPI) { value: Type.String({ description: "Value to set" }), }), async execute(_id, params, signal) { - return safeBrowserAction("fill", { selector: params.selector, value: params.value }, signal); + return run("fill", { selector: params.selector, value: params.value }, signal); }, }); } diff --git a/frontend/desktop/resources/pi-extensions/sitegeist-browser.ts b/frontend/desktop/resources/pi-extensions/sitegeist-browser.ts index f609066b0..5b6129c95 100644 --- a/frontend/desktop/resources/pi-extensions/sitegeist-browser.ts +++ b/frontend/desktop/resources/pi-extensions/sitegeist-browser.ts @@ -7,6 +7,7 @@ // Protocol: docs/sitegeist-relay-protocol.md. import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { Schema } from "effect"; import { Type, type Static, type TSchema } from "typebox"; type ToolResult = { @@ -14,40 +15,60 @@ type ToolResult = { details: Record; }; -type RelayResponse = { result?: unknown; error?: { code?: number; message?: string } }; - const DEFAULT_RELAY_URL = "http://127.0.0.1:7717"; const DEFAULT_TIMEOUT_MS = 120_000; +const RelayResponseSchema = Schema.Struct({ + result: Schema.optional(Schema.Unknown), + error: Schema.optional( + Schema.Struct({ + code: Schema.optional(Schema.Number), + message: Schema.optional(Schema.String), + }), + ), +}); +const RelayCapabilitiesSchema = Schema.Struct({ + methods: Schema.optional(Schema.Array(Schema.String)), +}); + +type RelayConfig = { + relayUrl: string; + sessionId: string; + timeoutMs: number; + token: string; +}; -const RELAY_URL = (process.env.SITEGEIST_RELAY_URL || DEFAULT_RELAY_URL).replace(/\/+$/, ""); -const RELAY_TOKEN = process.env.SITEGEIST_RELAY_TOKEN ?? ""; -const RELAY_SESSION_ID = - process.env.SITEGEIST_RELAY_SESSION_ID || - process.env.LOCAL_STUDIO_BROWSER_SESSION_ID || - "default"; -const TIMEOUT_MS = (() => { +function relayConfig(): RelayConfig { + const sessionId = + process.env.SITEGEIST_RELAY_SESSION_ID || process.env.LOCAL_STUDIO_BROWSER_SESSION_ID || ""; + if (!sessionId) throw new Error("Sitegeist browser session configuration is missing"); const value = Number(process.env.SITEGEIST_RELAY_TOOL_TIMEOUT_MS); - return Number.isFinite(value) && value > 0 ? Math.trunc(value) : DEFAULT_TIMEOUT_MS; -})(); + return { + relayUrl: (process.env.SITEGEIST_RELAY_URL || DEFAULT_RELAY_URL).replace(/\/+$/, ""), + sessionId, + timeoutMs: Number.isFinite(value) && value > 0 ? Math.trunc(value) : DEFAULT_TIMEOUT_MS, + token: process.env.SITEGEIST_RELAY_TOKEN ?? "", + }; +} async function callRelay( + config: RelayConfig, method: string, params: Record, signal?: AbortSignal, ): Promise { const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS); + const timeout = setTimeout(() => controller.abort(), config.timeoutMs); const abort = () => controller.abort(); signal?.addEventListener("abort", abort, { once: true }); if (signal?.aborted) controller.abort(); const headers: Record = { "Content-Type": "application/json", - "X-Sitegeist-Session": RELAY_SESSION_ID, + "X-Sitegeist-Session": config.sessionId, }; - if (RELAY_TOKEN) headers.Authorization = `Bearer ${RELAY_TOKEN}`; + if (config.token) headers.Authorization = `Bearer ${config.token}`; - const response = await fetch(`${RELAY_URL}/rpc`, { + const response = await fetch(`${config.relayUrl}/rpc`, { method: "POST", headers, body: JSON.stringify({ jsonrpc: "2.0", id: Date.now(), method, params }), @@ -57,7 +78,9 @@ async function callRelay( signal?.removeEventListener("abort", abort); }); - const body = (await response.json().catch(() => ({}))) as RelayResponse; + const body = Schema.decodeUnknownSync(RelayResponseSchema)( + await response.json().catch(() => ({})), + ); if (!response.ok || body.error) { throw new Error(body.error?.message || `sitegeist relay HTTP ${response.status}`); } @@ -212,19 +235,25 @@ const TOOLS = [ }), ] as const; +type RunToolInput = { + name: string; + method: string; + params: Record; + rpcParams: Record; +}; + async function runTool( - name: string, - method: string, - params: Record, - rpcParams: Record, + config: RelayConfig, + input: RunToolInput, signal?: AbortSignal, ): Promise { + const { method, name, params, rpcParams } = input; try { - const result = await callRelay(method, rpcParams, signal); + const result = await callRelay(config, method, rpcParams, signal); const text = typeof result === "string" ? result : JSON.stringify(result, null, 2); return { content: [{ type: "text", text }], - details: { method, params, data: result, relaySessionId: RELAY_SESSION_ID }, + details: { method, params, data: result, relaySessionId: config.sessionId }, }; } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -235,14 +264,13 @@ async function runTool( } } -async function relayCapabilities(): Promise | null> { +async function relayCapabilities(config: RelayConfig): Promise | null> { try { const controller = new AbortController(); - const result = await callRelay("relay.capabilities", {}, controller.signal); - const methods = (result as { methods?: unknown })?.methods; - return Array.isArray(methods) - ? new Set(methods.filter((m): m is string => typeof m === "string")) - : null; + const result = Schema.decodeUnknownSync(RelayCapabilitiesSchema)( + await callRelay(config, "relay.capabilities", {}, controller.signal), + ); + return result.methods ? new Set(result.methods) : null; } catch { return null; } @@ -252,7 +280,8 @@ export default async function registerSitegeistBrowserExtension(pi: ExtensionAPI // Capability discovery: register only the tools the connected extension // implements. If discovery fails (relay down), register everything and let // each call surface the relay error. - const supported = await relayCapabilities(); + const config = relayConfig(); + const supported = await relayCapabilities(config); for (const tool of TOOLS) { if (supported && !supported.has(tool.method)) continue; @@ -263,7 +292,16 @@ export default async function registerSitegeistBrowserExtension(pi: ExtensionAPI parameters: tool.parameters, execute(_id, params, signal) { const args = params as Record; - return runTool(tool.name, tool.method, args, tool.pick(params as never), signal); + return runTool( + config, + { + name: tool.name, + method: tool.method, + params: args, + rpcParams: tool.pick(params as never), + }, + signal, + ); }, }); } diff --git a/frontend/scripts/agent-browser-live-store.test.ts b/frontend/scripts/agent-browser-live-store.test.ts index 2820ff81c..241a646c5 100644 --- a/frontend/scripts/agent-browser-live-store.test.ts +++ b/frontend/scripts/agent-browser-live-store.test.ts @@ -9,6 +9,7 @@ import { const A = "http://page.test/a"; const B = "http://page.test/b"; const C = "http://page.test/c"; +const SESSION = "session-a"; function frame(url: string) { return { @@ -43,18 +44,20 @@ function harness(initialUrl: string) { const navigations: string[] = []; const transport: BrowserLiveTransport = { frame: async () => frame(hostUrl), - navigate: async (url) => { + navigate: async (_session, url) => { navigations.push(url); hostUrl = url; return { status: 200, body: { ok: true, data: { url } } }; }, }; + const store = createBrowserLiveStore({ pollIntervalMs: 2, transport }); + store.focus(SESSION); return { host: (url: string) => { hostUrl = url; }, navigations, - store: createBrowserLiveStore({ pollIntervalMs: 2, transport }), + store, }; } @@ -96,13 +99,14 @@ test("remount keeps location blocked until explicit navigation settles", async ( pollIntervalMs: 2, transport: { frame: async () => frame(hostUrl), - navigate: async (url) => { + navigate: async (_session, url) => { const response = await request.promise; hostUrl = url; return response; }, }, }); + store.focus(SESSION); const unsubscribeA = store.subscribeState(() => undefined); await waitFor(() => store.getStateSnapshot().hydrated); const navigation = store.navigate(A); @@ -133,6 +137,7 @@ test("navigation settles against the host redirect destination", async () => { }, }, }); + store.focus(SESSION); const unsubscribe = store.subscribeState(() => undefined); try { await waitFor(() => store.getStateSnapshot().hydrated); @@ -152,7 +157,7 @@ test("explicit navigation dispatches in issue order and finishes at the latest t pollIntervalMs: 2, transport: { frame: async () => frame(hostUrl), - navigate: async (url) => { + navigate: async (_session, url) => { dispatches.push(url); const request = Promise.withResolvers<{ status: number; body: unknown }>(); requests.set(url, request); @@ -162,6 +167,7 @@ test("explicit navigation dispatches in issue order and finishes at the latest t }, }, }); + store.focus(SESSION); const unsubscribe = store.subscribeState(() => undefined); try { await waitFor(() => store.getStateSnapshot().hydrated); @@ -197,12 +203,13 @@ test("first poll after navigation settlement accepts an immediate redirect", asy frames.push(request); return request.promise; }, - navigate: async (url) => { + navigate: async (_session, url) => { hostUrl = url; return { status: 200, body: { ok: true, data: { url } } }; }, }, }); + store.focus(SESSION); const unsubscribe = store.subscribeState(() => undefined); try { await waitFor(() => frames.length === 1); @@ -236,6 +243,7 @@ test("failed navigation releases location only to a post-settlement poll", async navigate: async () => ({ status: 500, body: { ok: false, error: "Navigation failed" } }), }, }); + store.focus(SESSION); const unsubscribe = store.subscribeState(() => undefined); try { await waitFor(() => frames.length === 1); @@ -255,3 +263,104 @@ test("failed navigation releases location only to a post-settlement poll", async unsubscribe(); } }); + +test("focused session keys every request and switching aborts old traffic", async () => { + const frames: Array<{ session: string; signal: AbortSignal }> = []; + const navigations: Array<{ session: string; url: string; signal: AbortSignal }> = []; + const store = createBrowserLiveStore({ + pollIntervalMs: 2, + transport: { + frame: async (session, signal) => { + frames.push({ session, signal }); + return frame(session === "session-a" ? A : B); + }, + navigate: async (session, url, signal) => { + navigations.push({ session, url, signal }); + return { status: 200, body: { ok: true, data: { url } } }; + }, + }, + }); + const unsubscribe = store.subscribeState(() => undefined); + try { + await Effect.runPromise(Effect.sleep(10)); + assert.equal(frames.length, 0); + await store.navigate(A); + assert.equal(navigations.length, 0); + store.focus("session-a"); + await waitFor(() => frames.some((entry) => entry.session === "session-a")); + await store.navigate(A); + assert.equal(navigations.at(-1)?.session, "session-a"); + const oldSignal = frames.find((entry) => entry.session === "session-a")?.signal; + assert.ok(oldSignal); + store.focus("session-b"); + assert.equal(oldSignal.aborted, true); + await waitFor(() => frames.some((entry) => entry.session === "session-b")); + await store.navigate(B); + assert.equal(navigations.at(-1)?.session, "session-b"); + const requestCount = frames.length + navigations.length; + store.focus(null); + await store.navigate(C); + await Effect.runPromise(Effect.sleep(10)); + assert.equal(frames.length + navigations.length, requestCount); + } finally { + unsubscribe(); + } +}); + +test("an aborted navigation cannot report into the newly focused session", async () => { + let oldNavigationStarted = false; + const store = createBrowserLiveStore({ + pollIntervalMs: 2, + transport: { + frame: async (_session, signal) => { + await Effect.runPromise(Effect.sleep(100)); + if (signal.aborted) throw new Error("frame aborted"); + return frame(A); + }, + navigate: (session, url, signal) => { + if (session !== "session-a") { + return Promise.resolve({ status: 200, body: { ok: true, data: { url } } }); + } + oldNavigationStarted = true; + return new Promise<{ status: number; body: unknown }>((_resolve, reject) => { + signal.addEventListener("abort", () => reject(new Error("old navigation aborted")), { + once: true, + }); + }); + }, + }, + }); + store.focus("session-a"); + const oldNavigation = store.navigate(A); + await waitFor(() => oldNavigationStarted); + store.focus("session-b"); + const currentNavigation = store.navigate(B); + await Promise.all([oldNavigation, currentNavigation]); + assert.equal(store.getStateSnapshot().navigationError, null); +}); + +test("malformed focus keys never start stateful transport", async () => { + let requests = 0; + const store = createBrowserLiveStore({ + pollIntervalMs: 2, + transport: { + frame: async () => { + requests += 1; + return frame(A); + }, + navigate: async () => { + requests += 1; + return { status: 200, body: { ok: true, data: { url: A } } }; + }, + }, + }); + store.focus("bad key"); + const unsubscribe = store.subscribeState(() => undefined); + try { + await store.navigate(A); + await Effect.runPromise(Effect.sleep(10)); + assert.equal(requests, 0); + } finally { + unsubscribe(); + } +}); diff --git a/frontend/scripts/agent-browser-tools-regressions.test.ts b/frontend/scripts/agent-browser-tools-regressions.test.ts index eb826daad..399c26833 100644 --- a/frontend/scripts/agent-browser-tools-regressions.test.ts +++ b/frontend/scripts/agent-browser-tools-regressions.test.ts @@ -64,12 +64,14 @@ test("visible navigation, state, history, viewport, and input use browser-host r assert.equal(liveStore.includes(`/api/agent/browser/${route}`), true, route); } for (const route of ["input", "viewport"]) { - assert.equal(surface.includes(`postBrowser("${route}"`), true, route); + assert.equal(surface.includes(`"${route}"`), true, route); } assert.equal(extension.includes("/api/agent/browser/${verb}"), true); assert.equal(panel.includes('fetch("/api/agent/browser/navigate"'), false); assert.equal(surface.includes("/api/agent/browser/navigate"), false); assert.equal(panel.includes("navigateBrowserHost(result.url)"), true); + assert.equal(extension.includes("LOCAL_STUDIO_BROWSER_SESSION_HEADER"), true); + assert.equal(extension.includes("sessionId: BROWSER_SESSION_ID"), false); }); test("browser navigation rejects files and protected destinations", () => { diff --git a/frontend/src/features/agent/ui/agent-browser-effects.ts b/frontend/src/features/agent/ui/agent-browser-effects.ts index 929b04a79..0bf03fe0e 100644 --- a/frontend/src/features/agent/ui/agent-browser-effects.ts +++ b/frontend/src/features/agent/ui/agent-browser-effects.ts @@ -1,4 +1,5 @@ import type { Dispatch, SetStateAction } from "react"; +import { Schema } from "effect"; import { useMountSubscription } from "@/hooks/use-mount-subscription"; export type LocalhostSite = { @@ -10,6 +11,23 @@ export type LocalhostSite = { current?: boolean; }; +const LocalhostSiteSchema = Schema.Struct({ + port: Schema.Number, + url: Schema.String, + displayUrl: Schema.String, + title: Schema.String, + process: Schema.optional(Schema.String), + current: Schema.optional(Schema.Boolean), +}); +const LocalhostSitesResponseSchema = Schema.Struct({ + sites: Schema.optional(Schema.Array(LocalhostSiteSchema)), + error: Schema.optional(Schema.String), +}); + +export function decodeLocalhostSitesResponse(input: unknown) { + return Schema.decodeUnknownSync(LocalhostSitesResponseSchema)(input); +} + type UseLocalhostSitesEffectsParams = { enabled: boolean; onLoadingChange: Dispatch>; @@ -30,9 +48,9 @@ export function useLocalhostSitesEffects({ onErrorChange(null); void fetch("/api/agent/browser/localhosts", { cache: "no-store" }) .then(async (response) => { - const payload = (await response.json()) as { sites?: LocalhostSite[]; error?: string }; + const payload = decodeLocalhostSitesResponse(await response.json()); if (!response.ok || payload.error) throw new Error(payload.error || "Failed to scan"); - if (!cancelled) onSitesChange(payload.sites ?? []); + if (!cancelled) onSitesChange([...(payload.sites ?? [])]); }) .catch((error) => { if (!cancelled) { diff --git a/frontend/src/features/agent/ui/agent-browser-live-store.ts b/frontend/src/features/agent/ui/agent-browser-live-store.ts index 211dc0431..0203746f8 100644 --- a/frontend/src/features/agent/ui/agent-browser-live-store.ts +++ b/frontend/src/features/agent/ui/agent-browser-live-store.ts @@ -2,6 +2,11 @@ import { useSyncExternalStore } from "react"; import { Effect, Fiber, Schedule, Schema, Semaphore } from "effect"; +import { + browserSessionHeaders, + decodeBrowserSessionKey, + type BrowserSessionKey, +} from "@local-studio/agent-runtime/browser-session-contract"; import { browserLocationUpdate } from "@/features/agent/ui/agent-browser-location"; export type BrowserPaneState = { @@ -29,13 +34,18 @@ type BrowserTransportResponse = { }; export type BrowserLiveTransport = { - frame: () => Promise; - navigate: (url: string) => Promise; + frame: (sessionKey: BrowserSessionKey, signal: AbortSignal) => Promise; + navigate: ( + sessionKey: BrowserSessionKey, + url: string, + signal: AbortSignal, + ) => Promise; }; export type BrowserLiveStore = { getFrameSnapshot: () => BrowserLiveFrameSnapshot; getStateSnapshot: () => BrowserLiveStateSnapshot; + focus: (sessionKey: string | null) => void; navigate: (url: string) => Promise; subscribeFrame: (listener: () => void) => () => void; subscribeState: (listener: () => void) => () => void; @@ -83,12 +93,18 @@ async function request(path: string, init?: RequestInit): Promise request("/api/agent/browser/frame", { cache: "no-store" }), - navigate: (url) => + frame: (sessionKey, signal) => + request("/api/agent/browser/frame", { + cache: "no-store", + headers: browserSessionHeaders(sessionKey), + signal, + }), + navigate: (sessionKey, url, signal) => request("/api/agent/browser/navigate", { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { "Content-Type": "application/json", ...browserSessionHeaders(sessionKey) }, body: JSON.stringify({ url }), + signal, }), }; @@ -119,6 +135,8 @@ export function createBrowserLiveStore({ let locationPollBarrier: number | null = null; let locationRevision = 0; let emittedUrl = ""; + let sessionKey: BrowserSessionKey | null = null; + let sessionAbort: AbortController | null = null; const navigationLock = Semaphore.makeUnsafe(1); const stateListeners = new Set<() => void>(); const frameListeners = new Set<() => void>(); @@ -173,10 +191,14 @@ export function createBrowserLiveStore({ if (frame) emitFrame(frame); }; - const pollOnce = async (activeGeneration: number) => { + const pollOnce = async ( + activeGeneration: number, + activeSession: BrowserSessionKey, + signal: AbortSignal, + ) => { const pollRequestSequence = (pollSequence += 1); - const response = await transport.frame(); - if (activeGeneration !== generation) return; + const response = await transport.frame(activeSession, signal); + if (activeGeneration !== generation || sessionKey !== activeSession) return; const payload = Schema.decodeUnknownSync(BrowserFrameResponseSchema)(response.body); if (response.status === 503) { const unavailable = payload.error ?? "Browser unavailable"; @@ -207,24 +229,35 @@ export function createBrowserLiveStore({ return true; }; - const stop = () => { - generation += 1; - const fiber = pollFiber; - pollFiber = null; - if (fiber) void Effect.runPromise(Fiber.interrupt(fiber)); + const reset = () => { + const notifyState = stateSnapshot !== EMPTY_STATE; + const notifyFrame = frameSnapshot !== EMPTY_FRAME; stateSnapshot = EMPTY_STATE; frameSnapshot = EMPTY_FRAME; locationRevision = 0; emittedUrl = ""; locationPollBarrier = null; + if (notifyState) for (const listener of stateListeners) listener(); + if (notifyFrame) for (const listener of frameListeners) listener(); + }; + + const stop = (clear = true) => { + generation += 1; + const fiber = pollFiber; + pollFiber = null; + if (fiber) void Effect.runPromise(Fiber.interrupt(fiber)); + if (clear) reset(); }; const start = () => { - if (pollFiber || stateListeners.size + frameListeners.size === 0) return; + if (pollFiber || !sessionKey || stateListeners.size + frameListeners.size === 0) return; const activeGeneration = (generation += 1); + const activeSession = sessionKey; + const signal = sessionAbort?.signal; + if (!signal) return; pollFiber = Effect.runFork( Effect.tryPromise({ - try: () => pollOnce(activeGeneration), + try: () => pollOnce(activeGeneration, activeSession, signal), catch: (error) => error, }).pipe( Effect.catch(() => Effect.void), @@ -247,14 +280,17 @@ export function createBrowserLiveStore({ const navigate = (url: string) => { const target = url.trim(); - if (!target) return Promise.resolve(); + const activeSession = sessionKey; + const signal = sessionAbort?.signal; + if (!target || !activeSession || !signal) return Promise.resolve(); const navigationRequestSequence = (navigationSequence += 1); if (stateSnapshot.navigationError) { emitState({ ...stateSnapshot, navigationError: null }); } const program = Effect.gen(function* () { + if (sessionKey !== activeSession || signal.aborted) return; const response = yield* Effect.tryPromise({ - try: () => transport.navigate(target), + try: () => transport.navigate(activeSession, target, signal), catch: (error) => error, }); const payload = yield* Schema.decodeUnknownEffect(BrowserActionResponseSchema)(response.body); @@ -263,10 +299,13 @@ export function createBrowserLiveStore({ new Error(payload.error ?? `Browser navigation failed with HTTP ${response.status}`), ); } - settleNavigation(navigationRequestSequence); + if (sessionKey === activeSession && !signal.aborted) { + settleNavigation(navigationRequestSequence); + } }).pipe( Effect.catch((error) => Effect.sync(() => { + if (sessionKey !== activeSession || signal.aborted) return; if (!settleNavigation(navigationRequestSequence)) return; emitState({ ...stateSnapshot, navigationError: errorMessage(error) }); }), @@ -275,9 +314,30 @@ export function createBrowserLiveStore({ return Effect.runPromise(navigationLock.withPermit(program)); }; + const focus = (nextSessionKey: string | null) => { + let next: BrowserSessionKey | null = null; + if (nextSessionKey !== null) { + try { + next = decodeBrowserSessionKey(nextSessionKey); + } catch { + next = null; + } + } + if (next === sessionKey) return; + sessionAbort?.abort(); + sessionAbort = next ? new AbortController() : null; + sessionKey = next; + stop(); + navigationSequence = 0; + settledNavigationSequence = 0; + pollSequence = 0; + start(); + }; + return { getFrameSnapshot: () => frameSnapshot, getStateSnapshot: () => stateSnapshot, + focus, navigate, subscribeFrame: (listener) => subscribe(frameListeners, listener), subscribeState: (listener) => subscribe(stateListeners, listener), @@ -286,6 +346,10 @@ export function createBrowserLiveStore({ const browserLiveStore = createBrowserLiveStore(); +export function focusBrowserLiveSession(sessionKey: string | null): void { + browserLiveStore.focus(sessionKey); +} + export function navigateBrowserHost(url: string): Promise { return browserLiveStore.navigate(url); } diff --git a/frontend/src/features/agent/ui/agent-browser-panel.tsx b/frontend/src/features/agent/ui/agent-browser-panel.tsx index df7e52f73..062a02a90 100644 --- a/frontend/src/features/agent/ui/agent-browser-panel.tsx +++ b/frontend/src/features/agent/ui/agent-browser-panel.tsx @@ -25,7 +25,11 @@ import { type TerminalOwnersSnapshot, } from "@/features/agent/ui/use-persistent-terminal-owners"; import { resolveBrowserInput } from "@/features/agent/tools/browser-url"; -import { navigateBrowserHost } from "@/features/agent/ui/agent-browser-live-store"; +import { + focusBrowserLiveSession, + navigateBrowserHost, +} from "@/features/agent/ui/agent-browser-live-store"; +import { useMountSubscription } from "@/hooks/use-mount-subscription"; import { useTools } from "@/features/agent/tools/context"; import type { ComputerTab } from "@/features/agent/tools/types"; import type { GitSummary, Project } from "@/features/agent/projects/types"; @@ -112,6 +116,7 @@ export function AgentBrowserPanel({ gitSummary, }: AgentBrowserPanelProps) { const tools = useTools(); + const browserSessionId = focusedSession?.id ?? null; const [sideChatSeed, setSideChatSeed] = useState(() => createSideChatSession(null, null, ""), ); @@ -127,6 +132,10 @@ export function AgentBrowserPanel({ tools.computer.open && tools.computer.tab === "terminal", terminalOwner, ); + useMountSubscription(() => { + focusBrowserLiveSession(browserSessionId); + return () => focusBrowserLiveSession(null); + }, [browserSessionId]); const visibleTerminalState = useMemo(() => { const owners = terminalState.owners; const activeOwnerKey = owners.some((owner) => owner.mountKey === terminalState.activeOwnerKey) @@ -176,7 +185,7 @@ export function AgentBrowserPanel({ return true; } tools.setBrowserUrl(result.url, result.url); - void navigateBrowserHost(result.url); + if (browserSessionId) void navigateBrowserHost(result.url); return true; }; const syncBrowserLocation = (value: string) => { diff --git a/frontend/src/features/agent/ui/agent-browser-screencast.tsx b/frontend/src/features/agent/ui/agent-browser-screencast.tsx index cd624c2fe..17b9a44f2 100644 --- a/frontend/src/features/agent/ui/agent-browser-screencast.tsx +++ b/frontend/src/features/agent/ui/agent-browser-screencast.tsx @@ -15,29 +15,47 @@ import { browserViewportPoint, } from "@/features/agent/ui/agent-browser-input"; import { useBrowserLiveFrame } from "@/features/agent/ui/agent-browser-live-store"; +import { browserSessionHeadersOption } from "@local-studio/agent-runtime/browser-session-contract"; type Props = { navigationError: string | null; + sessionId: string | null; }; const VIEWPORT_MIN = { width: 320, height: 240 }; const VIEWPORT_MAX = { width: 1920, height: 1200 }; const MOVE_THROTTLE_MS = 33; -function postBrowser(path: string, body: unknown): void { +function postBrowser( + sessionId: string | null, + path: string, + body: unknown, + signal: AbortSignal, +): void { + const headers = browserSessionHeadersOption(sessionId); + if (!headers) return; void fetch(`/api/agent/browser/${path}`, { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { "Content-Type": "application/json", ...headers }, body: JSON.stringify(body), + signal, }).catch(() => undefined); } -export function ScreencastSurface({ navigationError }: Props) { +export function ScreencastSurface({ navigationError, sessionId }: Props) { const [container, setContainer] = useState(null); const { frame } = useBrowserLiveFrame(); const viewportRef = useRef({ width: 1280, height: 800 }); const lastMoveAtRef = useRef(0); - // ── Viewport sync: match the headless viewport to the pane size ──────── + const requestAbortRef = useRef(new AbortController()); + + useMountSubscription(() => { + const controller = new AbortController(); + requestAbortRef.current.abort(); + requestAbortRef.current = controller; + return () => controller.abort(); + }, [sessionId]); + useMountSubscription(() => { if (!container) return; let timer: EffectTimer | null = null; @@ -51,7 +69,7 @@ export function ScreencastSurface({ navigationError }: Props) { ); if (width === viewportRef.current.width && height === viewportRef.current.height) return; viewportRef.current = { width, height }; - postBrowser("viewport", { width, height }); + postBrowser(sessionId, "viewport", { width, height }, requestAbortRef.current.signal); }; const observer = new ResizeObserver(() => { if (timer) timer.cancel(); @@ -63,9 +81,8 @@ export function ScreencastSurface({ navigationError }: Props) { if (timer) timer.cancel(); observer.disconnect(); }; - }, [container]); + }, [container, sessionId]); - // ── Input forwarding ──────────────────────────────────────────────────── const toViewport = (event: { clientX: number; clientY: number }) => { const rect = container?.getBoundingClientRect(); return browserViewportPoint(rect ?? null, viewportRef.current, event); @@ -75,26 +92,36 @@ export function ScreencastSurface({ navigationError }: Props) { container?.focus(); event.currentTarget.setPointerCapture(event.pointerId); const { x, y } = toViewport(event); - postBrowser("input", { - kind: "mouse", - type: "down", - x, - y, - button: browserMouseButton(event.button), - clickCount: Math.max(1, event.detail), - }); + postBrowser( + sessionId, + "input", + { + kind: "mouse", + type: "down", + x, + y, + button: browserMouseButton(event.button), + clickCount: Math.max(1, event.detail), + }, + requestAbortRef.current.signal, + ); }; const handlePointerUp = (event: ReactPointerEvent) => { const { x, y } = toViewport(event); - postBrowser("input", { - kind: "mouse", - type: "up", - x, - y, - button: browserMouseButton(event.button), - clickCount: Math.max(1, event.detail), - }); + postBrowser( + sessionId, + "input", + { + kind: "mouse", + type: "up", + x, + y, + button: browserMouseButton(event.button), + clickCount: Math.max(1, event.detail), + }, + requestAbortRef.current.signal, + ); }; const handlePointerMove = (event: ReactPointerEvent) => { @@ -102,19 +129,31 @@ export function ScreencastSurface({ navigationError }: Props) { if (now - lastMoveAtRef.current < MOVE_THROTTLE_MS) return; lastMoveAtRef.current = now; const { x, y } = toViewport(event); - postBrowser("input", { kind: "mouse", type: "move", x, y }); + postBrowser( + sessionId, + "input", + { kind: "mouse", type: "move", x, y }, + requestAbortRef.current.signal, + ); }; const handleWheel = (event: ReactWheelEvent) => { const { x, y } = toViewport(event); - postBrowser("input", { kind: "wheel", x, y, deltaX: event.deltaX, deltaY: event.deltaY }); + postBrowser( + sessionId, + "input", + { kind: "wheel", x, y, deltaX: event.deltaX, deltaY: event.deltaY }, + requestAbortRef.current.signal, + ); }; const handleKey = (type: "down" | "up") => (event: ReactKeyboardEvent) => { const inputs = browserKeyInputs(type, event); if (inputs.length === 0) return; event.preventDefault(); - for (const input of inputs) postBrowser("input", input); + for (const input of inputs) { + postBrowser(sessionId, "input", input, requestAbortRef.current.signal); + } }; return ( diff --git a/frontend/src/features/agent/ui/agent-browser.tsx b/frontend/src/features/agent/ui/agent-browser.tsx index 93a72820f..c24f4453c 100644 --- a/frontend/src/features/agent/ui/agent-browser.tsx +++ b/frontend/src/features/agent/ui/agent-browser.tsx @@ -1,6 +1,7 @@ "use client"; import { useCallback, useRef, useState, type FormEvent } from "react"; +import { Schema } from "effect"; import { ArrowLeftIcon, ArrowRightIcon, CloseIcon, ReloadIcon } from "@/ui/icons"; import { Alert } from "@/ui"; import { DEFAULT_BROWSER_URL } from "@/features/agent/tools/persistence"; @@ -9,13 +10,25 @@ import { useBrowserLiveState } from "@/features/agent/ui/agent-browser-live-stor import { useAgentBrowserEffects, useLocalhostSitesEffects, + decodeLocalhostSitesResponse, type LocalhostSite, } from "@/features/agent/ui/agent-browser-effects"; import { LocalhostStartPage } from "@/features/agent/ui/agent-browser-start-page"; import { ReadingView, type ReadablePage } from "@/features/agent/ui/agent-browser-reading-view"; import { useMountSubscription } from "@/hooks/use-mount-subscription"; +import { browserSessionHeadersOption } from "@local-studio/agent-runtime/browser-session-contract"; + +const ReadablePageResponseSchema = Schema.Struct({ + url: Schema.String, + title: Schema.String, + text: Schema.String, + markdown: Schema.optional(Schema.String), + contentType: Schema.optional(Schema.String), + error: Schema.optional(Schema.String), +}); type Props = { + sessionId: string | null; url: string; inputValue: string; onInputChange: (value: string) => void; @@ -26,6 +39,7 @@ type Props = { }; export function AgentBrowser({ + sessionId, url, inputValue, onInputChange, @@ -46,6 +60,7 @@ export function AgentBrowser({ const [localSitesLoading, setLocalSitesLoading] = useState(false); const [localSitesError, setLocalSitesError] = useState(null); const live = useBrowserLiveState(); + const sessionHeaders = browserSessionHeadersOption(sessionId); const onLocationChangeRef = useRef(onLocationChange); const navState = live.state; const showStartPage = !hasOpenedUrl && url === DEFAULT_BROWSER_URL; @@ -58,7 +73,7 @@ export function AgentBrowser({ const response = await fetch(`/api/agent/browser/fetch?url=${encodeURIComponent(target)}`, { cache: "no-store", }); - const payload = (await response.json()) as ReadablePage & { error?: string }; + const payload = Schema.decodeUnknownSync(ReadablePageResponseSchema)(await response.json()); if (!response.ok || payload.error) { throw new Error(payload.error || `HTTP ${response.status}`); } @@ -94,9 +109,16 @@ export function AgentBrowser({ if (live.unavailable) setReadingMode(true); }, [live.unavailable]); - const postLiveVerb = useCallback((verb: "back" | "forward" | "reload") => { - void fetch(`/api/agent/browser/${verb}`, { method: "POST" }).catch(() => undefined); - }, []); + const postLiveVerb = useCallback( + (verb: "back" | "forward" | "reload") => { + if (!sessionHeaders) return; + void fetch(`/api/agent/browser/${verb}`, { + method: "POST", + headers: sessionHeaders, + }).catch(() => undefined); + }, + [sessionHeaders], + ); const handleBack = () => { if (readingMode) return; postLiveVerb("back"); @@ -112,9 +134,9 @@ export function AgentBrowser({ setLocalSitesLoading(true); void fetch("/api/agent/browser/localhosts", { cache: "no-store" }) .then(async (response) => { - const payload = (await response.json()) as { sites?: LocalhostSite[]; error?: string }; + const payload = decodeLocalhostSitesResponse(await response.json()); if (!response.ok || payload.error) throw new Error(payload.error || "Failed to scan"); - setLocalSites(payload.sites ?? []); + setLocalSites([...(payload.sites ?? [])]); }) .catch((error) => setLocalSitesError(error instanceof Error ? error.message : "Failed to scan localhost"), @@ -245,8 +267,12 @@ export function AgentBrowser({ loading={readingLoading} onLinkClick={onNavigate} /> + ) : sessionHeaders ? ( + ) : ( - +
+ Select an agent session to enable the live browser. +
)}
diff --git a/frontend/src/features/agent/ui/computer-tab-panel.tsx b/frontend/src/features/agent/ui/computer-tab-panel.tsx index 22a45f626..e00d570a8 100644 --- a/frontend/src/features/agent/ui/computer-tab-panel.tsx +++ b/frontend/src/features/agent/ui/computer-tab-panel.tsx @@ -200,12 +200,14 @@ function SideChatTab({ function BrowserTab({ browserNavigationError, + focusedSession, onBrowserLocationChange, onNavigateBrowser, tools, }: ComputerTabPanelProps) { return ( { private readonly state = Promise.withResolvers(); readonly promise = this.state.promise; @@ -24,7 +26,10 @@ class Deferred { type Barrier = { release: Deferred; started: Deferred }; -const barrier = (): Barrier => ({ release: new Deferred(), started: new Deferred() }); +const barrier = (): Barrier => ({ + release: new Deferred(), + started: new Deferred(), +}); const state = (url: string): PageState => ({ canGoBack: false, @@ -56,7 +61,7 @@ class FakePage implements BrowserPage { return Promise.resolve({ data: `frame-${this.id}`, metadata: {} }); } - click(): Promise { + click(_selector: string): Promise { return Promise.resolve(true); } @@ -72,19 +77,19 @@ class FakePage implements BrowserPage { return Promise.resolve(); } - evaluate(): Promise { + evaluate(_expression: string): Promise { return Promise.resolve(undefined); } - fill(): Promise { + fill(_selector: string, _value: string): Promise { return Promise.resolve(true); } - goBack(): Promise { + goBack(_timeout: number): Promise { return Promise.resolve(); } - goForward(): Promise { + goForward(_timeout: number): Promise { return Promise.resolve(); } @@ -96,7 +101,7 @@ class FakePage implements BrowserPage { return page === this.raw; } - async navigate(url: string): Promise { + async navigate(url: string, _timeout: number): Promise { this.raw.navigationBarrier?.started.resolve(); await this.raw.navigationBarrier?.release.promise; if (this.closed) throw new Error("Target closed"); @@ -108,11 +113,11 @@ class FakePage implements BrowserPage { return Promise.resolve(this.raw.state); } - reload(): Promise { + reload(_timeout: number): Promise { return Promise.resolve(); } - screenshot(): Promise { + screenshot(_type: "png" | "jpeg", _quality?: number): Promise { return Promise.resolve(Buffer.from(this.id).toString("base64")); } @@ -120,7 +125,7 @@ class FakePage implements BrowserPage { return Promise.resolve(deltaY); } - setViewport(): Promise { + setViewport(_width: number, _height: number): Promise { return Promise.resolve(); } @@ -146,7 +151,7 @@ class FakeContext implements BrowserContextSurface { } pages(): RawPage[] { - return this.rawPages; + return this.rawPages.filter((page) => !page.closed); } close(): void { @@ -156,12 +161,13 @@ class FakeContext implements BrowserContextSurface { class FakeSession implements ManagedPlaywrightSession> { private isClosed = false; - private listeners = new Set<() => void>(); + private readonly listeners = new Set<() => void>(); constructor( readonly context: FakeContext, readonly generation: number, readonly mode: BrowserNetworkMode, + readonly scope: string, ) {} close(): Promise { @@ -183,10 +189,10 @@ class FakeSession implements ManagedPlaywrightSession { - readonly modes: BrowserNetworkMode[] = []; + readonly launches: Array<{ mode: BrowserNetworkMode; scope: string }> = []; readonly sessions: FakeSession[] = []; stops = 0; - private active: FakeSession | null = null; + private readonly active = new Map(); private generation = 0; private pageSerial = 0; private stopped = false; @@ -212,26 +218,27 @@ class FakeManager implements BrowserHostManager { return next; } - async ensure(mode: BrowserNetworkMode): Promise { + async ensure(mode: BrowserNetworkMode, scope: string): Promise { const pending = this.ensureBarriers.shift(); pending?.started.resolve(); await pending?.release.promise; if (this.stopped) throw new Error("Browser manager stopped"); - if (this.active?.mode === mode && !this.active.closed()) return this.active; - await this.active?.close(); + const active = this.active.get(scope); + if (active?.mode === mode && !active.closed()) return active; + await active?.close(); const context = new FakeContext( () => ({ closed: false, - id: `page-${++this.pageSerial}`, + id: `${scope}-page-${++this.pageSerial}`, navigationBarrier: this.navigationBarriers.shift() ?? null, state: state("about:blank"), }), this.pageCreationBarriers.shift() ?? null, ); - const session = new FakeSession(context, ++this.generation, mode); - this.active = session; + const session = new FakeSession(context, ++this.generation, mode, scope); + this.active.set(scope, session); this.sessions.push(session); - this.modes.push(mode); + this.launches.push({ mode, scope }); return session; } @@ -239,16 +246,32 @@ class FakeManager implements BrowserHostManager { return !this.stopped; } + async release(scope: string): Promise { + const active = this.active.get(scope); + await active?.close(); + if (this.active.get(scope) === active) this.active.delete(scope); + } + async stop(): Promise { + if (this.stopped) return; this.stopped = true; this.stops += 1; - await this.active?.close(); - this.active = null; + await Promise.all([...this.active.values()].map((session) => session.close())); + this.active.clear(); } } -const hostFor = (manager: FakeManager): BrowserHost => - new BrowserHost(manager, { attachPage: (page) => new FakePage(page) }); +const hostFor = ( + manager: FakeManager, + options: { + config?: { idleMs: number; maxSessions: number }; + now?: () => number; + } = {}, +): BrowserHost => + new BrowserHost(manager, { + attachPage: (page) => new FakePage(page), + ...options, + }); const activeRawPages = (manager: FakeManager): RawPage[] => manager.sessions.flatMap((session) => session.context.rawPages).filter((page) => !page.closed); @@ -257,17 +280,19 @@ async function concurrentStartup(first: "navigate" | "poll"): Promise { const manager = new FakeManager(); const host = hostFor(manager); const pending = manager.blockNextEnsure(); + const fixture = "https://public.test/visible"; const firstRequest = - first === "poll" ? host.pollFrame() : host.navigate("https://public.test/first"); + first === "poll" ? host.pollFrame(SESSION) : host.navigate(SESSION, fixture); await pending.started.promise; const secondRequest = - first === "poll" ? host.navigate("https://public.test/second") : host.pollFrame(); + first === "poll" ? host.navigate(SESSION, fixture) : host.pollFrame(SESSION); pending.release.resolve(); const results = await Promise.allSettled([firstRequest, secondRequest]); assert.deepEqual( results.map((result) => result.status), ["fulfilled", "fulfilled"], ); + assert.equal((await host.getUrl(SESSION)).url, fixture); assert.equal(manager.sessions.length, 1); assert.equal(activeRawPages(manager).length, 1); await host.stop(); @@ -281,79 +306,236 @@ test("first frame and navigation share one page when navigation starts first", a await concurrentStartup("navigate"); }); -test("cross-mode navigation preserves request order", async () => { +test("cross-mode navigation preserves order and replaces only that session context", async () => { const manager = new FakeManager(); const host = hostFor(manager); const pending = manager.blockNextNavigation(); - const first = host.navigate("https://public.test/first"); + const first = host.navigate(SESSION, "https://public.test/first"); await pending.started.promise; - const second = host.navigate("http://localhost:4173/second"); + const second = host.navigate(SESSION, "http://localhost:4173/second"); await Promise.resolve(); - assert.deepEqual(manager.modes, ["public"]); + assert.deepEqual(manager.launches, [{ mode: "public", scope: SESSION }]); pending.release.resolve(); assert.deepEqual(await Promise.all([first, second]), [ { title: "https://public.test/first", url: "https://public.test/first" }, { title: "http://localhost:4173/second", url: "http://localhost:4173/second" }, ]); - assert.deepEqual(manager.modes, ["public", "loopback"]); + assert.deepEqual(manager.launches, [ + { mode: "public", scope: SESSION }, + { mode: "loopback", scope: SESSION }, + ]); assert.equal(activeRawPages(manager).length, 1); await host.stop(); }); -test("trust-mode replacement clears pages from the revoked generation", async () => { +test("shutdown waits for in-flight navigation and closes every context once", async () => { const manager = new FakeManager(); const host = hostFor(manager); - await host.navigate("http://localhost:4173/private"); - const loopbackPage = manager.sessions[0]?.context.rawPages[0]; - assert.ok(loopbackPage); - await host.navigate("https://public.test/page"); - assert.equal(loopbackPage.closed, true); - assert.equal(activeRawPages(manager).length, 1); - assert.equal((await host.getUrl()).url, "https://public.test/page"); + const pending = manager.blockNextNavigation(); + const navigation = host.navigate(SESSION, "https://public.test/page"); + await pending.started.promise; + let stopped = false; + const stopping = host.stop().then(() => { + stopped = true; + }); + assert.equal(host.stop(), host.stop()); + await Promise.resolve(); + assert.equal(stopped, false); + pending.release.resolve(); + assert.deepEqual(await navigation, { + title: "https://public.test/page", + url: "https://public.test/page", + }); + await stopping; + assert.equal(manager.stops, 1); + assert.equal(activeRawPages(manager).length, 0); + await assert.rejects(host.page(SESSION), /Browser host stopped/u); +}); + +test("shutdown protects active manager and page creation", async () => { + for (const pending of ["ensure", "page"] as const) { + const manager = new FakeManager(); + const blocked = + pending === "ensure" ? manager.blockNextEnsure() : manager.blockNextPageCreation(); + const host = hostFor(manager); + const navigation = host.navigate(SESSION, "https://public.test/page"); + await blocked.started.promise; + const stopping = host.stop(); + blocked.release.resolve(); + assert.deepEqual(await navigation, { + title: "https://public.test/page", + url: "https://public.test/page", + }); + await stopping; + assert.equal(activeRawPages(manager).length, 0); + } +}); + +test("blocked top-level navigation never starts Playwright", async () => { + const manager = new FakeManager(); + const host = hostFor(manager); + await assert.rejects(host.navigate(SESSION, "http://10.0.0.1/private"), /blocked navigation/u); + assert.equal(manager.sessions.length, 0); await host.stop(); }); -test("stop during context creation prevents page publication", async () => { +test("different session keys own distinct contexts and reject cross-session page ids", async () => { const manager = new FakeManager(); const host = hostFor(manager); - const pending = manager.blockNextEnsure(); - const navigation = host.navigate("https://public.test/page"); - await pending.started.promise; - const stopping = host.stop(); - pending.release.resolve(); - await assert.rejects(navigation, /Browser host stopped/u); - await stopping; - assert.equal(manager.sessions[0]?.context.rawPages.length ?? 0, 0); - assert.equal(activeRawPages(manager).length, 0); + await Promise.all([ + host.navigate("session-a", "https://public.test/a"), + host.navigate("session-b", "https://public.test/b"), + ]); + const sessionA = manager.sessions.find((session) => session.scope === "session-a"); + const sessionB = manager.sessions.find((session) => session.scope === "session-b"); + const pageA = sessionA?.context.rawPages[0]; + const pageB = sessionB?.context.rawPages[0]; + assert.ok(pageA); + assert.ok(pageB); + assert.notEqual(sessionA?.context, sessionB?.context); + assert.notEqual(pageA.id, pageB.id); + assert.equal((await host.getUrl("session-a")).url, "https://public.test/a"); + assert.equal((await host.getUrl("session-b")).url, "https://public.test/b"); + await assert.rejects(host.getUrl("session-a", pageB.id), /does not belong to session/u); + await host.stop(); }); -test("stop during page creation prevents page publication", async () => { +test("fallback state and ordering are isolated by session", async () => { const manager = new FakeManager(); const host = hostFor(manager); - const pending = manager.blockNextPageCreation(); - const navigation = host.navigate("https://public.test/page"); + const pending = barrier(); + const first = host.withFallbackSession("session-a", async () => { + pending.started.resolve(); + await pending.release.promise; + return { + navigation: { mode: "public" as const, url: "https://public.test/a" }, + result: undefined, + }; + }); await pending.started.promise; - const stopping = host.stop(); + let secondStarted = false; + const second = host.withFallbackSession("session-a", async () => { + secondStarted = true; + return { + navigation: { mode: "public" as const, url: "https://public.test/second" }, + result: undefined, + }; + }); + await host.withFallbackSession("session-b", async () => ({ + navigation: { mode: "loopback", url: "http://localhost:4173/b" }, + result: undefined, + })); + assert.equal(secondStarted, false); pending.release.resolve(); - await assert.rejects(navigation, /Browser host stopped/u); - await stopping; - assert.equal(manager.sessions[0]?.context.rawPages.length, 1); - assert.equal(activeRawPages(manager).length, 0); + await Promise.all([first, second]); + const fallbackA = await host.withFallbackSession("session-a", async (navigation) => ({ + result: navigation, + })); + const fallbackB = await host.withFallbackSession("session-b", async (navigation) => ({ + result: navigation, + })); + assert.deepEqual(fallbackA, { + mode: "public", + url: "https://public.test/second", + }); + assert.deepEqual(fallbackB, { + mode: "loopback", + url: "http://localhost:4173/b", + }); + assert.equal(manager.sessions.length, 0); + await host.stop(); }); -test("stop is terminal and idempotent", async () => { +test("capacity evicts the least-recently-used idle session", async () => { const manager = new FakeManager(); - const host = hostFor(manager); - await host.navigate("https://public.test/page"); - await Promise.all([host.stop(), host.stop()]); - assert.equal(manager.stops, 1); - await assert.rejects(host.page(), /Browser host stopped/u); + let now = 0; + const host = hostFor(manager, { + config: { idleMs: 60_000, maxSessions: 2 }, + now: () => now, + }); + await host.navigate("session-a", "https://public.test/a"); + now = 10; + await host.navigate("session-b", "https://public.test/b"); + now = 20; + await host.getUrl("session-a"); + now = 30; + await host.navigate("session-c", "https://public.test/c"); + const activeScopes = manager.sessions + .filter((session) => !session.closed()) + .map((session) => session.scope) + .sort(); + assert.deepEqual(activeScopes, ["session-a", "session-c"]); + await host.stop(); }); -test("blocked top-level navigation never starts Playwright", async () => { +test("capacity fails closed while every session has in-flight work", async () => { + const manager = new FakeManager(); + const navigationA = manager.blockNextNavigation(); + const navigationB = manager.blockNextNavigation(); + const host = hostFor(manager, { + config: { idleMs: 60_000, maxSessions: 2 }, + }); + const activeA = host.navigate("session-a", "https://public.test/a"); + const activeB = host.navigate("session-b", "https://public.test/b"); + await Promise.all([navigationA.started.promise, navigationB.started.promise]); + await assert.rejects( + host.navigate("session-c", "https://public.test/c"), + /all sessions are active/u, + ); + navigationA.release.resolve(); + navigationB.release.resolve(); + await Promise.all([activeA, activeB]); + await host.stop(); +}); + +test("release waits for work, is idempotent, and serializes same-key recreation", async () => { const manager = new FakeManager(); + const navigation = manager.blockNextNavigation(); const host = hostFor(manager); - await assert.rejects(host.navigate("http://10.0.0.1/private"), /blocked navigation/u); - assert.equal(manager.sessions.length, 0); + const active = host.navigate(SESSION, "https://public.test/a"); + await navigation.started.promise; + let released = false; + const firstRelease = host.releaseSession(SESSION).then(() => { + released = true; + }); + const secondRelease = host.releaseSession(SESSION); + const recreated = host.navigate(SESSION, "https://public.test/recreated"); + await Promise.resolve(); + assert.equal(released, false); + assert.equal(manager.sessions.filter((session) => session.scope === SESSION).length, 1); + navigation.release.resolve(); + await active; + await Promise.all([firstRelease, secondRelease]); + assert.deepEqual(await recreated, { + title: "https://public.test/recreated", + url: "https://public.test/recreated", + }); + const scoped = manager.sessions.filter((session) => session.scope === SESSION); + assert.equal(scoped.length, 2); + assert.equal(scoped[0]?.closed(), true); + assert.equal(scoped[1]?.closed(), false); + await host.stop(); +}); + +test("idle cleanup skips active work and releases expired sessions", async () => { + const manager = new FakeManager(); + let now = 0; + const host = hostFor(manager, { + config: { idleMs: 60_000, maxSessions: 2 }, + now: () => now, + }); + await host.navigate("session-idle", "https://public.test/idle"); + now = 1; + const navigation = manager.blockNextNavigation(); + const active = host.navigate("session-active", "https://public.test/active"); + await navigation.started.promise; + now = 60_001; + await host.cleanupIdleSessions(); + const idle = manager.sessions.find((session) => session.scope === "session-idle"); + const busy = manager.sessions.find((session) => session.scope === "session-active"); + assert.equal(idle?.closed(), true); + assert.equal(busy?.closed(), false); + navigation.release.resolve(); + await active; await host.stop(); }); diff --git a/services/agent-runtime/src/browser-host/browser-host.ts b/services/agent-runtime/src/browser-host/browser-host.ts index 0dbb65c1f..5e17d3ae9 100644 --- a/services/agent-runtime/src/browser-host/browser-host.ts +++ b/services/agent-runtime/src/browser-host/browser-host.ts @@ -1,11 +1,14 @@ -import { Effect, Semaphore } from "effect"; +import { Deferred, Effect, Fiber, Schedule, Semaphore } from "effect"; import type { Page } from "playwright-core"; import { browserNavigation, + type BrowserNavigation, type BrowserNetworkMode, } from "../../../../shared/agent/sanitize-embedded-browser-url"; +import { decodeBrowserSessionKey, type BrowserSessionKey } from "../browser-session-contract"; import { getGlobalSingleton } from "../instances"; import { HostedPage, type PageState, type ScreencastFrame } from "./hosted-page"; +import { browserSessionConfig, type BrowserSessionConfig } from "./browser-session"; import { playwrightManager, type ManagedPlaywrightSession } from "./playwright"; export type { PageState, ScreencastFrame }; @@ -48,20 +51,28 @@ export type BrowserPage = { text: () => Promise; }; -export type BrowserHostManager = { +export type BrowserHostManager = { ensure: ( mode: BrowserNetworkMode, + scope: string, ) => Promise>>; isAvailable: () => boolean; + release: (scope: string) => Promise; stop: () => Promise; }; -export type BrowserHostOptions = { - attachPage: (page: RawPage) => BrowserPage; +type BrowserSessionHostOptions = { + attachPage?: (page: RawPage) => BrowserPage; }; -export class BrowserHost { - private pages = new Map>(); +export type BrowserHostOptions = BrowserSessionHostOptions & { + cleanupIntervalMs?: number; + config?: BrowserSessionConfig; + now?: () => number; +}; + +class BrowserSessionHost { + private readonly pages = new Map>(); private activeId: string | null = null; private activeGeneration = 0; private activeMode: BrowserNetworkMode | null = null; @@ -70,24 +81,20 @@ export class BrowserHost { private readonly navigationLock = Semaphore.makeUnsafe(1); private readonly transitionLock = Semaphore.makeUnsafe(1); - private readonly attachPage: (page: RawPage) => BrowserPage; - constructor( private readonly manager: BrowserHostManager, - { attachPage }: BrowserHostOptions, - ) { - this.attachPage = attachPage; - } + private readonly scope: BrowserSessionKey, + private readonly options: BrowserSessionHostOptions, + ) {} - isAvailable(): boolean { - return this.manager.isAvailable(); - } - - async page(pageId?: string): Promise> { + page(pageId?: string): Promise> { return this.pageForMode(this.activeMode ?? "public", pageId); } - private pageForMode(mode: BrowserNetworkMode, pageId?: string): Promise> { + private pageForMode( + mode: BrowserNetworkMode, + pageId?: string, + ): Promise> { return this.withPermit(this.transitionLock, () => this.pageUnlocked(mode, pageId)); } @@ -96,7 +103,9 @@ export class BrowserHost { pageId?: string, ): Promise> { this.assertRunning(); - const session = await this.manager.ensure(mode); + const attachPage = this.options.attachPage; + if (!attachPage) throw new Error("Browser page adapter unavailable"); + const session = await this.manager.ensure(mode, this.scope); this.assertRunning(); if (session.generation !== this.activeGeneration) { this.pages.clear(); @@ -106,39 +115,39 @@ export class BrowserHost { } const targetId = pageId ?? this.activeId; const cached = targetId ? this.pages.get(targetId) : undefined; + if (pageId && !cached) throw new Error("Browser page does not belong to session"); if (cached && !cached.closed) { this.activeId = cached.id; return cached; } if (cached) this.pages.delete(cached.id); - - const rawPage = - session.context - .pages() - .find((candidate) => - Array.from(this.pages.values()).every((hosted) => !hosted.matches(candidate)), - ) ?? (await session.context.newPage()); + const existing = session.context + .pages() + .find((candidate) => + Array.from(this.pages.values()).every((hosted) => !hosted.matches(candidate)), + ); + const rawPage = existing ?? (await session.context.newPage()); this.assertRunning(); - const hosted = this.attachPage(rawPage); + const hosted = attachPage(rawPage); this.pages.set(hosted.id, hosted); this.activeId = hosted.id; return hosted; } - async navigate(url: string, pageId?: string): Promise<{ url: string; title: string }> { + navigate(url: string, pageId?: string): Promise<{ url: string; title: string }> { const navigation = browserNavigation(normalizeUrl(url)); - if (!navigation) throw new Error("Browser network policy blocked navigation URL"); + if (!navigation) return Promise.reject(new Error("Browser network policy blocked navigation URL")); return this.withPermit(this.navigationLock, async () => { const page = await this.pageForMode(navigation.mode, pageId); await page.navigate(navigation.url, NAVIGATION_TIMEOUT_MS); const state = await page.readState(); - return { url: state.url, title: state.title }; + return { title: state.title, url: state.url }; }); } async getUrl(pageId?: string): Promise<{ url: string; title: string }> { const state = await (await this.page(pageId)).readState(); - return { url: state.url, title: state.title }; + return { title: state.title, url: state.url }; } async getState(pageId?: string): Promise { @@ -170,16 +179,14 @@ export class BrowserHost { } async click(args: { selector: string }, pageId?: string): Promise<{ found: boolean }> { - const page = await this.page(pageId); - return { found: await page.click(args.selector) }; + return { found: await (await this.page(pageId)).click(args.selector) }; } async fill( args: { selector: string; value: string }, pageId?: string, ): Promise<{ found: boolean }> { - const page = await this.page(pageId); - return { found: await page.fill(args.selector, args.value) }; + return { found: await (await this.page(pageId)).fill(args.selector, args.value) }; } async scroll( @@ -197,11 +204,17 @@ export class BrowserHost { return `data:image/png;base64,${data}`; } - async setViewport(width: number, height: number, pageId?: string): Promise { + async setViewport( + width: number, + height: number, + pageId?: string, + ): Promise { await (await this.page(pageId)).setViewport(width, height); } - async pollFrame(pageId?: string): Promise<{ frame: ScreencastFrame | null; state: PageState }> { + async pollFrame( + pageId?: string, + ): Promise<{ frame: ScreencastFrame | null; state: PageState }> { const page = await this.page(pageId); const [frame, state] = await Promise.all([page.captureFrame(), page.readState()]); return { frame, state }; @@ -215,14 +228,14 @@ export class BrowserHost { await (await this.page(pageId)).dispatchKey(args); } - stop(): Promise { + release(): Promise { if (this.stopping) return this.stopping; this.stopped = true; this.stopping = this.withPermit(this.transitionLock, async () => { this.pages.clear(); this.activeId = null; this.activeMode = null; - await this.manager.stop(); + await this.manager.release(this.scope); }); return this.stopping; } @@ -231,13 +244,391 @@ export class BrowserHost { if (this.stopped) throw new Error("Browser host stopped"); } - private withPermit(semaphore: Semaphore.Semaphore, task: () => Promise): Promise { + private withPermit( + semaphore: Semaphore.Semaphore, + task: () => Promise, + ): Promise { return Effect.runPromise( semaphore.withPermit(Effect.tryPromise({ try: task, catch: (error) => error })), ); } } +type SessionRecord = { + closing: Promise | null; + fallback: BrowserNavigation | null; + fallbackLock: Semaphore.Semaphore; + host: BrowserSessionHost; + inFlight: number; + key: BrowserSessionKey; + lastAccess: number; + releaseRequested: boolean; + releaseStarted: boolean; + released: Deferred.Deferred; +}; + +export type BrowserFallbackResult = { + navigation?: BrowserNavigation; + result: A; +}; + +type AcquireDecision = + | { type: "acquired"; record: SessionRecord } + | { type: "evict"; record: SessionRecord } + | { type: "wait"; record: SessionRecord }; + +export class BrowserHost { + private readonly sessions = new Map>(); + private readonly registryLock = Semaphore.makeUnsafe(1); + private readonly config: BrowserSessionConfig; + private readonly now: () => number; + private readonly cleanupIntervalMs: number; + private cleanupFiber: ReturnType | null = null; + private stopping: Promise | null = null; + private stopped = false; + + constructor( + private readonly manager: BrowserHostManager, + private readonly options: BrowserHostOptions = {}, + ) { + this.config = options.config ?? browserSessionConfig(); + this.now = options.now ?? Date.now; + this.cleanupIntervalMs = + options.cleanupIntervalMs ?? + Math.min(60_000, Math.max(1_000, Math.floor(this.config.idleMs / 2))); + } + + isAvailable(): boolean { + return this.manager.isAvailable(); + } + + private assertRunning(): void { + if (this.stopped) throw new Error("Browser host stopped"); + } + + private withPermit(task: () => Promise): Promise { + return Effect.runPromise( + this.registryLock.withPermit(Effect.tryPromise({ try: task, catch: (error) => error })), + ); + } + + private startCleanup(): void { + if (this.cleanupFiber || this.stopped) return; + this.cleanupFiber = Effect.runFork( + Effect.tryPromise({ + try: () => this.cleanupIdleSessions(), + catch: (error) => error, + }).pipe( + Effect.catch(() => Effect.void), + Effect.repeat(Schedule.spaced(this.cleanupIntervalMs)), + Effect.asVoid, + ), + ); + } + + private record(key: BrowserSessionKey): SessionRecord { + return { + closing: null, + fallback: null, + fallbackLock: Semaphore.makeUnsafe(1), + host: new BrowserSessionHost(this.manager, key, this.options), + inFlight: 0, + key, + lastAccess: this.now(), + releaseRequested: false, + releaseStarted: false, + released: Deferred.makeUnsafe(), + }; + } + + private lruIdleRecord(): SessionRecord | null { + return ( + [...this.sessions.values()] + .filter((record) => record.inFlight === 0 && !record.releaseRequested) + .sort((left, right) => + left.lastAccess === right.lastAccess + ? left.key.localeCompare(right.key) + : left.lastAccess - right.lastAccess, + )[0] ?? null + ); + } + + private acquireDecision(key: BrowserSessionKey): AcquireDecision { + this.assertRunning(); + const existing = this.sessions.get(key); + if (existing) { + if (existing.releaseRequested) return { type: "wait", record: existing }; + existing.inFlight += 1; + existing.lastAccess = this.now(); + return { type: "acquired", record: existing }; + } + if (this.sessions.size >= this.config.maxSessions) { + const idle = this.lruIdleRecord(); + if (idle) { + idle.releaseRequested = true; + idle.releaseStarted = true; + return { type: "evict", record: idle }; + } + const releasing = [...this.sessions.values()].find((record) => record.releaseRequested); + if (releasing) return { type: "wait", record: releasing }; + throw new Error("Browser session capacity reached while all sessions are active"); + } + const created = this.record(key); + created.inFlight = 1; + this.sessions.set(key, created); + return { type: "acquired", record: created }; + } + + private async acquire(key: BrowserSessionKey): Promise> { + this.startCleanup(); + for (;;) { + const decision = await this.withPermit(async () => this.acquireDecision(key)); + if (decision.type === "acquired") return decision.record; + if (decision.type === "evict") await this.closeRecord(decision.record); + else await Effect.runPromise(Deferred.await(decision.record.released)); + } + } + + private async closeRecord(record: SessionRecord): Promise { + record.closing ??= this.closeRecordOnce(record); + await record.closing; + } + + private async closeRecordOnce(record: SessionRecord): Promise { + let failure: unknown; + try { + await record.host.release(); + } catch (error) { + failure = error; + await this.manager.stop().catch(() => undefined); + } finally { + await this.withPermit(async () => { + if (this.sessions.get(record.key) === record) this.sessions.delete(record.key); + Deferred.doneUnsafe(record.released, Effect.void); + }); + } + if (failure) throw failure; + } + + private async finish(record: SessionRecord): Promise { + const close = await this.withPermit(async () => { + record.inFlight = Math.max(0, record.inFlight - 1); + record.lastAccess = this.now(); + if (!record.releaseRequested || record.inFlight > 0 || record.releaseStarted) return false; + record.releaseStarted = true; + return true; + }); + if (close) await this.closeRecord(record); + } + + private async withSession( + sessionKey: BrowserSessionKey, + task: (session: BrowserSessionHost, record: SessionRecord) => Promise, + ): Promise { + const key = decodeBrowserSessionKey(sessionKey); + const record = await this.acquire(key); + try { + return await task(record.host, record); + } finally { + await this.finish(record); + } + } + + private withFallbackPermit( + record: SessionRecord, + task: () => Promise, + ): Promise { + return Effect.runPromise( + record.fallbackLock.withPermit( + Effect.tryPromise({ try: task, catch: (error) => error }), + ), + ); + } + + page( + sessionKey: BrowserSessionKey, + pageId?: string, + ): Promise> { + return this.withSession(sessionKey, (session) => session.page(pageId)); + } + + navigate( + sessionKey: BrowserSessionKey, + url: string, + pageId?: string, + ): Promise<{ url: string; title: string }> { + return this.withSession(sessionKey, (session) => session.navigate(url, pageId)); + } + + getUrl( + sessionKey: BrowserSessionKey, + pageId?: string, + ): Promise<{ url: string; title: string }> { + return this.withSession(sessionKey, (session) => session.getUrl(pageId)); + } + + getState(sessionKey: BrowserSessionKey, pageId?: string): Promise { + return this.withSession(sessionKey, (session) => session.getState(pageId)); + } + + goBack(sessionKey: BrowserSessionKey, pageId?: string): Promise { + return this.withSession(sessionKey, (session) => session.goBack(pageId)); + } + + goForward(sessionKey: BrowserSessionKey, pageId?: string): Promise { + return this.withSession(sessionKey, (session) => session.goForward(pageId)); + } + + reload(sessionKey: BrowserSessionKey, pageId?: string): Promise { + return this.withSession(sessionKey, (session) => session.reload(pageId)); + } + + getText(sessionKey: BrowserSessionKey, pageId?: string): Promise { + return this.withSession(sessionKey, (session) => session.getText(pageId)); + } + + getHtml(sessionKey: BrowserSessionKey, pageId?: string): Promise { + return this.withSession(sessionKey, (session) => session.getHtml(pageId)); + } + + evaluate( + sessionKey: BrowserSessionKey, + expression: string, + pageId?: string, + ): Promise { + return this.withSession(sessionKey, (session) => session.evaluate(expression, pageId)); + } + + click( + sessionKey: BrowserSessionKey, + args: { selector: string }, + pageId?: string, + ): Promise<{ found: boolean }> { + return this.withSession(sessionKey, (session) => session.click(args, pageId)); + } + + fill( + sessionKey: BrowserSessionKey, + args: { selector: string; value: string }, + pageId?: string, + ): Promise<{ found: boolean }> { + return this.withSession(sessionKey, (session) => session.fill(args, pageId)); + } + + scroll( + sessionKey: BrowserSessionKey, + args: { deltaY: number; deltaX?: number }, + pageId?: string, + ): Promise<{ deltaX: number; deltaY: number; scrollY: number }> { + return this.withSession(sessionKey, (session) => session.scroll(args, pageId)); + } + + screenshot(sessionKey: BrowserSessionKey, pageId?: string): Promise { + return this.withSession(sessionKey, (session) => session.screenshot(pageId)); + } + + setViewport( + sessionKey: BrowserSessionKey, + width: number, + height: number, + pageId?: string, + ): Promise { + return this.withSession(sessionKey, (session) => + session.setViewport(width, height, pageId), + ); + } + + pollFrame( + sessionKey: BrowserSessionKey, + pageId?: string, + ): Promise<{ frame: ScreencastFrame | null; state: PageState }> { + return this.withSession(sessionKey, (session) => session.pollFrame(pageId)); + } + + dispatchMouse( + sessionKey: BrowserSessionKey, + args: MouseInput, + pageId?: string, + ): Promise { + return this.withSession(sessionKey, (session) => session.dispatchMouse(args, pageId)); + } + + dispatchKey( + sessionKey: BrowserSessionKey, + args: KeyInput, + pageId?: string, + ): Promise { + return this.withSession(sessionKey, (session) => session.dispatchKey(args, pageId)); + } + + withFallbackSession( + sessionKey: BrowserSessionKey, + task: (navigation: BrowserNavigation | null) => Promise>, + ): Promise { + return this.withSession(sessionKey, async (_session, record) => + this.withFallbackPermit(record, async () => { + const transition = await task(record.fallback); + if (transition.navigation) record.fallback = transition.navigation; + return transition.result; + }), + ); + } + + async releaseSession(sessionKey: BrowserSessionKey): Promise { + const key = decodeBrowserSessionKey(sessionKey); + const decision = await this.withPermit(async () => { + const record = this.sessions.get(key); + if (!record) return null; + record.releaseRequested = true; + if (record.inFlight > 0 || record.releaseStarted) return { close: false, record }; + record.releaseStarted = true; + return { close: true, record }; + }); + if (!decision) return; + if (decision.close) await this.closeRecord(decision.record); + await Effect.runPromise(Deferred.await(decision.record.released)); + } + + async cleanupIdleSessions(): Promise { + const expired = await this.withPermit(async () => { + const threshold = this.now() - this.config.idleMs; + return [...this.sessions.values()].filter((record) => { + if (record.inFlight > 0 || record.releaseRequested || record.lastAccess > threshold) { + return false; + } + record.releaseRequested = true; + record.releaseStarted = true; + return true; + }); + }); + await Promise.all(expired.map((record) => this.closeRecord(record))); + } + + stop(): Promise { + this.stopping ??= this.stopOnce(); + return this.stopping; + } + + private async stopOnce(): Promise { + this.stopped = true; + const cleanup = this.cleanupFiber; + this.cleanupFiber = null; + if (cleanup) await Effect.runPromise(Fiber.interrupt(cleanup)); + const records = await this.withPermit(async () => + [...this.sessions.values()].map((record) => { + record.releaseRequested = true; + if (record.inFlight === 0) record.releaseStarted = true; + return record; + }), + ); + await Promise.all( + records.filter((record) => record.releaseStarted).map((record) => this.closeRecord(record)), + ); + await Promise.all(records.map((record) => Effect.runPromise(Deferred.await(record.released)))); + await this.manager.stop(); + } +} + export type MouseInput = { type: "down" | "up" | "move" | "wheel"; x: number; @@ -248,7 +639,12 @@ export type MouseInput = { deltaY?: number; }; -export type KeyInput = { type: "down" | "up" | "char"; key: string; code: string; text?: string }; +export type KeyInput = { + type: "down" | "up" | "char"; + key: string; + code: string; + text?: string; +}; const clampDelta = (value: number): number => { if (!Number.isFinite(value)) return 0; diff --git a/services/agent-runtime/src/browser-host/browser-network-policy.test.ts b/services/agent-runtime/src/browser-host/browser-network-policy.test.ts index beb6c0edc..e443708c3 100644 --- a/services/agent-runtime/src/browser-host/browser-network-policy.test.ts +++ b/services/agent-runtime/src/browser-host/browser-network-policy.test.ts @@ -9,7 +9,7 @@ import { } from "../../../../shared/agent/sanitize-embedded-browser-url"; import { createBrowserNetworkPolicy, type BrowserHostResolver } from "./network-policy"; import { createPinningProxy, type PinnedDial } from "./pinning-proxy"; -import { playwrightArguments } from "./playwright"; +import { playwrightArguments, playwrightProxySettings } from "./playwright"; const addressCases: ReadonlyArray = [ ["8.8.8.8", "public"], @@ -100,8 +100,8 @@ test("derives public and explicit loopback navigation modes", () => { test("Playwright launch policy removes implicit bypasses and non-proxied transports", () => { const proxy = "http://127.0.0.1:4567"; - const args = playwrightArguments(proxy); - assert.ok(args.includes(`--proxy-server=${proxy}`)); + const args = playwrightArguments(); + assert.deepEqual(playwrightProxySettings(proxy), { bypass: "<-loopback>", server: proxy }); assert.ok(args.includes("--proxy-bypass-list=<-loopback>")); assert.ok(args.includes("--disable-quic")); assert.ok(args.includes("--force-webrtc-ip-handling-policy=disable_non_proxied_udp")); diff --git a/services/agent-runtime/src/browser-host/browser-request-classes.integration.test.ts b/services/agent-runtime/src/browser-host/browser-request-classes.integration.test.ts index a92d14c59..bda669254 100644 --- a/services/agent-runtime/src/browser-host/browser-request-classes.integration.test.ts +++ b/services/agent-runtime/src/browser-host/browser-request-classes.integration.test.ts @@ -1,9 +1,6 @@ import assert from "node:assert/strict"; -import { mkdtemp, rm } from "node:fs/promises"; import { createServer, type Server } from "node:http"; import { connect as netConnect } from "node:net"; -import os from "node:os"; -import path from "node:path"; import test from "node:test"; import { Schema } from "effect"; import { BrowserHost, type ScreencastFrame } from "./browser-host"; @@ -21,6 +18,7 @@ import { } from "./playwright"; const PUBLIC_ADDRESS = "8.8.8.8"; +const SESSION = "session-a"; const FrameColorSchema = Schema.Union([ Schema.Literal("red"), Schema.Literal("green"), @@ -111,24 +109,25 @@ function fixtureServer(blockedBase: string, loopbackBase: string, hosts: string[ }); } -function screencastServer(): Server { +function colorServer(): Server { return createServer((request, response) => { - const agentPage = request.url === "/b"; - const marker = agentPage ? "AGENT_B_SENTINEL" : "VISIBLE_A_SENTINEL"; - const background = agentPage ? "#00ff00" : "#ff0000"; + const green = request.url?.includes("owner=B") ?? false; + const marker = green ? "SESSION_B_SENTINEL" : "SESSION_A_SENTINEL"; + const background = green ? "#00ff00" : "#ff0000"; response.setHeader("content-type", "text/html"); response.end( - `${marker}

${marker}

`, + `${marker}

${marker}

`, ); }); } -async function classifyFrameColor( - host: BrowserHost, +async function classifyFrameColor( + host: BrowserHost, + session: string, frame: ScreencastFrame, ): Promise { const source = JSON.stringify(`data:image/jpeg;base64,${frame.data}`); - const result = await host.evaluate(`new Promise((resolve, reject) => { + const result = await host.evaluate(session, `new Promise((resolve, reject) => { const image = new Image(); image.onload = () => { const canvas = document.createElement("canvas"); @@ -193,30 +192,94 @@ function attemptedWebSocket(attempts: string[], port: number): boolean { }); } -test("static navigation refreshes the visible screencast frame", { timeout: 20_000 }, async () => { - const fixture = screencastServer(); +test("static navigation refreshes the visible session frame", { timeout: 20_000 }, async () => { + const fixture = colorServer(); const fixturePort = await listen(fixture); - const profile = await mkdtemp(path.join(os.tmpdir(), "local-studio-browser-frame-")); const policy = createBrowserNetworkPolicy(); const manager = new PlaywrightManager({ - launch: createPlaywrightSessionLauncher((mode) => path.join(profile, mode)), + launch: createPlaywrightSessionLauncher(), policy, resolveBinary: findBrowserBinary, }); const host = new BrowserHost(manager, { attachPage: HostedPage.attach }); try { - await host.navigate(`http://127.0.0.1:${fixturePort}/a`); - const first = (await host.pollFrame()).frame; + await host.navigate(SESSION, `http://127.0.0.1:${fixturePort}/?owner=A`); + const first = (await host.pollFrame(SESSION)).frame; assert.ok(first); - assert.equal(await classifyFrameColor(host, first), "red"); - await host.navigate(`http://127.0.0.1:${fixturePort}/b`); - const second = (await host.pollFrame()).frame; + assert.equal(await classifyFrameColor(host, SESSION, first), "red"); + await host.navigate(SESSION, `http://127.0.0.1:${fixturePort}/?owner=B`); + const second = (await host.pollFrame(SESSION)).frame; assert.ok(second); - assert.equal(await classifyFrameColor(host, second), "green"); - assert.equal(await host.getText(), "AGENT_B_SENTINEL"); + assert.equal(await classifyFrameColor(host, SESSION, second), "green"); + assert.equal(await host.getText(SESSION), "SESSION_B_SENTINEL"); } finally { await host.stop().catch(() => undefined); - await Promise.allSettled([close(fixture), rm(profile, { force: true, recursive: true })]); + await close(fixture); + } +}); + +test("two sessions isolate context state while sharing one Chromium process", { timeout: 20_000 }, async () => { + const fixture = colorServer(); + const fixturePort = await listen(fixture); + const manager = new PlaywrightManager({ + launch: createPlaywrightSessionLauncher(), + policy: createBrowserNetworkPolicy(), + resolveBinary: findBrowserBinary, + }); + const host = new BrowserHost(manager, { attachPage: HostedPage.attach }); + const sessionA = "session-a"; + const sessionB = "session-b"; + try { + await Promise.all([ + host.navigate(sessionA, `http://127.0.0.1:${fixturePort}/?owner=A`), + host.navigate(sessionB, `http://127.0.0.1:${fixturePort}/?owner=B`), + ]); + await host.evaluate( + sessionA, + `localStorage.setItem("owner", "A"); document.cookie = "owner=A"; document.querySelector("#input").value = "alpha"`, + ); + await host.evaluate( + sessionB, + `localStorage.setItem("owner", "B"); document.cookie = "owner=B"; document.querySelector("#input").value = ""`, + ); + await host.setViewport(sessionA, 640, 480); + await host.setViewport(sessionB, 900, 700); + assert.deepEqual( + await host.click(sessionA, { selector: "#input" }), + { found: true }, + ); + await host.dispatchKey(sessionA, { + type: "char", + key: "X", + code: "KeyX", + text: "X", + }); + const stateA = await host.evaluate( + sessionA, + `({ cookie: document.cookie, input: document.querySelector("#input").value, owner: localStorage.getItem("owner"), width: innerWidth })`, + ); + const stateB = await host.evaluate( + sessionB, + `({ cookie: document.cookie, input: document.querySelector("#input").value, owner: localStorage.getItem("owner"), width: innerWidth })`, + ); + assert.deepEqual(stateA, { cookie: "owner=A", input: "alphaX", owner: "A", width: 640 }); + assert.deepEqual(stateB, { cookie: "owner=B", input: "", owner: "B", width: 900 }); + const frameA = (await host.pollFrame(sessionA)).frame; + const frameB = (await host.pollFrame(sessionB)).frame; + assert.ok(frameA); + assert.ok(frameB); + assert.equal(await classifyFrameColor(host, sessionA, frameA), "red"); + assert.equal(await classifyFrameColor(host, sessionB, frameB), "green"); + assert.notEqual(await host.screenshot(sessionA), await host.screenshot(sessionB)); + const contextA = manager.current(sessionA)?.context; + const contextB = manager.current(sessionB)?.context; + assert.ok(contextA); + assert.ok(contextB); + assert.notEqual(contextA, contextB); + assert.equal(contextA.browser(), contextB.browser()); + } finally { + await host.stop().catch(() => undefined); + await close(fixture); } }); @@ -261,30 +324,33 @@ test( ), ); const proxies = { loopback: pinnedProxies[1], public: pinnedProxies[0] }; - const profile = await mkdtemp(path.join(os.tmpdir(), "local-studio-browser-policy-")); const manager = new PlaywrightManager({ createProxies: async () => proxies, - launch: createPlaywrightSessionLauncher((mode) => `${profile}-${mode}`), + launch: createPlaywrightSessionLauncher(), policy, resolveBinary: findBrowserBinary, }); const host = new BrowserHost(manager, { attachPage: HostedPage.attach }); try { assert.equal(manager.isAvailable(), true); - await host.navigate(`http://page.test:${fixturePort}/matrix`); - const publicGeneration = manager.current()?.generation; + await host.navigate(SESSION, `http://page.test:${fixturePort}/matrix`); + const publicGeneration = manager.current(SESSION)?.generation; assert.ok(publicGeneration); await host.evaluate( + SESSION, "new Promise((resolve, reject) => { const started = Date.now(); const poll = () => { if (window.workerStarted) resolve(true); else if (window.workerError || Date.now() - started > 5000) reject(new Error(window.workerError || 'worker timed out')); else setTimeout(poll, 50); }; poll(); })", ); - await host.evaluate("window.startDownload()"); + await host.evaluate(SESSION, "window.startDownload()"); await new Promise((resolve) => setTimeout(resolve, 500)); - await host.navigate(`http://page.test:${fixturePort}/redirect`).catch(() => undefined); - await host.navigate(`http://localhost:${fixturePort}/service-worker`); - const loopbackGeneration = manager.current()?.generation; + await host + .navigate(SESSION, `http://page.test:${fixturePort}/redirect`) + .catch(() => undefined); + await host.navigate(SESSION, `http://localhost:${fixturePort}/service-worker`); + const loopbackGeneration = manager.current(SESSION)?.generation; assert.ok(loopbackGeneration); assert.notEqual(loopbackGeneration, publicGeneration); const serviceWorkerState = await host.evaluate( + SESSION, "new Promise((resolve) => setTimeout(() => resolve({ ready: Boolean(window.serviceWorkerReady) }), 500))", ); assert.ok(serviceWorkerState && typeof serviceWorkerState === "object"); @@ -315,13 +381,15 @@ test( assert.ok( dials.some( (destination) => - destination.hostname === "page.test" && destination.address.address === PUBLIC_ADDRESS, + destination.hostname === "page.test" && + destination.address.address === PUBLIC_ADDRESS, ), ); assert.ok( dials.some( (destination) => - destination.hostname === "localhost" && destination.address.address === "127.0.0.1", + destination.hostname === "localhost" && + destination.address.address === "127.0.0.1", ), ); assert.ok(hosts.includes(`page.test:${fixturePort}`)); @@ -330,7 +398,6 @@ test( await host.stop().catch(() => undefined); await Promise.allSettled([proxies.public.close(), proxies.loopback.close()]); await Promise.allSettled([close(fixture), close(blocked)]); - await rm(profile, { force: true, recursive: true }); } }, ); diff --git a/services/agent-runtime/src/browser-host/browser-session.ts b/services/agent-runtime/src/browser-host/browser-session.ts new file mode 100644 index 000000000..6864c40dc --- /dev/null +++ b/services/agent-runtime/src/browser-host/browser-session.ts @@ -0,0 +1,31 @@ +import { Schema } from "effect"; + +const SessionLimitSchema = Schema.NumberFromString.pipe( + Schema.check( + Schema.isFinite(), + Schema.isInt(), + Schema.isGreaterThanOrEqualTo(1), + Schema.isLessThanOrEqualTo(32), + ), +); +const SessionIdleSchema = Schema.NumberFromString.pipe( + Schema.check( + Schema.isFinite(), + Schema.isInt(), + Schema.isGreaterThanOrEqualTo(60_000), + Schema.isLessThanOrEqualTo(86_400_000), + ), +); +const BrowserSessionConfigSchema = Schema.Struct({ + maxSessions: SessionLimitSchema, + idleMs: SessionIdleSchema, +}); + +export type BrowserSessionConfig = typeof BrowserSessionConfigSchema.Type; + +export function browserSessionConfig(env: NodeJS.ProcessEnv = process.env): BrowserSessionConfig { + return Schema.decodeUnknownSync(BrowserSessionConfigSchema)({ + maxSessions: env.LOCAL_STUDIO_BROWSER_MAX_SESSIONS ?? "8", + idleMs: env.LOCAL_STUDIO_BROWSER_SESSION_IDLE_MS ?? "900000", + }); +} diff --git a/services/agent-runtime/src/browser-host/playwright-manager-revocation.test.ts b/services/agent-runtime/src/browser-host/playwright-manager-revocation.test.ts index c18f2276d..e5b091e42 100644 --- a/services/agent-runtime/src/browser-host/playwright-manager-revocation.test.ts +++ b/services/agent-runtime/src/browser-host/playwright-manager-revocation.test.ts @@ -177,6 +177,23 @@ test("unexpected context closure permits a clean same-mode relaunch", async () = await manager.stop(); }); +test("scoped contexts coexist and release independently", async () => { + const fixture = new ManagerFixture(); + const manager = fixture.manager(); + const sessionA = await manager.ensure("public", "session-a"); + const sessionB = await manager.ensure("public", "session-b"); + assert.notEqual(sessionA.context, sessionB.context); + assert.equal(manager.current("session-a"), sessionA); + assert.equal(manager.current("session-b"), sessionB); + await manager.release("session-a"); + assert.equal(sessionA.closed(), true); + assert.equal(sessionB.closed(), false); + assert.equal(manager.current("session-a"), null); + assert.equal(manager.current("session-b"), sessionB); + await manager.stop(); + assert.equal(sessionB.closed(), true); +}); + test("stop closes the active context and both pinning proxies exactly once", async () => { const fixture = new ManagerFixture(); const manager = fixture.manager(); diff --git a/services/agent-runtime/src/browser-host/playwright.ts b/services/agent-runtime/src/browser-host/playwright.ts index a7f4b2944..1ad2d8551 100644 --- a/services/agent-runtime/src/browser-host/playwright.ts +++ b/services/agent-runtime/src/browser-host/playwright.ts @@ -1,9 +1,14 @@ import { execFileSync } from "node:child_process"; import { existsSync } from "node:fs"; -import os from "node:os"; import path from "node:path"; import { Effect, Semaphore } from "effect"; -import { chromium, type BrowserContext, type Route, type WebSocketRoute } from "playwright-core"; +import { + chromium, + type Browser, + type BrowserContext, + type Route, + type WebSocketRoute, +} from "playwright-core"; import type { BrowserNetworkMode } from "../../../../shared/agent/sanitize-embedded-browser-url"; import { getGlobalSingleton } from "../instances"; import { browserNetworkPolicy, type BrowserNetworkPolicy } from "./network-policy"; @@ -12,6 +17,7 @@ import { createBrowserPinningProxies, type PinningProxy } from "./pinning-proxy" const LAUNCH_TIMEOUT_MS = 15_000; const REVOCATION_TIMEOUT_MS = 5_000; const PROXY_BYPASS_LIST = "<-loopback>"; +const DEFAULT_SCOPE = "default"; type BrowserPinningProxies = Record; @@ -39,9 +45,6 @@ export type PlaywrightManagerOptions = { resolveBinary?: () => string | null; }; -const browserDataDirectory = (mode: BrowserNetworkMode): string => - path.join(os.tmpdir(), `local-studio-browser-profile-${mode}`); - const resolveOnPath = (binary: string): string | null => { try { const resolved = execFileSync("which", [binary], { encoding: "utf8" }).trim(); @@ -102,7 +105,7 @@ export const findBrowserBinary = (): string | null => { return platformBrowserCandidates().find((candidate) => existsSync(candidate)) ?? null; }; -export const playwrightArguments = (proxyUrl: string): string[] => [ +export const playwrightArguments = (): string[] => [ "--no-first-run", "--no-default-browser-check", "--disable-dev-shm-usage", @@ -113,10 +116,14 @@ export const playwrightArguments = (proxyUrl: string): string[] => [ "--disable-sync", "--no-pings", "--force-webrtc-ip-handling-policy=disable_non_proxied_udp", - `--proxy-server=${proxyUrl}`, `--proxy-bypass-list=${PROXY_BYPASS_LIST}`, ]; +export const playwrightProxySettings = (server: string): { bypass: string; server: string } => ({ + bypass: PROXY_BYPASS_LIST, + server, +}); + const guardRoute = async ( route: Route, mode: BrowserNetworkMode, @@ -164,25 +171,36 @@ const closeBrowserContext = async (context: BrowserContext): Promise => { } }; -export const createPlaywrightSessionLauncher = - ( - dataDirectoryForMode: (mode: BrowserNetworkMode) => string = browserDataDirectory, - ): LaunchPlaywrightSession => - async (executablePath, mode, proxy, policy) => { - const launch = (userDataDir: string): Promise => - chromium.launchPersistentContext(userDataDir, { - args: playwrightArguments(proxy.url), +export const createPlaywrightSessionLauncher = (): LaunchPlaywrightSession => { + let browser: Browser | null = null; + let launching: Promise | null = null; + const contexts = new Set(); + + const ensureBrowser = (executablePath: string): Promise => { + if (browser?.isConnected()) return Promise.resolve(browser); + launching ??= chromium + .launch({ + args: playwrightArguments(), executablePath, headless: true, - proxy: { server: proxy.url }, - serviceWorkers: "block", timeout: LAUNCH_TIMEOUT_MS, - viewport: { width: 1280, height: 800 }, + }) + .then((launched) => { + browser = launched; + return launched; + }) + .finally(() => { + launching = null; }); - const dataDirectory = dataDirectoryForMode(mode); - const context = await launch(dataDirectory).catch((error: unknown) => { - if (!String(error).includes("ProcessSingleton")) throw error; - return launch(`${dataDirectory}-${process.pid}`); + return launching; + }; + + return async (executablePath, mode, proxy, policy) => { + const activeBrowser = await ensureBrowser(executablePath); + const context = await activeBrowser.newContext({ + proxy: playwrightProxySettings(proxy.url), + serviceWorkers: "block", + viewport: { width: 1280, height: 800 }, }); try { await installNetworkGuards(context, mode, policy); @@ -190,12 +208,18 @@ export const createPlaywrightSessionLauncher = await closeBrowserContext(context).catch(() => undefined); throw error; } + contexts.add(context); let isClosed = false; const listeners = new Set<() => void>(); context.once("close", () => { isClosed = true; + contexts.delete(context); for (const listener of listeners) listener(); listeners.clear(); + if (contexts.size === 0 && browser === activeBrowser) { + browser = null; + void activeBrowser.close().catch(() => undefined); + } }); return { close: () => closeBrowserContext(context), @@ -205,6 +229,7 @@ export const createPlaywrightSessionLauncher = onClose: (listener) => listeners.add(listener), }; }; +}; const launchPlaywrightSession = createPlaywrightSessionLauncher(); @@ -217,7 +242,7 @@ const closeProxies = async (proxies: BrowserPinningProxies): Promise => { }; export class PlaywrightManager { - private active: ManagedPlaywrightSession | null = null; + private readonly active = new Map>(); private generation = 0; private poisoned: unknown = null; private proxies: BrowserPinningProxies | null = null; @@ -247,12 +272,19 @@ export class PlaywrightManager { return !this.stopped && this.poisoned === null && this.resolveBinary() !== null; } - ensure(mode: BrowserNetworkMode = "public"): Promise> { - return this.withPermit(() => this.ensureUnlocked(mode)); + ensure( + mode: BrowserNetworkMode = "public", + scope: string = DEFAULT_SCOPE, + ): Promise> { + return this.withPermit(() => this.ensureUnlocked(mode, scope)); } - current(): ManagedPlaywrightSession | null { - return this.active; + current(scope: string = DEFAULT_SCOPE): ManagedPlaywrightSession | null { + return this.active.get(scope) ?? null; + } + + release(scope: string = DEFAULT_SCOPE): Promise { + return this.withPermit(() => this.revokeActive(scope)); } stop(): Promise { @@ -261,11 +293,14 @@ export class PlaywrightManager { private async ensureUnlocked( mode: BrowserNetworkMode, + scope: string, ): Promise> { this.assertUsable(); - if (this.active?.closed()) this.active = null; - if (this.active?.mode === mode) return this.active; - if (this.active) await this.revokeActive(); + const active = this.active.get(scope); + if (active?.closed()) this.active.delete(scope); + const current = this.active.get(scope); + if (current?.mode === mode) return current; + if (current) await this.revokeActive(scope); const executablePath = this.resolveBinary(); if (!executablePath) { throw new Error("Browser unavailable: no Chromium found — set LOCAL_STUDIO_CHROME_PATH"); @@ -281,9 +316,9 @@ export class PlaywrightManager { onClose: (listener) => launched.onClose(listener), }; session.onClose(() => { - if (this.active === session) this.active = null; + if (this.active.get(scope) === session) this.active.delete(scope); }); - this.active = session; + this.active.set(scope, session); return session; } @@ -292,8 +327,8 @@ export class PlaywrightManager { return this.proxies; } - private async revokeActive(): Promise { - const session = this.active; + private async revokeActive(scope: string): Promise { + const session = this.active.get(scope); if (!session) return; try { await Effect.runPromise( @@ -305,7 +340,7 @@ export class PlaywrightManager { ), ); if (!session.closed()) throw new Error("Chromium termination was not confirmed"); - if (this.active === session) this.active = null; + if (this.active.get(scope) === session) this.active.delete(scope); } catch (error) { this.poisoned = error; throw error; @@ -316,10 +351,12 @@ export class PlaywrightManager { if (this.stopped) return; this.stopped = true; let failure: unknown = null; - try { - await this.revokeActive(); - } catch (error) { - failure = error; + for (const scope of [...this.active.keys()]) { + try { + await this.revokeActive(scope); + } catch (error) { + failure ??= error; + } } const proxies = this.proxies; this.proxies = null; diff --git a/services/agent-runtime/src/browser-runtime-session.test.ts b/services/agent-runtime/src/browser-runtime-session.test.ts new file mode 100644 index 000000000..2085793f1 --- /dev/null +++ b/services/agent-runtime/src/browser-runtime-session.test.ts @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { Effect } from "effect"; +import { BROWSER_SESSION_HEADER } from "./browser-session-contract"; +import { buildAgentSessionOptionsSync, withRuntimeEnvInjections } from "./pi-runtime-helpers"; + +test("runtime options inject the canonical focused browser session", () => { + const result = buildAgentSessionOptionsSync({ + options: { browserSessionId: "session-a", browserToolEnabled: true }, + processEnv: { LOCAL_STUDIO_FRONTEND_BASE: "http://127.0.0.1:3000" }, + }); + assert.equal(result.envInjections.LOCAL_STUDIO_BROWSER_SESSION_HEADER, BROWSER_SESSION_HEADER); + assert.equal(result.envInjections.LOCAL_STUDIO_BROWSER_SESSION_ID, "session-a"); + assert.equal(result.envInjections.SITEGEIST_RELAY_SESSION_ID, "session-a"); + assert.throws(() => + buildAgentSessionOptionsSync({ + options: { browserSessionId: "bad key", browserToolEnabled: true }, + processEnv: {}, + }), + ); + assert.throws(() => + buildAgentSessionOptionsSync({ options: { browserToolEnabled: true }, processEnv: {} }), + ); +}); + +test("runtime environment injection is serialized and restored exactly", async () => { + const env: NodeJS.ProcessEnv = { LOCAL_STUDIO_BROWSER_SESSION_ID: "original" }; + const observed: string[] = []; + const run = (sessionId: string) => + Effect.runPromise( + withRuntimeEnvInjections( + { LOCAL_STUDIO_BROWSER_SESSION_ID: sessionId, TEMPORARY_BROWSER_KEY: sessionId }, + Effect.gen(function* () { + observed.push(`${sessionId}:${env.LOCAL_STUDIO_BROWSER_SESSION_ID}`); + yield* Effect.sleep(5); + observed.push(`${sessionId}:${env.LOCAL_STUDIO_BROWSER_SESSION_ID}`); + }), + env, + ), + ); + await Promise.all([run("session-a"), run("session-b")]); + assert.deepEqual(observed, [ + "session-a:session-a", + "session-a:session-a", + "session-b:session-b", + "session-b:session-b", + ]); + assert.equal(env.LOCAL_STUDIO_BROWSER_SESSION_ID, "original"); + assert.equal(Object.hasOwn(env, "TEMPORARY_BROWSER_KEY"), false); +}); diff --git a/services/agent-runtime/src/browser-session-contract.ts b/services/agent-runtime/src/browser-session-contract.ts new file mode 100644 index 000000000..6bc732897 --- /dev/null +++ b/services/agent-runtime/src/browser-session-contract.ts @@ -0,0 +1,29 @@ +import { Schema } from "effect"; + +export const BROWSER_SESSION_HEADER = "x-local-studio-browser-session"; + +export const BrowserSessionKeySchema = Schema.String.pipe( + Schema.check( + Schema.isMinLength(1), + Schema.isMaxLength(128), + Schema.isPattern(/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u), + ), +); + +export type BrowserSessionKey = typeof BrowserSessionKeySchema.Type; + +export function decodeBrowserSessionKey(input: unknown): BrowserSessionKey { + return Schema.decodeUnknownSync(BrowserSessionKeySchema)(input); +} + +export function browserSessionHeaders(sessionKey: BrowserSessionKey): Record { + return { [BROWSER_SESSION_HEADER]: sessionKey }; +} + +export function browserSessionHeadersOption(input: unknown): Record | null { + try { + return browserSessionHeaders(decodeBrowserSessionKey(input)); + } catch { + return null; + } +} diff --git a/services/agent-runtime/src/http/browser-handlers.ts b/services/agent-runtime/src/http/browser-handlers.ts index d6b0869f5..22a538ab8 100644 --- a/services/agent-runtime/src/http/browser-handlers.ts +++ b/services/agent-runtime/src/http/browser-handlers.ts @@ -1,10 +1,20 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; +import { Schema } from "effect"; +import { + BROWSER_SESSION_HEADER, + decodeBrowserSessionKey, + type BrowserSessionKey, +} from "../browser-session-contract"; import { browserNavigation, type BrowserNavigation, } from "../../../../shared/agent/sanitize-embedded-browser-url"; -import { browserHost, type KeyInput, type MouseInput } from "../browser-host/browser-host"; +import { + BrowserHost, + browserHost, + type BrowserFallbackResult, +} from "../browser-host/browser-host"; import { fetchReadable } from "../browser-host/reader"; const ALLOWED_VERBS = new Set([ @@ -23,101 +33,159 @@ const ALLOWED_VERBS = new Set([ const UNAVAILABLE_ERROR = "Browser unavailable: no Chromium found — set LOCAL_STUDIO_CHROME_PATH"; -let lastFallback: BrowserNavigation | null = null; - type VerbResult = { ok: boolean; data?: unknown; error?: string }; +const VerbPayloadSchema = Schema.Record(Schema.String, Schema.Unknown); + +export function browserSessionKeyFromRequest(request: Request): BrowserSessionKey { + return decodeBrowserSessionKey(request.headers.get(BROWSER_SESSION_HEADER)); +} -export async function handleBrowserVerb(request: Request, verb: string): Promise { +function invalidBrowserSession(): Response { + return Response.json( + { ok: false, error: `A valid ${BROWSER_SESSION_HEADER} header is required` }, + { status: 400 }, + ); +} + +type BrowserSessionResult = + | { type: "invalid"; response: Response } + | { type: "valid"; session: BrowserSessionKey }; + +function requestBrowserSession(request: Request): BrowserSessionResult { + try { + return { type: "valid", session: browserSessionKeyFromRequest(request) }; + } catch { + return { type: "invalid", response: invalidBrowserSession() }; + } +} + +export async function handleBrowserVerb( + request: Request, + verb: string, + host: BrowserHost = browserHost, + reader: typeof fetchReadable = fetchReadable, +): Promise { + const sessionResult = requestBrowserSession(request); + if (sessionResult.type === "invalid") return sessionResult.response; + const { session } = sessionResult; if (!ALLOWED_VERBS.has(verb)) { return Response.json({ ok: false, error: `Unknown browser verb: ${verb}` }, { status: 400 }); } - const payload = await readPayload(request); try { - const result = await dispatchVerb(verb, payload); + const payload = await readPayload(request); + const result = await dispatchVerb(host, session, verb, payload, reader); return Response.json(result); } catch (error) { - return Response.json({ - ok: false, - error: error instanceof Error ? error.message : "Browser command failed", - }); + const payloadError = error instanceof BrowserPayloadError; + return Response.json( + { + ok: false, + error: error instanceof Error ? error.message : "Browser command failed", + }, + payloadError ? { status: 400 } : undefined, + ); } } +class BrowserPayloadError extends Error {} + async function readPayload(request: Request): Promise> { try { - const body = (await request.json()) as Record | null; - if (body && typeof body === "object") { - // sessionId was a renderer-bridge affinity hint; the host is global now. - const { sessionId: _sessionId, ...rest } = body; - return rest; + const text = await request.text(); + if (!text.trim()) return {}; + const body = Schema.decodeUnknownSync(VerbPayloadSchema)(JSON.parse(text)); + if (Object.hasOwn(body, "sessionId")) { + throw new BrowserPayloadError(`Use ${BROWSER_SESSION_HEADER} instead of body sessionId`); } - } catch { - // empty body is fine + return body; + } catch (error) { + if (error instanceof BrowserPayloadError) throw error; + throw new BrowserPayloadError("Invalid browser command JSON"); } - return {}; } -async function dispatchVerb(verb: string, payload: Record): Promise { - if (!browserHost.isAvailable()) return fallbackVerb(verb, payload); +async function dispatchVerb( + host: BrowserHost, + session: BrowserSessionKey, + verb: string, + payload: Record, + reader: typeof fetchReadable, +): Promise { + if (!host.isAvailable()) return fallbackVerb(host, session, verb, payload, reader); try { - return await runHostVerb(verb, payload); + return await runHostVerb(host, session, verb, payload); } catch (error) { // A launch/connection failure for the reading verbs still degrades to // reading mode rather than failing the tool call outright. - if (verb === "navigate" || verb === "get-text") return fallbackVerb(verb, payload); + if (verb === "navigate" || verb === "get-text") { + return fallbackVerb(host, session, verb, payload, reader); + } throw error; } } -async function runHostVerb(verb: string, payload: Record): Promise { +async function runHostVerb( + host: BrowserHost, + session: BrowserSessionKey, + verb: string, + payload: Record, +): Promise { switch (verb) { case "navigate": - return navigateVerb(payload); + return navigateVerb(host, session, payload); case "get-url": - return { ok: true, data: await browserHost.getUrl() }; + return { ok: true, data: await host.getUrl(session) }; case "get-text": - return { ok: true, data: { text: await browserHost.getText() } }; + return { ok: true, data: { text: await host.getText(session) } }; case "get-html": - return { ok: true, data: { html: await browserHost.getHtml() } }; + return { ok: true, data: { html: await host.getHtml(session) } }; case "screenshot": - return { ok: true, data: { dataUri: await browserHost.screenshot() } }; + return { ok: true, data: { dataUri: await host.screenshot(session) } }; case "click": - return selectorVerb(await browserHost.click({ selector: requireSelector(payload) })); + return selectorVerb(await host.click(session, { selector: requireSelector(payload) })); case "fill": return selectorVerb( - await browserHost.fill({ + await host.fill(session, { selector: requireSelector(payload), value: String(payload.value ?? ""), }), ); case "scroll": - return scrollVerb(payload); + return scrollVerb(host, session, payload); case "back": - await browserHost.goBack(); - return { ok: true, data: await browserHost.getState() }; + await host.goBack(session); + return { ok: true, data: await host.getState(session) }; case "forward": - await browserHost.goForward(); - return { ok: true, data: await browserHost.getState() }; + await host.goForward(session); + return { ok: true, data: await host.getState(session) }; case "reload": - await browserHost.reload(); - return { ok: true, data: await browserHost.getState() }; + await host.reload(session); + return { ok: true, data: await host.getState(session) }; default: return { ok: false, error: `Unsupported browser verb: ${verb}` }; } } -async function navigateVerb(payload: Record): Promise { +async function navigateVerb( + host: BrowserHost, + session: BrowserSessionKey, + payload: Record, +): Promise { // Pane rules: public web plus loopback (previewing local dev servers is the // pane's main job); other private ranges stay blocked. const navigation = browserNavigation(String(payload.url ?? "")); if (!navigation) return { ok: false, error: "valid public or localhost http(s) url required" }; - const result = await browserHost.navigate(navigation.url); + const result = await host.navigate(session, navigation.url); return { ok: true, data: result }; } -async function scrollVerb(payload: Record): Promise { +async function scrollVerb( + host: BrowserHost, + session: BrowserSessionKey, + payload: Record, +): Promise { const deltaY = Number(payload.deltaY ?? 0); - const result = await browserHost.scroll({ deltaY: Number.isFinite(deltaY) ? deltaY : 0 }); + const result = await host.scroll(session, { deltaY: Number.isFinite(deltaY) ? deltaY : 0 }); return { ok: true, data: { deltaY: result.deltaY, scrollY: result.scrollY } }; } @@ -140,28 +208,58 @@ function requireSelector(payload: Record): string { // without a url arg); every other verb returns the clear unavailable error. The // fallback honors pane rules (public + loopback) so local dev servers stay // previewable even when there's no headless Chromium to drive a full surface. -async function fallbackVerb(verb: string, payload: Record): Promise { +async function fallbackVerb( + host: BrowserHost, + session: BrowserSessionKey, + verb: string, + payload: Record, + reader: typeof fetchReadable, +): Promise { + if (verb !== "navigate" && verb !== "get-url" && verb !== "get-text" && verb !== "get-html") { + return { ok: false, error: UNAVAILABLE_ERROR }; + } + return host.withFallbackSession(session, (fallback) => + fallbackSessionVerb(verb, payload, fallback, reader), + ); +} + +async function fallbackSessionVerb( + verb: string, + payload: Record, + fallback: BrowserNavigation | null, + reader: typeof fetchReadable, +): Promise> { if (verb === "navigate") { const navigation = browserNavigation(String(payload.url ?? "")); - if (!navigation) return { ok: false, error: "valid public or localhost http(s) url required" }; - const reader = await fetchReadable(navigation.url, navigation.mode); - lastFallback = { mode: navigation.mode, url: reader.url }; - return { ok: true, data: { url: reader.url, title: reader.title, readingMode: true } }; + if (!navigation) { + return { result: { ok: false, error: "valid public or localhost http(s) url required" } }; + } + const result = await reader(navigation.url, navigation.mode); + return { + navigation: { mode: navigation.mode, url: result.url }, + result: { + ok: true, + data: { url: result.url, title: result.title, readingMode: true }, + }, + }; } if (verb === "get-url") { - return { ok: true, data: { url: lastFallback?.url ?? "", title: "" } }; + return { result: { ok: true, data: { url: fallback?.url ?? "", title: "" } } }; } if (verb === "get-text" || verb === "get-html") { const requested = browserNavigation(String(payload.url ?? "")); - const navigation = requested ?? lastFallback; - if (!navigation) return { ok: false, error: UNAVAILABLE_ERROR }; - const reader = await fetchReadable(navigation.url, navigation.mode); - lastFallback = { mode: navigation.mode, url: reader.url }; - return verb === "get-text" - ? { ok: true, data: { text: reader.text, readingMode: true } } - : { ok: true, data: { html: reader.markdown ?? reader.text, readingMode: true } }; + const navigation = requested ?? fallback; + if (!navigation) return { result: { ok: false, error: UNAVAILABLE_ERROR } }; + const result = await reader(navigation.url, navigation.mode); + return { + navigation: { mode: navigation.mode, url: result.url }, + result: + verb === "get-text" + ? { ok: true, data: { text: result.text, readingMode: true } } + : { ok: true, data: { html: result.markdown ?? result.text, readingMode: true } }, + }; } - return { ok: false, error: UNAVAILABLE_ERROR }; + return { result: { ok: false, error: UNAVAILABLE_ERROR } }; } export async function handleBrowserFetch(request: Request): Promise { @@ -185,12 +283,18 @@ export async function handleBrowserFetch(request: Request): Promise { // Next's standalone server buffers locally-built event streams, and polling // survives buffering proxies for remote deploys). -export async function handleBrowserFrame(): Promise { - if (!browserHost.isAvailable()) { +export async function handleBrowserFrame( + request: Request, + host: BrowserHost = browserHost, +): Promise { + const sessionResult = requestBrowserSession(request); + if (sessionResult.type === "invalid") return sessionResult.response; + const { session } = sessionResult; + if (!host.isAvailable()) { return Response.json({ ok: false, error: UNAVAILABLE_ERROR }, { status: 503 }); } try { - const { frame, state } = await browserHost.pollFrame(); + const { frame, state } = await host.pollFrame(session); return Response.json({ ok: true, data: { @@ -209,23 +313,65 @@ export async function handleBrowserFrame(): Promise { } } -type InputBody = - | ({ kind: "mouse" } & Omit & { type: MouseInput["type"] }) - | ({ kind: "wheel" } & Omit) - | ({ kind: "key" } & KeyInput); - -export async function handleBrowserInput(request: Request): Promise { - if (!browserHost.isAvailable()) { +const MouseButtonSchema = Schema.Union([ + Schema.Literal("left"), + Schema.Literal("right"), + Schema.Literal("middle"), +]); +const MouseTypeSchema = Schema.Union([ + Schema.Literal("down"), + Schema.Literal("up"), + Schema.Literal("move"), +]); +const KeyTypeSchema = Schema.Union([ + Schema.Literal("down"), + Schema.Literal("up"), + Schema.Literal("char"), +]); +const InputBodySchema = Schema.Union([ + Schema.Struct({ + kind: Schema.Literal("mouse"), + type: MouseTypeSchema, + x: Schema.Number, + y: Schema.Number, + button: Schema.optional(MouseButtonSchema), + clickCount: Schema.optional(Schema.Number), + }), + Schema.Struct({ + kind: Schema.Literal("wheel"), + x: Schema.Number, + y: Schema.Number, + deltaX: Schema.optional(Schema.Number), + deltaY: Schema.optional(Schema.Number), + }), + Schema.Struct({ + kind: Schema.Literal("key"), + type: KeyTypeSchema, + key: Schema.String, + code: Schema.String, + text: Schema.optional(Schema.String), + }), +]); +type InputBody = typeof InputBodySchema.Type; + +export async function handleBrowserInput( + request: Request, + host: BrowserHost = browserHost, +): Promise { + const sessionResult = requestBrowserSession(request); + if (sessionResult.type === "invalid") return sessionResult.response; + const { session } = sessionResult; + if (!host.isAvailable()) { return Response.json({ ok: false, error: "Browser unavailable" }, { status: 503 }); } let body: InputBody; try { - body = (await request.json()) as InputBody; + body = Schema.decodeUnknownSync(InputBodySchema)(await request.json()); } catch { return Response.json({ ok: false, error: "Invalid JSON" }, { status: 400 }); } try { - await dispatchInput(body); + await dispatchInput(host, session, body); return Response.json({ ok: true }); } catch (error) { return Response.json({ @@ -235,9 +381,13 @@ export async function handleBrowserInput(request: Request): Promise { } } -async function dispatchInput(body: InputBody): Promise { +async function dispatchInput( + host: BrowserHost, + session: BrowserSessionKey, + body: InputBody, +): Promise { if (body.kind === "key") { - await browserHost.dispatchKey({ + await host.dispatchKey(session, { type: body.type, key: body.key, code: body.code, @@ -246,7 +396,7 @@ async function dispatchInput(body: InputBody): Promise { return; } if (body.kind === "wheel") { - await browserHost.dispatchMouse({ + await host.dispatchMouse(session, { type: "wheel", x: Number(body.x) || 0, y: Number(body.y) || 0, @@ -255,7 +405,7 @@ async function dispatchInput(body: InputBody): Promise { }); return; } - await browserHost.dispatchMouse({ + await host.dispatchMouse(session, { type: body.type, x: Number(body.x) || 0, y: Number(body.y) || 0, @@ -386,12 +536,18 @@ export async function handleBrowserLocalhosts(request: Request): Promise { - if (!browserHost.isAvailable()) { +export async function handleBrowserState( + request: Request, + host: BrowserHost = browserHost, +): Promise { + const sessionResult = requestBrowserSession(request); + if (sessionResult.type === "invalid") return sessionResult.response; + const { session } = sessionResult; + if (!host.isAvailable()) { return Response.json({ ok: false, error: "Browser unavailable" }, { status: 503 }); } try { - return Response.json({ ok: true, data: await browserHost.getState() }); + return Response.json({ ok: true, data: await host.getState(session) }); } catch (error) { return Response.json({ ok: false, @@ -405,13 +561,21 @@ export async function handleBrowserState(): Promise { // Sets the headless Chromium viewport so it matches the visible panel's // dimensions. Body: { width, height }. -export async function handleBrowserViewport(request: Request): Promise { - if (!browserHost.isAvailable()) { +const ViewportBodySchema = Schema.Struct({ width: Schema.Number, height: Schema.Number }); + +export async function handleBrowserViewport( + request: Request, + host: BrowserHost = browserHost, +): Promise { + const sessionResult = requestBrowserSession(request); + if (sessionResult.type === "invalid") return sessionResult.response; + const { session } = sessionResult; + if (!host.isAvailable()) { return Response.json({ ok: false, error: "Browser unavailable" }, { status: 503 }); } - let body: { width?: unknown; height?: unknown }; + let body: typeof ViewportBodySchema.Type; try { - body = (await request.json()) as { width?: unknown; height?: unknown }; + body = Schema.decodeUnknownSync(ViewportBodySchema)(await request.json()); } catch { return Response.json({ ok: false, error: "Invalid JSON" }, { status: 400 }); } @@ -421,7 +585,7 @@ export async function handleBrowserViewport(request: Request): Promise return Response.json({ ok: false, error: "width and height are required" }, { status: 400 }); } try { - await browserHost.setViewport(width, height); + await host.setViewport(session, width, height); return Response.json({ ok: true, data: { width: Math.round(width), height: Math.round(height) }, diff --git a/services/agent-runtime/src/http/browser-session-handlers.test.ts b/services/agent-runtime/src/http/browser-session-handlers.test.ts new file mode 100644 index 000000000..49830c015 --- /dev/null +++ b/services/agent-runtime/src/http/browser-session-handlers.test.ts @@ -0,0 +1,310 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { BROWSER_SESSION_HEADER, decodeBrowserSessionKey } from "../browser-session-contract"; +import { BrowserHost, type BrowserHostManager } from "../browser-host/browser-host"; +import { fetchReadable, type ReaderResult } from "../browser-host/reader"; +import type { BrowserNetworkMode } from "../../../../shared/agent/sanitize-embedded-browser-url"; +import { + browserSessionKeyFromRequest, + handleBrowserFetch, + handleBrowserFrame, + handleBrowserInput, + handleBrowserLocalhosts, + handleBrowserState, + handleBrowserVerb, + handleBrowserViewport, +} from "./browser-handlers"; + +class CountingManager implements BrowserHostManager { + touches = 0; + + ensure(_mode: BrowserNetworkMode, _scope: string): Promise { + this.touches += 1; + return Promise.reject(new Error("Unexpected context creation")); + } + + isAvailable(): boolean { + this.touches += 1; + return false; + } + + release(_scope: string): Promise { + this.touches += 1; + return Promise.resolve(); + } + + stop(): Promise { + return Promise.resolve(); + } +} + +function request(path: string, method: "GET" | "POST", session?: string, body?: string): Request { + return new Request(`http://127.0.0.1${path}`, { + method, + headers: session === undefined ? undefined : { [BROWSER_SESSION_HEADER]: session }, + body: method === "POST" ? (body ?? "{}") : undefined, + }); +} + +function readable(url: string): ReaderResult { + return { contentType: "text/plain", text: url, title: url, url }; +} + +function deferredReadable(): { + fetch: typeof fetchReadable; + reject: (error: Error) => void; + resolve: (url: string) => void; + started: Promise; +} { + const completed = Promise.withResolvers(); + const started = Promise.withResolvers(); + return { + fetch: async () => { + started.resolve(); + return completed.promise; + }, + reject: completed.reject, + resolve: (url) => completed.resolve(readable(url)), + started: started.promise, + }; +} + +function navigateRequest(session: string, url: string): Request { + return request( + "/api/agent/browser/navigate", + "POST", + session, + JSON.stringify({ url }), + ); +} + +function getTextRequest(session: string): Request { + return request("/api/agent/browser/get-text", "POST", session); +} + +function getUrlRequest(session: string): Request { + return request("/api/agent/browser/get-url", "POST", session); +} + +async function statefulResponses(host: BrowserHost, session?: string): Promise { + const verbs = [ + "navigate", + "get-url", + "get-text", + "get-html", + "screenshot", + "click", + "scroll", + "fill", + "back", + "forward", + "reload", + ]; + return Promise.all([ + ...verbs.map((verb) => + handleBrowserVerb(request(`/api/agent/browser/${verb}`, "POST", session, "{}"), verb, host), + ), + handleBrowserFrame(request("/api/agent/browser/frame", "GET", session), host), + handleBrowserState(request("/api/agent/browser/state", "GET", session), host), + handleBrowserInput( + request( + "/api/agent/browser/input", + "POST", + session, + JSON.stringify({ kind: "mouse", type: "move", x: 1, y: 1 }), + ), + host, + ), + handleBrowserViewport( + request( + "/api/agent/browser/viewport", + "POST", + session, + JSON.stringify({ width: 800, height: 600 }), + ), + host, + ), + ]); +} + +test("every stateful endpoint rejects missing and malformed session headers before host access", async () => { + for (const value of [undefined, "", "bad key", "é", "a".repeat(129), "session-a,session-b"]) { + const manager = new CountingManager(); + const host = new BrowserHost(manager); + const responses = await statefulResponses(host, value); + assert.equal(responses.length, 15); + assert.deepEqual( + responses.map((response) => response.status), + Array(15).fill(400), + ); + assert.equal(manager.touches, 0); + await host.stop(); + } +}); + +test("session schema accepts exact boundaries and rejects unstable ASCII", () => { + assert.equal(decodeBrowserSessionKey("a"), "a"); + assert.equal(decodeBrowserSessionKey("a".repeat(128)), "a".repeat(128)); + for (const value of [ + null, + "", + "a".repeat(129), + " leading", + "trailing ", + "a/b", + "a\u0000b", + "☃", + ]) { + assert.throws(() => decodeBrowserSessionKey(value)); + } + assert.equal( + browserSessionKeyFromRequest(request("/api/agent/browser/state", "GET", "session-a")), + "session-a", + ); +}); + +test("body session affinity is rejected in favor of the canonical header", async () => { + const manager = new CountingManager(); + const host = new BrowserHost(manager); + const response = await handleBrowserVerb( + request( + "/api/agent/browser/navigate", + "POST", + "session-a", + JSON.stringify({ sessionId: "session-b", url: "https://example.com" }), + ), + "navigate", + host, + ); + assert.equal(response.status, 400); + assert.equal(manager.touches, 0); + await host.stop(); +}); + +test("stateless fetch ignores browser session headers", async () => { + const response = await handleBrowserFetch( + request("/api/agent/browser/fetch", "GET", "malformed session"), + ); + assert.equal(response.status, 400); + assert.deepEqual(await response.json(), { error: "url is required" }); +}); + +test("stateless localhost discovery ignores browser session headers", async () => { + const response = await handleBrowserLocalhosts( + request("/api/agent/browser/localhosts", "GET", "malformed session"), + ); + assert.equal(response.status, 200); +}); + +test("deferred fallback reads retain their session through idle cleanup", async () => { + const manager = new CountingManager(); + let now = 0; + const host = new BrowserHost(manager, { + config: { idleMs: 60_000, maxSessions: 1 }, + now: () => now, + }); + const initialUrl = "https://public.test/initial"; + await handleBrowserVerb( + navigateRequest("session-a", initialUrl), + "navigate", + host, + async (url) => readable(url), + ); + const reader = deferredReadable(); + const pending = handleBrowserVerb( + getTextRequest("session-a"), + "get-text", + host, + reader.fetch, + ); + await reader.started; + now = 60_001; + await host.cleanupIdleSessions(); + reader.reject(new Error("deferred failure")); + await pending; + const current = await handleBrowserVerb(getUrlRequest("session-a"), "get-url", host); + const body = await current.json(); + await host.stop(); + assert.deepEqual(body, { ok: true, data: { title: "", url: initialUrl } }); +}); + +test("release waits for a deferred fallback fetch", async () => { + const manager = new CountingManager(); + const host = new BrowserHost(manager); + const reader = deferredReadable(); + const pending = handleBrowserVerb( + navigateRequest("session-a", "https://public.test/pending"), + "navigate", + host, + reader.fetch, + ); + await reader.started; + let released = false; + const releasing = host.releaseSession("session-a").then(() => { + released = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + const releasedWhilePending = released; + reader.resolve("https://public.test/pending"); + await Promise.all([pending, releasing]); + await host.stop(); + assert.equal(releasedWhilePending, false); +}); + +test("a deferred fallback fetch holds its session capacity", async () => { + const manager = new CountingManager(); + const host = new BrowserHost(manager, { config: { idleMs: 60_000, maxSessions: 1 } }); + const reader = deferredReadable(); + const pending = handleBrowserVerb( + navigateRequest("session-a", "https://public.test/pending"), + "navigate", + host, + reader.fetch, + ); + await reader.started; + const blocked = await handleBrowserVerb(getUrlRequest("session-b"), "get-url", host); + const body = await blocked.json(); + reader.resolve("https://public.test/pending"); + await pending; + await host.stop(); + assert.deepEqual(body, { + error: "Browser session capacity reached while all sessions are active", + ok: false, + }); +}); + +test("later fallback navigation remains authoritative after an earlier fetch completes", async () => { + const manager = new CountingManager(); + const host = new BrowserHost(manager); + const firstUrl = "https://public.test/first"; + const secondUrl = "https://public.test/second"; + const firstReader = deferredReadable(); + let secondStarted = false; + const reader: typeof fetchReadable = async (url) => { + if (url === firstUrl) return firstReader.fetch(url); + secondStarted = true; + return readable(url); + }; + const first = handleBrowserVerb( + navigateRequest("session-a", firstUrl), + "navigate", + host, + reader, + ); + await firstReader.started; + const second = handleBrowserVerb( + navigateRequest("session-a", secondUrl), + "navigate", + host, + reader, + ); + await handleBrowserVerb(getUrlRequest("session-b"), "get-url", host); + const secondStartedBeforeFirstCompleted = secondStarted; + firstReader.resolve(firstUrl); + await Promise.all([first, second]); + const current = await handleBrowserVerb(getUrlRequest("session-a"), "get-url", host); + const body = await current.json(); + await host.stop(); + assert.equal(secondStartedBeforeFirstCompleted, false); + assert.deepEqual(body, { ok: true, data: { title: "", url: secondUrl } }); +}); diff --git a/services/agent-runtime/src/http/browser-standalone-session.test.ts b/services/agent-runtime/src/http/browser-standalone-session.test.ts new file mode 100644 index 000000000..2cba31c39 --- /dev/null +++ b/services/agent-runtime/src/http/browser-standalone-session.test.ts @@ -0,0 +1,131 @@ +import assert from "node:assert/strict"; +import { type ChildProcess, execFile, spawn } from "node:child_process"; +import { once } from "node:events"; +import { createServer } from "node:net"; +import { promisify } from "node:util"; +import test from "node:test"; +import { BROWSER_SESSION_HEADER } from "../browser-session-contract"; + +const execFileAsync = promisify(execFile); +const INVALID_SESSION = { + error: `A valid ${BROWSER_SESSION_HEADER} header is required`, + ok: false, +}; +const STATEFUL_REQUESTS: ReadonlyArray<{ + body?: string; + method: "GET" | "POST"; + path: string; +}> = [ + { body: "{}", method: "POST", path: "/api/agent/browser/get-url" }, + { body: "not-json", method: "POST", path: "/api/agent/browser/get-url" }, + { method: "GET", path: "/api/agent/browser/frame" }, + { body: "not-json", method: "POST", path: "/api/agent/browser/input" }, + { method: "GET", path: "/api/agent/browser/state" }, + { body: "not-json", method: "POST", path: "/api/agent/browser/viewport" }, +]; + +async function availablePort(): Promise { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + throw new Error("Temporary loopback server has no numeric port"); + } + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + return address.port; +} + +async function bundleStandalone(): Promise { + const command = process.platform === "win32" ? "npm.cmd" : "npm"; + await execFileAsync(command, ["run", "bundle", "--silent"], { cwd: process.cwd() }); +} + +async function startStandalone(port: number): Promise { + const child = spawn(process.execPath, ["dist/standalone.mjs"], { + cwd: process.cwd(), + env: { ...process.env, PORT: String(port) }, + stdio: ["ignore", "pipe", "pipe"], + }); + const ready = Promise.withResolvers(); + let output = ""; + const onData = (chunk: unknown) => { + output += String(chunk); + if (output.includes("[agent-runtime] listening")) ready.resolve(); + }; + const onError = (error: Error) => ready.reject(error); + const onExit = (code: number | null, signal: NodeJS.Signals | null) => + ready.reject(new Error(`Standalone exited before ready: ${code ?? signal ?? "unknown"}`)); + child.stdout?.on("data", onData); + child.stderr?.on("data", onData); + child.once("error", onError); + child.once("exit", onExit); + const timer = setTimeout( + () => ready.reject(new Error(`Standalone start timed out: ${output}`)), + 10_000, + ); + try { + await ready.promise; + return child; + } catch (error) { + child.kill("SIGTERM"); + throw error; + } finally { + clearTimeout(timer); + child.stdout?.off("data", onData); + child.stderr?.off("data", onData); + child.off("error", onError); + child.off("exit", onExit); + } +} + +async function stopStandalone(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = once(child, "exit"); + child.kill("SIGTERM"); + await exited; +} + +function sessionHeaders(session: string | undefined): HeadersInit { + return session === undefined ? {} : { [BROWSER_SESSION_HEADER]: session }; +} + +test("standalone browser routes reject invalid session keys before request parsing", async (context) => { + await bundleStandalone(); + const port = await availablePort(); + const child = await startStandalone(port); + context.after(() => stopStandalone(child)); + const origin = `http://127.0.0.1:${port}`; + for (const session of [undefined, "", "bad key"]) { + for (const request of STATEFUL_REQUESTS) { + const response = await fetch(`${origin}${request.path}`, { + ...(request.body === undefined ? {} : { body: request.body }), + headers: sessionHeaders(session), + method: request.method, + }); + assert.equal(response.status, 400); + assert.deepEqual(await response.json(), INVALID_SESSION); + } + } + const valid = await fetch(`${origin}/api/agent/browser/get-url`, { + body: "not-json", + headers: sessionHeaders("session-a"), + method: "POST", + }); + assert.equal(valid.status, 400); + assert.deepEqual(await valid.json(), { error: "Invalid browser command JSON", ok: false }); + const statelessFetch = await fetch(`${origin}/api/agent/browser/fetch`, { + headers: sessionHeaders("bad key"), + }); + assert.equal(statelessFetch.status, 400); + assert.deepEqual(await statelessFetch.json(), { error: "url is required" }); + const statelessLocalhosts = await fetch(`${origin}/api/agent/browser/localhosts`, { + headers: sessionHeaders("bad key"), + }); + assert.equal(statelessLocalhosts.status, 200); +}); diff --git a/services/agent-runtime/src/pi-runtime-helpers.ts b/services/agent-runtime/src/pi-runtime-helpers.ts index 82f60b5b7..bfb4c331b 100644 --- a/services/agent-runtime/src/pi-runtime-helpers.ts +++ b/services/agent-runtime/src/pi-runtime-helpers.ts @@ -2,7 +2,8 @@ import { existsSync, readFileSync } from "node:fs"; import { realpath, stat } from "node:fs/promises"; import { homedir } from "node:os"; import path from "node:path"; -import { Effect } from "effect"; +import { Effect, Semaphore } from "effect"; +import { BROWSER_SESSION_HEADER, decodeBrowserSessionKey } from "./browser-session-contract"; import { listProjectsFromStore } from "./projects-store"; import { hasEnabledConnectorsSync } from "./connectors-service"; @@ -322,13 +323,20 @@ function runtimeEnvInjections( ): Record { const frontendBase = env.LOCAL_STUDIO_FRONTEND_BASE ?? deriveFrontendBase(env); const relay = readSitegeistRelayEnv(env); + const browserSessionId = options.browserSessionId + ? decodeBrowserSessionKey(options.browserSessionId) + : ""; + if (options.browserToolEnabled === true && !browserSessionId) { + throw new Error("A valid browser session id is required when the browser tool is enabled"); + } return { - LOCAL_STUDIO_BROWSER_SESSION_ID: options.browserSessionId ?? "", + LOCAL_STUDIO_BROWSER_SESSION_HEADER: BROWSER_SESSION_HEADER, + LOCAL_STUDIO_BROWSER_SESSION_ID: browserSessionId, LOCAL_STUDIO_PLAN_SESSION_ID: options.planSessionId ?? "", LOCAL_STUDIO_FRONTEND_BASE: frontendBase, SITEGEIST_RELAY_URL: env.SITEGEIST_RELAY_URL ?? relay.SITEGEIST_RELAY_URL ?? "", SITEGEIST_RELAY_TOKEN: env.SITEGEIST_RELAY_TOKEN ?? relay.SITEGEIST_RELAY_TOKEN ?? "", - SITEGEIST_RELAY_SESSION_ID: options.browserSessionId ?? "", + SITEGEIST_RELAY_SESSION_ID: browserSessionId, }; } @@ -367,6 +375,50 @@ export function applyRuntimeEnvInjections( for (const [key, value] of Object.entries(envInjections)) env[key] = value; } +const runtimeEnvLock = Semaphore.makeUnsafe(1); + +type EnvironmentSnapshot = Record< + string, + { present: true; value: string | undefined } | { present: false } +>; + +function environmentSnapshot( + envInjections: Record, + env: NodeJS.ProcessEnv, +): EnvironmentSnapshot { + return Object.fromEntries( + Object.keys(envInjections).map((key) => [ + key, + Object.hasOwn(env, key) ? { present: true, value: env[key] } : { present: false }, + ]), + ); +} + +function restoreEnvironment(snapshot: EnvironmentSnapshot, env: NodeJS.ProcessEnv): void { + for (const [key, entry] of Object.entries(snapshot)) { + if (entry.present) env[key] = entry.value; + else delete env[key]; + } +} + +export function withRuntimeEnvInjections( + envInjections: Record, + effect: Effect.Effect, + env: NodeJS.ProcessEnv = process.env, +): Effect.Effect { + return runtimeEnvLock.withPermit( + Effect.acquireUseRelease( + Effect.sync(() => { + const snapshot = environmentSnapshot(envInjections, env); + applyRuntimeEnvInjections(envInjections, env); + return snapshot; + }), + () => effect, + (snapshot) => Effect.sync(() => restoreEnvironment(snapshot, env)), + ), + ); +} + export function buildAgentSessionOptions( input: AgentSessionOptionsInput, ): Promise { diff --git a/services/agent-runtime/src/pi-runtime.ts b/services/agent-runtime/src/pi-runtime.ts index 5ddb3dd87..2d2f25905 100644 --- a/services/agent-runtime/src/pi-runtime.ts +++ b/services/agent-runtime/src/pi-runtime.ts @@ -11,10 +11,10 @@ import { import { Effect } from "effect"; import type { AgentImageInput } from "../../../shared/agent/agent-image-input"; import { - applyRuntimeEnvInjections, buildAgentSessionOptionsSync, runtimeOptionsFingerprint, resolveAgentCwdEffect, + withRuntimeEnvInjections, type RuntimeStartOptions, } from "./pi-runtime-helpers"; import { refreshPiModels, resolvePiModelSelection } from "./pi-runtime-models"; @@ -140,86 +140,88 @@ class PiSdkSession extends EventEmitter implements PiAgentSession { }); const sessionOptions = buildAgentSessionOptionsSync({ options }); - applyRuntimeEnvInjections(sessionOptions.envInjections); const sessionManager = SessionManager.create(resolvedCwd); const resumeFile = desiredSessionId ? findSessionFile(resolvedCwd, desiredSessionId) : null; if (resumeFile) sessionManager.setSessionFile(resumeFile); const resuming = Boolean(resumeFile); - const runtime = yield* Effect.tryPromise({ - try: () => - createAgentSessionRuntime( - ({ cwd, agentDir, sessionManager, sessionStartEvent }) => - Effect.runPromise( - Effect.gen(function* () { - const services = yield* Effect.tryPromise({ - try: () => - createAgentSessionServices({ - cwd, - agentDir, - modelRuntime: sharedModelRuntime, - resourceLoaderOptions: { - noExtensions: true, - additionalSkillPaths: sessionOptions.skills, - additionalExtensionPaths: sessionOptions.extensionPaths, - additionalPromptTemplatePaths: sessionOptions.promptTemplatePaths, - }, - }), - catch: (error) => error, - }); - const model = services.modelRuntime.getModel(providerId, backendModelId); - if (!model) { - return yield* Effect.fail( - new Error( - `Model '${providerId}/${backendModelId}' is not available to the SDK runtime.`, - ), + const runtime = yield* withRuntimeEnvInjections( + sessionOptions.envInjections, + Effect.tryPromise({ + try: () => + createAgentSessionRuntime( + ({ cwd, agentDir, sessionManager, sessionStartEvent }) => + Effect.runPromise( + Effect.gen(function* () { + const services = yield* Effect.tryPromise({ + try: () => + createAgentSessionServices({ + cwd, + agentDir, + modelRuntime: sharedModelRuntime, + resourceLoaderOptions: { + noExtensions: true, + additionalSkillPaths: sessionOptions.skills, + additionalExtensionPaths: sessionOptions.extensionPaths, + additionalPromptTemplatePaths: sessionOptions.promptTemplatePaths, + }, + }), + catch: (error) => error, + }); + const model = services.modelRuntime.getModel(providerId, backendModelId); + if (!model) { + return yield* Effect.fail( + new Error( + `Model '${providerId}/${backendModelId}' is not available to the SDK runtime.`, + ), + ); + } + const created = yield* Effect.tryPromise({ + try: () => + createAgentSessionFromServices({ + services, + sessionManager, + sessionStartEvent, + model, + thinkingLevel: selectedModel.reasoning ? "high" : undefined, + }), + catch: (error) => error, + }); + const extensionErrors = services.resourceLoader + .getExtensions() + .errors.map(({ path, error }) => ({ + type: "error" as const, + message: `Failed to load extension "${path}": ${error}`, + path, + })); + const diagnostics = [...services.diagnostics, ...extensionErrors]; + diagnosticsMap().set( + agentDir, + diagnostics.map((d) => ({ + type: d.type as PiResourceDiagnostic["type"], + message: d.message, + path: "path" in d ? (d as { path?: string }).path : undefined, + })), ); - } - const created = yield* Effect.tryPromise({ - try: () => - createAgentSessionFromServices({ - services, - sessionManager, - sessionStartEvent, - model, - thinkingLevel: selectedModel.reasoning ? "high" : undefined, - }), - catch: (error) => error, - }); - const extensionErrors = services.resourceLoader - .getExtensions() - .errors.map(({ path, error }) => ({ - type: "error" as const, - message: `Failed to load extension "${path}": ${error}`, - path, - })); - const diagnostics = [...services.diagnostics, ...extensionErrors]; - diagnosticsMap().set( - agentDir, - diagnostics.map((d) => ({ - type: d.type as PiResourceDiagnostic["type"], - message: d.message, - path: "path" in d ? (d as { path?: string }).path : undefined, - })), - ); - return { - ...created, - services, - diagnostics, - }; - }), - ), - { - cwd: resolvedCwd, - agentDir, - sessionManager, - sessionStartEvent: { - type: "session_start", - reason: resuming ? "resume" : "startup", + return { + ...created, + services, + diagnostics, + }; + }), + ), + { + cwd: resolvedCwd, + agentDir, + sessionManager, + sessionStartEvent: { + type: "session_start", + reason: resuming ? "resume" : "startup", + }, }, - }, - ), - catch: (error) => error, - }); + ), + catch: (error) => error, + }), + ); this.runtime = runtime; this.agentDir = agentDir; diff --git a/services/agent-runtime/src/server.ts b/services/agent-runtime/src/server.ts index 26e768086..f4edea416 100644 --- a/services/agent-runtime/src/server.ts +++ b/services/agent-runtime/src/server.ts @@ -88,10 +88,10 @@ app.post("/api/agent/providers/:providerId/logout", (c) => ); app.get("/api/agent/browser/fetch", (c) => handleBrowserFetch(c.req.raw)); -app.get("/api/agent/browser/frame", () => handleBrowserFrame()); +app.get("/api/agent/browser/frame", (c) => handleBrowserFrame(c.req.raw)); app.post("/api/agent/browser/input", (c) => handleBrowserInput(c.req.raw)); app.get("/api/agent/browser/localhosts", (c) => handleBrowserLocalhosts(c.req.raw)); -app.get("/api/agent/browser/state", () => handleBrowserState()); +app.get("/api/agent/browser/state", (c) => handleBrowserState(c.req.raw)); app.post("/api/agent/browser/viewport", (c) => handleBrowserViewport(c.req.raw)); app.post("/api/agent/browser/:verb", (c) => handleBrowserVerb(c.req.raw, c.req.param("verb")));