diff --git a/Makefile b/Makefile index 204106b..3656804 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: help \ - build build-release check check-lib install clean run run-mcp run-mcp-http \ + build build-release check check-lib install clean run run-batch-capped run-mcp run-mcp-http \ docker-build docker-build-no-cache docker-build-dev \ docker-push-dev docker-push-dev-hub docker-push-dev-ghcr-personal docker-push-dev-ghcr-itc \ docker-push-release docker-push-release-hub \ @@ -41,6 +41,8 @@ help: @echo " make install cargo install --path evaluator --features apps" @echo " make run ARGS='…' cargo run --bin evaluator --features apps -- …" @echo " e.g. make run ARGS='evaluate --url http://127.0.0.1:8765/demo1shop.html --fn window.eval'" + @echo " make run-batch-capped same as run, wrapped in systemd --scope MemoryMax=$(MEMORY_MAX)" + @echo " e.g. make run-batch-capped ARGS='batch -p archive/test.csv -f window.eval'" @echo " make run-mcp cargo run --bin evaluator-mcp (stdio)" @echo " make run-mcp-http cargo run --bin evaluator-mcp -- --http" @echo " (needs: npm run build + Chromium / PUPPETEER_EXECUTABLE_PATH)" @@ -82,6 +84,13 @@ clean: run: cd evaluator && $(CARGO) run $(CARGO_FLAGS) --bin $(CLI_BIN) -- $(if $(strip $(ARGS)),$(ARGS),--help) +# Cap batch RAM so the OOM killer hits the job scope, not Cursor/desktop. +MEMORY_MAX ?= 4G +run-batch-capped: + @test -n "$(strip $(ARGS))" || (echo 'usage: make run-batch-capped ARGS="batch -p …"' >&2; exit 2) + systemd-run --user --scope -p MemoryMax=$(MEMORY_MAX) --quiet -- \ + $(MAKE) run ARGS='$(ARGS)' + run-mcp: cd evaluator && $(CARGO) run $(CARGO_FLAGS) --bin evaluator-mcp -- $(ARGS) diff --git a/apps/evaluator/server.ts b/apps/evaluator/server.ts index 14985e9..07bb482 100644 --- a/apps/evaluator/server.ts +++ b/apps/evaluator/server.ts @@ -219,20 +219,21 @@ async function runBatchCli(argv: string[]): Promise { process.exit(2); } void concurrency; // one browser, sequential (plan) - const outcomes = await runBatch({ + // Never take/emit screenshots in batch — base64 PNGs OOM the host. + const count = await runBatch({ urls, fn, + screenshot: false, onSite: (site, outcome) => { console.log( JSON.stringify({ url: site, results: outcome.results, - screenshot: outcome.screenshot, }) ); }, }); - console.error(`[evaluate] batch done count=${outcomes.length}`); + console.error(`[evaluate] batch done count=${count}`); } async function main(): Promise { diff --git a/libs/util/puppeteer/src/lib/browser-engine.ts b/libs/util/puppeteer/src/lib/browser-engine.ts index c020b35..b53dff3 100644 --- a/libs/util/puppeteer/src/lib/browser-engine.ts +++ b/libs/util/puppeteer/src/lib/browser-engine.ts @@ -2,8 +2,18 @@ import type { Message } from '@evaluator/shared-types'; import type { Observable } from 'rxjs'; import type { ConsoleHit } from './console-hit'; +export type GotoOpts = { + /** Default true. Batch must pass false — PNG base64 retains hundreds of MB. */ + screenshot?: boolean; + /** + * Default `networkidle` (single evaluate / UI). Batch should use `load` — + * storefronts never go idle and keep Chromium thrashing for minutes. + */ + waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit'; +}; + export interface EvaluateSession { - goto(message: Message): Promise; + goto(message: Message, opts?: GotoOpts): Promise; close(): Promise; readonly results: Observable; } diff --git a/libs/util/puppeteer/src/lib/playwright.engine.ts b/libs/util/puppeteer/src/lib/playwright.engine.ts index 3dc47c3..759784a 100644 --- a/libs/util/puppeteer/src/lib/playwright.engine.ts +++ b/libs/util/puppeteer/src/lib/playwright.engine.ts @@ -35,59 +35,69 @@ export class PlaywrightEngine implements EvaluateSession { }); } - async goto(message: Message): Promise { + async goto( + message: Message, + opts?: { screenshot?: boolean; waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit' } + ): Promise { const page = await this.getNewPage(message); if (!page) { return; } - this.setListener(page); - let aborted = false; - let url = ''; - this.ws?.send(JSON.stringify('request')); - console.error(getHostname(message.url.trim())); - const hostname = getHostname(message.url.trim()); - await page.route('**/*', async (route) => { - const req = route.request(); - if ( - req.isNavigationRequest() && - req.frame() === page.mainFrame() && - !req.url().includes(hostname) - ) { - aborted = true; - url = req.url(); - console.error(req.url(), message.url); - this.ws?.send(JSON.stringify('aborted before redirection to ' + req.url())); - await route.abort(); - } else { - await route.continue(); - } - }); - this.ws?.send(JSON.stringify('set request interception')); - this.ws?.send(JSON.stringify('server message.url ' + message.url.trim())); - console.error('server message.url', message.url.trim()); - let error = false; - await page - .goto(message.url.trim(), { - timeout: EVALUATE_TIMEOUT_MS, - waitUntil: 'networkidle', - }) - .catch((err: Error) => { - this.ws?.send(JSON.stringify('error ' + err.toString())); - console.error(message.url, url, err); - error = true; + const takeScreenshot = opts?.screenshot !== false; + const waitUntil = opts?.waitUntil ?? 'networkidle'; + try { + this.setListener(page); + let aborted = false; + let url = ''; + this.ws?.send(JSON.stringify('request')); + console.error(getHostname(message.url.trim())); + const hostname = getHostname(message.url.trim()); + await page.route('**/*', async (route) => { + const req = route.request(); + if ( + req.isNavigationRequest() && + req.frame() === page.mainFrame() && + !req.url().includes(hostname) + ) { + aborted = true; + url = req.url(); + console.error(req.url(), message.url); + this.ws?.send(JSON.stringify('aborted before redirection to ' + req.url())); + await route.abort(); + } else { + await route.continue(); + } }); - if (!aborted && !error) { - this.ws?.send(JSON.stringify('server tries screenshot')); - console.error('server tries screenshot', message.url.trim()); - const buffer = await page.screenshot({ type: 'png' }); - const base64 = buffer.toString('base64'); - this.ws?.send(JSON.stringify('screenshot done')); - console.error('server screenshot'); - if (base64) { - return JSON.stringify(`data:image/png;base64,${base64}`); + this.ws?.send(JSON.stringify('set request interception')); + this.ws?.send(JSON.stringify('server message.url ' + message.url.trim())); + console.error('server message.url', message.url.trim()); + let error = false; + await page + .goto(message.url.trim(), { + timeout: EVALUATE_TIMEOUT_MS, + waitUntil, + }) + .catch((err: Error) => { + this.ws?.send(JSON.stringify('error ' + err.toString())); + console.error(message.url, url, err); + error = true; + }); + if (!aborted && !error && takeScreenshot) { + this.ws?.send(JSON.stringify('server tries screenshot')); + console.error('server tries screenshot', message.url.trim()); + const buffer = await page.screenshot({ type: 'png' }); + const base64 = buffer.toString('base64'); + this.ws?.send(JSON.stringify('screenshot done')); + console.error('server screenshot'); + if (base64) { + return JSON.stringify(`data:image/png;base64,${base64}`); + } } + return; + } finally { + // One context per site — must close or batch leaks Chromium contexts until OOM. + await page.context().close().catch(() => undefined); } - return; } private async getNewPage(message: Message) { @@ -100,14 +110,20 @@ export class PlaywrightEngine implements EvaluateSession { return; } this.ws?.send(JSON.stringify('get browser')); - const context = await browser.newContext({ userAgent: EVALUATE_USER_AGENT }); - const page = await context.newPage(); - this.ws?.send(JSON.stringify('new page done')); - const tpl = buildEvalTemplate(message); - await this.sqliteService.insert(message); - this.ws?.send(JSON.stringify('evaluate Document')); - await page.addInitScript(tpl); - return page; + try { + const context = await browser.newContext({ userAgent: EVALUATE_USER_AGENT }); + const page = await context.newPage(); + this.ws?.send(JSON.stringify('new page done')); + const tpl = buildEvalTemplate(message); + await this.sqliteService.insert(message); + this.ws?.send(JSON.stringify('evaluate Document')); + await page.addInitScript(tpl); + return page; + } catch (err) { + console.error('getNewPage failed', err); + this.ws?.send(JSON.stringify('getNewPage err ' + String(err))); + return; + } } async close() { diff --git a/libs/util/puppeteer/src/lib/puppeteer.engine.ts b/libs/util/puppeteer/src/lib/puppeteer.engine.ts index d2ce65c..3eb38b3 100644 --- a/libs/util/puppeteer/src/lib/puppeteer.engine.ts +++ b/libs/util/puppeteer/src/lib/puppeteer.engine.ts @@ -35,57 +35,75 @@ export class PuppeteerEngine implements EvaluateSession { }); } - async goto(message: Message): Promise { + async goto( + message: Message, + opts?: { screenshot?: boolean; waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit' } + ): Promise { const page = await this.getNewPage(message); if (!page) { return; } - this.setListener(page); - let aborted = false; - let url = ''; - this.ws?.send(JSON.stringify('request')); - console.error(getHostname(message.url.trim())); - page.on('request', (req) => { - if ( - req.isNavigationRequest() && - req.frame() === page.mainFrame() && - !req.url().includes(getHostname(message.url.trim())) - ) { - aborted = true; - url = req.url(); - console.error(req.url(), message.url); - this.ws?.send(JSON.stringify('aborted before redirection to ' + req.url())); - req.abort('aborted'); - } else { - req.continue(); - } - }); - this.ws?.send(JSON.stringify('set request interception')); - await page.setRequestInterception(true); - this.ws?.send(JSON.stringify('server message.url ' + message.url.trim())); - console.error('server message.url', message.url.trim()); - let error = false; - await page - .goto(message.url.trim(), { - timeout: EVALUATE_TIMEOUT_MS, - waitUntil: ['domcontentloaded', 'networkidle0'], - }) - .catch((err) => { - this.ws?.send(JSON.stringify('error ' + err.toString())); - console.error(message.url, url, err); - error = true; + const takeScreenshot = opts?.screenshot !== false; + // Default matches historical UI/evaluate behavior (domcontentloaded + networkidle0). + // Puppeteer has no 'commit'; map it to 'domcontentloaded'. + const waitUntil: Array<'load' | 'domcontentloaded' | 'networkidle0'> = + opts?.waitUntil === 'load' + ? ['load'] + : opts?.waitUntil === 'domcontentloaded' || opts?.waitUntil === 'commit' + ? ['domcontentloaded'] + : opts?.waitUntil === 'networkidle' + ? ['domcontentloaded', 'networkidle0'] + : ['domcontentloaded', 'networkidle0']; + try { + this.setListener(page); + let aborted = false; + let url = ''; + this.ws?.send(JSON.stringify('request')); + console.error(getHostname(message.url.trim())); + page.on('request', (req) => { + if ( + req.isNavigationRequest() && + req.frame() === page.mainFrame() && + !req.url().includes(getHostname(message.url.trim())) + ) { + aborted = true; + url = req.url(); + console.error(req.url(), message.url); + this.ws?.send(JSON.stringify('aborted before redirection to ' + req.url())); + req.abort('aborted'); + } else { + req.continue(); + } }); - if (!aborted && !error) { - this.ws?.send(JSON.stringify('server tries screenshot')); - console.error('server tries screenshot', message.url.trim()); - const base64 = (await page.screenshot({ encoding: 'base64' })) as string; - this.ws?.send(JSON.stringify('screenshot done')); - console.error('server screenshot'); - if (base64) { - return JSON.stringify(`data:image/png;base64,${base64}`); + this.ws?.send(JSON.stringify('set request interception')); + await page.setRequestInterception(true); + this.ws?.send(JSON.stringify('server message.url ' + message.url.trim())); + console.error('server message.url', message.url.trim()); + let error = false; + await page + .goto(message.url.trim(), { + timeout: EVALUATE_TIMEOUT_MS, + waitUntil, + }) + .catch((err) => { + this.ws?.send(JSON.stringify('error ' + err.toString())); + console.error(message.url, url, err); + error = true; + }); + if (!aborted && !error && takeScreenshot) { + this.ws?.send(JSON.stringify('server tries screenshot')); + console.error('server tries screenshot', message.url.trim()); + const base64 = (await page.screenshot({ encoding: 'base64' })) as string; + this.ws?.send(JSON.stringify('screenshot done')); + console.error('server screenshot'); + if (base64) { + return JSON.stringify(`data:image/png;base64,${base64}`); + } } + return; + } finally { + await page.close().catch(() => undefined); } - return; } private async getNewPage(message: Message) { @@ -98,14 +116,20 @@ export class PuppeteerEngine implements EvaluateSession { return browser; } this.ws?.send(JSON.stringify('get browser')); - const page = await browser.newPage(); - this.ws?.send(JSON.stringify('new page done')); - const tpl = buildEvalTemplate(message); - await this.sqliteService.insert(message); - this.ws?.send(JSON.stringify('evaluate Document')); - await page.evaluateOnNewDocument(tpl); - await page.setUserAgent(EVALUATE_USER_AGENT); - return page; + try { + const page = await browser.newPage(); + this.ws?.send(JSON.stringify('new page done')); + const tpl = buildEvalTemplate(message); + await this.sqliteService.insert(message); + this.ws?.send(JSON.stringify('evaluate Document')); + await page.evaluateOnNewDocument(tpl); + await page.setUserAgent(EVALUATE_USER_AGENT); + return page; + } catch (err) { + console.error('getNewPage failed', err); + this.ws?.send(JSON.stringify('getNewPage err ' + String(err))); + return; + } } async close() { diff --git a/libs/util/puppeteer/src/lib/run-evaluate.ts b/libs/util/puppeteer/src/lib/run-evaluate.ts index 88fc843..9a3c191 100644 --- a/libs/util/puppeteer/src/lib/run-evaluate.ts +++ b/libs/util/puppeteer/src/lib/run-evaluate.ts @@ -29,6 +29,11 @@ export type RunBatchOpts = { fn?: string; /** Reserved; batch uses one browser and processes sites sequentially. */ concurrency?: number; + /** + * Default false. Screenshots are huge base64 strings; enabling them in batch + * will OOM the host when combined with context leaks. + */ + screenshot?: boolean; onSite?: (site: string, outcome: RunEvaluateOutcome) => void; }; @@ -108,12 +113,14 @@ export async function runEvaluate(opts: RunEvaluateOpts): Promise { +export async function runBatch(opts: RunBatchOpts): Promise { const urls = opts.urls.map((u) => u.trim()).filter(Boolean); if (urls.length === 0) { - return []; + return 0; } for (const url of urls) { if (!isValidHttpUrl(url)) { @@ -125,7 +132,8 @@ export async function runBatch(opts: RunBatchOpts): Promise