Skip to content

feat: optional evaluate screenshots and capped batch Make target #68

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gRoussac merged 2 commits into from
Aug 5, 2026
Merged

feat: optional evaluate screenshots and capped batch Make target #68

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c615be5
feat: optional evaluate screenshots and capped batch Make target.
Interchouette Aug 5, 2026
040f9e3
fix: use runBatch count return in batch CLI.
Interchouette Aug 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
.PHONY: help \
build build-release check check-lib install clean run run-mcp run-mcp-http \
build build-release check check-lib install clean run run-batch-capped run-mcp run-mcp-http \
docker-build docker-build-no-cache docker-build-dev \
docker-push-dev docker-push-dev-hub docker-push-dev-ghcr-personal docker-push-dev-ghcr-itc \
docker-push-release docker-push-release-hub \
Expand Down Expand Up @@ -41,6 +41,8 @@ help:
@echo " make install cargo install --path evaluator --features apps"
@echo " make run ARGS='…' cargo run --bin evaluator --features apps -- …"
@echo " e.g. make run ARGS='evaluate --url http://127.0.0.1:8765/demo1shop.html --fn window.eval'"
@echo " make run-batch-capped same as run, wrapped in systemd --scope MemoryMax=$(MEMORY_MAX)"
@echo " e.g. make run-batch-capped ARGS='batch -p archive/test.csv -f window.eval'"
@echo " make run-mcp cargo run --bin evaluator-mcp (stdio)"
@echo " make run-mcp-http cargo run --bin evaluator-mcp -- --http"
@echo " (needs: npm run build + Chromium / PUPPETEER_EXECUTABLE_PATH)"
Expand Down Expand Up @@ -82,6 +84,13 @@ clean:
run:
cd evaluator && $(CARGO) run $(CARGO_FLAGS) --bin $(CLI_BIN) -- $(if $(strip $(ARGS)),$(ARGS),--help)

# Cap batch RAM so the OOM killer hits the job scope, not Cursor/desktop.
MEMORY_MAX ?= 4G
run-batch-capped:
@test -n "$(strip $(ARGS))" || (echo 'usage: make run-batch-capped ARGS="batch -p …"' >&2; exit 2)
systemd-run --user --scope -p MemoryMax=$(MEMORY_MAX) --quiet -- \
$(MAKE) run ARGS='$(ARGS)'

run-mcp:
cd evaluator && $(CARGO) run $(CARGO_FLAGS) --bin evaluator-mcp -- $(ARGS)

Expand Down
7 changes: 4 additions & 3 deletions apps/evaluator/server.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,20 +219,21 @@ async function runBatchCli(argv: string[]): Promise<void> {
process.exit(2);
}
void concurrency; // one browser, sequential (plan)
const outcomes = await runBatch({
// Never take/emit screenshots in batch — base64 PNGs OOM the host.
const count = await runBatch({
urls,
fn,
screenshot: false,
onSite: (site, outcome) => {
console.log(
JSON.stringify({
url: site,
results: outcome.results,
screenshot: outcome.screenshot,
})
);
},
});
console.error(`[evaluate] batch done count=${outcomes.length}`);
console.error(`[evaluate] batch done count=${count}`);
}

async function main(): Promise<void> {
Expand Down
12 changes: 11 additions & 1 deletion libs/util/puppeteer/src/lib/browser-engine.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,18 @@ import type { Message } from '@evaluator/shared-types';
import type { Observable } from 'rxjs';
import type { ConsoleHit } from './console-hit';

export type GotoOpts = {
/** Default true. Batch must pass false — PNG base64 retains hundreds of MB. */
screenshot?: boolean;
/**
* Default `networkidle` (single evaluate / UI). Batch should use `load` —
* storefronts never go idle and keep Chromium thrashing for minutes.
*/
waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit';
};

export interface EvaluateSession {
goto(message: Message): Promise<string | undefined>;
goto(message: Message, opts?: GotoOpts): Promise<string | undefined>;
close(): Promise<void>;
readonly results: Observable<ConsoleHit>;
}
Expand Down
124 changes: 70 additions & 54 deletions libs/util/puppeteer/src/lib/playwright.engine.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,59 +35,69 @@ export class PlaywrightEngine implements EvaluateSession {
});
}

async goto(message: Message): Promise<string | undefined> {
async goto(
message: Message,
opts?: { screenshot?: boolean; waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit' }
): Promise<string | undefined> {
const page = await this.getNewPage(message);
if (!page) {
return;
}
this.setListener(page);
let aborted = false;
let url = '';
this.ws?.send(JSON.stringify('request'));
console.error(getHostname(message.url.trim()));
const hostname = getHostname(message.url.trim());
await page.route('**/*', async (route) => {
const req = route.request();
if (
req.isNavigationRequest() &&
req.frame() === page.mainFrame() &&
!req.url().includes(hostname)
) {
aborted = true;
url = req.url();
console.error(req.url(), message.url);
this.ws?.send(JSON.stringify('aborted before redirection to ' + req.url()));
await route.abort();
} else {
await route.continue();
}
});
this.ws?.send(JSON.stringify('set request interception'));
this.ws?.send(JSON.stringify('server message.url ' + message.url.trim()));
console.error('server message.url', message.url.trim());
let error = false;
await page
.goto(message.url.trim(), {
timeout: EVALUATE_TIMEOUT_MS,
waitUntil: 'networkidle',
})
.catch((err: Error) => {
this.ws?.send(JSON.stringify('error ' + err.toString()));
console.error(message.url, url, err);
error = true;
const takeScreenshot = opts?.screenshot !== false;
const waitUntil = opts?.waitUntil ?? 'networkidle';
try {
this.setListener(page);
let aborted = false;
let url = '';
this.ws?.send(JSON.stringify('request'));
console.error(getHostname(message.url.trim()));
const hostname = getHostname(message.url.trim());
await page.route('**/*', async (route) => {
const req = route.request();
if (
req.isNavigationRequest() &&
req.frame() === page.mainFrame() &&
!req.url().includes(hostname)
) {
aborted = true;
url = req.url();
console.error(req.url(), message.url);
this.ws?.send(JSON.stringify('aborted before redirection to ' + req.url()));
await route.abort();
} else {
await route.continue();
}
});
if (!aborted && !error) {
this.ws?.send(JSON.stringify('server tries screenshot'));
console.error('server tries screenshot', message.url.trim());
const buffer = await page.screenshot({ type: 'png' });
const base64 = buffer.toString('base64');
this.ws?.send(JSON.stringify('screenshot done'));
console.error('server screenshot');
if (base64) {
return JSON.stringify(`data:image/png;base64,${base64}`);
this.ws?.send(JSON.stringify('set request interception'));
this.ws?.send(JSON.stringify('server message.url ' + message.url.trim()));
console.error('server message.url', message.url.trim());
let error = false;
await page
.goto(message.url.trim(), {
timeout: EVALUATE_TIMEOUT_MS,
waitUntil,
})
.catch((err: Error) => {
this.ws?.send(JSON.stringify('error ' + err.toString()));
console.error(message.url, url, err);
error = true;
});
if (!aborted && !error && takeScreenshot) {
this.ws?.send(JSON.stringify('server tries screenshot'));
console.error('server tries screenshot', message.url.trim());
const buffer = await page.screenshot({ type: 'png' });
const base64 = buffer.toString('base64');
this.ws?.send(JSON.stringify('screenshot done'));
console.error('server screenshot');
if (base64) {
return JSON.stringify(`data:image/png;base64,${base64}`);
}
}
return;
} finally {
// One context per site — must close or batch leaks Chromium contexts until OOM.
await page.context().close().catch(() => undefined);
}
return;
}

private async getNewPage(message: Message) {
Expand All @@ -100,14 +110,20 @@ export class PlaywrightEngine implements EvaluateSession {
return;
}
this.ws?.send(JSON.stringify('get browser'));
const context = await browser.newContext({ userAgent: EVALUATE_USER_AGENT });
const page = await context.newPage();
this.ws?.send(JSON.stringify('new page done'));
const tpl = buildEvalTemplate(message);
await this.sqliteService.insert(message);
this.ws?.send(JSON.stringify('evaluate Document'));
await page.addInitScript(tpl);
return page;
try {
const context = await browser.newContext({ userAgent: EVALUATE_USER_AGENT });
const page = await context.newPage();
this.ws?.send(JSON.stringify('new page done'));
const tpl = buildEvalTemplate(message);
await this.sqliteService.insert(message);
this.ws?.send(JSON.stringify('evaluate Document'));
await page.addInitScript(tpl);
return page;
} catch (err) {
console.error('getNewPage failed', err);
this.ws?.send(JSON.stringify('getNewPage err ' + String(err)));
return;
}
}

async close() {
Expand Down
128 changes: 76 additions & 52 deletions libs/util/puppeteer/src/lib/puppeteer.engine.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,57 +35,75 @@ export class PuppeteerEngine implements EvaluateSession {
});
}

async goto(message: Message): Promise<string | undefined> {
async goto(
message: Message,
opts?: { screenshot?: boolean; waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit' }
): Promise<string | undefined> {
const page = await this.getNewPage(message);
if (!page) {
return;
}
this.setListener(page);
let aborted = false;
let url = '';
this.ws?.send(JSON.stringify('request'));
console.error(getHostname(message.url.trim()));
page.on('request', (req) => {
if (
req.isNavigationRequest() &&
req.frame() === page.mainFrame() &&
!req.url().includes(getHostname(message.url.trim()))
) {
aborted = true;
url = req.url();
console.error(req.url(), message.url);
this.ws?.send(JSON.stringify('aborted before redirection to ' + req.url()));
req.abort('aborted');
} else {
req.continue();
}
});
this.ws?.send(JSON.stringify('set request interception'));
await page.setRequestInterception(true);
this.ws?.send(JSON.stringify('server message.url ' + message.url.trim()));
console.error('server message.url', message.url.trim());
let error = false;
await page
.goto(message.url.trim(), {
timeout: EVALUATE_TIMEOUT_MS,
waitUntil: ['domcontentloaded', 'networkidle0'],
})
.catch((err) => {
this.ws?.send(JSON.stringify('error ' + err.toString()));
console.error(message.url, url, err);
error = true;
const takeScreenshot = opts?.screenshot !== false;
// Default matches historical UI/evaluate behavior (domcontentloaded + networkidle0).
// Puppeteer has no 'commit'; map it to 'domcontentloaded'.
const waitUntil: Array<'load' | 'domcontentloaded' | 'networkidle0'> =
opts?.waitUntil === 'load'
? ['load']
: opts?.waitUntil === 'domcontentloaded' || opts?.waitUntil === 'commit'
? ['domcontentloaded']
: opts?.waitUntil === 'networkidle'
? ['domcontentloaded', 'networkidle0']
: ['domcontentloaded', 'networkidle0'];
try {
this.setListener(page);
let aborted = false;
let url = '';
this.ws?.send(JSON.stringify('request'));
console.error(getHostname(message.url.trim()));
page.on('request', (req) => {
if (
req.isNavigationRequest() &&
req.frame() === page.mainFrame() &&
!req.url().includes(getHostname(message.url.trim()))
) {
aborted = true;
url = req.url();
console.error(req.url(), message.url);
this.ws?.send(JSON.stringify('aborted before redirection to ' + req.url()));
req.abort('aborted');
} else {
req.continue();
}
});
if (!aborted && !error) {
this.ws?.send(JSON.stringify('server tries screenshot'));
console.error('server tries screenshot', message.url.trim());
const base64 = (await page.screenshot({ encoding: 'base64' })) as string;
this.ws?.send(JSON.stringify('screenshot done'));
console.error('server screenshot');
if (base64) {
return JSON.stringify(`data:image/png;base64,${base64}`);
this.ws?.send(JSON.stringify('set request interception'));
await page.setRequestInterception(true);
this.ws?.send(JSON.stringify('server message.url ' + message.url.trim()));
console.error('server message.url', message.url.trim());
let error = false;
await page
.goto(message.url.trim(), {
timeout: EVALUATE_TIMEOUT_MS,
waitUntil,
})
.catch((err) => {
this.ws?.send(JSON.stringify('error ' + err.toString()));
console.error(message.url, url, err);
error = true;
});
if (!aborted && !error && takeScreenshot) {
this.ws?.send(JSON.stringify('server tries screenshot'));
console.error('server tries screenshot', message.url.trim());
const base64 = (await page.screenshot({ encoding: 'base64' })) as string;
this.ws?.send(JSON.stringify('screenshot done'));
console.error('server screenshot');
if (base64) {
return JSON.stringify(`data:image/png;base64,${base64}`);
}
}
return;
} finally {
await page.close().catch(() => undefined);
}
return;
}

private async getNewPage(message: Message) {
Expand All @@ -98,14 +116,20 @@ export class PuppeteerEngine implements EvaluateSession {
return browser;
}
this.ws?.send(JSON.stringify('get browser'));
const page = await browser.newPage();
this.ws?.send(JSON.stringify('new page done'));
const tpl = buildEvalTemplate(message);
await this.sqliteService.insert(message);
this.ws?.send(JSON.stringify('evaluate Document'));
await page.evaluateOnNewDocument(tpl);
await page.setUserAgent(EVALUATE_USER_AGENT);
return page;
try {
const page = await browser.newPage();
this.ws?.send(JSON.stringify('new page done'));
const tpl = buildEvalTemplate(message);
await this.sqliteService.insert(message);
this.ws?.send(JSON.stringify('evaluate Document'));
await page.evaluateOnNewDocument(tpl);
await page.setUserAgent(EVALUATE_USER_AGENT);
return page;
} catch (err) {
console.error('getNewPage failed', err);
this.ws?.send(JSON.stringify('getNewPage err ' + String(err)));
return;
}
}

async close() {
Expand Down
Loading