diff --git a/bench/README.md b/bench/README.md index f860f1e..4e6b3d4 100644 --- a/bench/README.md +++ b/bench/README.md @@ -44,6 +44,32 @@ This screen runs every fixture in local diff-file mode. It does no forge I/O. Th Inspect the fixture IDs and source before a live run in [`fixtures/cases.ts`](fixtures/cases.ts). A selected-case screen requires `--case ` together with `--screen-profile `, which supplies the exact provider and price contract. A live screen is development evidence, not a hosted admission or a comparison with another model. +## Expanded clean bank + +The 25-case clean bank combines the 13 admission clean cases with 12 supplemental cases in [`fixtures/clean-screen.ts`](fixtures/clean-screen.ts). The [clean-screen entrypoint](src/clean-screen.ts) passes the exported `cleanScreenCases` to the existing `runLive` API. Default screens and admission retain the 70-case corpus and its attested evaluator inputs. The supplemental cases cover authorization, expiry, asynchronous ordering, cancellation, retry limits, pagination, integer precision, defaults, lock release, SQL parameters, array ownership, and partial updates. Behavioral tests exercise both versions of each executable module, including rejection paths and boundary values. + +Live diff-file screening sends the diff. Supplemental source comments therefore contain the argument and behavior contracts; the `Object.hasOwn` case also supplies package metadata declaring Node.js 22 or later. Metadata outside the diff is not evidence available to this screen. + +After the build and dependency setup above, select a checked-in profile: + +| `REVIEW_MODEL` | `SCREEN_PROFILE` | +| --- | --- | +| `openai/gpt-5.6-luna` | [clean-screen-luna.json](clean-screen-luna.json) | +| `z-ai/glm-5.2` | [clean-screen-glm.json](clean-screen-glm.json) | + +Each profile contains one generator, an empty `scorerChain`, the canonical `providerGenerationModels` identity, and explicit provider route and price bounds. The screen disables the scorer when the `scorerModel` option is absent. For Luna, set `export REVIEW_MODEL=openai/gpt-5.6-luna SCREEN_PROFILE=clean-screen-luna.json`; use the GLM row for its run. With the model credential in the environment and GNU `timeout` installed, run from `bench/`: + +```sh +POSTIL_LLM_REQUEST_TIMEOUT_SECS=30 POSTIL_LLM_TOTAL_TIMEOUT_SECS=60 \ +timeout 780s bun run src/clean-screen.ts +``` + +Each invocation retains a separate report and per-case evidence. Its `supplementalScreen` field records a separate framed SHA-256 digest of the supplemental fixture module and entrypoint; `summary.evaluatorSha256` identifies the default evaluator. The process fails if every case is unavailable; partial reports retain unavailable cases for inspection. The CLI safety cap is 180 seconds per case. Report final review silence, final findings, suppressed findings with reasons, and unavailable cases separately, with the 13 legacy and 12 supplemental cases identified. A silent final review can contain suppressed model findings. + +The clean-bank-v2 evidence identifies the [measured fixture and evaluator source](https://github.com/postil-dev/postil-cli/tree/a7e7c67235519fff79c6e82c44550ac29255dcdc) and its [exact invocation](https://github.com/postil-dev/postil-cli/blob/a7e7c67235519fff79c6e82c44550ac29255dcdc/bench/README.md#expanded-clean-bank). The command above uses the same 25 fixture payloads with the default evaluator source digest; its reports are separate evidence. + +Compare models only when the selected cases, fixture hash, evaluator hash, binary hash, retry settings, and concurrency match. Provider routes remain explicit. Evidence identifies the fixture/evaluator source by immutable commit and the measured executable by SHA-256; use `POSTIL_BIN` to select that executable. A different build produces separate evidence. This authored bank is not held-out validation, and one observation per fixture does not establish a stable false-positive rate. + ## Managed qualification Managed qualification exercises an ordered generator and scorer pair through the mock forge and a real provider. It requires an exact pair, provider identity and route, three complete repeats, and a release build whose embedded profile matches the worktree. Run the manual [managed admission workflow](../.github/workflows/bench-live.yml) for the attested hosted path. `bun run verify-admission` validates checked-in admission evidence. diff --git a/bench/clean-screen-glm.json b/bench/clean-screen-glm.json new file mode 100644 index 0000000..77da03b --- /dev/null +++ b/bench/clean-screen-glm.json @@ -0,0 +1,22 @@ +{ + "benchmarkProviderIdentity": "openrouter:managed-routing", + "upstreamProviderIdentity": "Z.AI", + "upstreamProviderRoute": "z-ai/fp8", + "apiBase": "https://openrouter.ai:443/api/v1", + "apiFormat": "openai-compatible", + "generatorChain": [ + "z-ai/glm-5.2" + ], + "consensus": 1, + "scorerChain": [], + "providerGenerationModels": { + "z-ai/glm-5.2": "z-ai/glm-5.2-20260616" + }, + "modelPriceBounds": [ + { + "model": "z-ai/glm-5.2", + "inputMicrosPerMillionTokens": 1400000, + "outputMicrosPerMillionTokens": 4400000 + } + ] +} \ No newline at end of file diff --git a/bench/clean-screen-luna.json b/bench/clean-screen-luna.json new file mode 100644 index 0000000..33215d2 --- /dev/null +++ b/bench/clean-screen-luna.json @@ -0,0 +1,22 @@ +{ + "benchmarkProviderIdentity": "openrouter:managed-routing", + "upstreamProviderIdentity": "Azure", + "upstreamProviderRoute": "azure/eu", + "apiBase": "https://openrouter.ai:443/api/v1", + "apiFormat": "openai-compatible", + "generatorChain": [ + "openai/gpt-5.6-luna" + ], + "consensus": 1, + "scorerChain": [], + "providerGenerationModels": { + "openai/gpt-5.6-luna": "openai/gpt-5.6-luna-20260709" + }, + "modelPriceBounds": [ + { + "model": "openai/gpt-5.6-luna", + "inputMicrosPerMillionTokens": 220000, + "outputMicrosPerMillionTokens": 1320000 + } + ] +} \ No newline at end of file diff --git a/bench/fixtures/cases.test.ts b/bench/fixtures/cases.test.ts index ad05725..c1cd5ea 100644 --- a/bench/fixtures/cases.test.ts +++ b/bench/fixtures/cases.test.ts @@ -1,5 +1,137 @@ import { describe, expect, test } from "bun:test"; -import { makeDiff } from "./cases"; +import { cases, makeDiff } from "./cases"; +import { supplementalCleanCases, supplementalCleanSpecs } from "./clean-screen"; +import { benchmarkCase, parseUnifiedDiffFiles } from "../src/harness"; + +test("supplemental modules supply complete evidence and unique clean targets", () => { + const ids = [...cases, ...supplementalCleanCases].map((input) => input.id); + expect(new Set(ids).size).toBe(ids.length); + expect(supplementalCleanCases).toHaveLength(12); + for (const input of supplementalCleanCases) { + const parsed = benchmarkCase.parse(input); + expect(parsed.groundTruth.findings).toEqual([]); + expect(parsed.modelOutput.findings).toEqual([]); + const file = parseUnifiedDiffFiles(parsed.diff).find((file) => file.path === parsed.allowedContext.files[0].path)!; + expect(file.after.replace(/^ /gm, "").trim()).toBe(parsed.allowedContext.files[0].content.trim()); + expect(parsed.diff).toContain(parsed.allowedContext.docs[0].content); + } +}); + +for (const version of ["before", "after"] as const) { + const program = (id: string): any => { + const spec = supplementalCleanSpecs.find((entry) => entry.id === `clean-${id}`)!; + return new Function(`${spec[version].replace("export ", "")}; return run;`)(); + }; + describe(`supplemental clean behavior: ${version}`, () => { + test("tenant checks allow the owner and reject a different tenant", () => { + const run = program("tenant-guard-extraction"); + expect(run({ tenantId: "a" }, { tenantId: "a", name: "report" })).toBe("report"); + expect(() => run({ tenantId: "b" }, { tenantId: "a", name: "report" })).toThrow("Forbidden"); + }); + test("expiry includes the boundary and preserves falsy cached values", () => { + const run = program("cache-expiry-boundary"); + expect(run(null, 10)).toBeNull(); + expect(run({ expiresAt: 10, value: false }, 9)).toBe(false); + expect(run({ expiresAt: 10, value: "stale" }, 10)).toBeNull(); + expect(run({ expiresAt: 10, value: "stale" }, 11)).toBeNull(); + }); + test("concurrent reads preserve order and propagate rejection", async () => { + const run = program("concurrent-result-order"); + const pending = new Map void>(); + const result = run([1, 2], (id: number) => new Promise((resolve) => pending.set(id, resolve))); + expect([...pending.keys()]).toEqual([1, 2]); + pending.get(2)!("second"); pending.get(1)!("first"); + expect(await result).toEqual(["first", "second"]); + expect(await run([], () => { throw new Error("unexpected read"); })).toEqual([]); + await expect(run([1], async () => { throw new Error("read failed"); })).rejects.toThrow("read failed"); + }); + test("cancellation reaches the request with the original signal", async () => { + const run = program("abort-signal-forwarding"); + const controller = new AbortController(); + const result = run((_url: string, options: RequestInit) => { + expect(options.method).toBe("GET"); + expect(options.signal).toBe(controller.signal); + return new Promise((_resolve, reject) => options.signal!.addEventListener("abort", () => reject(new Error("aborted")), { once: true })); + }, "https://example.test/items", controller.signal); + controller.abort(); + await expect(result).rejects.toThrow("aborted"); + }); + test("retry budget stops at the boundary and propagates failures", async () => { + const run = program("retry-limit-extraction"); + let calls = 0; + const retry = async () => { calls++; return "retried"; }; + expect(await run(2, retry, 3)).toBe("retried"); + expect(await run(3, retry, 3)).toBe("exhausted"); + expect(await run(4, retry, 3)).toBe("exhausted"); + expect(calls).toBe(1); + await expect(run(0, async () => { throw new Error("retry failed"); }, 3)).rejects.toThrow("retry failed"); + }); + test("pagination excludes the cursor without gaps or repeats", () => { + const run = program("pagination-exclusive-cursor"); + const rows = [1, 3, 5, 8].map((id) => ({ id })); + expect(run(rows, 1, 2)).toEqual([{ id: 3 }, { id: 5 }]); + expect(run(rows, 5, 2)).toEqual([{ id: 8 }]); + expect(run(rows, 8, 2)).toEqual([]); + expect(run([], 0, 2)).toEqual([]); + }); + test("serialization retains exact integer digits", () => { + const run = program("bigint-json-serialization"); + for (const value of [0n, 9007199254740993n, -9007199254740993n]) { + expect(JSON.parse(run(value))).toEqual({ totalMicros: String(value) }); + } + }); + test("configuration distinguishes zero from missing values", () => { + const run = program("config-preserves-zero"); + expect(run({})).toBe(3); expect(run({ retries: null })).toBe(3); + expect(run({ retries: 0 })).toBe(0); expect(run({ retries: 5 })).toBe(5); + }); + test("lock stays held until the write settles on success and failure", async () => { + const run = program("lock-finally-release"); + for (const fails of [false, true]) { + const events: string[] = []; + let finish!: () => void; + const result = run(async () => { events.push("lock"); return () => events.push("release"); }, async () => { + events.push("write"); + await new Promise((resolve) => { finish = resolve; }); + events.push("settled"); + if (fails) throw new Error("write failed"); + return 42; + }); + await Promise.resolve(); + expect(events).toEqual(["lock", "write"]); + finish(); + if (fails) await expect(result).rejects.toThrow("write failed"); + else expect(await result).toBe(42); + expect(events).toEqual(["lock", "write", "settled", "release"]); + } + }); + test("untrusted query input remains a separate bound parameter", async () => { + const run = program("parameterized-query-extraction"); + const input = "' OR 1=1 --"; + const result = await run({ query: async (sql: string, params: unknown[]) => { + expect(sql).toBe("SELECT id FROM people WHERE name = $1"); + expect(params).toEqual([input]); + return [{ id: 7 }]; + } }, input); + expect(result).toEqual([{ id: 7 }]); + }); + test("sorting preserves caller ownership and descending numeric order", () => { + const run = program("input-array-copy-sort"); + const input = Object.freeze([2, -1, 10, 2]); + expect(run(input)).toEqual([10, 2, 2, -1]); + expect(input).toEqual([2, -1, 10, 2]); + expect(run([])).toEqual([]); + }); + test("partial updates distinguish absence, null, and inherited properties", () => { + const run = program("optional-field-presence"); + const current = Object.freeze({ displayName: "Name", id: 1 }); + expect(run(current, {})).toEqual(current); + expect(run(current, { displayName: null })).toEqual({ displayName: null, id: 1 }); + expect(run(current, { displayName: "" })).toEqual({ displayName: "", id: 1 }); + expect(run(current, Object.create({ displayName: "inherited" }))).toEqual(current); + }); + }); +} describe("makeDiff", () => { test("uses one-line hunk counts for one-line fixtures", () => { diff --git a/bench/fixtures/clean-screen.ts b/bench/fixtures/clean-screen.ts new file mode 100644 index 0000000..1e6b6cf --- /dev/null +++ b/bench/fixtures/clean-screen.ts @@ -0,0 +1,126 @@ +import { createHash } from "node:crypto"; +import { cases, makeDiff } from "./cases"; +import { parseUnifiedDiffFiles, type BenchmarkCaseInput } from "../src/harness"; + +// Supplemental screening cases do not change the release admission corpus. +// Complete executable modules keep callers and relevant contracts in evidence. +function cleanProgram( + id: string, path: string, before: string, replacement: [string, string], + policy: string, labels: string[], index: number, prefixDiff?: string, +) { + if (!before.includes(replacement[0])) throw new Error(`Missing change in ${id}`); + const documentedBefore = `/** ${policy} */\n${before}`; + const after = documentedBefore.replace(replacement[0], replacement[1]); + return { + id: `clean-${id}`, name: id.replaceAll("-", " "), pullNumber: 100 + index, + path, line: 1, before: documentedBefore, after, allowedFileContent: after, policy, prefixDiff, + scoringLabels: [...labels, "clean", "supplemental-clean"], + admission: { classification: "clean" as const, contractRule: id }, + }; +} + +export const supplementalCleanSpecs = [ + cleanProgram("tenant-guard-extraction", "src/projects/read.js", `export function run(actor, project) { + if (actor.tenantId !== project.tenantId) throw new Error('Forbidden'); + return project.name; +}`, [ + "if (actor.tenantId !== project.tenantId) throw new Error('Forbidden');", + "const sameTenant = actor.tenantId === project.tenantId;\n if (!sameTenant) throw new Error('Forbidden');", + ], "Actors and projects have required string tenantId fields. A read requires the same tenant.", ["authorization", "multi-tenant"], 0), + cleanProgram("cache-expiry-boundary", "src/cache/read.js", `export function run(entry, now) { + if (!entry || now >= entry.expiresAt) return null; + return entry.value; +}`, [ + "if (!entry || now >= entry.expiresAt) return null;", + "if (!entry) return null;\n const expired = entry.expiresAt <= now;\n if (expired) return null;", + ], "Cache timestamps are finite epoch milliseconds. Entries expire at expiresAt, including equality.", ["cache", "boundary"], 1), + cleanProgram("concurrent-result-order", "src/catalog/load.js", `export async function run(ids, read) { + return await Promise.all(ids.map(async (id) => await read(id))); +}`, [ + "ids.map(async (id) => await read(id))", "ids.map((id) => read(id))", + ], "read performs an independent read and returns a promise. Results retain input order regardless of completion order.", ["concurrency", "ordering"], 2), + cleanProgram("abort-signal-forwarding", "src/client/request.js", `export async function run(fetcher, url, signal) { + return await fetcher(url, { method: 'GET', signal }); +}`, [ + "return await fetcher(url, { method: 'GET', signal });", + "const options = { method: 'GET', signal };\n return await fetcher(url, options);", + ], "The caller owns cancellation. Requests forward its AbortSignal unchanged.", ["cancellation", "http"], 3), + cleanProgram("retry-limit-extraction", "src/jobs/retry.js", `export async function run(attempt, retry, maximumAttempts) { + if (attempt < maximumAttempts) return await retry(); + return 'exhausted'; +}`, [ + "if (attempt < maximumAttempts) return await retry();", + "const canRetry = attempt < maximumAttempts;\n if (canRetry) return await retry();", + ], "attempt is the number of attempts already used. The caller supplies positive integer maximumAttempts.", ["retries", "boundary"], 4), + cleanProgram("pagination-exclusive-cursor", "src/feed/page.js", `export function run(rows, cursor, limit) { + return rows.filter((row) => row.id > cursor).slice(0, limit); +}`, [ + "return rows.filter((row) => row.id > cursor).slice(0, limit);", + "const remaining = rows.filter((row) => cursor < row.id);\n return remaining.slice(0, limit);", + ], "Rows have unique numeric IDs sorted ascending. The cursor is exclusive and limit is a positive integer.", ["pagination", "boundary"], 5), + cleanProgram("bigint-json-serialization", "src/billing/serialize.js", `export function run(totalMicros) { + return JSON.stringify({ totalMicros: totalMicros.toString() }); +}`, [ + "return JSON.stringify({ totalMicros: totalMicros.toString() });", + "const decimal = totalMicros.toString(10);\n return JSON.stringify({ totalMicros: decimal });", + ], "totalMicros is a bigint. The wire field is a base-ten string, including values above Number.MAX_SAFE_INTEGER.", ["serialization", "precision"], 6), + cleanProgram("config-preserves-zero", "src/config/retries.js", `export function run(config) { + return config.retries === undefined || config.retries === null ? 3 : config.retries; +}`, [ + "config.retries === undefined || config.retries === null ? 3 : config.retries", + "config.retries ?? 3", + ], "retries is an optional nonnegative integer or null. Zero disables retries; null and undefined select three.", ["configuration", "defaults"], 7), + cleanProgram("lock-finally-release", "src/jobs/locked.js", `export async function run(lock, write) { + const release = await lock(); + try { return await write(); } + finally { release(); } +}`, [ + "try { return await write(); }", + "try {\n const result = await write();\n return result;\n }", + ], "lock resolves to a synchronous release function. The lock covers the entire write and releases on success or failure.", ["resource-lifecycle", "concurrency"], 8), + cleanProgram("parameterized-query-extraction", "src/search/find.js", `export async function run(database, name) { + return await database.query('SELECT id FROM people WHERE name = $1', [name]); +}`, [ + "return await database.query('SELECT id FROM people WHERE name = $1', [name]);", + "const statement = 'SELECT id FROM people WHERE name = $1';\n const parameters = [name];\n return await database.query(statement, parameters);", + ], "database.query binds positional parameters separately from SQL. name is arbitrary user input.", ["sql", "injection"], 9), + cleanProgram("input-array-copy-sort", "src/ranking/sort.js", `export function run(scores) { + return scores.slice().sort((a, b) => b - a); +}`, ["scores.slice()", "[...scores]"], + "scores is a dense array of finite numbers. Sorting returns a descending copy and leaves the input unchanged.", ["mutation", "ordering"], 10), + cleanProgram("optional-field-presence", "src/profile/patch.js", `export function run(current, patch) { + return Object.prototype.hasOwnProperty.call(patch, 'displayName') + ? { ...current, displayName: patch.displayName } : { ...current }; +}`, ["Object.prototype.hasOwnProperty.call(patch, 'displayName')", "Object.hasOwn(patch, 'displayName')"], + "The runtime supports Object.hasOwn. An absent displayName preserves the value; an own null value explicitly clears it. Inherited properties are ignored.", ["partial-update", "property-presence"], 11, + makeDiff("package.json", [{ line: 1, + before: '{"private":true,"type":"module","engines":{"node":">=22"}}', + after: '{"private":true,"type":"module","engines":{"node":">=22.0.0"}}', + }])), +]; + +export const supplementalCleanCases: BenchmarkCaseInput[] = supplementalCleanSpecs.map((spec) => { + const diff = (spec.prefixDiff ?? "") + makeDiff(spec.path, [{ line: spec.line, before: spec.before, after: spec.after }]); + const additionalFiles = parseUnifiedDiffFiles(diff) + .filter((file) => file.path !== spec.path) + .map((file) => ({ path: file.path, content: file.after })); + return { + id: spec.id, name: spec.name, repo: "benchmark/example-fixtures", + pullNumber: spec.pullNumber, + headSha: createHash("sha1").update(String(spec.pullNumber)).digest("hex"), + diff, primaryChange: { path: spec.path, line: spec.line }, + allowedContext: { + files: [{ path: spec.path, content: spec.allowedFileContent }, ...additionalFiles], + docs: [{ path: "review-policy.md", content: spec.policy }], + }, + disallowedSources: [], scoringLabels: spec.scoringLabels, admission: spec.admission, + groundTruth: { findings: [] }, guardrails: { forbiddenPromptSubstrings: [] }, + modelOutput: { summary: "", findings: [] }, + expectations: { minFindings: 0, maxFindings: 0, requiredFindings: [] }, + }; +}); + +export const cleanScreenCases: BenchmarkCaseInput[] = [ + ...cases.filter((input) => input.admission?.classification === "clean"), + ...supplementalCleanCases, +]; diff --git a/bench/src/clean-screen.test.ts b/bench/src/clean-screen.test.ts new file mode 100644 index 0000000..87d4cce --- /dev/null +++ b/bench/src/clean-screen.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { cases } from "../fixtures/cases"; +import { cleanScreenCases, supplementalCleanCases } from "../fixtures/clean-screen"; +import { benchmarkCase } from "./harness"; +import { selectLiveScreeningCases } from "./run"; +import { cleanScreenExitCode, cleanScreenOptions, cleanScreenSourceIdentity } from "./clean-screen"; +import type { LiveReport } from "./live"; + +test("clean screening preserves the measured corpus without extending default selection", () => { + expect(cases).toHaveLength(70); + expect(cases.filter((input) => input.admission?.classification === "clean")).toHaveLength(13); + expect(selectLiveScreeningCases(cases, [])).toEqual(cases); + expect(() => selectLiveScreeningCases(cases, [supplementalCleanCases[0].id])).toThrow("unknown --case"); + expect(cleanScreenCases).toHaveLength(25); + expect(createHash("sha256").update(JSON.stringify(cleanScreenCases.map((input) => benchmarkCase.parse(input)))).digest("hex")) + .toBe("3ccdc2b617cd4325e993d9e8462adb28205669b5cb25aea1b42341b737418c7c"); +}); + +test("clean screen fixes the matched execution settings and requires an explicit profile", () => { + const options = cleanScreenOptions({ + REVIEW_MODEL: "test/model", SCREEN_PROFILE: "profile.json", POSTIL_BIN: "postil", + REVIEW_SCORER_MODEL: "ignored/scorer", BENCH_CONCURRENCY: "99", + POSTIL_BENCH_BOUNDED: "1", + }, "clean-test"); + expect(options).toEqual({ + binary: "postil", model: "test/model", screenProfilePath: "profile.json", + concurrency: 3, retries: 0, timeoutMs: 180_000, bounded: false, + selectedCaseIds: cleanScreenCases.map((input) => input.id), runId: "clean-test", + }); + expect(options.scorerModel).toBeUndefined(); + expect(() => cleanScreenOptions({}, "clean-test")).toThrow("REVIEW_MODEL and SCREEN_PROFILE"); + expect(() => cleanScreenOptions({ REVIEW_MODEL: "test/model" }, "clean-test")).toThrow("SCREEN_PROFILE"); +}); + +test("supplemental source identity is separate from the attested evaluator inputs", async () => { + const identity = await cleanScreenSourceIdentity(); + const defaultSources = JSON.parse(await readFile(resolve(import.meta.dir, "../evaluator-contract-sources.json"), "utf8")); + expect(identity.version).toBe(1); + expect(identity.sourcePaths).toEqual(["bench/fixtures/clean-screen.ts", "bench/src/clean-screen.ts"]); + expect(identity.sourceSha256).toMatch(/^[a-f0-9]{64}$/); + expect(identity.sourcePaths.every((path) => !defaultSources.includes(path))).toBe(true); +}); + +test("unavailable cases remain distinct and only an entirely unavailable screen fails", () => { + const report = (scored: boolean[]) => ({ results: scored.map((value) => ({ scored: value })) }) as LiveReport; + expect(cleanScreenExitCode(report([true, true]))).toBe(0); + expect(cleanScreenExitCode(report([true, false]))).toBe(0); + expect(cleanScreenExitCode(report([false, false]))).toBe(1); +}); + +test("the entrypoint rejects missing configuration before inference", async () => { + const child = Bun.spawn(["bun", resolve(import.meta.dir, "clean-screen.ts")], { + env: { PATH: process.env.PATH }, stdout: "pipe", stderr: "pipe", + }); + expect(await child.exited).toBe(1); + expect(await new Response(child.stderr).text()).toContain("Set REVIEW_MODEL and SCREEN_PROFILE"); +}); diff --git a/bench/src/clean-screen.ts b/bench/src/clean-screen.ts new file mode 100644 index 0000000..ed9f17c --- /dev/null +++ b/bench/src/clean-screen.ts @@ -0,0 +1,55 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { cleanScreenCases } from "../fixtures/clean-screen"; +import { formatLiveReport, runLive, type LiveOptions, type LiveReport } from "./live"; + +const sourcePaths = ["bench/fixtures/clean-screen.ts", "bench/src/clean-screen.ts"]; + +export async function cleanScreenSourceIdentity() { + const hash = createHash("sha256"); + for (const path of sourcePaths) { + hash.update(path).update("\0"); + hash.update(await readFile(resolve(import.meta.dir, "../..", path))).update("\0"); + } + return { version: 1, sourcePaths, sourceSha256: hash.digest("hex") }; +} + +export function cleanScreenOptions( + environment: Record, + runId: string, +): LiveOptions { + const model = environment.REVIEW_MODEL?.trim(); + const screenProfilePath = environment.SCREEN_PROFILE?.trim(); + if (!model || !screenProfilePath) throw new Error("Set REVIEW_MODEL and SCREEN_PROFILE"); + return { + binary: environment.POSTIL_BIN ?? resolve(import.meta.dir, "../../target/release/postil"), + model, screenProfilePath, + concurrency: 3, retries: 0, timeoutMs: 180_000, bounded: false, + selectedCaseIds: cleanScreenCases.map((input) => input.id), runId, + }; +} + +// Match the default live screen: retain partial reports and fail if no case scores. +export function cleanScreenExitCode(report: LiveReport): number { + return report.results.length > 0 && !report.results.some((result) => result.scored) ? 1 : 0; +} + +async function main() { + const options = cleanScreenOptions(process.env, `clean-bank-${crypto.randomUUID()}`); + const supplementalScreen = await cleanScreenSourceIdentity(); + const report = await runLive(cleanScreenCases, options); + const output = resolve(import.meta.dir, "../.runs", `${options.runId}.json`); + await writeFile(output, JSON.stringify({ ...report, supplementalScreen }, null, 2), { + flag: "wx", mode: 0o600, + }); + console.log(formatLiveReport(report)); + process.exitCode = cleanScreenExitCode(report); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +}