diff --git a/.github/workflows/bench-live.yml b/.github/workflows/bench-live.yml index e1f03c6..3f0628a 100644 --- a/.github/workflows/bench-live.yml +++ b/.github/workflows/bench-live.yml @@ -10,7 +10,7 @@ on: description: Comma-separated OpenRouter model ids required: true default: >- - deepseek/deepseek-v4-pro,moonshotai/kimi-k2.6,qwen/qwen3-32b,mistralai/mistral-small-3.2-24b-instruct,google/gemma-3-27b-it + deepseek/deepseek-v4-pro,google/gemini-3.1-flash-lite,moonshotai/kimi-k2.7-code,mistralai/mistral-large-2512 cases: description: Case selection (currently informational; "all" runs the full set) required: false @@ -19,6 +19,14 @@ on: description: Abort if the projected upper-bound total cost exceeds this many USD required: false default: "15" + scorer_model: + description: REVIEW_SCORER_MODEL override for live-models bench (scorer-eval uses hardcoded defaults) + required: false + default: "" + run_scorer_eval: + description: Run scorer calibration eval instead of live-models bench + required: false + default: "false" concurrency: group: bench-live @@ -49,31 +57,65 @@ jobs: # from fixture diff sizes. Aborts before spending anything if the matrix # would exceed the cap, or if any model has unknown pricing. - name: Cost guardrail + if: ${{ inputs.run_scorer_eval != 'true' }} working-directory: bench run: bun run cost-guard --cap "$POSTIL_BENCH_COST_CAP_USD" - name: Run live-models bench + if: ${{ inputs.run_scorer_eval != 'true' }} working-directory: bench env: POSTIL_BENCH_MODE: live + REVIEW_SCORER_MODEL: ${{ inputs.scorer_model }} POSTIL_API_KEY: ${{ secrets.POSTIL_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }} LLM_API_KEY: ${{ secrets.LLM_API_KEY }} run: bun run src/run.ts --json-out "$GITHUB_WORKSPACE/bench-live-report.json" + - name: Run scorer calibration eval + if: ${{ inputs.run_scorer_eval == 'true' }} + working-directory: bench + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: bun run scorer-eval --json-out "$GITHUB_WORKSPACE/scorer-eval-report.json" + - name: Write step summary if: always() working-directory: bench run: | report="$GITHUB_WORKSPACE/bench-live-report.json" + scorer_report="$GITHUB_WORKSPACE/scorer-eval-report.json" { - echo '## Live-models bench' - if [ -f "$report" ]; then + if [ -f "$scorer_report" ]; then + echo '## Scorer calibration eval' echo '' echo '```' bun -e ' - const r = require(process.env.GITHUB_WORKSPACE + "/bench-live-report.json"); + const r = await Bun.file(process.env.GITHUB_WORKSPACE + "/scorer-eval-report.json").json(); + const pad = (s, w) => String(s).length >= w ? String(s) : String(s) + " ".repeat(w - String(s).length); + console.log([pad("model",36),pad("structured",11),pad("true kept",10),pad("false down",11),pad("mean true",10),pad("mean false",11),pad("pass",5)].join(" ")); + for (const a of r.models) { + console.log([ + pad(a.id,36), + pad(String(a.structuredFailures),11), + pad(`${a.trueFindingHighConfidence}/${a.trueFindingCases}`,10), + pad(`${a.falseFindingDownscored}/${a.falseFindingCases}`,11), + pad(a.meanTrueConfidence.toFixed(2),10), + pad(a.meanFalseConfidence.toFixed(2),11), + pad(a.passed ? "yes" : "no",5), + ].join(" ")); + } + console.log(""); + console.log("Endpoint " + r.apiBase + " | generated " + r.generatedAt); + ' + echo '```' + elif [ -f "$report" ]; then + echo '## Live-models bench' + echo '' + echo '```' + bun -e ' + const r = await Bun.file(process.env.GITHUB_WORKSPACE + "/bench-live-report.json").json(); const pad = (s, w) => String(s).length >= w ? String(s) : String(s) + " ".repeat(w - String(s).length); console.log([pad("model",40),pad("detect",8),pad("FP",5),pad("cases",6),pad("$/review",12),pad("total $",12),pad("mean ms",9)].join(" ")); for (const a of r.modelAggregates) { @@ -101,9 +143,33 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" - name: Upload report artifact - if: always() + if: always() && inputs.run_scorer_eval != 'true' uses: actions/upload-artifact@v4 with: name: bench-live-report path: bench-live-report.json if-no-files-found: warn + + - name: Upload scorer eval artifact + if: always() && inputs.run_scorer_eval == 'true' + uses: actions/upload-artifact@v4 + with: + name: scorer-eval-report + path: scorer-eval-report.json + if-no-files-found: warn + + - name: Upload live-model raw run artifacts + if: always() && inputs.run_scorer_eval != 'true' + uses: actions/upload-artifact@v4 + with: + name: bench-live-raw-runs + path: bench/.runs/live-models + if-no-files-found: warn + + - name: Upload scorer eval raw run artifacts + if: always() && inputs.run_scorer_eval == 'true' + uses: actions/upload-artifact@v4 + with: + name: scorer-eval-raw-runs + path: bench/.runs/scorer-eval + if-no-files-found: warn diff --git a/README.md b/README.md index 2b3d8f6..bb39720 100644 --- a/README.md +++ b/README.md @@ -144,10 +144,10 @@ gate: model: name: deepseek/deepseek-v4-pro cascade: - - mistralai/mistral-small-3.2-24b-instruct - - google/gemma-3-27b-it - - qwen/qwen3-32b - scorer: openai/gpt-5-mini + - google/gemini-3.1-flash-lite + - moonshotai/kimi-k2.7-code + - mistralai/mistral-large-2512 + scorer: anthropic/claude-haiku-4.5 apiBase: https://openrouter.ai/api/v1 # ignored from config by default; see note below consensus: 1 # >1: only findings multiple models agree on survive ``` diff --git a/bench/README.md b/bench/README.md index 2f07716..01767ae 100644 --- a/bench/README.md +++ b/bench/README.md @@ -128,6 +128,32 @@ runs the live bench with any configured `POSTIL_API_KEY`, `OPENROUTER_API_KEY`, `MODEL_API_KEY`, or `LLM_API_KEY` secret, uploads the JSON report as an artifact, and prints the per-model table to the job step summary. +## Scorer calibration live eval (opt-in, mocked generator + real scorer) + +The independent scorer has a different job from the primary review generator: +it receives already-generated findings, without the generator's confidence or +kind, and calibrates each finding's confidence and kind against local diff +context. `bun run scorer-eval` evaluates that role directly by mocking the +primary generator with fixed findings and proxying only scorer requests to the +real OpenRouter endpoint. + +```sh +cargo build --quiet --release +cd bench +export MODEL_API_KEY=... # or LLM_API_KEY / OPENROUTER_API_KEY +POSTIL_SCORER_EVAL_MODELS=anthropic/claude-haiku-4.5,openai/gpt-5-mini \ + bun run scorer-eval --json-out scorer-eval-report.json +``` + +The eval uses 12 existing fixtures: six seeded defect findings copied from +mock-mode ground truth, and six deliberately injected false findings on clean +fixtures. A scorer passes the structured-output floor when every case records +the requested `scorerModel` and no `scorerError`; it passes calibration when all +true findings remain confident `risk` findings and at least half of injected +false findings are down-scored below 0.6 confidence or retyped as +`uncertainty`. The manual `Bench (live models)` workflow can run this path with +`run_scorer_eval=true` and uploads `scorer-eval-report.json`. + ## Diff-file live mode (opt-in, single model, no forge) This live mode runs the real release binary against the same fixtures with a diff --git a/bench/package.json b/bench/package.json index d8bd1b1..62c2ad5 100644 --- a/bench/package.json +++ b/bench/package.json @@ -7,6 +7,7 @@ "bench": "bun run src/run.ts", "bench:live": "bun run src/run.ts --live", "bench:live-models": "POSTIL_BENCH_MODE=live bun run src/run.ts", + "scorer-eval": "bun run src/scorer-eval.ts", "cost-guard": "bun run src/cost-guard.ts" }, "dependencies": { diff --git a/bench/src/livemodels.ts b/bench/src/livemodels.ts index c6d4b4e..2fc63d3 100644 --- a/bench/src/livemodels.ts +++ b/bench/src/livemodels.ts @@ -267,6 +267,10 @@ function liveEnv( GITHUB_TOKEN: "benchmark-github-token", REVIEW_MODEL: model, }; + const scorerModel = process.env.REVIEW_SCORER_MODEL?.trim(); + if (scorerModel) { + env.REVIEW_SCORER_MODEL = scorerModel; + } // Forward the selected inference-key variable without logging or placing the // value on argv. Neutral aliases are also mirrored into POSTIL_API_KEY so // older binaries can run from the same benchmark harness. diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts new file mode 100644 index 0000000..2746bd4 --- /dev/null +++ b/bench/src/scorer-eval.test.ts @@ -0,0 +1,255 @@ +import { describe, expect, test } from "bun:test"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { cases as fixtureInputs } from "../fixtures/cases"; +import { benchmarkCase } from "./harness"; +import { + FALSE_FINDING_CASES, + GENERATOR_MODEL, + TRUE_FINDING_CASES, + aggregate, + falseFinding, + firstAddedLine, + formatReport, + isolatedEnv, + parseModels, + selectEvalCases, + startScorerProxy, + trueFinding, + type ScorerEvalCase, + type ScorerEvalReport, +} from "./scorer-eval"; + +const fixtures = fixtureInputs.map((input) => benchmarkCase.parse(input)); + +function fixture(id: string) { + const c = fixtures.find((candidate) => candidate.id === id); + if (!c) throw new Error(`missing fixture ${id}`); + return c; +} + +function result(overrides: Partial): ScorerEvalCase { + return { + id: "case", + name: "Case", + scenario: "trueFinding", + model: "scorer/model", + envelopeProduced: true, + scorerModel: "scorer/model", + scorerError: null, + scorerConfidence: 0.9, + scorerKind: "risk", + finalConfidence: 0.9, + finalKind: "risk", + passed: true, + reason: "ok", + durationMs: 1000, + promptTokens: 10, + completionTokens: 5, + ...overrides, + }; +} + +function listen(server: ReturnType): Promise { + return new Promise((resolvePromise, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + const address = server.address() as AddressInfo; + resolvePromise(`http://127.0.0.1:${address.port}`); + }); + }); +} + +function close(server: ReturnType): Promise { + return new Promise((resolvePromise, reject) => { + server.close((err) => (err ? reject(err) : resolvePromise())); + }); +} + +function requestBody(req: IncomingMessage): Promise { + return new Promise((resolvePromise, reject) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + req.on("end", () => resolvePromise(Buffer.concat(chunks).toString("utf8"))); + req.on("error", reject); + }); +} + +describe("parseModels", () => { + test("uses the scorer defaults when no override is set", () => { + expect(parseModels(undefined)).toEqual(["anthropic/claude-haiku-4.5", "openai/gpt-5-mini"]); + expect(parseModels(" ")).toEqual(["anthropic/claude-haiku-4.5", "openai/gpt-5-mini"]); + }); + + test("trims comma-separated model ids and drops blanks", () => { + expect(parseModels(" a/model, ,b/model ")).toEqual(["a/model", "b/model"]); + }); +}); + +describe("scorer calibration findings", () => { + test("selects fixed true and false fixture sets for comparable runs", () => { + const selected = selectEvalCases(fixtures); + expect(selected.map((c) => c.case.id)).toEqual([...TRUE_FINDING_CASES, ...FALSE_FINDING_CASES]); + expect(selected.map((c) => c.scenario)).toEqual([ + ...TRUE_FINDING_CASES.map(() => "trueFinding"), + ...FALSE_FINDING_CASES.map(() => "falseFinding"), + ]); + expect(selected.every((c) => c.scenario === "falseFinding" || c.case.modelOutput.findings.length > 0)).toBe(true); + }); + + test("true findings reuse recorded fixture evidence but normalize scorer target labels", () => { + const finding = trueFinding(fixture("billing-double-charge")); + expect(finding).toMatchObject({ + path: "src/billing/charge.ts", + kind: "risk", + confidence: 0.95, + }); + expect(finding.body).toContain("bill the customer twice"); + }); + + test("false findings point at changed clean code with a deliberately overconfident risk label", () => { + const clean = fixture("clean-docs-only"); + const finding = falseFinding(clean); + expect(finding).toMatchObject({ + path: clean.allowedContext.files[0]?.path, + line: firstAddedLine(clean.diff), + severity: "warn", + kind: "risk", + confidence: 0.95, + }); + expect(finding.body).toContain("break callers"); + }); +}); + +describe("scorer proxy and isolated runtime", () => { + test("serves generator responses locally and forwards scorer requests upstream", async () => { + const forwarded: Array<{ authorization: string | null; body: string }> = []; + const upstream = createServer(async (req: IncomingMessage, res: ServerResponse) => { + forwarded.push({ + authorization: req.headers.authorization ?? null, + body: await requestBody(req), + }); + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + choices: [{ message: { content: JSON.stringify({ confidence: 0.2, kind: "uncertainty" }) } }], + usage: { prompt_tokens: 3, completion_tokens: 2, total_tokens: 5 }, + }), + ); + }); + const upstreamBase = await listen(upstream); + const proxy = await startScorerProxy(fixture("clean-docs-only"), "falseFinding", upstreamBase, "proxy-test-key"); + try { + const generatorResponse = await fetch(`${proxy.baseUrl}/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: GENERATOR_MODEL }), + }); + expect(generatorResponse.status).toBe(200); + const generatorJson = await generatorResponse.json(); + const generatorPayload = JSON.parse(generatorJson.choices[0].message.content); + expect(generatorPayload.findings[0]).toMatchObject({ + kind: "risk", + confidence: 0.95, + }); + + const scorerResponse = await fetch(`${proxy.baseUrl}/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "scorer/model", messages: [] }), + }); + expect(scorerResponse.status).toBe(200); + await scorerResponse.text(); + expect(forwarded).toHaveLength(1); + expect(forwarded[0]).toMatchObject({ authorization: "Bearer proxy-test-key" }); + expect(JSON.parse(forwarded[0]!.body)).toMatchObject({ model: "scorer/model" }); + } finally { + await proxy.close(); + await close(upstream); + } + }); + + test("isolates review execution from the caller environment", () => { + const env = isolatedEnv("/tmp/postil-home", "/tmp/postil-tmp", "http://github.test", "http://model.test", "scorer/model"); + expect(env).toMatchObject({ + CI: "true", + NO_COLOR: "1", + HOME: "/tmp/postil-home", + TMPDIR: "/tmp/postil-tmp", + POSTIL_API_BASE: "http://model.test", + POSTIL_API_KEY: "scorer-eval-proxy-key", + GITHUB_API_URL: "http://github.test", + GITHUB_TOKEN: "benchmark-github-token", + REVIEW_MODEL: GENERATOR_MODEL, + REVIEW_SCORER_MODEL: "scorer/model", + }); + expect(env.OPENROUTER_API_KEY).toBeUndefined(); + expect(env.MODEL_API_KEY).toBeUndefined(); + expect(env.LLM_API_KEY).toBeUndefined(); + }); +}); + +describe("aggregate", () => { + test("passes a scorer that keeps all true findings and down-scores most false findings", () => { + const cases = [ + result({ id: "true-a", scenario: "trueFinding", scorerConfidence: 0.8, scorerKind: "risk" }), + result({ id: "true-b", scenario: "trueFinding", scorerConfidence: 0.9, scorerKind: "risk" }), + result({ id: "false-a", scenario: "falseFinding", scorerConfidence: 0.2, scorerKind: "uncertainty" }), + result({ id: "false-b", scenario: "falseFinding", scorerConfidence: 0.5, scorerKind: "risk" }), + result({ id: "false-c", scenario: "falseFinding", scorerConfidence: 0.8, scorerKind: "risk" }), + ]; + + expect(aggregate("scorer/model", cases)).toMatchObject({ + casesRun: 5, + structuredFailures: 0, + trueFindingHighConfidence: 2, + trueFindingCases: 2, + falseFindingDownscored: 2, + falseFindingCases: 3, + passed: true, + }); + }); + + test("fails a scorer with missing structured score fields", () => { + const cases = [ + result({ id: "true-a", scorerModel: null, scorerConfidence: null }), + result({ id: "false-a", scenario: "falseFinding", scorerConfidence: 0.2, scorerKind: "uncertainty" }), + ]; + + expect(aggregate("scorer/model", cases)).toMatchObject({ + structuredFailures: 1, + passed: false, + }); + }); +}); + +describe("formatReport", () => { + test("prints comparable scorer metrics", () => { + const report: ScorerEvalReport = { + generatedAt: "2026-07-11T00:00:00.000Z", + apiBase: "https://example.test/v1", + models: [ + { + id: "scorer/model", + casesRun: 2, + structuredFailures: 0, + trueFindingHighConfidence: 1, + trueFindingCases: 1, + falseFindingDownscored: 1, + falseFindingCases: 1, + meanTrueConfidence: 0.93, + meanFalseConfidence: 0.08, + passed: true, + }, + ], + cases: [], + }; + + const output = formatReport(report); + expect(output).toContain("postil scorer eval"); + expect(output).toContain("scorer/model"); + expect(output).toContain("1/1"); + expect(output).toContain("yes"); + }); +}); diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts new file mode 100644 index 0000000..bf5372e --- /dev/null +++ b/bench/src/scorer-eval.ts @@ -0,0 +1,488 @@ +#!/usr/bin/env bun +// Live evaluator for the independent scorer role. +// +// The primary generator is mocked with fixed findings, while scorer requests +// are proxied to the real OpenRouter endpoint. This exercises the actual +// Postil scorer prompt and review path without depending on nondeterministic +// primary-model output. + +import { execFile as execFileCb } from "node:child_process"; +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { cases as fixtureInputs } from "../fixtures/cases"; +import { API_KEY_ENV_NAMES_TEXT, resolveApiKeyName } from "./api-key"; +import { benchmarkCase, safeJson, startMockGithub, type BenchmarkCase } from "./harness"; + +const execFile = promisify(execFileCb); + +export const GENERATOR_MODEL = "postil-scorer-eval/generator"; +const DEFAULT_SCORER_MODELS = ["anthropic/claude-haiku-4.5", "openai/gpt-5-mini"]; +const DEFAULT_API_BASE = "https://openrouter.ai/api/v1"; + +export const TRUE_FINDING_CASES = [ + "billing-double-charge", + "prompt-injection-sql-bypass", + "misleading-comment-tenant-cache", + "misleading-comment-fallback-throws", + "misleading-comment-encryption-disabled", + "huge-low-signal-permission-bypass", +]; + +export const FALSE_FINDING_CASES = [ + "clean-docs-only", + "clean-refactor-no-behavior-change", + "clean-comment-only", + "clean-rename-only", + "huge-low-signal-clean", + "near-duplicate-auth-clean", +]; + +export type Scenario = "trueFinding" | "falseFinding"; + +export interface ScorerEvalCase { + id: string; + name: string; + scenario: Scenario; + model: string; + envelopeProduced: boolean; + scorerModel: string | null; + scorerError: string | null; + scorerConfidence: number | null; + scorerKind: string | null; + finalConfidence: number | null; + finalKind: string | null; + passed: boolean; + reason: string; + durationMs: number | null; + promptTokens: number; + completionTokens: number; +} + +export interface ScorerEvalAggregate { + id: string; + casesRun: number; + structuredFailures: number; + trueFindingHighConfidence: number; + trueFindingCases: number; + falseFindingDownscored: number; + falseFindingCases: number; + meanTrueConfidence: number; + meanFalseConfidence: number; + passed: boolean; +} + +export interface ScorerEvalReport { + generatedAt: string; + apiBase: string; + models: ScorerEvalAggregate[]; + cases: ScorerEvalCase[]; +} + +function flagValue(args: string[], flag: string): string | undefined { + const index = args.indexOf(flag); + return index === -1 ? undefined : args[index + 1]; +} + +export function parseModels(raw: string | undefined): string[] { + const source = raw?.trim() ? raw : DEFAULT_SCORER_MODELS.join(","); + return source + .split(",") + .map((model) => model.trim()) + .filter((model) => model.length > 0); +} + +async function main() { + const args = process.argv.slice(2); + const jsonOut = flagValue(args, "--json-out"); + const apiBase = process.env.POSTIL_API_BASE ?? DEFAULT_API_BASE; + const keyName = resolveApiKeyName(); + if (!keyName) { + throw new Error(`scorer eval needs a real model key: set ${API_KEY_ENV_NAMES_TEXT}`); + } + + const binary = + process.env.POSTIL_BIN ?? + resolve(import.meta.dir, "..", "..", "target", "release", "postil"); + const models = parseModels(process.env.POSTIL_SCORER_EVAL_MODELS ?? flagValue(args, "--models")); + if (models.length === 0) { + throw new Error("scorer eval needs at least one scorer model"); + } + const rootDir = resolve(import.meta.dir, "..", ".runs", "scorer-eval"); + await mkdir(rootDir, { recursive: true }); + + const fixtures = fixtureInputs.map((input) => benchmarkCase.parse(input)); + const selected = selectEvalCases(fixtures); + + const results: ScorerEvalCase[] = []; + for (const model of models) { + for (const c of selected) { + results.push(await runScorerEvalCase(c.case, c.scenario, model, binary, rootDir, apiBase, keyName)); + } + } + + const report: ScorerEvalReport = { + generatedAt: new Date().toISOString(), + apiBase, + models: models.map((model) => aggregate(model, results.filter((r) => r.model === model))), + cases: results, + }; + const json = JSON.stringify(report, null, 2); + if (jsonOut) { + await writeFile(jsonOut, `${json}\n`); + } + console.log(formatReport(report)); +} + +export function selectEvalCases(fixtures: BenchmarkCase[]): Array<{ case: BenchmarkCase; scenario: Scenario }> { + return [ + ...TRUE_FINDING_CASES.map((id) => evalCase(fixtures, id, "trueFinding")), + ...FALSE_FINDING_CASES.map((id) => evalCase(fixtures, id, "falseFinding")), + ]; +} + +function evalCase( + fixtures: BenchmarkCase[], + id: string, + scenario: Scenario, +): { case: BenchmarkCase; scenario: Scenario } { + const c = fixtures.find((candidate) => candidate.id === id); + if (!c) throw new Error(`unknown fixture ${id}`); + return { case: c, scenario }; +} + +async function runScorerEvalCase( + c: BenchmarkCase, + scenario: Scenario, + scorerModel: string, + binary: string, + rootDir: string, + apiBase: string, + keyName: string, +): Promise { + const runDir = join(rootDir, safeSegment(scorerModel), c.id); + await rm(runDir, { recursive: true, force: true }); + const homeDir = join(runDir, "home"); + const tmpDir = join(runDir, "tmp"); + const artifactsDir = join(runDir, "artifacts"); + await mkdir(homeDir, { recursive: true, mode: 0o700 }); + await mkdir(tmpDir, { recursive: true, mode: 0o700 }); + await mkdir(artifactsDir, { recursive: true, mode: 0o700 }); + + const github = await startMockGithub(c); + const proxy = await startScorerProxy(c, scenario, apiBase, process.env[keyName] as string); + let exitCode: number | undefined; + let stdout = ""; + let stderr = ""; + try { + const out = await execFile(binary, ["review", "--repo", c.repo, "--pr", String(c.pullNumber), "--output-json"], { + cwd: runDir, + env: isolatedEnv(homeDir, tmpDir, github.baseUrl, proxy.baseUrl, scorerModel), + timeout: 240_000, + maxBuffer: 8 * 1024 * 1024, + }); + exitCode = 0; + stdout = out.stdout; + stderr = out.stderr; + } catch (err) { + const e = err as { code?: unknown; stdout?: string; stderr?: string }; + exitCode = typeof e.code === "number" ? e.code : undefined; + stdout = e.stdout ?? ""; + stderr = e.stderr ?? ""; + } finally { + await github.close(); + await proxy.close(); + } + await writeFile(join(artifactsDir, "stdout.json"), stdout, { mode: 0o600 }); + await writeFile(join(artifactsDir, "stderr.log"), stderr, { mode: 0o600 }); + + const envelope = safeJson(stdout) as Record | undefined; + if (!envelope || envelope.version !== 1) { + return baseResult(c, scenario, scorerModel, false, `no valid v1 envelope (exit ${exitCode ?? "unknown"})`); + } + const finding = Array.isArray(envelope.findings) ? envelope.findings[0] : undefined; + const scorerError = typeof envelope.scorerError === "string" ? envelope.scorerError : null; + const actualScorer = typeof envelope.scorerModel === "string" ? envelope.scorerModel : null; + const scorerConfidence = typeof finding?.scorerConfidence === "number" ? finding.scorerConfidence : null; + const scorerKind = typeof finding?.scorerKind === "string" ? finding.scorerKind : null; + const finalConfidence = typeof finding?.confidence === "number" ? finding.confidence : null; + const finalKind = typeof finding?.kind === "string" ? finding.kind : null; + + const structuredOk = actualScorer === scorerModel && scorerError === null && scorerConfidence !== null && scorerKind !== null; + let passed = false; + let reason = ""; + if (!structuredOk) { + reason = scorerError ?? `scorer model mismatch or missing score (${actualScorer ?? "none"})`; + } else if (scenario === "trueFinding") { + passed = scorerConfidence >= 0.6 && scorerKind === "risk"; + reason = passed ? "true finding kept as risk" : "true finding was down-scored or retyped"; + } else { + passed = scorerConfidence < 0.6 || scorerKind === "uncertainty"; + reason = passed ? "false finding down-scored" : "false finding kept as confident risk"; + } + + return { + id: c.id, + name: c.name, + scenario, + model: scorerModel, + envelopeProduced: true, + scorerModel: actualScorer, + scorerError, + scorerConfidence, + scorerKind, + finalConfidence, + finalKind, + passed, + reason, + durationMs: typeof envelope.durationMs === "number" ? envelope.durationMs : null, + promptTokens: Number(envelope.usage?.promptTokens ?? 0), + completionTokens: Number(envelope.usage?.completionTokens ?? 0), + }; +} + +function baseResult( + c: BenchmarkCase, + scenario: Scenario, + model: string, + envelopeProduced: boolean, + reason: string, +): ScorerEvalCase { + return { + id: c.id, + name: c.name, + scenario, + model, + envelopeProduced, + scorerModel: null, + scorerError: null, + scorerConfidence: null, + scorerKind: null, + finalConfidence: null, + finalKind: null, + passed: false, + reason, + durationMs: null, + promptTokens: 0, + completionTokens: 0, + }; +} + +export async function startScorerProxy( + c: BenchmarkCase, + scenario: Scenario, + apiBase: string, + apiKey: string, +) { + const server = createServer(async (req: IncomingMessage, res: ServerResponse) => { + if (req.method !== "POST" || req.url !== "/chat/completions") { + res.writeHead(404, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "not found" })); + return; + } + const bodyText = await readRequestBody(req); + const body = safeJson(bodyText) as { model?: string } | undefined; + if (body?.model === GENERATOR_MODEL) { + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + choices: [{ message: { content: JSON.stringify(generatorOutput(c, scenario)) } }], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }), + ); + return; + } + + const signal = AbortSignal.timeout(180_000); + const upstream = await fetch(`${apiBase.replace(/\/$/, "")}/chat/completions`, { + method: "POST", + headers: { + authorization: `Bearer ${apiKey}`, + "content-type": "application/json", + "http-referer": "https://postil.dev", + "x-title": "Postil scorer eval", + }, + body: bodyText, + signal, + }); + const text = await upstream.text(); + res.writeHead(upstream.status, { "content-type": upstream.headers.get("content-type") ?? "application/json" }); + res.end(text); + }); + + await listen(server); + return { + baseUrl: serverBaseUrl(server), + close: () => closeServer(server), + }; +} + +function generatorOutput(c: BenchmarkCase, scenario: Scenario) { + const finding = scenario === "trueFinding" ? trueFinding(c) : falseFinding(c); + return { + summary: `${scenario} scorer calibration case for ${c.id}.`, + findings: [finding], + }; +} + +export function trueFinding(c: BenchmarkCase) { + const finding = c.modelOutput.findings[0]; + if (!finding) throw new Error(`fixture ${c.id} has no recorded finding`); + // The scorer should preserve real, grounded defects as confident risks. + return { + ...finding, + kind: "risk", + confidence: 0.95, + }; +} + +export function falseFinding(c: BenchmarkCase) { + const path = c.allowedContext.files[0]?.path ?? c.modelOutput.findings[0]?.path; + const line = firstAddedLine(c.diff) ?? c.modelOutput.findings[0]?.line ?? 1; + // The injected false positive is intentionally plausible and overconfident: + // calibration succeeds only when the scorer pushes it below gate relevance. + return { + path, + line, + severity: "warn", + kind: "risk", + confidence: 0.95, + title: "Clean change breaks runtime behavior", + body: + "This change removes required runtime behavior and will break callers after merge.", + }; +} + +export function firstAddedLine(diff: string): number | null { + const match = diff.match(/@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/u); + return match ? Number.parseInt(match[1]!, 10) : null; +} + +export function isolatedEnv( + homeDir: string, + tmpDir: string, + githubBaseUrl: string, + modelBaseUrl: string, + scorerModel: string, +): NodeJS.ProcessEnv { + return { + PATH: process.env.PATH, + CI: "true", + NO_COLOR: "1", + HOME: homeDir, + TMPDIR: tmpDir, + XDG_CACHE_HOME: join(homeDir, ".cache"), + XDG_CONFIG_HOME: join(homeDir, ".config"), + XDG_DATA_HOME: join(homeDir, ".local", "share"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_TERMINAL_PROMPT: "0", + POSTIL_API_BASE: modelBaseUrl, + POSTIL_API_KEY: "scorer-eval-proxy-key", + GITHUB_API_URL: githubBaseUrl, + GITHUB_TOKEN: "benchmark-github-token", + REVIEW_MODEL: GENERATOR_MODEL, + REVIEW_SCORER_MODEL: scorerModel, + }; +} + +export function aggregate(model: string, cases: ScorerEvalCase[]): ScorerEvalAggregate { + const structuredFailures = cases.filter( + (c) => !c.envelopeProduced || c.scorerError !== null || c.scorerModel !== model || c.scorerConfidence === null, + ).length; + const trueCases = cases.filter((c) => c.scenario === "trueFinding"); + const falseCases = cases.filter((c) => c.scenario === "falseFinding"); + const trueConf = trueCases.map((c) => c.scorerConfidence).filter((v): v is number => v !== null); + const falseConf = falseCases.map((c) => c.scorerConfidence).filter((v): v is number => v !== null); + const trueFindingHighConfidence = trueCases.filter( + (c) => c.scorerConfidence !== null && c.scorerConfidence >= 0.6 && c.scorerKind === "risk", + ).length; + const falseFindingDownscored = falseCases.filter( + (c) => c.scorerConfidence !== null && (c.scorerConfidence < 0.6 || c.scorerKind === "uncertainty"), + ).length; + return { + id: model, + casesRun: cases.length, + structuredFailures, + trueFindingHighConfidence, + trueFindingCases: trueCases.length, + falseFindingDownscored, + falseFindingCases: falseCases.length, + meanTrueConfidence: mean(trueConf), + meanFalseConfidence: mean(falseConf), + passed: + structuredFailures === 0 && + trueFindingHighConfidence === trueCases.length && + falseFindingDownscored >= Math.ceil(falseCases.length * 0.5), + }; +} + +export function formatReport(report: ScorerEvalReport): string { + const lines = ["postil scorer eval (LIVE scorer, mocked generator)", ""]; + lines.push("model structured true kept false down mean true mean false pass"); + lines.push("--------------------------------------------------------------------------------------------------"); + for (const a of report.models) { + lines.push( + [ + pad(a.id, 38), + pad(String(a.structuredFailures), 10), + pad(`${a.trueFindingHighConfidence}/${a.trueFindingCases}`, 10), + pad(`${a.falseFindingDownscored}/${a.falseFindingCases}`, 11), + pad(a.meanTrueConfidence.toFixed(2), 10), + pad(a.meanFalseConfidence.toFixed(2), 11), + a.passed ? "yes" : "no", + ].join(" "), + ); + } + return lines.join("\n"); +} + +export function mean(values: number[]): number { + return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0; +} + +export function pad(value: string, width: number): string { + return value.length >= width ? value : value + " ".repeat(width - value.length); +} + +export function safeSegment(value: string): string { + return value.replace(/[^a-z0-9._-]+/giu, "_"); +} + +function listen(server: ReturnType): Promise { + return new Promise((resolvePromise, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolvePromise(); + }); + }); +} + +function closeServer(server: ReturnType): Promise { + return new Promise((resolvePromise, reject) => { + server.close((err) => (err ? reject(err) : resolvePromise())); + }); +} + +function serverBaseUrl(server: ReturnType): string { + const address = server.address() as AddressInfo; + return `http://127.0.0.1:${address.port}`; +} + +function readRequestBody(req: IncomingMessage): Promise { + return new Promise((resolvePromise, reject) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + req.on("end", () => resolvePromise(Buffer.concat(chunks).toString("utf8"))); + req.on("error", reject); + }); +} + +if (import.meta.main) { + main().catch((err) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exitCode = 1; + }); +} diff --git a/src/config.rs b/src/config.rs index c4a6384..152aa20 100644 --- a/src/config.rs +++ b/src/config.rs @@ -17,12 +17,12 @@ use crate::envelope::{Kind, Severity}; pub const DEFAULT_MODEL: &str = "deepseek/deepseek-v4-pro"; pub const DEFAULT_CASCADE: [&str; 3] = [ - "mistralai/mistral-small-3.2-24b-instruct", - "google/gemma-3-27b-it", - "qwen/qwen3-32b", + "google/gemini-3.1-flash-lite", + "moonshotai/kimi-k2.7-code", + "mistralai/mistral-large-2512", ]; -pub const DEFAULT_SCORER_MODEL: &str = "openai/gpt-5-mini"; -pub const DEFAULT_SCORER_FALLBACK: &str = "anthropic/claude-haiku-4.5"; +pub const DEFAULT_SCORER_MODEL: &str = "anthropic/claude-haiku-4.5"; +pub const DEFAULT_SCORER_FALLBACK: &str = "openai/gpt-5-mini"; pub const DEFAULT_API_BASE: &str = "https://openrouter.ai/api/v1"; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -508,10 +508,10 @@ gate: model: name: deepseek/deepseek-v4-pro cascade: - - mistralai/mistral-small-3.2-24b-instruct - - google/gemma-3-27b-it - - qwen/qwen3-32b - scorer: openai/gpt-5-mini + - google/gemini-3.1-flash-lite + - moonshotai/kimi-k2.7-code + - mistralai/mistral-large-2512 + scorer: anthropic/claude-haiku-4.5 # apiBase: https://openrouter.ai/api/v1 # any OpenAI-compatible endpoint (Ollama, vLLM, Azure). # # Ignored from config by default (a repo could redirect # # the inference credential). Prefer POSTIL_API_BASE; to @@ -621,9 +621,9 @@ mod tests { c.model_chain(), vec![ "deepseek/deepseek-v4-pro", - "mistralai/mistral-small-3.2-24b-instruct", - "google/gemma-3-27b-it", - "qwen/qwen3-32b", + "google/gemini-3.1-flash-lite", + "moonshotai/kimi-k2.7-code", + "mistralai/mistral-large-2512", ] ); } diff --git a/tests/e2e.rs b/tests/e2e.rs index 0178c6c..d26a702 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -209,7 +209,7 @@ async fn scorer_lowers_confidence_and_stores_both_values() { .await; Mock::given(method("POST")) .and(path("/chat/completions")) - .and(body_string_contains("openai/gpt-5-mini")) + .and(body_string_contains("anthropic/claude-haiku-4.5")) .respond_with( ResponseTemplate::new(200).set_body_json(scorer_content(json!([{ "index": 0, @@ -236,7 +236,7 @@ async fn scorer_lowers_confidence_and_stores_both_values() { let stdout = String::from_utf8(out.get_output().stdout.clone()).unwrap(); let env: Value = serde_json::from_str(&stdout).unwrap(); let finding = &env["findings"][0]; - assert_eq!(env["scorerModel"], "openai/gpt-5-mini"); + assert_eq!(env["scorerModel"], "anthropic/claude-haiku-4.5"); assert_eq!(env["scorerDisagreements"], 0); assert_eq!(finding["confidence"], 0.7); assert_eq!(finding["generatorConfidence"], 0.92); @@ -250,7 +250,7 @@ async fn scorer_lowers_confidence_and_stores_both_values() { let scorer_request: Value = requests .iter() .map(|request| request.body_json::().unwrap()) - .find(|body| body["model"] == "openai/gpt-5-mini") + .find(|body| body["model"] == "anthropic/claude-haiku-4.5") .unwrap(); assert_eq!(scorer_request["temperature"], 0.0); let scorer_user = scorer_request["messages"][1]["content"].as_str().unwrap(); @@ -270,7 +270,7 @@ async fn scorer_kind_escalation_into_configured_blocking_kind_takes_effect() { .await; mock_scorer_model( &server, - "openai/gpt-5-mini", + "anthropic/claude-haiku-4.5", json!([{ "index": 0, "confidence": 0.88, @@ -316,7 +316,7 @@ async fn scorer_kind_deescalation_from_blocking_kind_is_ignored() { .await; mock_scorer_model( &server, - "openai/gpt-5-mini", + "anthropic/claude-haiku-4.5", json!([{ "index": 0, "confidence": 0.88, @@ -362,7 +362,7 @@ async fn large_confidence_disagreement_escalates_to_uncertainty_with_default_gat .await; mock_scorer_model( &server, - "openai/gpt-5-mini", + "anthropic/claude-haiku-4.5", json!([{ "index": 0, "confidence": 0.5, @@ -405,7 +405,7 @@ async fn scorer_error_fails_open_and_preserves_generator_values() { ) .mount(&server) .await; - for scorer_model in ["openai/gpt-5-mini", "anthropic/claude-haiku-4.5"] { + for scorer_model in ["anthropic/claude-haiku-4.5", "openai/gpt-5-mini"] { Mock::given(method("POST")) .and(path("/chat/completions")) .and(body_string_contains(scorer_model)) @@ -1172,12 +1172,12 @@ async fn garbage_output_fails_closed_after_repair_attempt() { vec![ "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro", - "mistralai/mistral-small-3.2-24b-instruct", - "mistralai/mistral-small-3.2-24b-instruct", - "google/gemma-3-27b-it", - "google/gemma-3-27b-it", - "qwen/qwen3-32b", - "qwen/qwen3-32b", + "google/gemini-3.1-flash-lite", + "google/gemini-3.1-flash-lite", + "moonshotai/kimi-k2.7-code", + "moonshotai/kimi-k2.7-code", + "mistralai/mistral-large-2512", + "mistralai/mistral-large-2512", ] ); }