From 8b04d9a9b465c711c250aa2203b7636910b99b72 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Sat, 13 Jun 2026 09:43:34 +0000 Subject: [PATCH 1/2] bench(live): bounded concurrency and one transient retry Run live bench cases through a bounded worker pool (default 6, configurable via --concurrency or BENCH_CONCURRENCY) instead of strictly sequentially. Each case keeps its own isolated run directory, and results are written into index-aligned slots and sorted by case index before reporting, so report ordering stays deterministic regardless of completion order. --concurrency 1 restores sequential execution. Retry a case once (after a short backoff) when its first attempt fails with a transient provider error: a non-zero exit whose stderr carries an HTTP 5xx/429, rate-limit, timeout, or connection signature, or a run that produced no valid v1 envelope at all. A valid envelope (including a gate-failing exit or one with findings/false positives) is a normal result and is never retried. A case that fails twice is recorded as an error, as before. Mock mode and the mock CI path are untouched. --- bench/src/live.ts | 130 +++++++++++++++++++++++++++++++++++++++++++--- bench/src/run.ts | 17 ++++-- 2 files changed, 138 insertions(+), 9 deletions(-) diff --git a/bench/src/live.ts b/bench/src/live.ts index ed06aa8..3524e9e 100644 --- a/bench/src/live.ts +++ b/bench/src/live.ts @@ -36,6 +36,18 @@ const execFile = promisify(execFileCb); export const DEFAULT_LIVE_MODEL = "deepseek/deepseek-v4-pro"; +/** Default number of cases run concurrently. Live inference is I/O-bound on the + * provider, so a small pool cuts wall-clock time without overloading the API. + * Override with --concurrency or BENCH_CONCURRENCY. */ +export const DEFAULT_LIVE_CONCURRENCY = 6; + +/** Each case that fails with a transient/provider error is retried this many + * extra times (so one retry total). A second failure is recorded as an error. */ +const DEFAULT_LIVE_RETRIES = 1; + +/** Backoff before the single retry of a transiently-failed case. */ +const RETRY_BACKOFF_MS = 2_000; + /** A defect is detected when a finding hits the right file and is within this * many lines of the ground-truth line. Matches the manual fixture run. */ const LINE_TOLERANCE = 3; @@ -74,6 +86,10 @@ export interface LiveOptions { rootDir?: string; /** Per-case timeout (default 180s; live inference is slow). */ timeoutMs?: number; + /** Cases run concurrently (default DEFAULT_LIVE_CONCURRENCY). */ + concurrency?: number; + /** Extra attempts on a transient/provider failure (default 1 = one retry). */ + retries?: number; } interface GroundTruth { @@ -106,6 +122,9 @@ export interface LiveCaseResult { completionTokens: number; exitCode: number | undefined; error?: string; + /** Captured stderr of the binary run. Used internally to classify transient + * provider failures for retry; not part of the persisted report schema. */ + stderr?: string; } export interface LiveSummary { @@ -158,11 +177,105 @@ export async function runLive( await assertBinary(options.binary); const rootDir = options.rootDir ?? resolve(import.meta.dir, "..", ".runs"); - const results: LiveCaseResult[] = []; - for (const [index, c] of cases.entries()) { - results.push(await runLiveCase(c, index, rootDir, options)); + + // Bounded worker pool: a small fixed number of workers pull case indices off a + // shared cursor until the queue drains. Each case writes its result into the + // slot for its original index, so completion order never affects the report. + const results = new Array(cases.length); + const concurrency = Math.max(1, Math.min(options.concurrency ?? DEFAULT_LIVE_CONCURRENCY, cases.length || 1)); + let cursor = 0; + const worker = async (): Promise => { + for (;;) { + const index = cursor++; + if (index >= cases.length) return; + results[index] = await runLiveCaseWithRetry(cases[index]!, index, rootDir, options); + } + }; + await Promise.all(Array.from({ length: concurrency }, () => worker())); + + // Results are already index-aligned; sorting by case index is belt-and-braces + // so the written report is deterministically ordered regardless of the pool. + // Strip the internal `stderr` field so the persisted report schema is intact. + const ordered = results + .map((result, index) => ({ result, index })) + .sort((a, b) => a.index - b.index) + .map(({ result }) => { + const { stderr: _stderr, ...rest } = result; + return rest; + }); + return { summary: summarize(ordered, options), results: ordered }; +} + +/** + * Runs a case, retrying once (by default) after a short backoff if the first + * attempt fails with a transient/provider error: a non-zero exit whose stderr + * carries an HTTP-5xx/429/timeout/connection signature, or no valid v1 envelope + * at all (empty/garbled output). A valid envelope with findings is a normal + * result and is never retried. A case that fails every attempt is returned as + * the last attempt's error result, which summarize() already counts as an error. + */ +async function runLiveCaseWithRetry( + c: ReturnType, + index: number, + rootDir: string, + options: LiveOptions, +): Promise { + const maxRetries = options.retries ?? DEFAULT_LIVE_RETRIES; + let last: LiveCaseResult | undefined; + for (let attempt = 0; attempt <= maxRetries; attempt++) { + last = await runLiveCase(c, index, rootDir, options); + // Scored => a valid envelope was produced; that is a normal result (even if + // it has findings or false positives), so never retry it. + if (last.scored) return last; + if (attempt < maxRetries && isTransientFailure(last)) { + await sleep(RETRY_BACKOFF_MS); + continue; + } + return last; } - return { summary: summarize(results, options), results }; + return last!; +} + +/** Signatures of a transient provider/transport failure in stderr: HTTP 5xx and + * 429, rate-limit/timeout/connection wording. Used to decide whether a failed + * (unscored) case is worth one retry. */ +const TRANSIENT_STDERR = new RegExp( + [ + "\\b(5\\d{2}|429)\\b", // HTTP 5xx / 429 status + "rate.?limit", + "too many requests", + "timed? ?out", + "timeout", + "temporarily unavailable", + "service unavailable", + "bad gateway", + "gateway time-?out", + "overloaded", + "connection (reset|refused|closed|error)", + "econnreset", + "econnrefused", + "etimedout", + "socket hang up", + "network", + ].join("|"), + "i", +); + +/** True when an unscored case looks like a transient provider failure: either + * the binary emitted a recognizable transient signature on stderr, or it + * produced no valid envelope at all (empty/garbled output, e.g. a dropped + * response). Both are worth one retry; a deterministic parse-shaped failure that + * is not provider-side will just fail again and be recorded as an error. */ +function isTransientFailure(result: LiveCaseResult): boolean { + if (result.scored) return false; + if (result.stderr && TRANSIENT_STDERR.test(result.stderr)) return true; + // No valid envelope (empty/invalid output) — typically a dropped or truncated + // provider response; retry once. + return result.error?.startsWith("no valid v1 envelope") ?? false; +} + +function sleep(ms: number): Promise { + return new Promise((res) => setTimeout(res, ms)); } async function assertBinary(binary: string): Promise { @@ -212,6 +325,7 @@ async function runLiveCase( await writeFile(diffPath, c.diff, { mode: 0o600 }); let stdout = ""; + let stderr = ""; let exitCode: number | undefined; try { const out = await execFile( @@ -226,14 +340,18 @@ async function runLiveCase( ); exitCode = 0; stdout = out.stdout; + stderr = out.stderr; } catch (err) { // Exit 1 with a valid envelope is the gate failing on an error-severity - // finding, not a transport failure — keep the stdout and score it. - const e = err as { code?: unknown; stdout?: string }; + // finding, not a transport failure — keep the stdout and score it. The + // stderr is captured so a genuine transport failure can be retried. + const e = err as { code?: unknown; stdout?: string; stderr?: string; message?: string }; exitCode = typeof e.code === "number" ? e.code : undefined; stdout = e.stdout ?? ""; + stderr = e.stderr ?? e.message ?? ""; } base.exitCode = exitCode; + base.stderr = stderr; await writeFile(join(runDir, "stdout.json"), stdout, { mode: 0o600 }); const parsed = envelopeV1.safeParse(safeJson(stdout)); diff --git a/bench/src/run.ts b/bench/src/run.ts index 480831a..3fcc536 100644 --- a/bench/src/run.ts +++ b/bench/src/run.ts @@ -11,7 +11,7 @@ // measures detection ability. Requires POSTIL_API_KEY in the environment. // // bun run bench:live # or: BENCH_LIVE=1 bun run src/run.ts -// bun run bench --live [--json] [--json-out ] [--model ] +// bun run bench --live [--json] [--json-out ] [--model ] [--concurrency ] // // Environment: // POSTIL_BIN path to the postil binary (default ../target/release/postil) @@ -19,18 +19,28 @@ // POSTIL_API_KEY required in live mode; never logged or printed // REVIEW_MODEL model id for live mode (else --model, else default) // BENCH_LIVE set to 1 to select live mode +// BENCH_CONCURRENCY live-mode case parallelism (else --concurrency, else default 6) import { mkdir, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { cases } from "../fixtures/cases"; import { formatReport, runBenchmark } from "./harness"; -import { DEFAULT_LIVE_MODEL, formatLiveReport, runLive } from "./live"; +import { DEFAULT_LIVE_CONCURRENCY, DEFAULT_LIVE_MODEL, formatLiveReport, runLive } from "./live"; function flagValue(args: string[], flag: string): string | undefined { const index = args.indexOf(flag); return index === -1 ? undefined : args[index + 1]; } +/** Resolve live-mode concurrency from BENCH_CONCURRENCY, then --concurrency, + * then the default. Non-positive or non-numeric inputs fall back to the default. */ +function liveConcurrency(args: string[]): number { + const raw = process.env.BENCH_CONCURRENCY ?? flagValue(args, "--concurrency"); + if (raw === undefined) return DEFAULT_LIVE_CONCURRENCY; + const n = Number.parseInt(raw, 10); + return Number.isFinite(n) && n > 0 ? n : DEFAULT_LIVE_CONCURRENCY; +} + async function main() { const args = process.argv.slice(2); const json = args.includes("--json"); @@ -42,7 +52,8 @@ async function main() { if (live) { const model = process.env.REVIEW_MODEL ?? flagValue(args, "--model") ?? DEFAULT_LIVE_MODEL; - const report = await runLive(cases, { binary, model }); + const concurrency = liveConcurrency(args); + const report = await runLive(cases, { binary, model, concurrency }); await writeReport(jsonOut, JSON.stringify(report, null, 2)); console.log(json ? JSON.stringify(report, null, 2) : formatLiveReport(report)); return; From 8b51d2dba52692f7dcef2504e053b9e00df5ef23 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Sat, 13 Jun 2026 09:43:37 +0000 Subject: [PATCH 2/2] docs(bench): document live --concurrency and one-retry behavior --- bench/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/bench/README.md b/bench/README.md index c805fdb..5615f3b 100644 --- a/bench/README.md +++ b/bench/README.md @@ -53,6 +53,7 @@ is written to any repo. export POSTIL_API_KEY=... # required; never logged or printed bun run bench:live # or: bun run bench --live # REVIEW_MODEL or --model overrides the model (default deepseek/deepseek-v4-pro) +# --concurrency or BENCH_CONCURRENCY sets case parallelism (default 6) ``` It refuses to run without `POSTIL_API_KEY` and never reads or prints the key @@ -61,6 +62,24 @@ external provider. Every live run writes a timestamped JSON report under `.runs/` (gitignored); `--json-out ` writes an additional copy and `--json` prints the report as JSON. +### Concurrency and retries + +Cases run through a bounded worker pool, `--concurrency ` (or +`BENCH_CONCURRENCY`, default 6) at a time, instead of strictly sequentially. +Each case still gets its own isolated run directory, and results are sorted by +case index before the report is written, so the output is byte-for-byte +deterministic in ordering regardless of completion order. Set `--concurrency 1` +to fall back to fully sequential execution. + +Each case is retried **once** (after a short backoff) when its first attempt +fails with a transient provider error: a non-zero exit whose stderr carries an +HTTP 5xx/429, rate-limit, timeout, or connection signature, or a run that +produced no valid v1 envelope at all (empty/garbled output, typically a dropped +response). A valid envelope is always treated as a normal result and is never +retried — including a gate-failing exit (exit 1 with a scored envelope) or one +that merely reports findings or false positives. A case that fails on both +attempts is recorded as an error and excluded from scoring, exactly as before. + ### What live mode scores - **Detection rate**: a defect counts as detected when a finding matches the