From 956e1a9ce137dc003bd721dec0f3ffdf652f45f0 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Sat, 13 Jun 2026 07:51:52 +0000 Subject: [PATCH 1/2] bench: report severity match exact and within one tier in live mode The live-mode detection report scored severity by strict equality only, which understated quality: many warn<->error differences are defensible judgment calls on the fixture labels rather than model errors (see the severity-tier calibration investigation), so a strict-equality metric reads as failure on arguable disagreements. Score severity two ways and report both. severityMatch keeps the original strict equality. severityMatchWithinOneTier treats adjacent tiers on the info < warn < error scale (info<->warn, warn<->error) as a match and counts only the two-tier info<->error gap as a real mismatch. Both numbers appear in the report JSON and the summary line (severity match (exact) and (+/-1 tier)), and the per-case detail still shows truth vs found severity, flagging adjacent calls distinctly from true mismatches. Neither figure is a peer-comparison claim. Mock pass/fail behaviour is unaffected. --- bench/src/live.ts | 64 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/bench/src/live.ts b/bench/src/live.ts index f960e61..ed06aa8 100644 --- a/bench/src/live.ts +++ b/bench/src/live.ts @@ -20,6 +20,10 @@ // match is tracked among detections; clean cases should be silent; any finding // in a clean case and any non-matching finding in a defect case is a false // positive. +// +// Severity match is reported two ways: exact (strict equality) and within one +// tier (info<->warn and warn<->error treated as a match). See the severity-tier +// helpers below for the rationale. import { execFile as execFileCb } from "node:child_process"; import { createHash } from "node:crypto"; @@ -36,6 +40,31 @@ export const DEFAULT_LIVE_MODEL = "deepseek/deepseek-v4-pro"; * many lines of the ground-truth line. Matches the manual fixture run. */ const LINE_TOLERANCE = 3; +/** Severity tiers, ordered low to high (mirrors src/envelope.rs: info < warn < + * error). Used to compute the +/-1-tier adjacency tolerance below. */ +const SEVERITY_ORDER = ["info", "warn", "error"] as const; + +function severityRank(severity: string | null): number { + return severity === null + ? -1 + : SEVERITY_ORDER.indexOf(severity as (typeof SEVERITY_ORDER)[number]); +} + +/** + * True when found and truth severities are within one tier of each other on the + * info < warn < error scale. Adjacent pairs (info<->warn, warn<->error) count as + * a match; only the two-tier gap info<->error is a real mismatch. This reflects + * the severity-calibration finding that warn<->error is frequently a defensible + * judgment call rather than a model error (see .cache/severity-investigation.md). + * It is reported alongside the strict exact metric, never as a replacement. + * Unknown/missing severities never match. */ +function severityWithinOneTier(found: string | null, truth: string | null): boolean { + const f = severityRank(found); + const t = severityRank(truth); + if (f < 0 || t < 0) return false; + return Math.abs(f - t) <= 1; +} + export interface LiveOptions { /** Path to the postil binary (a release build). */ binary: string; @@ -65,7 +94,10 @@ export interface LiveCaseResult { silent: boolean | null; truthSeverity: string | null; foundSeverity: string | null; + /** Strict equality of found vs truth severity (the original metric). */ severityMatch: boolean | null; + /** Found and truth severity within one tier (info<->warn, warn<->error). */ + severityMatchWithinOneTier: boolean | null; confidence: number | null; /** Findings beyond the matched true positive (false positives in this case). */ falsePositives: number; @@ -86,7 +118,10 @@ export interface LiveSummary { scoredCases: number; detected: number; detectionRate: string; - severityMatchAmongDetected: string; + /** Strict severity equality among detections (the original metric). */ + severityMatchExact: string; + /** Severity within one tier among detections (warn<->error is defensible). */ + severityMatchWithinOneTier: string; silentOnClean: string; falsePositives: number; confidenceOfDetections: { @@ -162,6 +197,7 @@ async function runLiveCase( truthSeverity: truth.severity, foundSeverity: null, severityMatch: null, + severityMatchWithinOneTier: null, confidence: null, falsePositives: 0, durationMs: null, @@ -240,7 +276,10 @@ function scoreCase(base: LiveCaseResult, truth: GroundTruth, env: Envelope): Liv base.detected = Boolean(match); if (match) { base.foundSeverity = match.severity; + // Exact: strict equality (the original, strict metric). Within-one-tier: + // adjacent severities count as a match (see severityWithinOneTier). base.severityMatch = match.severity === truth.severity; + base.severityMatchWithinOneTier = severityWithinOneTier(match.severity, truth.severity); base.confidence = match.confidence; } // Every finding that is not the matched true positive is a false positive. @@ -268,7 +307,8 @@ function summarize(results: LiveCaseResult[], options: LiveOptions): LiveSummary const scored = results.filter((r) => r.scored); const detected = defects.filter((r) => r.detected === true); - const sevMatched = detected.filter((r) => r.severityMatch === true); + const sevMatchedExact = detected.filter((r) => r.severityMatch === true); + const sevMatchedWithinOneTier = detected.filter((r) => r.severityMatchWithinOneTier === true); const silentClean = cleans.filter((r) => r.silent === true); const falsePositives = results.reduce((sum, r) => sum + r.falsePositives, 0); @@ -294,7 +334,8 @@ function summarize(results: LiveCaseResult[], options: LiveOptions): LiveSummary scoredCases: scored.length, detected: detected.length, detectionRate: `${detected.length}/${defects.length}`, - severityMatchAmongDetected: `${sevMatched.length}/${detected.length}`, + severityMatchExact: `${sevMatchedExact.length}/${detected.length}`, + severityMatchWithinOneTier: `${sevMatchedWithinOneTier.length}/${detected.length}`, silentOnClean: `${silentClean.length}/${cleans.length}`, falsePositives, confidenceOfDetections: { @@ -331,7 +372,8 @@ export function formatLiveReport(report: LiveReport): string { const s = report.summary; const lines: string[] = [ `postil bench (LIVE mode): model ${s.model}`, - `Detection ${s.detectionRate} defects | severity-match ${s.severityMatchAmongDetected} | ` + + `Detection ${s.detectionRate} defects | severity match (exact) ${s.severityMatchExact} | ` + + `severity match (+/-1 tier) ${s.severityMatchWithinOneTier} | ` + `silent-on-clean ${s.silentOnClean} | false-positives ${s.falsePositives}`, ]; const c = s.confidenceOfDetections; @@ -349,6 +391,9 @@ export function formatLiveReport(report: LiveReport): string { lines.push(`Cases without a valid envelope: ${s.errors} (excluded from scoring)`); } lines.push( + "Severity match (exact) is strict equality; (+/-1 tier) treats adjacent", + "info<->warn and warn<->error as a match, since warn<->error is often a", + "defensible judgment call. Neither is a peer-comparison claim.", "Note: single model, one run per case, diff-only (no repo context). A measured", "baseline for this CLI — NOT a peer comparison.", "", @@ -363,7 +408,7 @@ export function formatLiveReport(report: LiveReport): string { } else { const tag = r.detected ? "HIT " : "MISS"; const sev = r.detected - ? `${r.truthSeverity}->${r.foundSeverity}${r.severityMatch ? "" : " (mismatch)"} conf=${fmt(r.confidence)}` + ? `${r.truthSeverity}->${r.foundSeverity}${severityTag(r)} conf=${fmt(r.confidence)}` : `${r.truthSeverity}`; const fp = r.falsePositives > 0 ? ` +${r.falsePositives} FP` : ""; lines.push(`${tag} ${r.id}: ${sev}${fp}`); @@ -372,6 +417,15 @@ export function formatLiveReport(report: LiveReport): string { return lines.join("\n"); } +/** Per-case severity annotation: exact match shows nothing; an adjacent-tier + * (defensible) call is flagged distinctly from a true two-tier mismatch so the + * per-case detail keeps truth-vs-found fully visible. */ +function severityTag(r: LiveCaseResult): string { + if (r.severityMatch) return ""; + if (r.severityMatchWithinOneTier) return " (+/-1 tier)"; + return " (mismatch)"; +} + function fmt(v: number | null): string { return v === null ? "n/a" : v.toFixed(2); } From 7539d45f4f1d2cb1a7792e660b558712b1616449 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Sat, 13 Jun 2026 07:51:59 +0000 Subject: [PATCH 2/2] bench: document both live severity metrics in README Explain why live mode reports severity match both exact and within one tier: exact is the strict metric, while the +/-1-tier band reflects that warn<->error is frequently a defensible judgment call (per the severity-calibration finding, the consistent mismatches are reasonable alternative calls and the direction is bidirectional, so it is not a one-way miscalibration). State explicitly that neither metric is a peer-comparison claim and that the per-case detail keeps truth-vs-found severity visible. --- bench/README.md | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/bench/README.md b/bench/README.md index f3a980e..c805fdb 100644 --- a/bench/README.md +++ b/bench/README.md @@ -65,15 +65,33 @@ external provider. Every live run writes a timestamped JSON report under - **Detection rate**: a defect counts as detected when a finding matches the ground-truth path with line within +/-3. -- **Severity match** among detections. +- **Severity match (exact)** among detections: strict equality between the found + severity and the fixture's ground-truth severity. +- **Severity match (+/-1 tier)** among detections: a wider band that treats + adjacent tiers on the `info < warn < error` scale (i.e. `info`<->`warn` and + `warn`<->`error`) as a match, counting only the two-tier `info`<->`error` gap + as a real mismatch. - **Silence on clean PRs**: a clean case should produce no findings. - **False positives**: any finding in a clean case, and any non-matching finding in a defect case. - **Confidence distribution** of true detections, and per-case duration / token usage. +Both severity numbers are reported, and the per-case detail always shows the +truth-vs-found severity so nothing is hidden. The exact figure is the strict +metric; the +/-1-tier figure exists because `warn`<->`error` is frequently a +defensible judgment call rather than a model error — the severity-calibration +investigation found the consistent "mismatches" are reasonable alternative +calls on judgment-heavy fixture labels (in several cases the model is correctly +applying the prompt's own severity rule while the fixture is the aggressive +party), and the mismatch direction is bidirectional, so it is not a one-way +miscalibration a prompt change could fix. The +/-1-tier band measures +agreement-up-to-a-defensible-disagreement; it is a softer view of the same +detections, not a different or better result. + These numbers are a **measured baseline for this CLI** — a single model, one run -per case, diff-only with no repository context or policy docs. They are **not a -peer comparison**: no competitor has been run on the same fixtures, and LLM -nondeterminism means the rate can move a few points run to run. Treat them as -internal evidence, not a published benchmark. +per case, diff-only with no repository context or policy docs. **Neither +severity metric is a peer-comparison claim**: no competitor has been run on the +same fixtures, and LLM nondeterminism means the rates (severity match most of +all) can move a few points run to run. Treat them as internal evidence, not a +published benchmark.