Replace default review model cascade

.github/workflows/bench-live.yml

@@ -10,7 +10,7 @@ on:
         description: Comma-separated OpenRouter model ids
         required: true
         default: >-
-          deepseek/deepseek-v4-pro,moonshotai/kimi-k2.6,qwen/qwen3-32b,mistralai/mistral-small-3.2-24b-instruct,google/gemma-3-27b-it
+          deepseek/deepseek-v4-pro,google/gemini-3.1-flash-lite,moonshotai/kimi-k2.7-code,mistralai/mistral-large-2512
       cases:
         description: Case selection (currently informational; "all" runs the full set)
         required: false
@@ -19,6 +19,14 @@ on:
         description: Abort if the projected upper-bound total cost exceeds this many USD
         required: false
         default: "15"
+      scorer_model:
+        description: REVIEW_SCORER_MODEL override for live-models bench (scorer-eval uses hardcoded defaults)
+        required: false
+        default: ""
+      run_scorer_eval:
+        description: Run scorer calibration eval instead of live-models bench
+        required: false
+        default: "false"
 
 concurrency:
   group: bench-live
@@ -49,31 +57,65 @@ jobs:
       # from fixture diff sizes. Aborts before spending anything if the matrix
       # would exceed the cap, or if any model has unknown pricing.
       - name: Cost guardrail
+        if: ${{ inputs.run_scorer_eval != 'true' }}
         working-directory: bench
         run: bun run cost-guard --cap "$POSTIL_BENCH_COST_CAP_USD"
 
       - name: Run live-models bench
+        if: ${{ inputs.run_scorer_eval != 'true' }}
         working-directory: bench
         env:
           POSTIL_BENCH_MODE: live
+          REVIEW_SCORER_MODEL: ${{ inputs.scorer_model }}
           POSTIL_API_KEY: ${{ secrets.POSTIL_API_KEY }}
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
           MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }}
           LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
         run: bun run src/run.ts --json-out "$GITHUB_WORKSPACE/bench-live-report.json"
 
+      - name: Run scorer calibration eval
+        if: ${{ inputs.run_scorer_eval == 'true' }}
+        working-directory: bench
+        env:
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+        run: bun run scorer-eval --json-out "$GITHUB_WORKSPACE/scorer-eval-report.json"
+
       - name: Write step summary
         if: always()
         working-directory: bench
         run: |
           report="$GITHUB_WORKSPACE/bench-live-report.json"
+          scorer_report="$GITHUB_WORKSPACE/scorer-eval-report.json"
           {
-            echo '## Live-models bench'
-            if [ -f "$report" ]; then
+            if [ -f "$scorer_report" ]; then
+              echo '## Scorer calibration eval'
               echo ''
               echo '```'
               bun -e '
-                const r = require(process.env.GITHUB_WORKSPACE + "/bench-live-report.json");
+                const r = await Bun.file(process.env.GITHUB_WORKSPACE + "/scorer-eval-report.json").json();
+                const pad = (s, w) => String(s).length >= w ? String(s) : String(s) + " ".repeat(w - String(s).length);
+                console.log([pad("model",36),pad("structured",11),pad("true kept",10),pad("false down",11),pad("mean true",10),pad("mean false",11),pad("pass",5)].join(" "));
+                for (const a of r.models) {
+                  console.log([
+                    pad(a.id,36),
+                    pad(String(a.structuredFailures),11),
+                    pad(`${a.trueFindingHighConfidence}/${a.trueFindingCases}`,10),
+                    pad(`${a.falseFindingDownscored}/${a.falseFindingCases}`,11),
+                    pad(a.meanTrueConfidence.toFixed(2),10),
+                    pad(a.meanFalseConfidence.toFixed(2),11),
+                    pad(a.passed ? "yes" : "no",5),
+                  ].join(" "));
+                }
+                console.log("");
+                console.log("Endpoint " + r.apiBase + " | generated " + r.generatedAt);
+              '
+              echo '```'
+            elif [ -f "$report" ]; then
+              echo '## Live-models bench'
+              echo ''
+              echo '```'
+              bun -e '
+                const r = await Bun.file(process.env.GITHUB_WORKSPACE + "/bench-live-report.json").json();
                 const pad = (s, w) => String(s).length >= w ? String(s) : String(s) + " ".repeat(w - String(s).length);
                 console.log([pad("model",40),pad("detect",8),pad("FP",5),pad("cases",6),pad("$/review",12),pad("total $",12),pad("mean ms",9)].join(" "));
                 for (const a of r.modelAggregates) {
@@ -101,9 +143,33 @@ jobs:
           } >> "$GITHUB_STEP_SUMMARY"
 
       - name: Upload report artifact
-        if: always()
+        if: always() && inputs.run_scorer_eval != 'true'
         uses: actions/upload-artifact@v4
         with:
           name: bench-live-report
           path: bench-live-report.json
           if-no-files-found: warn
+
+      - name: Upload scorer eval artifact
+        if: always() && inputs.run_scorer_eval == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: scorer-eval-report
+          path: scorer-eval-report.json
+          if-no-files-found: warn
+
+      - name: Upload live-model raw run artifacts
+        if: always() && inputs.run_scorer_eval != 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: bench-live-raw-runs
+          path: bench/.runs/live-models
+          if-no-files-found: warn
+
+      - name: Upload scorer eval raw run artifacts
+        if: always() && inputs.run_scorer_eval == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: scorer-eval-raw-runs
+          path: bench/.runs/scorer-eval
+          if-no-files-found: warn

README.md

@@ -144,10 +144,10 @@ gate:
 model:
   name: deepseek/deepseek-v4-pro
   cascade:
-    - mistralai/mistral-small-3.2-24b-instruct
-    - google/gemma-3-27b-it
-    - qwen/qwen3-32b
-  scorer: openai/gpt-5-mini
+    - google/gemini-3.1-flash-lite
+    - moonshotai/kimi-k2.7-code
+    - mistralai/mistral-large-2512
+  scorer: anthropic/claude-haiku-4.5
   apiBase: https://openrouter.ai/api/v1    # ignored from config by default; see note below
   consensus: 1            # >1: only findings multiple models agree on survive
 ```

bench/README.md

@@ -128,6 +128,32 @@ runs the live bench with any configured `POSTIL_API_KEY`, `OPENROUTER_API_KEY`,
 `MODEL_API_KEY`, or `LLM_API_KEY` secret, uploads the JSON report as an artifact,
 and prints the per-model table to the job step summary.
 
+## Scorer calibration live eval (opt-in, mocked generator + real scorer)
+
+The independent scorer has a different job from the primary review generator:
+it receives already-generated findings, without the generator's confidence or
+kind, and calibrates each finding's confidence and kind against local diff
+context. `bun run scorer-eval` evaluates that role directly by mocking the
+primary generator with fixed findings and proxying only scorer requests to the
+real OpenRouter endpoint.
+
+```sh
+cargo build --quiet --release
+cd bench
+export MODEL_API_KEY=...          # or LLM_API_KEY / OPENROUTER_API_KEY
+POSTIL_SCORER_EVAL_MODELS=anthropic/claude-haiku-4.5,openai/gpt-5-mini \
+  bun run scorer-eval --json-out scorer-eval-report.json
+```
+
+The eval uses 12 existing fixtures: six seeded defect findings copied from
+mock-mode ground truth, and six deliberately injected false findings on clean
+fixtures. A scorer passes the structured-output floor when every case records
+the requested `scorerModel` and no `scorerError`; it passes calibration when all
+true findings remain confident `risk` findings and at least half of injected
+false findings are down-scored below 0.6 confidence or retyped as
+`uncertainty`. The manual `Bench (live models)` workflow can run this path with
+`run_scorer_eval=true` and uploads `scorer-eval-report.json`.
+
 ## Diff-file live mode (opt-in, single model, no forge)
 
 This live mode runs the real release binary against the same fixtures with a

bench/package.json

@@ -7,6 +7,7 @@
     "bench": "bun run src/run.ts",
     "bench:live": "bun run src/run.ts --live",
     "bench:live-models": "POSTIL_BENCH_MODE=live bun run src/run.ts",
+    "scorer-eval": "bun run src/scorer-eval.ts",
     "cost-guard": "bun run src/cost-guard.ts"
   },
   "dependencies": {

bench/src/livemodels.ts

@@ -267,6 +267,10 @@ function liveEnv(
     GITHUB_TOKEN: "benchmark-github-token",
     REVIEW_MODEL: model,
   };
+  const scorerModel = process.env.REVIEW_SCORER_MODEL?.trim();
+  if (scorerModel) {
+    env.REVIEW_SCORER_MODEL = scorerModel;
+  }
   // Forward the selected inference-key variable without logging or placing the
   // value on argv. Neutral aliases are also mirrored into POSTIL_API_KEY so
   // older binaries can run from the same benchmark harness.