Benchmark the model the release actually ships

.github/workflows/release.yml

@@ -116,7 +116,6 @@ jobs:
       - name: Run diff-file live benchmark
         working-directory: bench
         env:
-          REVIEW_MODEL: z-ai/glm-5.2
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
         run: bun run bench:live -- --run-id "release-${{ github.ref_name }}" --json-out "${{ runner.temp }}/bench-live-report.json"
 

bench/src/livemodels.test.ts

@@ -1323,7 +1323,10 @@ describe("managed admission workflow", () => {
     ).text();
     expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?\n  bench-live:\n/u);
     expect(release).toMatch(/bench-live:\n\s+needs: validate-tag\n/u);
-    expect(release).toContain("REVIEW_MODEL: z-ai/glm-5.2");
+    // The gate must score whatever the binary ships. Restating the id here
+    // pinned it to one model, so the release that changed the shipped default
+    // still benchmarked the previous one and passed against its old baseline.
+    expect(release).not.toContain("REVIEW_MODEL:");
     expect(release).toContain("OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}");
     expect(release).toContain("bun run bench:live --");
     expect(release).toContain("bun run bench:compare --");

bench/src/run.test.ts

@@ -1,5 +1,7 @@
 import { afterEach, describe, expect, test } from "bun:test";
 import { link, mkdir, mkdtemp, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises";
+import { readFileSync, rmSync, writeFileSync } from "node:fs";
+import { randomUUID } from "node:crypto";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 import {
@@ -18,6 +20,7 @@ import {
   parseLiveModelsFailureReport,
   prepareExplicitOutputs,
   selectLiveScreeningCases,
+  shippedDefaultModel,
   validateScreeningEnvironment,
   validateModeSpecificFlags,
   validateLiveScreenContract,
@@ -477,3 +480,30 @@ describe("benchmark output lifecycle", () => {
     expect(await Bun.file(report).exists()).toBe(false);
   });
 });
+
+describe("shipped default model", () => {
+  test("reads the generator the binary bakes in", () => {
+    const path = join(tmpdir(), `postil-config-${randomUUID()}.toml`);
+    writeFileSync(path, 'version = 3\ndefault_model = "provider/shipped"\ncascade = []\n');
+    expect(shippedDefaultModel(path)).toBe("provider/shipped");
+    rmSync(path);
+  });
+
+  test("treats an empty default as absent so the caller still errors", () => {
+    const path = join(tmpdir(), `postil-config-${randomUUID()}.toml`);
+    writeFileSync(path, 'version = 3\ndefault_model = ""\n');
+    expect(shippedDefaultModel(path)).toBeUndefined();
+    rmSync(path);
+  });
+
+  test("matches the repository config, so the release gate cannot score another model", () => {
+    // The release workflow used to restate the id in YAML and kept scoring the
+    // previous generator after the shipped default moved.
+    const workflow = readFileSync(
+      resolve(import.meta.dir, "..", "..", ".github", "workflows", "release.yml"),
+      "utf8",
+    );
+    expect(workflow).not.toContain("REVIEW_MODEL:");
+    expect(shippedDefaultModel()).toBeDefined();
+  });
+});

bench/src/run.ts

@@ -50,6 +50,7 @@
 //   --run-id <id>           optional live-screen artifact namespace
 
 import { createHash, randomUUID } from "node:crypto";
+import { readFileSync } from "node:fs";
 import { mkdir, readFile, realpath, rename, rm, stat, writeFile } from "node:fs/promises";
 import { basename, dirname, resolve } from "node:path";
 import { cases } from "../fixtures/cases";
@@ -219,6 +220,20 @@ function liveConcurrency(args: string[]): number {
   return Number.isFinite(n) && n > 0 ? n : DEFAULT_LIVE_CONCURRENCY;
 }
 
+/** The generator the binary under test actually ships, read from the same
+ * `config.toml` the build bakes in. Restating the id in a caller lets the
+ * benchmark certify a model the release does not contain: the release gate
+ * carried `REVIEW_MODEL: z-ai/glm-5.2` in workflow YAML and kept scoring that
+ * model after the shipped default moved, passing against the old baseline. */
+export function shippedDefaultModel(
+  configPath = resolve(import.meta.dir, "..", "..", "config.toml"),
+): string | undefined {
+  const raw = readFileSync(configPath, "utf8");
+  const match = /^\s*default_model\s*=\s*"([^"]*)"/mu.exec(raw);
+  const model = match?.[1]?.trim();
+  return model === undefined || model.length === 0 ? undefined : model;
+}
+
 export function generatedLiveScreenRunId(now = new Date(), uuid = randomUUID()): string {
   const stamp = now.toISOString().replace(/[:.]/gu, "-");
   return `screen-${stamp}-${uuid}`;
@@ -312,7 +327,8 @@ async function main() {
   }
 
   if (live) {
-    const model = process.env.REVIEW_MODEL ?? flagValue(args, "--model");
+    const model = process.env.REVIEW_MODEL ?? flagValue(args, "--model") ??
+      shippedDefaultModel();
     if (!model?.trim()) {
       throw new Error("live benchmark needs an explicit model: set REVIEW_MODEL or --model");
     }