diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b61c945..b33b24e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -116,7 +116,6 @@ jobs: - name: Run diff-file live benchmark working-directory: bench env: - REVIEW_MODEL: z-ai/glm-5.2 OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} run: bun run bench:live -- --run-id "release-${{ github.ref_name }}" --json-out "${{ runner.temp }}/bench-live-report.json" diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 43bd532..f39949f 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -1323,7 +1323,10 @@ describe("managed admission workflow", () => { ).text(); expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?\n bench-live:\n/u); expect(release).toMatch(/bench-live:\n\s+needs: validate-tag\n/u); - expect(release).toContain("REVIEW_MODEL: z-ai/glm-5.2"); + // The gate must score whatever the binary ships. Restating the id here + // pinned it to one model, so the release that changed the shipped default + // still benchmarked the previous one and passed against its old baseline. + expect(release).not.toContain("REVIEW_MODEL:"); expect(release).toContain("OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}"); expect(release).toContain("bun run bench:live --"); expect(release).toContain("bun run bench:compare --"); diff --git a/bench/src/run.test.ts b/bench/src/run.test.ts index b7695a7..3dbaa30 100644 --- a/bench/src/run.test.ts +++ b/bench/src/run.test.ts @@ -1,5 +1,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { link, mkdir, mkdtemp, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { randomUUID } from "node:crypto"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { @@ -18,6 +20,7 @@ import { parseLiveModelsFailureReport, prepareExplicitOutputs, selectLiveScreeningCases, + shippedDefaultModel, validateScreeningEnvironment, validateModeSpecificFlags, validateLiveScreenContract, @@ -477,3 +480,30 @@ describe("benchmark output lifecycle", () => { expect(await Bun.file(report).exists()).toBe(false); }); }); + +describe("shipped default model", () => { + test("reads the generator the binary bakes in", () => { + const path = join(tmpdir(), `postil-config-${randomUUID()}.toml`); + writeFileSync(path, 'version = 3\ndefault_model = "provider/shipped"\ncascade = []\n'); + expect(shippedDefaultModel(path)).toBe("provider/shipped"); + rmSync(path); + }); + + test("treats an empty default as absent so the caller still errors", () => { + const path = join(tmpdir(), `postil-config-${randomUUID()}.toml`); + writeFileSync(path, 'version = 3\ndefault_model = ""\n'); + expect(shippedDefaultModel(path)).toBeUndefined(); + rmSync(path); + }); + + test("matches the repository config, so the release gate cannot score another model", () => { + // The release workflow used to restate the id in YAML and kept scoring the + // previous generator after the shipped default moved. + const workflow = readFileSync( + resolve(import.meta.dir, "..", "..", ".github", "workflows", "release.yml"), + "utf8", + ); + expect(workflow).not.toContain("REVIEW_MODEL:"); + expect(shippedDefaultModel()).toBeDefined(); + }); +}); diff --git a/bench/src/run.ts b/bench/src/run.ts index 82a3333..446d444 100644 --- a/bench/src/run.ts +++ b/bench/src/run.ts @@ -50,6 +50,7 @@ // --run-id optional live-screen artifact namespace import { createHash, randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; import { mkdir, readFile, realpath, rename, rm, stat, writeFile } from "node:fs/promises"; import { basename, dirname, resolve } from "node:path"; import { cases } from "../fixtures/cases"; @@ -219,6 +220,20 @@ function liveConcurrency(args: string[]): number { return Number.isFinite(n) && n > 0 ? n : DEFAULT_LIVE_CONCURRENCY; } +/** The generator the binary under test actually ships, read from the same + * `config.toml` the build bakes in. Restating the id in a caller lets the + * benchmark certify a model the release does not contain: the release gate + * carried `REVIEW_MODEL: z-ai/glm-5.2` in workflow YAML and kept scoring that + * model after the shipped default moved, passing against the old baseline. */ +export function shippedDefaultModel( + configPath = resolve(import.meta.dir, "..", "..", "config.toml"), +): string | undefined { + const raw = readFileSync(configPath, "utf8"); + const match = /^\s*default_model\s*=\s*"([^"]*)"/mu.exec(raw); + const model = match?.[1]?.trim(); + return model === undefined || model.length === 0 ? undefined : model; +} + export function generatedLiveScreenRunId(now = new Date(), uuid = randomUUID()): string { const stamp = now.toISOString().replace(/[:.]/gu, "-"); return `screen-${stamp}-${uuid}`; @@ -312,7 +327,8 @@ async function main() { } if (live) { - const model = process.env.REVIEW_MODEL ?? flagValue(args, "--model"); + const model = process.env.REVIEW_MODEL ?? flagValue(args, "--model") ?? + shippedDefaultModel(); if (!model?.trim()) { throw new Error("live benchmark needs an explicit model: set REVIEW_MODEL or --model"); }