From 5dd8332cb4581657f7301bd23daefc66e04709ff Mon Sep 17 00:00:00 2001 From: Mao Nakamoto <41178744+maonakamoto@users.noreply.github.com> Date: Wed, 26 Aug 2026 23:29:02 +0200 Subject: [PATCH] fix(ci): the audit called a pricing table a retired model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OrangeCat has nothing wrong with it. It renders a tier comparison — models: ['Claude 3.5 Sonnet', 'GPT-4o', 'Gemini 2.0 Flash'] — human-readable copy that never reaches an API. Every entry but one carries a space and was filtered out; `GPT-4o` was not, so it slipped the shape filter, landed nearest an OpenRouter marker, and got announced as a retired pin. That is the one failure a daily gate cannot afford. Six repos really are broken by dead pins right now, and a report that puts a healthy repo alongside them teaches the reader to skim — taking the real six with it. This fleet already has that scar: gates that fail red on good code are what trained `--no-verify`. The rule is measured, not guessed. On 2026-08-26 Groq listed 14 model ids and OpenRouter 416, and not one of those 430 contained a capital letter. So an id with a capital cannot be a RETIRED id at those vendors — it was never one of their ids at all. "Retired" is the wrong diagnosis, not just the wrong target. Such an id is reported as `unattributed` — listed, not judged. Never dropped. For `GPT-4o` that bucket is also literally right: it IS an OpenAI product name, and OpenAI is a vendor this audit does not query. The rule stays where the evidence is. Together AI really does serve ids with capitals (`meta-llama/Llama-3.3-70B-Instruct-Turbo-Free` is in this fleet), so it applies only to the two catalogues that were counted, and a test pins that. Live sweep before: 12 retired across 6 repos. After: 11 across 5, with OrangeCat correctly absent and every other finding unchanged. Self-test: 36 checks, no network, no key, no checkout. The new block asserts both halves — the label is not called retired, AND the genuinely retired pin sitting in the same file is still judged. Suppressing a false positive is only worth doing if the true positive beside it survives. Co-Authored-By: Claude Opus 5 --- scripts/ci/model-pin-audit.mjs | 34 +++++++++++++++ scripts/ci/test-model-pin-audit.mjs | 65 ++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/scripts/ci/model-pin-audit.mjs b/scripts/ci/model-pin-audit.mjs index 296cc0d..5a1e582 100755 --- a/scripts/ci/model-pin-audit.mjs +++ b/scripts/ci/model-pin-audit.mjs @@ -278,9 +278,43 @@ export function collate(findings) { * look" as "nothing is there" reports every pin as retired and invents an * outage, which is worse than silence because someone acts on it. */ +/** + * Vendors whose entire published catalogue is lowercase. + * + * MEASURED, not assumed: on 2026-08-26 Groq listed 14 ids and OpenRouter 416, + * and not one of those 430 contained a capital letter. So an id carrying a + * capital cannot be a RETIRED id at these two — it was never one of their ids + * at all, and "retired" is the wrong diagnosis rather than merely the wrong + * target. + * + * This rule exists because the audit reported a retired pin in a repo that had + * nothing wrong. OrangeCat renders a pricing table: + * + * models: ['Claude 3.5 Sonnet', 'GPT-4o', 'Gemini 2.0 Flash'] + * + * — human-readable marketing copy that never reaches an API. `GPT-4o` was the + * only entry without a space, so it slipped the shape filter, landed nearest an + * OpenRouter marker, and was announced as a retired model pin. That is the + * failure mode a daily gate can least afford: cry wolf on a healthy repo and + * the reader learns to skim the report, taking the next real outage with it. + * + * Note where this lands such an id: `unattributed` — reported, not judged. + * Never dropped. For `GPT-4o` that bucket is also literally correct, since it + * IS an OpenAI product name and OpenAI is a vendor we do not query. + */ +const LOWERCASE_ONLY_VENDORS = new Set(["groq", "openrouter"]); + +/** Could this id ever have been served by this vendor? */ +export function possibleAt(vendorId, id) { + return !(LOWERCASE_ONLY_VENDORS.has(vendorId) && /[A-Z]/.test(id)); +} + export function judge(findings, live) { return findings.map((f) => { if (!f.vendor) return { ...f, state: "unattributed" }; + // Attribution put it here, but the vendor could never have served it, so + // the attribution is what is wrong. Report it; do not assert a retirement. + if (!possibleAt(f.vendor, f.id)) return { ...f, vendor: null, state: "unattributed" }; const set = live.get(f.vendor); if (!set) return { ...f, state: "unchecked" }; return { ...f, state: set.has(f.id) ? "ok" : "gone" }; diff --git a/scripts/ci/test-model-pin-audit.mjs b/scripts/ci/test-model-pin-audit.mjs index d050b08..0288b2b 100755 --- a/scripts/ci/test-model-pin-audit.mjs +++ b/scripts/ci/test-model-pin-audit.mjs @@ -20,7 +20,7 @@ * * Fixtures are inline strings — no network, no gh, no checkout, no key. */ -import { extractPins, attribute, collate, judge, looksLikeModelId } from "./model-pin-audit.mjs"; +import { extractPins, attribute, collate, judge, looksLikeModelId, possibleAt } from "./model-pin-audit.mjs"; let failures = 0; function check(name, actual, expected) { @@ -268,5 +268,68 @@ console.log("\nvendor confusion (regression)"); check("the xAI pin is unchecked, not retired", judged.filter((j) => j.state === "unchecked").map((j) => j.id), ["grok-3-mini"]); } +// ── a display label is not a pin ──────────────────────────────────── +// +// Regression: the audit announced a RETIRED model in OrangeCat, a repo with +// nothing wrong. The string was marketing copy in a pricing table — every other +// entry contained a space and was filtered out, `GPT-4o` did not. Reporting a +// healthy repo as broken is how a daily gate loses its reader. +// +// Both halves are asserted. Suppressing the false one is only worth doing if +// the real retired pin in the very same file is still caught. + +console.log("\ndisplay labels vs real pins"); + +const ORANGECAT_UI = ` +import { OPENROUTER_API_URL } from './constants' + +export const TIERS = { + standard: { + title: 'Standard Tier', + models: ['Claude 3.5 Sonnet', 'GPT-4o', 'Gemini 2.0 Flash'], + }, +} + +// the actual call, further down the same file +const model = 'openai/gpt-oss-20b:free' +`; + +{ + check("an uppercase id is impossible at openrouter", possibleAt("openrouter", "GPT-4o"), false); + check("and impossible at groq", possibleAt("groq", "Llama-3.3-70B"), false); + check("but lowercase ids stay possible", possibleAt("openrouter", "openai/gpt-oss-20b:free"), true); + // Uppercase is only impossible where the catalogue was measured. Together's + // real ids DO carry capitals, so the rule must not spread to every vendor. + check("uppercase is not ruled out at vendors we did not measure", possibleAt("together", "meta-llama/Llama-3.3-70B-Instruct-Turbo-Free"), true); + + const pins = extractPins(ORANGECAT_UI); + const findings = pins.map((p) => ({ + repo: "orangecat", + path: "src/lib/ai-guidance.ts", + line: p.line, + id: p.id, + vendor: attribute(ORANGECAT_UI, p.line), + })); + const judged = judge(findings, new Map([["openrouter", OR_LIVE]])); + + check( + "the display label is not reported as retired", + judged.filter((j) => j.state === "gone").map((j) => j.id).includes("GPT-4o"), + false, + ); + check( + "it is listed as unjudged rather than dropped", + judged.filter((j) => j.state === "unattributed").map((j) => j.id).includes("GPT-4o"), + true, + ); + // OR_LIVE contains openai/gpt-oss-20b:free, so the real pin here is live — + // what matters is that it was judged at all, not silenced alongside the label. + check( + "the real pin in the same file is still judged against the catalogue", + judged.find((j) => j.id === "openai/gpt-oss-20b:free")?.state, + "ok", + ); +} + console.log(failures ? `\n✗ ${failures} failure(s)` : "\n✓ all checks pass"); process.exit(failures ? 1 : 0);