Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions scripts/ci/model-pin-audit.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -278,9 +278,43 @@ export function collate(findings) {
* look" as "nothing is there" reports every pin as retired and invents an
* outage, which is worse than silence because someone acts on it.
*/
/**
* Vendors whose entire published catalogue is lowercase.
*
* MEASURED, not assumed: on 2026-08-26 Groq listed 14 ids and OpenRouter 416,
* and not one of those 430 contained a capital letter. So an id carrying a
* capital cannot be a RETIRED id at these two — it was never one of their ids
* at all, and "retired" is the wrong diagnosis rather than merely the wrong
* target.
*
* This rule exists because the audit reported a retired pin in a repo that had
* nothing wrong. OrangeCat renders a pricing table:
*
* models: ['Claude 3.5 Sonnet', 'GPT-4o', 'Gemini 2.0 Flash']
*
* — human-readable marketing copy that never reaches an API. `GPT-4o` was the
* only entry without a space, so it slipped the shape filter, landed nearest an
* OpenRouter marker, and was announced as a retired model pin. That is the
* failure mode a daily gate can least afford: cry wolf on a healthy repo and
* the reader learns to skim the report, taking the next real outage with it.
*
* Note where this lands such an id: `unattributed` — reported, not judged.
* Never dropped. For `GPT-4o` that bucket is also literally correct, since it
* IS an OpenAI product name and OpenAI is a vendor we do not query.
*/
const LOWERCASE_ONLY_VENDORS = new Set(["groq", "openrouter"]);

/** Could this id ever have been served by this vendor? */
export function possibleAt(vendorId, id) {
return !(LOWERCASE_ONLY_VENDORS.has(vendorId) && /[A-Z]/.test(id));
}

export function judge(findings, live) {
return findings.map((f) => {
if (!f.vendor) return { ...f, state: "unattributed" };
// Attribution put it here, but the vendor could never have served it, so
// the attribution is what is wrong. Report it; do not assert a retirement.
if (!possibleAt(f.vendor, f.id)) return { ...f, vendor: null, state: "unattributed" };
const set = live.get(f.vendor);
if (!set) return { ...f, state: "unchecked" };
return { ...f, state: set.has(f.id) ? "ok" : "gone" };
Expand Down
65 changes: 64 additions & 1 deletion scripts/ci/test-model-pin-audit.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
*
* Fixtures are inline strings — no network, no gh, no checkout, no key.
*/
import { extractPins, attribute, collate, judge, looksLikeModelId } from "./model-pin-audit.mjs";
import { extractPins, attribute, collate, judge, looksLikeModelId, possibleAt } from "./model-pin-audit.mjs";

let failures = 0;
function check(name, actual, expected) {
Expand Down Expand Up @@ -268,5 +268,68 @@ console.log("\nvendor confusion (regression)");
check("the xAI pin is unchecked, not retired", judged.filter((j) => j.state === "unchecked").map((j) => j.id), ["grok-3-mini"]);
}

// ── a display label is not a pin ────────────────────────────────────
//
// Regression: the audit announced a RETIRED model in OrangeCat, a repo with
// nothing wrong. The string was marketing copy in a pricing table — every other
// entry contained a space and was filtered out, `GPT-4o` did not. Reporting a
// healthy repo as broken is how a daily gate loses its reader.
//
// Both halves are asserted. Suppressing the false one is only worth doing if
// the real retired pin in the very same file is still caught.

console.log("\ndisplay labels vs real pins");

const ORANGECAT_UI = `
import { OPENROUTER_API_URL } from './constants'

export const TIERS = {
standard: {
title: 'Standard Tier',
models: ['Claude 3.5 Sonnet', 'GPT-4o', 'Gemini 2.0 Flash'],
},
}

// the actual call, further down the same file
const model = 'openai/gpt-oss-20b:free'
`;

{
check("an uppercase id is impossible at openrouter", possibleAt("openrouter", "GPT-4o"), false);
check("and impossible at groq", possibleAt("groq", "Llama-3.3-70B"), false);
check("but lowercase ids stay possible", possibleAt("openrouter", "openai/gpt-oss-20b:free"), true);
// Uppercase is only impossible where the catalogue was measured. Together's
// real ids DO carry capitals, so the rule must not spread to every vendor.
check("uppercase is not ruled out at vendors we did not measure", possibleAt("together", "meta-llama/Llama-3.3-70B-Instruct-Turbo-Free"), true);

const pins = extractPins(ORANGECAT_UI);
const findings = pins.map((p) => ({
repo: "orangecat",
path: "src/lib/ai-guidance.ts",
line: p.line,
id: p.id,
vendor: attribute(ORANGECAT_UI, p.line),
}));
const judged = judge(findings, new Map([["openrouter", OR_LIVE]]));

check(
"the display label is not reported as retired",
judged.filter((j) => j.state === "gone").map((j) => j.id).includes("GPT-4o"),
false,
);
check(
"it is listed as unjudged rather than dropped",
judged.filter((j) => j.state === "unattributed").map((j) => j.id).includes("GPT-4o"),
true,
);
// OR_LIVE contains openai/gpt-oss-20b:free, so the real pin here is live —
// what matters is that it was judged at all, not silenced alongside the label.
check(
"the real pin in the same file is still judged against the catalogue",
judged.find((j) => j.id === "openai/gpt-oss-20b:free")?.state,
"ok",
);
}

console.log(failures ? `\n✗ ${failures} failure(s)` : "\n✓ all checks pass");
process.exit(failures ? 1 : 0);