From e25966cb4ff71ed112ac488e84455a376bba484a Mon Sep 17 00:00:00 2001 From: Mao Nakamoto <41178744+maonakamoto@users.noreply.github.com> Date: Thu, 27 Aug 2026 01:59:27 +0200 Subject: [PATCH] fix(ci): read model maps, and stop reporting things that were never pins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repairing six repos today was a live test of this audit, and it failed in both directions at once: it missed retired ids in shapes it could not parse, and reported ids that were never pins. Six faults, each found by acting on the previous report and looking at what it pointed to. MISSED — ids the audit could not see 1. Model MAPS. The walker only opened on `[`, and two repos keep their ids in object literals — on opposite sides of the colon. Hirnli: `{ '8b': 'llama-3.1-8b-instant' }`, ids as values. OrangeCat: `{ 'llama-3.3-70b-versatile': { contextWindow: ... } }`, ids as keys, including `DEFAULT_GROQ_MODEL` — the baseline every free non-BYOK user gets. Both registries were entirely retired and both read as clean. 2. `\bmodels?\b` does not match `GROQ_MODELS`. The underscore before it is a word character, so there is no boundary there — and almost every map in this fleet is named exactly that way. This was why (1) still found nothing after the walker was taught about braces. 3. `modelId` is not `model`. Kivvi's fallback is `const FALLBACK_MODEL: ModelSelection = { providerId: "groq", modelId: "..." }` — singular, so no collection walker covers it, and the single-id pattern was anchored on the bare word. Found only by re-running the live sweep after fix (4) and noticing a real finding had disappeared. REPORTED — things that were never pins 4. `for (const model of models) {` opened a region over an entire loop body, so the request headers inside were read as ids and a `Content-Type: application/json` was reported as a retired Groq model. The opener now requires a declaration (`[:=]` right after the token) and a PLURAL token — `function supportsReasoningEffort(model: string): boolean {` is a declaration by the first rule alone, and its `startsWith("qwen/")` prefixes were being reported too. 5. The audit was reading ids out of COMMENTS. A model list is exactly where someone documents the id they just replaced, usually in backticks — so Kivvi's `// replaces `meta-llama/llama-3.2-3b-instruct:free`, retired` was reported as a live pin in the very commit that removed it. Comments are stripped from a region before extraction. An audit reporting on prose rather than code is the failure it exists to catch elsewhere. 6. Two misattributions, both "a true statement about the wrong vendor": - `gpt-4o-mini` under an `openai:` key was called a retired OPENROUTER model. It is not an OpenRouter id at all; there it would be `openai/gpt-4o-mini`. Nearest-marker-above found OpenRouter's base URL in the block before. A provider-keyed record now names its own rows via `keyMarker`, anchored to a line starting with the key and a colon — necessary, because bare `openai` appears inside `api.groq.com/openai/v1`. - `OLLAMA_MODEL=llama3.2` was called a retired Groq model. Ollama is now a listed non-queryable vendor, so its tags are attributed and reported unchecked. What the operator pulled locally is not a question Groq can answer. Also: a size alias is not a model id. Reading maps turned Hirnli's `{ '70b': ..., '8b': ... }` keys into findings — `70b` has a digit and no space, so it looked like an id, and `8b` escaped only by being two characters long. A vendor id always carries a separator: a slash, a hyphen or a dot. Nothing in either live catalogue is a single unseparated word. Net effect on the live sweep, across the same 34 repos: before this session 12 retired, 5 repos, several of them wrong after coverage (#47) 16 retired, and hirnli appeared for the first time now 2 retired, both real, both in Kivvi The intervening repairs account for most of the drop; this commit accounts for the false ones. Every remaining line is a genuine retired id in a genuine call path. Self-test: 68 -> 102 checks, still no network, no key, no checkout. Every fixture is the real code that fooled it — Hirnli's alias map, OrangeCat's registry and provider record, Kivvi's typed array and fallback, the loop body, the comment. Co-Authored-By: Claude Opus 5 --- scripts/ci/model-pin-audit.mjs | 153 ++++++++++++++--- scripts/ci/test-model-pin-audit.mjs | 250 ++++++++++++++++++++++++++++ 2 files changed, 380 insertions(+), 23 deletions(-) diff --git a/scripts/ci/model-pin-audit.mjs b/scripts/ci/model-pin-audit.mjs index 52caabc..d818f17 100755 --- a/scripts/ci/model-pin-audit.mjs +++ b/scripts/ci/model-pin-audit.mjs @@ -94,6 +94,9 @@ export const VENDORS = [ baseUrl: "https://api.groq.com/openai/v1", keyEnv: "GROQ_API_KEY", markers: [/groq/i], + // A provider-keyed record — `groq: { defaultModel: '...' }` — names the + // vendor for everything inside it, and sits closer to the pin than any URL. + keyMarker: /^[ \t]*['"]?groq['"]?\s*:/m, }, { id: "openrouter", @@ -101,16 +104,48 @@ export const VENDORS = [ baseUrl: "https://openrouter.ai/api/v1", keyEnv: "OPENROUTER_API_KEY", markers: [/openrouter/i], + keyMarker: /^[ \t]*['"]?openrouter['"]?\s*:/m, }, // Not queryable here — no catalogue call is wired for these. They are listed // so their ids are ATTRIBUTED and reported unchecked, rather than falling to // whichever queryable vendor happens to sit nearest in the file. Markers are // deliberately specific: bare /openai/ would match Groq's own // `api.groq.com/openai/v1` path and every `openai/gpt-oss-*` id it serves. - { id: "xai", queryable: false, markers: [/api\.x\.ai/i, /\bXAI_API_KEY\b/, /\bgrok\b/i] }, - { id: "anthropic", queryable: false, markers: [/api\.anthropic\.com/i, /\bANTHROPIC_API_KEY\b/] }, - { id: "openai", queryable: false, markers: [/api\.openai\.com/i, /\bOPENAI_API_KEY\b/] }, - { id: "google", queryable: false, markers: [/generativelanguage\.googleapis/i, /\bGEMINI_API_KEY\b/] }, + // Ollama runs on the user's own machine, so "is this id still served" is not + // a question with a fleet-wide answer — whatever the operator pulled is what + // exists. It is listed so its ids are ATTRIBUTED rather than falling to the + // nearest cloud vendor above them. + // + // Without this, evig's `.env.example` line `OLLAMA_MODEL=llama3.2` was + // attributed to Groq — the nearest marker above it — and reported RETIRED. + // `llama3.2` is a perfectly valid Ollama tag on a healthy line; Groq simply + // never served anything by that name. Same class as the pricing-table false + // positive: a true statement about the wrong vendor. + { + id: "ollama", + queryable: false, + markers: [/\bOLLAMA_[A-Z_]+\b/, /\bollama\b/i, /localhost:11434/, /127\.0\.0\.1:11434/], + }, + // + // `keyMarker` is how a provider-keyed record names its own rows. OrangeCat + // writes: + // + // export const PROVIDER_BASE_URLS = { openai: '...', openrouter: '...' } + // export const PROVIDER_RUNTIME = { + // openai: { baseUrl: ..., defaultModel: 'gpt-4o-mini' }, + // + // The pin belongs to OpenAI, but the nearest marker above it was OpenRouter's + // URL in the block before — so `gpt-4o-mini` was reported as a retired + // OpenRouter model. It is not an OpenRouter id at all; there it would be + // `openai/gpt-4o-mini`. The bare key `openai:` is deliberately not in + // `markers`, because `api.groq.com/openai/v1` contains that word — anchoring + // it to the start of a line followed by a colon is what makes it safe. + { id: "xai", queryable: false, markers: [/api\.x\.ai/i, /\bXAI_API_KEY\b/, /\bgrok\b/i], keyMarker: /^[ \t]*['"]?xai['"]?\s*:/m }, + { id: "anthropic", queryable: false, markers: [/api\.anthropic\.com/i, /\bANTHROPIC_API_KEY\b/], keyMarker: /^[ \t]*['"]?anthropic['"]?\s*:/m }, + { id: "openai", queryable: false, markers: [/api\.openai\.com/i, /\bOPENAI_API_KEY\b/], keyMarker: /^[ \t]*['"]?openai['"]?\s*:/m }, + { id: "google", queryable: false, markers: [/generativelanguage\.googleapis/i, /\bGEMINI_API_KEY\b/], keyMarker: /^[ \t]*['"]?google['"]?\s*:/m }, + { id: "together", queryable: false, markers: [/api\.together\.xyz/i, /\bTOGETHER_API_KEY\b/], keyMarker: /^[ \t]*['"]?together['"]?\s*:/m }, + { id: "deepseek", queryable: false, markers: [/api\.deepseek\.com/i, /\bDEEPSEEK_API_KEY\b/], keyMarker: /^[ \t]*['"]?deepseek['"]?\s*:/m }, ]; /** @@ -241,45 +276,87 @@ export function looksLikeModelId(s) { if (/\.(ts|tsx|js|mjs|cjs|json|css|scss|md|png|jpe?g|svg|ico|txt|ya?ml)$/i.test(s)) return false; if (/^[A-Z][A-Z0-9_]*$/.test(s)) return false; // SCREAMING_CASE is an env name // Model ids essentially always carry a version digit or a vendor/ prefix. + // A vendor id always carries a separator: a slash for routed ids + // (`openai/gpt-oss-120b`), or a hyphen or dot within the name + // (`llama-3.3-70b-versatile`, `llama3.2`, `codex-4`). Nothing in either live + // catalogue is a single unseparated word. + // + // Without this, reading model MAPS turned their keys into findings: Hirnli's + // alias table is `{ '70b': '...', '8b': '...' }`, and `70b` has a digit and no + // space, so it read as a pin and would have been reported retired at Groq. It + // is a size alias. `8b` escaped only by being two characters long, which is + // not a rule anyone should rely on. + if (!/[/.-]/.test(s)) return false; + return /\d/.test(s) || s.includes("/"); } /** - * The lines making up each `models = [ ... ]` array, with real line numbers. + * The lines making up each `models` collection, with real line numbers. * * A tiny bracket walker rather than a regex, because the shapes in this fleet - * defeat any single pattern: `models: ['a']`, `models = [`, and Kivvi's - * `models: AIModel[] = [` all mean the same thing, and the last one hid two - * retired ids from the audit for as long as this was a regex. - * - * The opening bracket is the LAST one on the declaring line, which is what - * makes the annotated form work — in `models: AIModel[] = [`, the first `[` - * belongs to the type and closes immediately, so anchoring to it would read an - * empty array and report nothing. Exactly the silent miss this replaces. + * defeat any single pattern. All four of these are one repo's way of saying the + * same thing, and every one of them hid a retired id at some point: + * + * models: ['a', 'b'] an inline array + * models: AIModel[] = [ an array behind a TS annotation + * GROQ_MODELS = { '8b': 'llama-...' } a map whose VALUES are ids + * GROQ_MODELS = { 'llama-...': { ... } } a map whose KEYS are ids + * + * Maps were the second discovery and cost two more repos. Hirnli kept its ids + * as map values and Orangecat as map keys — including `DEFAULT_GROQ_MODEL`, + * the baseline every free user gets — and an array-only walker read straight + * past both. So the opener is `[` or `{`, and every quoted string in the region + * is a candidate regardless of which side of the colon it sits on. + * + * The opening bracket is the LAST one on the declaring line. That is what makes + * the annotated form work: in `models: AIModel[] = [`, the first `[` belongs to + * the type and closes immediately, so anchoring to it reads an empty array and + * reports nothing — a silent miss, the worst output an audit has. */ export function modelListRegions(text) { const lines = text.split("\n"); const regions = []; + const CLOSER = { "[": "]", "{": "}" }; for (let i = 0; i < lines.length; i++) { - // Greedy on purpose: it backtracks to the LAST `[` within reach. - const opener = /\bmodels?\b[^\n]{0,80}\[/i.exec(lines[i]); + // Greedy on purpose: it backtracks to the LAST opener within reach. + // + // `\w*` before `models` is load-bearing. A bare `\bmodels?\b` does NOT + // match `GROQ_MODELS`, because the underscore before it is a word character + // so there is no boundary there — which is precisely why two repos' model + // maps read as empty. Almost every map in this fleet is named that way. + // + // The `[:=]` immediately after the token is what makes this a DECLARATION + // rather than any line that mentions models. Without it, + // `for (const model of models) {` opened a region over the whole loop body + // — and the request headers inside were then read as model ids, so a + // `Content-Type: application/json` was reported as a retired Groq model. + // PLURAL only. A singular `model:` is a parameter or a single-id property, + // and `function supportsReasoningEffort(model: string): boolean {` is a + // declaration by the rule above — it opened a region over the function body + // and read the `startsWith("qwen/")` prefixes inside as retired ids. + // Collections are plural; single ids are matched on one line by the + // patterns in extractPins. + const opener = /\b[a-z0-9_]*models\b\s*[:=][^\n]{0,80}[[{]/i.exec(lines[i]); if (!opener) continue; + const open = opener[0].at(-1); + const close = CLOSER[open]; const region = []; let depth = 0; let opened = false; - // A models array running past 200 lines is not a models array. + // A models collection running past 200 lines is not a models collection. for (let j = i; j < lines.length && j < i + 200; j++) { const segment = j === i ? lines[j].slice(opener.index + opener[0].length - 1) : lines[j]; region.push({ line: j + 1, text: segment }); for (const ch of segment) { - if (ch === "[") { + if (ch === open) { depth++; opened = true; - } else if (ch === "]") { + } else if (ch === close) { depth--; } } @@ -314,8 +391,22 @@ export function extractPins(text) { if (!found.has(id)) found.set(id, line); }; - // 1a. model: 'x' — a single id on one line - for (const m of text.matchAll(/\bmodels?\s*[:=]\s*['"`]([^'"`\n]{0,120})['"`]/gi)) { + // 1a. model: 'x' — a single id on one line. + // + // The `id|name` suffix is not decoration. Kivvi's fallback reads + // + // const FALLBACK_MODEL: ModelSelection = { + // providerId: "groq", + // modelId: "llama-3.3-70b-versatile", + // }; + // + // and `modelId` is not `model`, so a pattern anchored on the bare word walked + // straight past a retired id sitting in the app's default model selection. + // The declaration above it is singular, so the collection walker does not + // cover it either — this line is the only thing that does. + for (const m of text.matchAll( + /\b[a-z0-9_]*model(?:s|id|_id|name|_name)?\b\s*[:=]\s*['"`]([^'"`\n]{0,120})['"`]/gi, + )) { remember(m[1], m.index); } @@ -337,7 +428,15 @@ export function extractPins(text) { // attribution is measured in lines from the pin. for (const region of modelListRegions(text)) { for (const { line, text: lineText } of region) { - for (const q of lineText.matchAll(/['"`]([^'"`\n]+)['"`]/g)) rememberAt(q[1], line); + // Comments are stripped first. A model list is exactly where someone + // documents the id they just replaced, and such notes are usually written + // in backticks — so without this, a comment reading "replaces + // `meta-llama/llama-3.2-3b-instruct:free`, which was retired" reports + // that id as a live pin, in the very commit that removed it. The audit + // would be reporting on prose instead of code: the exact failure it + // exists to catch elsewhere. + const code = lineText.replace(/\/\*[\s\S]*?\*\//g, "").replace(/\/\/.*$/, ""); + for (const q of code.matchAll(/['"`]([^'"`\n]+)['"`]/g)) rememberAt(q[1], line); } } @@ -383,8 +482,14 @@ export function attribute(text, line, vendors = VENDORS, maxDistance = MAX_ATTRI const distances = (vendor) => { let above = Infinity; let below = Infinity; + // `keyMarker` counts as a mention: in a provider-keyed record, the row's + // own key is the most local and most reliable statement of which vendor a + // pin belongs to — closer than any base URL, and unambiguous. + const mentions = (l) => + vendor.markers.some((re) => re.test(l)) || (vendor.keyMarker?.test(l) ?? false); + for (let i = 0; i < lines.length; i++) { - if (!vendor.markers.some((re) => re.test(lines[i]))) continue; + if (!mentions(lines[i])) continue; const d = i + 1 - line; if (d <= 0) above = Math.min(above, -d); else below = Math.min(below, d); @@ -406,7 +511,9 @@ export function attribute(text, line, vendors = VENDORS, maxDistance = MAX_ATTRI if (fromBelow.length > 1 && fromBelow[0].below < fromBelow[1].below) return fromBelow[0].id; } - const named = vendors.filter((v) => v.markers.some((re) => re.test(text))); + const named = vendors.filter( + (v) => v.markers.some((re) => re.test(text)) || (v.keyMarker?.test(text) ?? false), + ); return named.length === 1 ? named[0].id : null; } diff --git a/scripts/ci/test-model-pin-audit.mjs b/scripts/ci/test-model-pin-audit.mjs index ae093a3..282ee27 100755 --- a/scripts/ci/test-model-pin-audit.mjs +++ b/scripts/ci/test-model-pin-audit.mjs @@ -485,5 +485,255 @@ export class GroqProvider extends OpenAICompatibleProvider { ); } +// ── model MAPS, not just arrays ───────────────────────────────────── +// +// Third discovery in the same sweep, and it cost two more repos. +// +// Once arrays were readable, two repos still reported nothing, because their +// ids live in object literals — and on OPPOSITE sides of the colon. Hirnli +// keeps them as map values, Orangecat as map keys, including +// `DEFAULT_GROQ_MODEL`, the baseline every free non-BYOK user gets. Both were +// entirely retired and both read as clean. +// +// So the walker takes `{` as readily as `[`, and every quoted string in the +// region is a candidate regardless of which side of the colon it is on. + +console.log("\nmodel maps"); + +// Hirnli's shape: the id is the VALUE. +const HIRNLI_ALIAS_MAP = ` +const GROQ_API_URL = 'https://api.groq.com/openai/v1/chat/completions'; + +/** Available models with different rate limits */ +export const GROQ_MODELS = { + '70b': 'llama-3.3-70b-versatile', // 12k TPM, best quality + '8b': 'llama-3.1-8b-instant', // 20k TPM, faster, good for triage +} as const; +`; + +// Orangecat's shape: the id is the KEY, and the value is a nested object. +const ORANGECAT_REGISTRY = ` +// Groq's best free models +const GROQ_MODELS = { + // Fast, capable model - great for chat + 'llama-3.3-70b-versatile': { + name: 'Llama 3.3 70B Versatile', + contextWindow: 128000, + maxOutputTokens: 32768, + }, + 'llama-3.1-8b-instant': { + name: 'Llama 3.1 8B Instant', + contextWindow: 128000, + maxOutputTokens: 8192, + }, +} as const; + +const GROQ_API_URL = 'https://api.groq.com/openai/v1'; +`; + +{ + const ids = extractPins(HIRNLI_ALIAS_MAP).map((p) => p.id).sort(); + check( + "ids that are map VALUES are found", + ids, + ["llama-3.1-8b-instant", "llama-3.3-70b-versatile"], + ); + // '70b' and '8b' are the keys here. They are aliases, not ids, and must not + // be reported as pins — `looksLikeModelId` is what keeps them out. + check("the size aliases beside them are not mistaken for ids", ids.includes("8b"), false); +} + +{ + const pins = extractPins(ORANGECAT_REGISTRY); + const ids = pins.map((p) => p.id).sort(); + check( + "ids that are map KEYS are found", + ids, + ["llama-3.1-8b-instant", "llama-3.3-70b-versatile"], + ); + // Nested objects inside the map must not end the region early. + check("a nested object does not truncate the map", ids.length, 2); + // Human-readable names sit in the same nested objects. + check("display names inside the map are not pins", ids.includes("Llama 3.3 70B Versatile"), false); + + for (const pin of pins) { + check(`${pin.id} in a map attributes to groq`, attribute(ORANGECAT_REGISTRY, pin.line), "groq"); + } + + const judged = judge( + pins.map((p) => ({ repo: "orangecat", path: "src/services/ai/groq.ts", line: p.line, id: p.id, vendor: "groq" })), + new Map([["groq", GROQ_LIVE]]), + ); + check("and both are judged retired against the live catalogue", judged.filter((j) => j.state === "gone").length, 2); +} + +{ + // The walker must not mistake an ordinary import for a model collection. + check("an import naming models opens no region", modelListRegions(`import { getAllModels } from "./providers";`).length, 0); +} + +// A size alias is not a model id. +{ + console.log("\nseparators"); + check("a bare size token is not an id", looksLikeModelId("70b"), false); + check("nor is the two-character one", looksLikeModelId("8b"), false); + check("routed ids are ids", looksLikeModelId("openai/gpt-oss-120b"), true); + check("hyphenated ids are ids", looksLikeModelId("llama-3.3-70b-versatile"), true); + check("dotted ids are ids", looksLikeModelId("llama3.2"), true); + check("short hyphenated ids are ids", looksLikeModelId("codex-4"), true); +} + +// A loop over models is not a declaration of models. +// +// Regression from the map support: `{` as an opener made +// `for (const model of models) {` open a region across the whole loop body, so +// the request headers inside it were read as ids and a Content-Type header was +// reported as a retired Groq model. Widening a matcher is exactly when to check +// what it now swallows. +{ + console.log("\ndeclarations only"); + + const LOOP = ` + const models = groqModels(); + for (const model of models) { + const response = await fetch(API_CONFIG.GROQ_API_URL, { + headers: { + Authorization: \`Bearer \${key}\`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ model, messages }), + }); + } + `; + + check("a for-of over models opens no region", modelListRegions(LOOP).length, 0); + check("so a content-type header is not a model", extractPins(LOOP).map((p) => p.id), []); + + // The declaration forms must still open one. + check("a const array declaration still opens", modelListRegions(`const models = [`).length, 1); + check("an annotated declaration still opens", modelListRegions(` models: AIModel[] = [`).length, 1); + check("a map declaration still opens", modelListRegions(`export const GROQ_MODELS = {`).length, 1); + check("an inline property still opens", modelListRegions(`models: ['a/b-1'],`).length, 1); +} + +// `modelId` is not `model`. +// +// Regression found by re-running the live sweep after tightening the collection +// walker to plural-only: a real retired id in Kivvi disappeared from the report. +// Its declaration is singular (`const FALLBACK_MODEL: ModelSelection = {`), so +// the walker correctly ignores it, and the single-id pattern was anchored on the +// bare word `model` — which does not match `modelId`. Nothing covered it. +{ + console.log("\nsingle-id property shapes"); + + const KIVVI_FALLBACK = ` +const STORAGE_KEY = "kivvi-selected-model"; + +// Fallback default — used before API loads or when stored model is unavailable +const FALLBACK_MODEL: ModelSelection = { + providerId: "groq", + modelId: "llama-3.3-70b-versatile", +}; +`; + + const ids = extractPins(KIVVI_FALLBACK).map((p) => p.id); + check("a modelId property is a pin", ids.includes("llama-3.3-70b-versatile"), true); + check("the storage key beside it is not", ids.includes("kivvi-selected-model"), false); + + check("model_id also reads", extractPins(`model_id = "openai/gpt-oss-120b"`).map((p) => p.id), ["openai/gpt-oss-120b"]); + check("modelName also reads", extractPins(`modelName: 'openai/gpt-oss-20b'`).map((p) => p.id), ["openai/gpt-oss-20b"]); + + // A parameter annotation is still not a pin. + check("a typed parameter is not a pin", extractPins(`function f(model: string): boolean { return model.startsWith("qwen/"); }`).map((p) => p.id), []); +} + +// A provider-keyed record names its own rows. +// +// OrangeCat's `gpt-4o-mini` was reported as a RETIRED OPENROUTER model. It is +// not an OpenRouter id at all — there it would be `openai/gpt-4o-mini` — it is +// OpenAI's, sitting under an `openai:` key. The nearest marker above it was +// OpenRouter's base URL in the block before, so nearest-marker-above put it with +// the wrong vendor and then judged it against a catalogue that was never going +// to list it. +// +// The bare word `openai` cannot be a general marker: `api.groq.com/openai/v1` +// contains it, and every `openai/gpt-oss-*` id Groq serves. Anchoring to a line +// that STARTS with the key and a colon is what makes it safe. +{ + console.log("provider-keyed records"); + + const ORANGECAT_RUNTIME = ` +export const PROVIDER_BASE_URLS = { + openai: 'https://api.openai.com/v1', + groq: 'https://api.groq.com/openai/v1', + openrouter: 'https://openrouter.ai/api/v1', +} as const; + +export const PROVIDER_RUNTIME = { + openai: { + baseUrl: PROVIDER_BASE_URLS.openai, + defaultModel: 'gpt-4o-mini', + }, + openrouter: { + baseUrl: PROVIDER_BASE_URLS.openrouter, + defaultModel: 'nvidia/nemotron-3-super-120b-a12b:free', + }, +}; +`; + + const pins = extractPins(ORANGECAT_RUNTIME); + const at = (id) => pins.find((p) => p.id === id); + + check("both defaults are extracted", pins.length >= 2, true); + check( + "an id under an openai: key belongs to openai", + attribute(ORANGECAT_RUNTIME, at("gpt-4o-mini").line), + "openai", + ); + check( + "and the one under openrouter: belongs to openrouter", + attribute(ORANGECAT_RUNTIME, at("nvidia/nemotron-3-super-120b-a12b:free").line), + "openrouter", + ); + + // OpenAI is not queryable, so its pin must be reported UNCHECKED — never + // judged against a catalogue that could not list it. + const judged = judge( + pins.map((p) => ({ repo: "orangecat", path: "src/config/ai-provider-runtime.ts", line: p.line, id: p.id, vendor: attribute(ORANGECAT_RUNTIME, p.line) })), + new Map([["openrouter", new Set(["nvidia/nemotron-3-super-120b-a12b:free"])]]), + ); + check("the openai pin is unchecked, not retired", judged.find((j) => j.id === "gpt-4o-mini")?.state, "unchecked"); + check("the openrouter pin is confirmed live", judged.find((j) => j.id === "nvidia/nemotron-3-super-120b-a12b:free")?.state, "ok"); +} + +// Ollama is local: its tags are attributed, never judged. +{ + console.log("\nlocal providers"); + + const EVIG_ENV = ` +# Groq (cloud) +GROQ_API_KEY= +GROQ_MODEL=openai/gpt-oss-120b + +# Ollama URL (local LLM - for local embeddings only) +OLLAMA_URL=http://localhost:11434 +OLLAMA_MODEL=llama3.2 +`; + + const pins = extractPins(EVIG_ENV); + const ollamaPin = pins.find((p) => p.id === "llama3.2"); + check("the ollama tag is extracted", Boolean(ollamaPin), true); + // Before this, the nearest marker above was GROQ_MODEL and `llama3.2` was + // reported as a retired GROQ model. It is a valid Ollama tag on a healthy + // line; Groq simply never served anything by that name. + check("and attributed to ollama, not groq", attribute(EVIG_ENV, ollamaPin.line), "ollama"); + + const judged = judge( + [{ repo: "evig", path: ".env.example", line: ollamaPin.line, id: "llama3.2", vendor: "ollama" }], + new Map([["groq", GROQ_LIVE]]), + ); + check("a local tag is unchecked, never retired", judged[0].state, "unchecked"); +} + console.log(failures ? `\n✗ ${failures} failure(s)` : "\n✓ all checks pass"); process.exit(failures ? 1 : 0);