diff --git a/scripts/ci/model-pin-audit.mjs b/scripts/ci/model-pin-audit.mjs index 52caabc..d818f17 100755 --- a/scripts/ci/model-pin-audit.mjs +++ b/scripts/ci/model-pin-audit.mjs @@ -94,6 +94,9 @@ export const VENDORS = [ baseUrl: "https://api.groq.com/openai/v1", keyEnv: "GROQ_API_KEY", markers: [/groq/i], + // A provider-keyed record — `groq: { defaultModel: '...' }` — names the + // vendor for everything inside it, and sits closer to the pin than any URL. + keyMarker: /^[ \t]*['"]?groq['"]?\s*:/m, }, { id: "openrouter", @@ -101,16 +104,48 @@ export const VENDORS = [ baseUrl: "https://openrouter.ai/api/v1", keyEnv: "OPENROUTER_API_KEY", markers: [/openrouter/i], + keyMarker: /^[ \t]*['"]?openrouter['"]?\s*:/m, }, // Not queryable here — no catalogue call is wired for these. They are listed // so their ids are ATTRIBUTED and reported unchecked, rather than falling to // whichever queryable vendor happens to sit nearest in the file. Markers are // deliberately specific: bare /openai/ would match Groq's own // `api.groq.com/openai/v1` path and every `openai/gpt-oss-*` id it serves. - { id: "xai", queryable: false, markers: [/api\.x\.ai/i, /\bXAI_API_KEY\b/, /\bgrok\b/i] }, - { id: "anthropic", queryable: false, markers: [/api\.anthropic\.com/i, /\bANTHROPIC_API_KEY\b/] }, - { id: "openai", queryable: false, markers: [/api\.openai\.com/i, /\bOPENAI_API_KEY\b/] }, - { id: "google", queryable: false, markers: [/generativelanguage\.googleapis/i, /\bGEMINI_API_KEY\b/] }, + // Ollama runs on the user's own machine, so "is this id still served" is not + // a question with a fleet-wide answer — whatever the operator pulled is what + // exists. It is listed so its ids are ATTRIBUTED rather than falling to the + // nearest cloud vendor above them. + // + // Without this, evig's `.env.example` line `OLLAMA_MODEL=llama3.2` was + // attributed to Groq — the nearest marker above it — and reported RETIRED. + // `llama3.2` is a perfectly valid Ollama tag on a healthy line; Groq simply + // never served anything by that name. Same class as the pricing-table false + // positive: a true statement about the wrong vendor. + { + id: "ollama", + queryable: false, + markers: [/\bOLLAMA_[A-Z_]+\b/, /\bollama\b/i, /localhost:11434/, /127\.0\.0\.1:11434/], + }, + // + // `keyMarker` is how a provider-keyed record names its own rows. OrangeCat + // writes: + // + // export const PROVIDER_BASE_URLS = { openai: '...', openrouter: '...' } + // export const PROVIDER_RUNTIME = { + // openai: { baseUrl: ..., defaultModel: 'gpt-4o-mini' }, + // + // The pin belongs to OpenAI, but the nearest marker above it was OpenRouter's + // URL in the block before — so `gpt-4o-mini` was reported as a retired + // OpenRouter model. It is not an OpenRouter id at all; there it would be + // `openai/gpt-4o-mini`. The bare key `openai:` is deliberately not in + // `markers`, because `api.groq.com/openai/v1` contains that word — anchoring + // it to the start of a line followed by a colon is what makes it safe. + { id: "xai", queryable: false, markers: [/api\.x\.ai/i, /\bXAI_API_KEY\b/, /\bgrok\b/i], keyMarker: /^[ \t]*['"]?xai['"]?\s*:/m }, + { id: "anthropic", queryable: false, markers: [/api\.anthropic\.com/i, /\bANTHROPIC_API_KEY\b/], keyMarker: /^[ \t]*['"]?anthropic['"]?\s*:/m }, + { id: "openai", queryable: false, markers: [/api\.openai\.com/i, /\bOPENAI_API_KEY\b/], keyMarker: /^[ \t]*['"]?openai['"]?\s*:/m }, + { id: "google", queryable: false, markers: [/generativelanguage\.googleapis/i, /\bGEMINI_API_KEY\b/], keyMarker: /^[ \t]*['"]?google['"]?\s*:/m }, + { id: "together", queryable: false, markers: [/api\.together\.xyz/i, /\bTOGETHER_API_KEY\b/], keyMarker: /^[ \t]*['"]?together['"]?\s*:/m }, + { id: "deepseek", queryable: false, markers: [/api\.deepseek\.com/i, /\bDEEPSEEK_API_KEY\b/], keyMarker: /^[ \t]*['"]?deepseek['"]?\s*:/m }, ]; /** @@ -241,45 +276,87 @@ export function looksLikeModelId(s) { if (/\.(ts|tsx|js|mjs|cjs|json|css|scss|md|png|jpe?g|svg|ico|txt|ya?ml)$/i.test(s)) return false; if (/^[A-Z][A-Z0-9_]*$/.test(s)) return false; // SCREAMING_CASE is an env name // Model ids essentially always carry a version digit or a vendor/ prefix. + // A vendor id always carries a separator: a slash for routed ids + // (`openai/gpt-oss-120b`), or a hyphen or dot within the name + // (`llama-3.3-70b-versatile`, `llama3.2`, `codex-4`). Nothing in either live + // catalogue is a single unseparated word. + // + // Without this, reading model MAPS turned their keys into findings: Hirnli's + // alias table is `{ '70b': '...', '8b': '...' }`, and `70b` has a digit and no + // space, so it read as a pin and would have been reported retired at Groq. It + // is a size alias. `8b` escaped only by being two characters long, which is + // not a rule anyone should rely on. + if (!/[/.-]/.test(s)) return false; + return /\d/.test(s) || s.includes("/"); } /** - * The lines making up each `models = [ ... ]` array, with real line numbers. + * The lines making up each `models` collection, with real line numbers. * * A tiny bracket walker rather than a regex, because the shapes in this fleet - * defeat any single pattern: `models: ['a']`, `models = [`, and Kivvi's - * `models: AIModel[] = [` all mean the same thing, and the last one hid two - * retired ids from the audit for as long as this was a regex. - * - * The opening bracket is the LAST one on the declaring line, which is what - * makes the annotated form work — in `models: AIModel[] = [`, the first `[` - * belongs to the type and closes immediately, so anchoring to it would read an - * empty array and report nothing. Exactly the silent miss this replaces. + * defeat any single pattern. All four of these are one repo's way of saying the + * same thing, and every one of them hid a retired id at some point: + * + * models: ['a', 'b'] an inline array + * models: AIModel[] = [ an array behind a TS annotation + * GROQ_MODELS = { '8b': 'llama-...' } a map whose VALUES are ids + * GROQ_MODELS = { 'llama-...': { ... } } a map whose KEYS are ids + * + * Maps were the second discovery and cost two more repos. Hirnli kept its ids + * as map values and Orangecat as map keys — including `DEFAULT_GROQ_MODEL`, + * the baseline every free user gets — and an array-only walker read straight + * past both. So the opener is `[` or `{`, and every quoted string in the region + * is a candidate regardless of which side of the colon it sits on. + * + * The opening bracket is the LAST one on the declaring line. That is what makes + * the annotated form work: in `models: AIModel[] = [`, the first `[` belongs to + * the type and closes immediately, so anchoring to it reads an empty array and + * reports nothing — a silent miss, the worst output an audit has. */ export function modelListRegions(text) { const lines = text.split("\n"); const regions = []; + const CLOSER = { "[": "]", "{": "}" }; for (let i = 0; i < lines.length; i++) { - // Greedy on purpose: it backtracks to the LAST `[` within reach. - const opener = /\bmodels?\b[^\n]{0,80}\[/i.exec(lines[i]); + // Greedy on purpose: it backtracks to the LAST opener within reach. + // + // `\w*` before `models` is load-bearing. A bare `\bmodels?\b` does NOT + // match `GROQ_MODELS`, because the underscore before it is a word character + // so there is no boundary there — which is precisely why two repos' model + // maps read as empty. Almost every map in this fleet is named that way. + // + // The `[:=]` immediately after the token is what makes this a DECLARATION + // rather than any line that mentions models. Without it, + // `for (const model of models) {` opened a region over the whole loop body + // — and the request headers inside were then read as model ids, so a + // `Content-Type: application/json` was reported as a retired Groq model. + // PLURAL only. A singular `model:` is a parameter or a single-id property, + // and `function supportsReasoningEffort(model: string): boolean {` is a + // declaration by the rule above — it opened a region over the function body + // and read the `startsWith("qwen/")` prefixes inside as retired ids. + // Collections are plural; single ids are matched on one line by the + // patterns in extractPins. + const opener = /\b[a-z0-9_]*models\b\s*[:=][^\n]{0,80}[[{]/i.exec(lines[i]); if (!opener) continue; + const open = opener[0].at(-1); + const close = CLOSER[open]; const region = []; let depth = 0; let opened = false; - // A models array running past 200 lines is not a models array. + // A models collection running past 200 lines is not a models collection. for (let j = i; j < lines.length && j < i + 200; j++) { const segment = j === i ? lines[j].slice(opener.index + opener[0].length - 1) : lines[j]; region.push({ line: j + 1, text: segment }); for (const ch of segment) { - if (ch === "[") { + if (ch === open) { depth++; opened = true; - } else if (ch === "]") { + } else if (ch === close) { depth--; } } @@ -314,8 +391,22 @@ export function extractPins(text) { if (!found.has(id)) found.set(id, line); }; - // 1a. model: 'x' — a single id on one line - for (const m of text.matchAll(/\bmodels?\s*[:=]\s*['"`]([^'"`\n]{0,120})['"`]/gi)) { + // 1a. model: 'x' — a single id on one line. + // + // The `id|name` suffix is not decoration. Kivvi's fallback reads + // + // const FALLBACK_MODEL: ModelSelection = { + // providerId: "groq", + // modelId: "llama-3.3-70b-versatile", + // }; + // + // and `modelId` is not `model`, so a pattern anchored on the bare word walked + // straight past a retired id sitting in the app's default model selection. + // The declaration above it is singular, so the collection walker does not + // cover it either — this line is the only thing that does. + for (const m of text.matchAll( + /\b[a-z0-9_]*model(?:s|id|_id|name|_name)?\b\s*[:=]\s*['"`]([^'"`\n]{0,120})['"`]/gi, + )) { remember(m[1], m.index); } @@ -337,7 +428,15 @@ export function extractPins(text) { // attribution is measured in lines from the pin. for (const region of modelListRegions(text)) { for (const { line, text: lineText } of region) { - for (const q of lineText.matchAll(/['"`]([^'"`\n]+)['"`]/g)) rememberAt(q[1], line); + // Comments are stripped first. A model list is exactly where someone + // documents the id they just replaced, and such notes are usually written + // in backticks — so without this, a comment reading "replaces + // `meta-llama/llama-3.2-3b-instruct:free`, which was retired" reports + // that id as a live pin, in the very commit that removed it. The audit + // would be reporting on prose instead of code: the exact failure it + // exists to catch elsewhere. + const code = lineText.replace(/\/\*[\s\S]*?\*\//g, "").replace(/\/\/.*$/, ""); + for (const q of code.matchAll(/['"`]([^'"`\n]+)['"`]/g)) rememberAt(q[1], line); } } @@ -383,8 +482,14 @@ export function attribute(text, line, vendors = VENDORS, maxDistance = MAX_ATTRI const distances = (vendor) => { let above = Infinity; let below = Infinity; + // `keyMarker` counts as a mention: in a provider-keyed record, the row's + // own key is the most local and most reliable statement of which vendor a + // pin belongs to — closer than any base URL, and unambiguous. + const mentions = (l) => + vendor.markers.some((re) => re.test(l)) || (vendor.keyMarker?.test(l) ?? false); + for (let i = 0; i < lines.length; i++) { - if (!vendor.markers.some((re) => re.test(lines[i]))) continue; + if (!mentions(lines[i])) continue; const d = i + 1 - line; if (d <= 0) above = Math.min(above, -d); else below = Math.min(below, d); @@ -406,7 +511,9 @@ export function attribute(text, line, vendors = VENDORS, maxDistance = MAX_ATTRI if (fromBelow.length > 1 && fromBelow[0].below < fromBelow[1].below) return fromBelow[0].id; } - const named = vendors.filter((v) => v.markers.some((re) => re.test(text))); + const named = vendors.filter( + (v) => v.markers.some((re) => re.test(text)) || (v.keyMarker?.test(text) ?? false), + ); return named.length === 1 ? named[0].id : null; } diff --git a/scripts/ci/test-model-pin-audit.mjs b/scripts/ci/test-model-pin-audit.mjs index ae093a3..282ee27 100755 --- a/scripts/ci/test-model-pin-audit.mjs +++ b/scripts/ci/test-model-pin-audit.mjs @@ -485,5 +485,255 @@ export class GroqProvider extends OpenAICompatibleProvider { ); } +// ── model MAPS, not just arrays ───────────────────────────────────── +// +// Third discovery in the same sweep, and it cost two more repos. +// +// Once arrays were readable, two repos still reported nothing, because their +// ids live in object literals — and on OPPOSITE sides of the colon. Hirnli +// keeps them as map values, Orangecat as map keys, including +// `DEFAULT_GROQ_MODEL`, the baseline every free non-BYOK user gets. Both were +// entirely retired and both read as clean. +// +// So the walker takes `{` as readily as `[`, and every quoted string in the +// region is a candidate regardless of which side of the colon it is on. + +console.log("\nmodel maps"); + +// Hirnli's shape: the id is the VALUE. +const HIRNLI_ALIAS_MAP = ` +const GROQ_API_URL = 'https://api.groq.com/openai/v1/chat/completions'; + +/** Available models with different rate limits */ +export const GROQ_MODELS = { + '70b': 'llama-3.3-70b-versatile', // 12k TPM, best quality + '8b': 'llama-3.1-8b-instant', // 20k TPM, faster, good for triage +} as const; +`; + +// Orangecat's shape: the id is the KEY, and the value is a nested object. +const ORANGECAT_REGISTRY = ` +// Groq's best free models +const GROQ_MODELS = { + // Fast, capable model - great for chat + 'llama-3.3-70b-versatile': { + name: 'Llama 3.3 70B Versatile', + contextWindow: 128000, + maxOutputTokens: 32768, + }, + 'llama-3.1-8b-instant': { + name: 'Llama 3.1 8B Instant', + contextWindow: 128000, + maxOutputTokens: 8192, + }, +} as const; + +const GROQ_API_URL = 'https://api.groq.com/openai/v1'; +`; + +{ + const ids = extractPins(HIRNLI_ALIAS_MAP).map((p) => p.id).sort(); + check( + "ids that are map VALUES are found", + ids, + ["llama-3.1-8b-instant", "llama-3.3-70b-versatile"], + ); + // '70b' and '8b' are the keys here. They are aliases, not ids, and must not + // be reported as pins — `looksLikeModelId` is what keeps them out. + check("the size aliases beside them are not mistaken for ids", ids.includes("8b"), false); +} + +{ + const pins = extractPins(ORANGECAT_REGISTRY); + const ids = pins.map((p) => p.id).sort(); + check( + "ids that are map KEYS are found", + ids, + ["llama-3.1-8b-instant", "llama-3.3-70b-versatile"], + ); + // Nested objects inside the map must not end the region early. + check("a nested object does not truncate the map", ids.length, 2); + // Human-readable names sit in the same nested objects. + check("display names inside the map are not pins", ids.includes("Llama 3.3 70B Versatile"), false); + + for (const pin of pins) { + check(`${pin.id} in a map attributes to groq`, attribute(ORANGECAT_REGISTRY, pin.line), "groq"); + } + + const judged = judge( + pins.map((p) => ({ repo: "orangecat", path: "src/services/ai/groq.ts", line: p.line, id: p.id, vendor: "groq" })), + new Map([["groq", GROQ_LIVE]]), + ); + check("and both are judged retired against the live catalogue", judged.filter((j) => j.state === "gone").length, 2); +} + +{ + // The walker must not mistake an ordinary import for a model collection. + check("an import naming models opens no region", modelListRegions(`import { getAllModels } from "./providers";`).length, 0); +} + +// A size alias is not a model id. +{ + console.log("\nseparators"); + check("a bare size token is not an id", looksLikeModelId("70b"), false); + check("nor is the two-character one", looksLikeModelId("8b"), false); + check("routed ids are ids", looksLikeModelId("openai/gpt-oss-120b"), true); + check("hyphenated ids are ids", looksLikeModelId("llama-3.3-70b-versatile"), true); + check("dotted ids are ids", looksLikeModelId("llama3.2"), true); + check("short hyphenated ids are ids", looksLikeModelId("codex-4"), true); +} + +// A loop over models is not a declaration of models. +// +// Regression from the map support: `{` as an opener made +// `for (const model of models) {` open a region across the whole loop body, so +// the request headers inside it were read as ids and a Content-Type header was +// reported as a retired Groq model. Widening a matcher is exactly when to check +// what it now swallows. +{ + console.log("\ndeclarations only"); + + const LOOP = ` + const models = groqModels(); + for (const model of models) { + const response = await fetch(API_CONFIG.GROQ_API_URL, { + headers: { + Authorization: \`Bearer \${key}\`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ model, messages }), + }); + } + `; + + check("a for-of over models opens no region", modelListRegions(LOOP).length, 0); + check("so a content-type header is not a model", extractPins(LOOP).map((p) => p.id), []); + + // The declaration forms must still open one. + check("a const array declaration still opens", modelListRegions(`const models = [`).length, 1); + check("an annotated declaration still opens", modelListRegions(` models: AIModel[] = [`).length, 1); + check("a map declaration still opens", modelListRegions(`export const GROQ_MODELS = {`).length, 1); + check("an inline property still opens", modelListRegions(`models: ['a/b-1'],`).length, 1); +} + +// `modelId` is not `model`. +// +// Regression found by re-running the live sweep after tightening the collection +// walker to plural-only: a real retired id in Kivvi disappeared from the report. +// Its declaration is singular (`const FALLBACK_MODEL: ModelSelection = {`), so +// the walker correctly ignores it, and the single-id pattern was anchored on the +// bare word `model` — which does not match `modelId`. Nothing covered it. +{ + console.log("\nsingle-id property shapes"); + + const KIVVI_FALLBACK = ` +const STORAGE_KEY = "kivvi-selected-model"; + +// Fallback default — used before API loads or when stored model is unavailable +const FALLBACK_MODEL: ModelSelection = { + providerId: "groq", + modelId: "llama-3.3-70b-versatile", +}; +`; + + const ids = extractPins(KIVVI_FALLBACK).map((p) => p.id); + check("a modelId property is a pin", ids.includes("llama-3.3-70b-versatile"), true); + check("the storage key beside it is not", ids.includes("kivvi-selected-model"), false); + + check("model_id also reads", extractPins(`model_id = "openai/gpt-oss-120b"`).map((p) => p.id), ["openai/gpt-oss-120b"]); + check("modelName also reads", extractPins(`modelName: 'openai/gpt-oss-20b'`).map((p) => p.id), ["openai/gpt-oss-20b"]); + + // A parameter annotation is still not a pin. + check("a typed parameter is not a pin", extractPins(`function f(model: string): boolean { return model.startsWith("qwen/"); }`).map((p) => p.id), []); +} + +// A provider-keyed record names its own rows. +// +// OrangeCat's `gpt-4o-mini` was reported as a RETIRED OPENROUTER model. It is +// not an OpenRouter id at all — there it would be `openai/gpt-4o-mini` — it is +// OpenAI's, sitting under an `openai:` key. The nearest marker above it was +// OpenRouter's base URL in the block before, so nearest-marker-above put it with +// the wrong vendor and then judged it against a catalogue that was never going +// to list it. +// +// The bare word `openai` cannot be a general marker: `api.groq.com/openai/v1` +// contains it, and every `openai/gpt-oss-*` id Groq serves. Anchoring to a line +// that STARTS with the key and a colon is what makes it safe. +{ + console.log("provider-keyed records"); + + const ORANGECAT_RUNTIME = ` +export const PROVIDER_BASE_URLS = { + openai: 'https://api.openai.com/v1', + groq: 'https://api.groq.com/openai/v1', + openrouter: 'https://openrouter.ai/api/v1', +} as const; + +export const PROVIDER_RUNTIME = { + openai: { + baseUrl: PROVIDER_BASE_URLS.openai, + defaultModel: 'gpt-4o-mini', + }, + openrouter: { + baseUrl: PROVIDER_BASE_URLS.openrouter, + defaultModel: 'nvidia/nemotron-3-super-120b-a12b:free', + }, +}; +`; + + const pins = extractPins(ORANGECAT_RUNTIME); + const at = (id) => pins.find((p) => p.id === id); + + check("both defaults are extracted", pins.length >= 2, true); + check( + "an id under an openai: key belongs to openai", + attribute(ORANGECAT_RUNTIME, at("gpt-4o-mini").line), + "openai", + ); + check( + "and the one under openrouter: belongs to openrouter", + attribute(ORANGECAT_RUNTIME, at("nvidia/nemotron-3-super-120b-a12b:free").line), + "openrouter", + ); + + // OpenAI is not queryable, so its pin must be reported UNCHECKED — never + // judged against a catalogue that could not list it. + const judged = judge( + pins.map((p) => ({ repo: "orangecat", path: "src/config/ai-provider-runtime.ts", line: p.line, id: p.id, vendor: attribute(ORANGECAT_RUNTIME, p.line) })), + new Map([["openrouter", new Set(["nvidia/nemotron-3-super-120b-a12b:free"])]]), + ); + check("the openai pin is unchecked, not retired", judged.find((j) => j.id === "gpt-4o-mini")?.state, "unchecked"); + check("the openrouter pin is confirmed live", judged.find((j) => j.id === "nvidia/nemotron-3-super-120b-a12b:free")?.state, "ok"); +} + +// Ollama is local: its tags are attributed, never judged. +{ + console.log("\nlocal providers"); + + const EVIG_ENV = ` +# Groq (cloud) +GROQ_API_KEY= +GROQ_MODEL=openai/gpt-oss-120b + +# Ollama URL (local LLM - for local embeddings only) +OLLAMA_URL=http://localhost:11434 +OLLAMA_MODEL=llama3.2 +`; + + const pins = extractPins(EVIG_ENV); + const ollamaPin = pins.find((p) => p.id === "llama3.2"); + check("the ollama tag is extracted", Boolean(ollamaPin), true); + // Before this, the nearest marker above was GROQ_MODEL and `llama3.2` was + // reported as a retired GROQ model. It is a valid Ollama tag on a healthy + // line; Groq simply never served anything by that name. + check("and attributed to ollama, not groq", attribute(EVIG_ENV, ollamaPin.line), "ollama"); + + const judged = judge( + [{ repo: "evig", path: ".env.example", line: ollamaPin.line, id: "llama3.2", vendor: "ollama" }], + new Map([["groq", GROQ_LIVE]]), + ); + check("a local tag is unchecked, never retired", judged[0].state, "unchecked"); +} + console.log(failures ? `\n✗ ${failures} failure(s)` : "\n✓ all checks pass"); process.exit(failures ? 1 : 0);