diff --git a/scripts/ci/model-pin-audit.mjs b/scripts/ci/model-pin-audit.mjs index 5a1e582..52caabc 100755 --- a/scripts/ci/model-pin-audit.mjs +++ b/scripts/ci/model-pin-audit.mjs @@ -121,19 +121,103 @@ export const VENDORS = [ * which mentions no vendor in its path and was invisible to the first version * of this filter. A pin does not have to live in a file called `provider.ts`. */ -const LIKELY_PATH = - /(^|\/)\.env\.example$|(^|\/)env\.(ts|js|mjs)$|(^|\/)(lib|src|app|apps|packages|config)\/.*(provider|model|llm|chat|ai|openai|anthropic)[^/]*\.(ts|js|mjs)$/i; +/** + * Words that make a path segment worth opening — a DIRECTORY name as readily as + * a file name. + * + * The previous version of this filter required the word to appear in the + * FILENAME, and the cost of that was measured rather than imagined. Kivvi keeps + * its provider clients in `packages/ai/src/providers/`: + * + * packages/ai/src/providers/anthropic.ts ← scanned + * packages/ai/src/providers/groq.ts ← never opened + * packages/ai/src/providers/openrouter.ts ← never opened + * packages/ai/src/providers/index.ts ← never opened + * + * Two files in the same directory, one seen and one not, decided entirely by + * which vendor names someone had typed into a regex. Three retired ids lived in + * the unopened ones, so the audit reported Kivvi as having 2 dead pins when it + * had 6 — and understating a repo is worse than missing it outright, because + * the number looks like an answer. + * + * Matching is by TOKEN, never substring. `ai` as a substring appears in `mail`, + * `chain`, `domain`, `detail` and `maintenance`; as a token it appears in `ai`, + * `ai-guidance` and `open-ai`. Substring matching here would have quietly + * traded this blind spot for a flood of irrelevant files, and the cap would then + * have dropped real candidates to make room. + */ +const AI_TOKENS = new Set([ + // the concern + "ai", "llm", "gpt", "model", "models", "provider", "providers", "chain", + "chat", "chats", "completion", "completions", "prompt", "prompts", + "agent", "agents", "embedding", "embeddings", "inference", + // the vendors — every one of these is a plausible file name, and the list + // being short is exactly what caused the miss above + "groq", "openrouter", "openai", "anthropic", "claude", "xai", "grok", + "gemini", "google", "ollama", "mistral", "together", "deepseek", "cohere", + "nvidia", "perplexity", "fireworks", "replicate", +]); + +/** Directories that can contain application source. */ +const SOURCE_ROOT = /^(lib|src|app|apps|packages|config|server|services|api)$/i; + +/** Does this one path segment name an AI concern or a vendor? */ +export function segmentNamesAI(segment) { + const base = segment.replace(/\.(ts|js|mjs|tsx|jsx)$/i, "").toLowerCase(); + return base.split(/[^a-z0-9]+/).some((token) => AI_TOKENS.has(token)); +} -const POSSIBLE_PATH = - /(^|\/)(lib|src|app|apps|packages|config)\/.*(constants?|config|settings|defaults)[^/]*\.(ts|js|mjs)$/i; +/** + * A file that almost certainly decides which model gets called. + * + * The AI word may sit anywhere below the source root — `ai/providers/groq.ts` + * qualifies on its directory alone, which is the whole point. + */ +export function isLikelyPath(path) { + if (/(^|\/)\.env\.example$/i.test(path)) return true; + if (/(^|\/)env\.(ts|js|mjs)$/i.test(path)) return true; + if (!/\.(ts|js|mjs)$/i.test(path)) return false; + + const segments = path.split("/"); + const rootAt = segments.findIndex((seg) => SOURCE_ROOT.test(seg)); + if (rootAt < 0) return false; + return segments.slice(rootAt + 1).some(segmentNamesAI); +} -const CANDIDATE_PATH = new RegExp(`(${LIKELY_PATH.source})|(${POSSIBLE_PATH.source})`, "i"); +/** + * The long tail a name-based filter misses. Botsmann kept its model id in + * `lib/constants.ts`, which mentions no vendor and no AI concern anywhere in + * its path. A pin does not have to live in a file called `provider.ts`. + */ +export function isPossiblePath(path) { + return /(^|\/)(lib|src|app|apps|packages|config)\/.*(constants?|config|settings|defaults)[^/]*\.(ts|js|mjs)$/i.test( + path, + ); +} + +/** Worth opening at all. */ +export function isCandidatePath(path) { + return isLikelyPath(path) || isPossiblePath(path); +} /** Never open these, whatever they are named. */ const SKIP_PATH = /(^|\/)(node_modules|dist|build|\.next|coverage|__tests__|__fixtures__)\/|(^|\/)\.claude\/worktrees\//; -const MAX_FILES_PER_REPO = 90; +/** + * How many candidate files one repo may cost. + * + * Raised 90 -> 160 because the coverage ledger stopped being a caveat and + * became a finding: on 2026-08-27 it reported OrangeCat opening 90 of 133 with + * 4 likely-AI files dropped, and FleetCrown 90 of 109 with 15 dropped. Ranking + * puts likely files first, so shedding generic config is harmless — shedding + * fifteen files that name an AI concern is a blind spot, and in the two largest + * repos in the fleet. + * + * 160 clears both with headroom. The ledger stays, because the next repo to + * outgrow the cap should say so rather than quietly report a smaller number. + */ +const MAX_FILES_PER_REPO = 160; /** A vendor named 40+ lines from a pin is not describing that pin. */ const MAX_ATTRIBUTION_DISTANCE = 40; @@ -160,6 +244,54 @@ export function looksLikeModelId(s) { return /\d/.test(s) || s.includes("/"); } +/** + * The lines making up each `models = [ ... ]` array, with real line numbers. + * + * A tiny bracket walker rather than a regex, because the shapes in this fleet + * defeat any single pattern: `models: ['a']`, `models = [`, and Kivvi's + * `models: AIModel[] = [` all mean the same thing, and the last one hid two + * retired ids from the audit for as long as this was a regex. + * + * The opening bracket is the LAST one on the declaring line, which is what + * makes the annotated form work — in `models: AIModel[] = [`, the first `[` + * belongs to the type and closes immediately, so anchoring to it would read an + * empty array and report nothing. Exactly the silent miss this replaces. + */ +export function modelListRegions(text) { + const lines = text.split("\n"); + const regions = []; + + for (let i = 0; i < lines.length; i++) { + // Greedy on purpose: it backtracks to the LAST `[` within reach. + const opener = /\bmodels?\b[^\n]{0,80}\[/i.exec(lines[i]); + if (!opener) continue; + + const region = []; + let depth = 0; + let opened = false; + + // A models array running past 200 lines is not a models array. + for (let j = i; j < lines.length && j < i + 200; j++) { + const segment = j === i ? lines[j].slice(opener.index + opener[0].length - 1) : lines[j]; + region.push({ line: j + 1, text: segment }); + + for (const ch of segment) { + if (ch === "[") { + depth++; + opened = true; + } else if (ch === "]") { + depth--; + } + } + if (opened && depth <= 0) break; + } + + regions.push(region); + } + + return regions; +} + /** * Pull candidate model ids out of one file's text, with the line each sits on. * @@ -177,9 +309,36 @@ export function extractPins(text) { if (!found.has(id)) found.set(id, lineOf(index)); }; - // 1. model: 'x' | models: ['a', 'b'] - for (const m of text.matchAll(/\bmodels?\s*[:=]\s*(\[[\s\S]{0,400}?\]|['"`][^'"`\n]{0,120}['"`])/gi)) { - for (const q of m[1].matchAll(/['"`]([^'"`\n]+)['"`]/g)) remember(q[1], m.index); + const rememberAt = (id, line) => { + if (!looksLikeModelId(id)) return; + if (!found.has(id)) found.set(id, line); + }; + + // 1a. model: 'x' — a single id on one line + for (const m of text.matchAll(/\bmodels?\s*[:=]\s*['"`]([^'"`\n]{0,120})['"`]/gi)) { + remember(m[1], m.index); + } + + // 1b. a models ARRAY, walked by bracket depth rather than matched by regex. + // + // This used to be `models?\s*[:=]\s*\[[\s\S]{0,400}?\]`, and it missed a + // whole package. Kivvi declares its list as: + // + // models: AIModel[] = [ + // + // A TypeScript type annotation sits between the key and the array, so the + // pattern never fired, and two retired Groq ids inside were invisible. The + // 400-character window was the second half of the same problem: a list of + // richly described models runs well past it, so even a matching array was + // read only as far as its first two entries. + // + // Walking the brackets has neither limit. It also reports each id's OWN line + // instead of the line the array opened on, which matters because vendor + // attribution is measured in lines from the pin. + for (const region of modelListRegions(text)) { + for (const { line, text: lineText } of region) { + for (const q of lineText.matchAll(/['"`]([^'"`\n]+)['"`]/g)) rememberAt(q[1], line); + } } // 2. GROQ_MODEL: z.string().default('x') | const DEFAULT_MODEL = 'x' @@ -338,8 +497,8 @@ let privateVisible = true; function rank(paths, repoName) { const sorted = [...paths].sort((a, b) => { - const ta = LIKELY_PATH.test(a) ? 0 : 1; - const tb = LIKELY_PATH.test(b) ? 0 : 1; + const ta = isLikelyPath(a) ? 0 : 1; + const tb = isLikelyPath(b) ? 0 : 1; return ta - tb || a.length - b.length; }); if (sorted.length > MAX_FILES_PER_REPO) { @@ -347,7 +506,7 @@ function rank(paths, repoName) { // tier is opened first, so a truncation that sheds only generic config // files has not touched the audit's real coverage — and saying which it was // is the difference between a caveat and an alarm. - const likelyDropped = sorted.slice(MAX_FILES_PER_REPO).filter((p) => LIKELY_PATH.test(p)).length; + const likelyDropped = sorted.slice(MAX_FILES_PER_REPO).filter((p) => isLikelyPath(p)).length; truncated.push({ repo: repoName, seen: sorted.length, opened: MAX_FILES_PER_REPO, likelyDropped }); } return sorted.slice(0, MAX_FILES_PER_REPO); @@ -388,7 +547,7 @@ async function remoteFiles(repo) { return []; // empty repo, or no access — not a finding } const all = (tree.tree ?? []) - .filter((n) => n.type === "blob" && !SKIP_PATH.test("/" + n.path) && CANDIDATE_PATH.test(n.path)) + .filter((n) => n.type === "blob" && !SKIP_PATH.test("/" + n.path) && isCandidatePath(n.path)) .map((n) => n.path); const paths = rank(all, repo.name); @@ -426,7 +585,7 @@ function walk(dir, depth, acc) { if (e.isDirectory()) { if (/^(node_modules|dist|build|\.next|coverage|\.git|\.claude)$/.test(e.name)) continue; walk(full, depth - 1, acc); - } else if (CANDIDATE_PATH.test(full) && !SKIP_PATH.test(full)) { + } else if (isCandidatePath(full) && !SKIP_PATH.test(full)) { try { if (statSync(full).size < 400_000) acc.push(full); } catch { /* raced */ } diff --git a/scripts/ci/test-model-pin-audit.mjs b/scripts/ci/test-model-pin-audit.mjs index 0288b2b..ae093a3 100755 --- a/scripts/ci/test-model-pin-audit.mjs +++ b/scripts/ci/test-model-pin-audit.mjs @@ -20,7 +20,19 @@ * * Fixtures are inline strings — no network, no gh, no checkout, no key. */ -import { extractPins, attribute, collate, judge, looksLikeModelId, possibleAt } from "./model-pin-audit.mjs"; +import { + extractPins, + attribute, + collate, + judge, + looksLikeModelId, + possibleAt, + isLikelyPath, + isPossiblePath, + isCandidatePath, + segmentNamesAI, + modelListRegions, +} from "./model-pin-audit.mjs"; let failures = 0; function check(name, actual, expected) { @@ -331,5 +343,147 @@ const model = 'openai/gpt-oss-20b:free' ); } +// ── which files get opened ─────────────────────────────────────────── +// +// Regression, and the expensive kind: a MISS, not a false alarm. +// +// The filter used to require the AI word in the FILENAME. Kivvi keeps its +// provider clients side by side in packages/ai/src/providers/, and the audit +// opened `anthropic.ts` while never opening `groq.ts` next to it — decided +// entirely by which vendor names happened to be in a regex. Three retired ids +// sat in the unopened files, so the report said Kivvi had 2 dead pins when it +// had 6. +// +// Understating a repo is worse than skipping it. A skipped repo is absent; an +// understated one prints a number that reads like an answer. + +console.log("\nwhich files get opened"); + +{ + // The four files from the miss, verbatim. + check("a vendor-named file under an ai/ directory is likely", isLikelyPath("packages/ai/src/providers/groq.ts"), true); + check("...and so is its openrouter sibling", isLikelyPath("packages/ai/src/providers/openrouter.ts"), true); + check("...and the index.ts beside them", isLikelyPath("packages/ai/src/providers/index.ts"), true); + check("...and the one that already worked still works", isLikelyPath("packages/ai/src/providers/anthropic.ts"), true); + + check("the app-side caller stays likely", isLikelyPath("apps/web/lib/ai/call-provider.ts"), true); + check("env schemas are always opened", isLikelyPath("src/lib/env.ts"), true); + check("so are .env.example files", isLikelyPath(".env.example"), true); + + // Botsmann's pin lived here, in a path naming no vendor and no AI concern. + check("a bare constants file is not likely", isLikelyPath("lib/constants.ts"), false); + check("...but is still a candidate, via the possible tier", isPossiblePath("lib/constants.ts"), true); + check("...so it does get opened", isCandidatePath("lib/constants.ts"), true); +} + +// Matching is by token, never substring — this is the half that keeps the fix +// from turning one blind spot into a flood. `ai` is a substring of all of these, +// and the cap would then drop real candidates to make room for them. +{ + check("mail is not ai", segmentNamesAI("mail.ts"), false); + check("domain is not ai", segmentNamesAI("domain.ts"), false); + check("detail is not ai", segmentNamesAI("detail.ts"), false); + check("maintenance is not ai", segmentNamesAI("maintenance.ts"), false); + check("captain is not ai", segmentNamesAI("captain.ts"), false); + + check("but ai is ai", segmentNamesAI("ai"), true); + check("and ai-guidance is ai", segmentNamesAI("ai-guidance.ts"), true); + check("and call-provider is a provider", segmentNamesAI("call-provider.ts"), true); + check("and llm-client is an llm", segmentNamesAI("llm-client.ts"), true); + + check("an unrelated util is not opened", isCandidatePath("src/lib/format-date.ts"), false); + check("nor is a mailer", isCandidatePath("src/lib/mail.ts"), false); +} + +// ── model arrays the old regex could not read ───────────────────────────── +// +// Second half of the Kivvi miss. Even once the file was being opened, nothing +// came out of it: the extractor matched `models` followed directly by `[`, and +// Kivvi writes a TypeScript type annotation in between. +// +// models: AIModel[] = [ +// +// The first `[` on that line belongs to `AIModel[]` and closes immediately, so +// anchoring to it reads an empty array — a silent nothing, which is the worst +// possible output for an audit. Two retired Groq ids sat inside. + +console.log("\nmodel arrays"); + +const KIVVI_TYPED_ARRAY = ` +import type { AIModel } from "../types"; + +/** Uses OpenAI-compatible API at https://api.groq.com/openai/v1. */ +export class GroqProvider extends OpenAICompatibleProvider { + id = "groq"; + name = "Groq"; + + models: AIModel[] = [ + { + id: "llama-3.3-70b-versatile", + name: "Llama 3.3 70B", + contextWindow: 128000, + supportsTools: true, + costPer1kInput: 0, + }, + { + id: "llama-3.1-8b-instant", + name: "Llama 3.1 8B Instant", + contextWindow: 128000, + supportsTools: true, + costPer1kInput: 0, + }, + ]; + + protected baseUrl = "https://api.groq.com/openai/v1"; +} +`; + +{ + const pins = extractPins(KIVVI_TYPED_ARRAY); + const ids = pins.map((p) => p.id).sort(); + check( + "a type-annotated models array is read at all", + ids, + ["llama-3.1-8b-instant", "llama-3.3-70b-versatile"], + ); + + // The SECOND entry is the one a fixed-width window loses. The old pattern + // read at most 400 characters after `models`, and a richly described list + // runs past that long before it ends. + check("the second entry is not lost to a character budget", ids.length, 2); + + // Human-readable names sit in the same objects and must not be mistaken for + // ids — they are filtered by shape, not by position. + check("display names in the same object are not pins", ids.includes("Llama 3.3 70B"), false); + + // Each id reports its OWN line, not the line the array opened on. Vendor + // attribution is measured in lines from the pin, so a whole array collapsed + // onto one line would attribute every entry from the same neighbourhood. + const lines = pins.map((p) => p.line); + check("each id carries its own line number", new Set(lines).size, 2); + + for (const pin of pins) { + check(`${pin.id} attributes to groq`, attribute(KIVVI_TYPED_ARRAY, pin.line), "groq"); + } + + const regions = modelListRegions(KIVVI_TYPED_ARRAY); + check("exactly one models array is found in the file", regions.length, 1); +} + +{ + // The plain shapes must keep working. + check("a one-line string form still reads", extractPins(`const model = 'openai/gpt-oss-120b'`).map((p) => p.id), ["openai/gpt-oss-120b"]); + check( + "a plain inline array still reads", + extractPins(`models: ['openai/gpt-oss-120b', 'openai/gpt-oss-20b']`).map((p) => p.id).sort(), + ["openai/gpt-oss-120b", "openai/gpt-oss-20b"], + ); + check( + "an assignment without an annotation still reads", + extractPins(`const models = [\n "openai/gpt-oss-120b",\n]`).map((p) => p.id), + ["openai/gpt-oss-120b"], + ); +} + console.log(failures ? `\n✗ ${failures} failure(s)` : "\n✓ all checks pass"); process.exit(failures ? 1 : 0);