From 4611abc1be0e8b0eaacf2d23c100ec194c33fe0b Mon Sep 17 00:00:00 2001 From: Mao Nakamoto <41178744+maonakamoto@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:20:33 +0200 Subject: [PATCH] fix(ci): the audit was reporting a fraction and calling it a total MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kivvi has seven retired model ids. The audit reported two. Understating a repo is worse than skipping one. A skipped repo is visibly absent; an understated repo prints a number that reads like an answer, and the five pins it left out were in `packages/ai` — Kivvi's actual provider chain. Three independent causes, each found by chasing the previous one. 1. The AI word had to appear in the FILENAME. Kivvi keeps its provider clients side by side: packages/ai/src/providers/anthropic.ts ← scanned packages/ai/src/providers/groq.ts ← never opened packages/ai/src/providers/openrouter.ts ← never opened packages/ai/src/providers/index.ts ← never opened Two files in one directory, one seen and one not, decided entirely by which vendor names someone had typed into a regex. A directory called `ai/` counted for nothing. Path matching now looks at every segment below the source root, and the vendor list is no longer four names long. Matching is by TOKEN, never substring: `ai` is inside `mail`, `chain`, `domain`, `detail` and `maintenance`, and substring matching would have traded one blind spot for a flood that the file cap then drops real candidates to make room for. Tested in both directions. 2. Opening the files was not enough — nothing came out of them. `extractPins` matched `models` followed directly by `[`, and Kivvi writes models: AIModel[] = [ where a TypeScript type annotation sits in between. The first `[` on that line belongs to `AIModel[]` and closes immediately, so the pattern read an empty array: a silent nothing, the worst output an audit can produce. The same regex also capped the array at 400 characters, losing later entries of any richly described list. Replaced with a bracket walker that has neither limit and reports each id's own line — which also improves attribution, since that is measured in lines from the pin. 3. The coverage ledger stopped being a caveat and became a finding: OrangeCat opened 90 of 133 candidates with 4 likely-AI files dropped, FleetCrown 90 of 109 with 15 dropped. The two largest repos in the fleet, both partly blind. Cap raised 90 -> 160, which clears both with headroom. The ledger stays, so the next repo to outgrow it says so. What the fleet actually looks like, before -> after: 12 retired pins -> 16, and the composition changed more than the count. Newly visible: - hirnli — 2 dead pins, a repo that had never appeared in a report at all - orangecat:src/lib/ai/form-prefill-service.ts:23 — genuinely broken, and it is the repo this audit cleared as a false positive earlier today. It was right that the pricing-table string was not a pin, and wrong that the repo was fine. - kivvi packages/ai — 5 more, including a `google/gemini-2.0-flash-001` that no previous run had seen - evig src/lib/hirn/providers — 2 more Every one of them was retired before this commit. None of them was new. Self-test: 57 -> 76 checks, still no network, no key, no checkout. The fixtures are the real paths and the real declaration, so this specific miss cannot come back quietly. Co-Authored-By: Claude Opus 5 --- scripts/ci/model-pin-audit.mjs | 187 +++++++++++++++++++++++++--- scripts/ci/test-model-pin-audit.mjs | 156 ++++++++++++++++++++++- 2 files changed, 328 insertions(+), 15 deletions(-) diff --git a/scripts/ci/model-pin-audit.mjs b/scripts/ci/model-pin-audit.mjs index 5a1e582..52caabc 100755 --- a/scripts/ci/model-pin-audit.mjs +++ b/scripts/ci/model-pin-audit.mjs @@ -121,19 +121,103 @@ export const VENDORS = [ * which mentions no vendor in its path and was invisible to the first version * of this filter. A pin does not have to live in a file called `provider.ts`. */ -const LIKELY_PATH = - /(^|\/)\.env\.example$|(^|\/)env\.(ts|js|mjs)$|(^|\/)(lib|src|app|apps|packages|config)\/.*(provider|model|llm|chat|ai|openai|anthropic)[^/]*\.(ts|js|mjs)$/i; +/** + * Words that make a path segment worth opening — a DIRECTORY name as readily as + * a file name. + * + * The previous version of this filter required the word to appear in the + * FILENAME, and the cost of that was measured rather than imagined. Kivvi keeps + * its provider clients in `packages/ai/src/providers/`: + * + * packages/ai/src/providers/anthropic.ts ← scanned + * packages/ai/src/providers/groq.ts ← never opened + * packages/ai/src/providers/openrouter.ts ← never opened + * packages/ai/src/providers/index.ts ← never opened + * + * Two files in the same directory, one seen and one not, decided entirely by + * which vendor names someone had typed into a regex. Three retired ids lived in + * the unopened ones, so the audit reported Kivvi as having 2 dead pins when it + * had 6 — and understating a repo is worse than missing it outright, because + * the number looks like an answer. + * + * Matching is by TOKEN, never substring. `ai` as a substring appears in `mail`, + * `chain`, `domain`, `detail` and `maintenance`; as a token it appears in `ai`, + * `ai-guidance` and `open-ai`. Substring matching here would have quietly + * traded this blind spot for a flood of irrelevant files, and the cap would then + * have dropped real candidates to make room. + */ +const AI_TOKENS = new Set([ + // the concern + "ai", "llm", "gpt", "model", "models", "provider", "providers", "chain", + "chat", "chats", "completion", "completions", "prompt", "prompts", + "agent", "agents", "embedding", "embeddings", "inference", + // the vendors — every one of these is a plausible file name, and the list + // being short is exactly what caused the miss above + "groq", "openrouter", "openai", "anthropic", "claude", "xai", "grok", + "gemini", "google", "ollama", "mistral", "together", "deepseek", "cohere", + "nvidia", "perplexity", "fireworks", "replicate", +]); + +/** Directories that can contain application source. */ +const SOURCE_ROOT = /^(lib|src|app|apps|packages|config|server|services|api)$/i; + +/** Does this one path segment name an AI concern or a vendor? */ +export function segmentNamesAI(segment) { + const base = segment.replace(/\.(ts|js|mjs|tsx|jsx)$/i, "").toLowerCase(); + return base.split(/[^a-z0-9]+/).some((token) => AI_TOKENS.has(token)); +} -const POSSIBLE_PATH = - /(^|\/)(lib|src|app|apps|packages|config)\/.*(constants?|config|settings|defaults)[^/]*\.(ts|js|mjs)$/i; +/** + * A file that almost certainly decides which model gets called. + * + * The AI word may sit anywhere below the source root — `ai/providers/groq.ts` + * qualifies on its directory alone, which is the whole point. + */ +export function isLikelyPath(path) { + if (/(^|\/)\.env\.example$/i.test(path)) return true; + if (/(^|\/)env\.(ts|js|mjs)$/i.test(path)) return true; + if (!/\.(ts|js|mjs)$/i.test(path)) return false; + + const segments = path.split("/"); + const rootAt = segments.findIndex((seg) => SOURCE_ROOT.test(seg)); + if (rootAt < 0) return false; + return segments.slice(rootAt + 1).some(segmentNamesAI); +} -const CANDIDATE_PATH = new RegExp(`(${LIKELY_PATH.source})|(${POSSIBLE_PATH.source})`, "i"); +/** + * The long tail a name-based filter misses. Botsmann kept its model id in + * `lib/constants.ts`, which mentions no vendor and no AI concern anywhere in + * its path. A pin does not have to live in a file called `provider.ts`. + */ +export function isPossiblePath(path) { + return /(^|\/)(lib|src|app|apps|packages|config)\/.*(constants?|config|settings|defaults)[^/]*\.(ts|js|mjs)$/i.test( + path, + ); +} + +/** Worth opening at all. */ +export function isCandidatePath(path) { + return isLikelyPath(path) || isPossiblePath(path); +} /** Never open these, whatever they are named. */ const SKIP_PATH = /(^|\/)(node_modules|dist|build|\.next|coverage|__tests__|__fixtures__)\/|(^|\/)\.claude\/worktrees\//; -const MAX_FILES_PER_REPO = 90; +/** + * How many candidate files one repo may cost. + * + * Raised 90 -> 160 because the coverage ledger stopped being a caveat and + * became a finding: on 2026-08-27 it reported OrangeCat opening 90 of 133 with + * 4 likely-AI files dropped, and FleetCrown 90 of 109 with 15 dropped. Ranking + * puts likely files first, so shedding generic config is harmless — shedding + * fifteen files that name an AI concern is a blind spot, and in the two largest + * repos in the fleet. + * + * 160 clears both with headroom. The ledger stays, because the next repo to + * outgrow the cap should say so rather than quietly report a smaller number. + */ +const MAX_FILES_PER_REPO = 160; /** A vendor named 40+ lines from a pin is not describing that pin. */ const MAX_ATTRIBUTION_DISTANCE = 40; @@ -160,6 +244,54 @@ export function looksLikeModelId(s) { return /\d/.test(s) || s.includes("/"); } +/** + * The lines making up each `models = [ ... ]` array, with real line numbers. + * + * A tiny bracket walker rather than a regex, because the shapes in this fleet + * defeat any single pattern: `models: ['a']`, `models = [`, and Kivvi's + * `models: AIModel[] = [` all mean the same thing, and the last one hid two + * retired ids from the audit for as long as this was a regex. + * + * The opening bracket is the LAST one on the declaring line, which is what + * makes the annotated form work — in `models: AIModel[] = [`, the first `[` + * belongs to the type and closes immediately, so anchoring to it would read an + * empty array and report nothing. Exactly the silent miss this replaces. + */ +export function modelListRegions(text) { + const lines = text.split("\n"); + const regions = []; + + for (let i = 0; i < lines.length; i++) { + // Greedy on purpose: it backtracks to the LAST `[` within reach. + const opener = /\bmodels?\b[^\n]{0,80}\[/i.exec(lines[i]); + if (!opener) continue; + + const region = []; + let depth = 0; + let opened = false; + + // A models array running past 200 lines is not a models array. + for (let j = i; j < lines.length && j < i + 200; j++) { + const segment = j === i ? lines[j].slice(opener.index + opener[0].length - 1) : lines[j]; + region.push({ line: j + 1, text: segment }); + + for (const ch of segment) { + if (ch === "[") { + depth++; + opened = true; + } else if (ch === "]") { + depth--; + } + } + if (opened && depth <= 0) break; + } + + regions.push(region); + } + + return regions; +} + /** * Pull candidate model ids out of one file's text, with the line each sits on. * @@ -177,9 +309,36 @@ export function extractPins(text) { if (!found.has(id)) found.set(id, lineOf(index)); }; - // 1. model: 'x' | models: ['a', 'b'] - for (const m of text.matchAll(/\bmodels?\s*[:=]\s*(\[[\s\S]{0,400}?\]|['"`][^'"`\n]{0,120}['"`])/gi)) { - for (const q of m[1].matchAll(/['"`]([^'"`\n]+)['"`]/g)) remember(q[1], m.index); + const rememberAt = (id, line) => { + if (!looksLikeModelId(id)) return; + if (!found.has(id)) found.set(id, line); + }; + + // 1a. model: 'x' — a single id on one line + for (const m of text.matchAll(/\bmodels?\s*[:=]\s*['"`]([^'"`\n]{0,120})['"`]/gi)) { + remember(m[1], m.index); + } + + // 1b. a models ARRAY, walked by bracket depth rather than matched by regex. + // + // This used to be `models?\s*[:=]\s*\[[\s\S]{0,400}?\]`, and it missed a + // whole package. Kivvi declares its list as: + // + // models: AIModel[] = [ + // + // A TypeScript type annotation sits between the key and the array, so the + // pattern never fired, and two retired Groq ids inside were invisible. The + // 400-character window was the second half of the same problem: a list of + // richly described models runs well past it, so even a matching array was + // read only as far as its first two entries. + // + // Walking the brackets has neither limit. It also reports each id's OWN line + // instead of the line the array opened on, which matters because vendor + // attribution is measured in lines from the pin. + for (const region of modelListRegions(text)) { + for (const { line, text: lineText } of region) { + for (const q of lineText.matchAll(/['"`]([^'"`\n]+)['"`]/g)) rememberAt(q[1], line); + } } // 2. GROQ_MODEL: z.string().default('x') | const DEFAULT_MODEL = 'x' @@ -338,8 +497,8 @@ let privateVisible = true; function rank(paths, repoName) { const sorted = [...paths].sort((a, b) => { - const ta = LIKELY_PATH.test(a) ? 0 : 1; - const tb = LIKELY_PATH.test(b) ? 0 : 1; + const ta = isLikelyPath(a) ? 0 : 1; + const tb = isLikelyPath(b) ? 0 : 1; return ta - tb || a.length - b.length; }); if (sorted.length > MAX_FILES_PER_REPO) { @@ -347,7 +506,7 @@ function rank(paths, repoName) { // tier is opened first, so a truncation that sheds only generic config // files has not touched the audit's real coverage — and saying which it was // is the difference between a caveat and an alarm. - const likelyDropped = sorted.slice(MAX_FILES_PER_REPO).filter((p) => LIKELY_PATH.test(p)).length; + const likelyDropped = sorted.slice(MAX_FILES_PER_REPO).filter((p) => isLikelyPath(p)).length; truncated.push({ repo: repoName, seen: sorted.length, opened: MAX_FILES_PER_REPO, likelyDropped }); } return sorted.slice(0, MAX_FILES_PER_REPO); @@ -388,7 +547,7 @@ async function remoteFiles(repo) { return []; // empty repo, or no access — not a finding } const all = (tree.tree ?? []) - .filter((n) => n.type === "blob" && !SKIP_PATH.test("/" + n.path) && CANDIDATE_PATH.test(n.path)) + .filter((n) => n.type === "blob" && !SKIP_PATH.test("/" + n.path) && isCandidatePath(n.path)) .map((n) => n.path); const paths = rank(all, repo.name); @@ -426,7 +585,7 @@ function walk(dir, depth, acc) { if (e.isDirectory()) { if (/^(node_modules|dist|build|\.next|coverage|\.git|\.claude)$/.test(e.name)) continue; walk(full, depth - 1, acc); - } else if (CANDIDATE_PATH.test(full) && !SKIP_PATH.test(full)) { + } else if (isCandidatePath(full) && !SKIP_PATH.test(full)) { try { if (statSync(full).size < 400_000) acc.push(full); } catch { /* raced */ } diff --git a/scripts/ci/test-model-pin-audit.mjs b/scripts/ci/test-model-pin-audit.mjs index 0288b2b..ae093a3 100755 --- a/scripts/ci/test-model-pin-audit.mjs +++ b/scripts/ci/test-model-pin-audit.mjs @@ -20,7 +20,19 @@ * * Fixtures are inline strings — no network, no gh, no checkout, no key. */ -import { extractPins, attribute, collate, judge, looksLikeModelId, possibleAt } from "./model-pin-audit.mjs"; +import { + extractPins, + attribute, + collate, + judge, + looksLikeModelId, + possibleAt, + isLikelyPath, + isPossiblePath, + isCandidatePath, + segmentNamesAI, + modelListRegions, +} from "./model-pin-audit.mjs"; let failures = 0; function check(name, actual, expected) { @@ -331,5 +343,147 @@ const model = 'openai/gpt-oss-20b:free' ); } +// ── which files get opened ─────────────────────────────────────────── +// +// Regression, and the expensive kind: a MISS, not a false alarm. +// +// The filter used to require the AI word in the FILENAME. Kivvi keeps its +// provider clients side by side in packages/ai/src/providers/, and the audit +// opened `anthropic.ts` while never opening `groq.ts` next to it — decided +// entirely by which vendor names happened to be in a regex. Three retired ids +// sat in the unopened files, so the report said Kivvi had 2 dead pins when it +// had 6. +// +// Understating a repo is worse than skipping it. A skipped repo is absent; an +// understated one prints a number that reads like an answer. + +console.log("\nwhich files get opened"); + +{ + // The four files from the miss, verbatim. + check("a vendor-named file under an ai/ directory is likely", isLikelyPath("packages/ai/src/providers/groq.ts"), true); + check("...and so is its openrouter sibling", isLikelyPath("packages/ai/src/providers/openrouter.ts"), true); + check("...and the index.ts beside them", isLikelyPath("packages/ai/src/providers/index.ts"), true); + check("...and the one that already worked still works", isLikelyPath("packages/ai/src/providers/anthropic.ts"), true); + + check("the app-side caller stays likely", isLikelyPath("apps/web/lib/ai/call-provider.ts"), true); + check("env schemas are always opened", isLikelyPath("src/lib/env.ts"), true); + check("so are .env.example files", isLikelyPath(".env.example"), true); + + // Botsmann's pin lived here, in a path naming no vendor and no AI concern. + check("a bare constants file is not likely", isLikelyPath("lib/constants.ts"), false); + check("...but is still a candidate, via the possible tier", isPossiblePath("lib/constants.ts"), true); + check("...so it does get opened", isCandidatePath("lib/constants.ts"), true); +} + +// Matching is by token, never substring — this is the half that keeps the fix +// from turning one blind spot into a flood. `ai` is a substring of all of these, +// and the cap would then drop real candidates to make room for them. +{ + check("mail is not ai", segmentNamesAI("mail.ts"), false); + check("domain is not ai", segmentNamesAI("domain.ts"), false); + check("detail is not ai", segmentNamesAI("detail.ts"), false); + check("maintenance is not ai", segmentNamesAI("maintenance.ts"), false); + check("captain is not ai", segmentNamesAI("captain.ts"), false); + + check("but ai is ai", segmentNamesAI("ai"), true); + check("and ai-guidance is ai", segmentNamesAI("ai-guidance.ts"), true); + check("and call-provider is a provider", segmentNamesAI("call-provider.ts"), true); + check("and llm-client is an llm", segmentNamesAI("llm-client.ts"), true); + + check("an unrelated util is not opened", isCandidatePath("src/lib/format-date.ts"), false); + check("nor is a mailer", isCandidatePath("src/lib/mail.ts"), false); +} + +// ── model arrays the old regex could not read ───────────────────────────── +// +// Second half of the Kivvi miss. Even once the file was being opened, nothing +// came out of it: the extractor matched `models` followed directly by `[`, and +// Kivvi writes a TypeScript type annotation in between. +// +// models: AIModel[] = [ +// +// The first `[` on that line belongs to `AIModel[]` and closes immediately, so +// anchoring to it reads an empty array — a silent nothing, which is the worst +// possible output for an audit. Two retired Groq ids sat inside. + +console.log("\nmodel arrays"); + +const KIVVI_TYPED_ARRAY = ` +import type { AIModel } from "../types"; + +/** Uses OpenAI-compatible API at https://api.groq.com/openai/v1. */ +export class GroqProvider extends OpenAICompatibleProvider { + id = "groq"; + name = "Groq"; + + models: AIModel[] = [ + { + id: "llama-3.3-70b-versatile", + name: "Llama 3.3 70B", + contextWindow: 128000, + supportsTools: true, + costPer1kInput: 0, + }, + { + id: "llama-3.1-8b-instant", + name: "Llama 3.1 8B Instant", + contextWindow: 128000, + supportsTools: true, + costPer1kInput: 0, + }, + ]; + + protected baseUrl = "https://api.groq.com/openai/v1"; +} +`; + +{ + const pins = extractPins(KIVVI_TYPED_ARRAY); + const ids = pins.map((p) => p.id).sort(); + check( + "a type-annotated models array is read at all", + ids, + ["llama-3.1-8b-instant", "llama-3.3-70b-versatile"], + ); + + // The SECOND entry is the one a fixed-width window loses. The old pattern + // read at most 400 characters after `models`, and a richly described list + // runs past that long before it ends. + check("the second entry is not lost to a character budget", ids.length, 2); + + // Human-readable names sit in the same objects and must not be mistaken for + // ids — they are filtered by shape, not by position. + check("display names in the same object are not pins", ids.includes("Llama 3.3 70B"), false); + + // Each id reports its OWN line, not the line the array opened on. Vendor + // attribution is measured in lines from the pin, so a whole array collapsed + // onto one line would attribute every entry from the same neighbourhood. + const lines = pins.map((p) => p.line); + check("each id carries its own line number", new Set(lines).size, 2); + + for (const pin of pins) { + check(`${pin.id} attributes to groq`, attribute(KIVVI_TYPED_ARRAY, pin.line), "groq"); + } + + const regions = modelListRegions(KIVVI_TYPED_ARRAY); + check("exactly one models array is found in the file", regions.length, 1); +} + +{ + // The plain shapes must keep working. + check("a one-line string form still reads", extractPins(`const model = 'openai/gpt-oss-120b'`).map((p) => p.id), ["openai/gpt-oss-120b"]); + check( + "a plain inline array still reads", + extractPins(`models: ['openai/gpt-oss-120b', 'openai/gpt-oss-20b']`).map((p) => p.id).sort(), + ["openai/gpt-oss-120b", "openai/gpt-oss-20b"], + ); + check( + "an assignment without an annotation still reads", + extractPins(`const models = [\n "openai/gpt-oss-120b",\n]`).map((p) => p.id), + ["openai/gpt-oss-120b"], + ); +} + console.log(failures ? `\n✗ ${failures} failure(s)` : "\n✓ all checks pass"); process.exit(failures ? 1 : 0);