From e94e82ea3cb870ed0aea4ec8f0117030abf20d94 Mon Sep 17 00:00:00 2001 From: Mao Nakamoto <41178744+maonakamoto@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:04:13 +0200 Subject: [PATCH] feat(ci): ask the vendor, daily, whether our model ids still exist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AOZ reported "KI-Assistent nicht konfiguriert. Bitte GROQ_API_KEY setzen" on a deployment whose key was valid. Groq had retired the whole llama-3.x family and the pinned id answered 404. Production had been failing exactly as long as the demo and nobody knew, because "not configured" is the only thing the app can say. Six pins across five repos were dead the same morning — botsmann, evig, kivvi, orangecat, truthseeker — three of them serving live traffic. None of it was visible from inside a repo: every gate was green, because the code is correct and the vendor changed underneath it. No lint rule, type check or unit test can see this, which is exactly the gap the central audits exist to cover. So this asks the only authority that knows. One GET /models per vendor, ZERO tokens, which is what makes it schedulable — and "somebody is supposed to remember" is the mechanism that already failed for eight days in ai-ration's own consumer. It calls ai-ration's checkCatalog rather than growing a second vendor query here; the package takes the chain as an argument, so it generalised to arbitrary ids with no change. That makes dotfiles its second adopter, which is the medicine this file prescribes to everyone else. Three findings survive as tests because the first live runs produced them: - xAI's `grok-3-mini` was filed under Groq — one letter apart, and xAI's URL sat 6 lines from the pin while Groq's sat 18. Attribution now follows the marker ABOVE the pin, since that is the branch that owns it, and vendors we cannot query still own their ids so they report UNCHECKED instead of being absorbed by the nearest vendor we can. - fleetcrown's computed `${link.provider.id}/${link.model}` was reported retired. An interpolated id is the absence of a pin. - botsmann's pin lives in lib/constants.ts, which no AI-shaped filename filter opens. Candidates are now two-tiered and ranked, and truncation is REPORTED — a bounded sweep that stays quiet about what it skipped reads exactly like one that found nothing. An unreadable catalogue reports UNCHECKED, never GONE. Treating "I could not look" as "nothing is there" would mark every pin retired and invent a fleet-wide outage somebody then acts on. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 10 + .github/workflows/model-pins.yml | 95 +++++ SHARED.md | 7 +- scripts/ci/model-pin-audit.mjs | 565 ++++++++++++++++++++++++++++ scripts/ci/test-model-pin-audit.mjs | 272 +++++++++++++ 5 files changed, 947 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/model-pins.yml create mode 100755 scripts/ci/model-pin-audit.mjs create mode 100755 scripts/ci/test-model-pin-audit.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6af7d9..afed102 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,16 @@ jobs: - name: The auto-merge sweep behaves run: bash scripts/ci/test-auto-merge-sweep.sh + # This detector reports on repos nobody is watching, so its own failure + # mode is silence: a clean report from a broken audit is worse than no + # audit, because it prints a ✓. Both sides are pinned — the real AOZ + # regression is still caught, corrected code stays quiet — plus the two + # false positives the first live run produced (xAI's `grok-3-mini` filed + # under Groq, and a computed `${...}` id read as a pin). No network, no + # key, no checkout, so it runs here as well as in the daily sweep. + - name: The model-pin audit still detects, and still stays quiet + run: node scripts/ci/test-model-pin-audit.mjs + # Drift guard. The fleet is on v7; templates handing out v4 is exactly # how this repo fell behind the repos it governs. - name: No stale action versions diff --git a/.github/workflows/model-pins.yml b/.github/workflows/model-pins.yml new file mode 100644 index 0000000..be1a042 --- /dev/null +++ b/.github/workflows/model-pins.yml @@ -0,0 +1,95 @@ +# Is any model id this fleet pins still served by its vendor? +# +# On 2026-08-26 the AOZ assistant reported "KI-Assistent nicht konfiguriert" on +# a deployment whose key was valid. Groq had retired the whole llama-3.x family. +# Production had been failing exactly as long as the demo, and nobody knew, +# because "not configured" is the only thing the app can say. +# +# Six pins across five repos were dead the same morning. None of it was +# detectable from inside a repo: every gate was green, because the code is +# correct and the vendor changed underneath it. +# +# Daily, not weekly. A retirement is decided by someone outside this fleet and +# lands without warning — unlike the UI defects next door, which arrive with a +# design change. The check costs ZERO tokens (one GET /models per vendor), so +# the only argument against running it often is noise, and a silent day is one +# line in a job summary. +name: Model pins + +on: + schedule: + # 07:10 UTC, before the working day — a retirement found at 09:00 is a + # morning's work; one found by a user is an outage of unknown age. + - cron: '10 7 * * *' + workflow_dispatch: + inputs: + strict: + description: 'Fail the run when pins are retired' + type: boolean + default: false + pull_request: + paths: + - 'scripts/ci/model-pin-audit.mjs' + - 'scripts/ci/test-model-pin-audit.mjs' + - '.github/workflows/model-pins.yml' + +permissions: + contents: read + +jobs: + audit: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 22 + + # The detector's own test needs no network, no key and no checkout, so it + # runs on pull_request too — where the sweep below cannot, because a fork + # has no secrets and an unreadable catalogue is deliberately not a pass. + - name: Self-test the detector + run: node scripts/ci/test-model-pin-audit.mjs + + # dotfiles has no package.json on purpose. Install ai-ration into a + # scratch dir and point the audit at it — the same shape ui-defects.yml + # uses for playwright. The vendor query lives in ai-ration precisely so + # this repo does not grow a second copy of it. + - name: Install ai-ration + if: github.event_name != 'pull_request' + run: | + mkdir -p "$RUNNER_TEMP/air" && cd "$RUNNER_TEMP/air" + npm init -y >/dev/null + npm i --no-audit --no-fund github:maonakamoto/ai-ration#v0.2.1 >/dev/null + + - name: Audit the fleet's pins + if: github.event_name != 'pull_request' + env: + AI_RATION_FROM: ${{ runner.temp }}/air + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Absent keys are handled, not fatal: the vendor is reported UNCHECKED + # rather than clean. "I could not look" is not "nothing is wrong". + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: | + set -uo pipefail + flag=--warn-only + if [ "${{ inputs.strict }}" = "true" ]; then flag=""; fi + + # Into the job summary, not just the log. A finding nobody scrolls to + # is the same as no finding — which is how eight days passed. + { + echo '## Model pins' + echo + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + set +e + node scripts/ci/model-pin-audit.mjs $flag 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + status=${PIPESTATUS[0]} + set -e + + echo '```' >> "$GITHUB_STEP_SUMMARY" + exit "$status" diff --git a/SHARED.md b/SHARED.md index 6c602da..c4f43ae 100644 --- a/SHARED.md +++ b/SHARED.md @@ -28,7 +28,9 @@ the inventory underneath it is **generated**, and the number it produces is a | [`limitkit`](https://github.com/maonakamoto/limitkit) | `npm i github:maonakamoto/limitkit#v0.1.0` | the fleet's **12 hand-rolled rate limiters** (this file's own "next extraction" row). Sliding/fixed windows over an injectable two-method `Store`; **bounded** memory default (the unbounded-Map leak is impossible by construction); standard `X-RateLimit-*` + `Retry-After` headers — what orangecat's ADR-0002 specified seven months before anything enforced it; `clientIp()`. Refusals count nothing, so a hammered key recovers. Ships no middleware and **no limit values** — how many attempts a route allows is app semantics, asserted locally. | **Adopted:** `ai-forms` — fleetcrown, evig, aoz-housing, surf-your-life. -`ai-ration` — fleetcrown. `threadkit` — **nobody yet**. +`ai-ration` — fleetcrown, **and this repo** (`model-pin-audit.mjs` calls +`checkCatalog`; the audit needed exactly the vendor query the package owns, so +writing a second one here would have been this file's own sin). `threadkit` — **nobody yet**. `limitkit` — fleetcrown (proving consumer; its old limiter had the unbounded Map). **Next adopter should be orangecat** — it closes ADR-0002 by making its Upstash client a 12-line `Store` adapter and deleting three of its four @@ -73,6 +75,7 @@ fleet-wide checker. | Script | Answers | |---|---| | `scripts/ci/verify-floor-audit.sh` | does every repo's `verify` actually run lint + typecheck + test? | +| `scripts/ci/model-pin-audit.mjs` | is any model id the fleet pins no longer served by its vendor? Zero tokens — one `GET /models` per vendor — so it runs DAILY. Uses `ai-ration`'s `checkCatalog` rather than a second vendor query. Self-tested by `scripts/ci/test-model-pin-audit.mjs`, which pins both sides and both real defects the first run produced: xAI's `grok-3-mini` misfiled under Groq, and a computed `${...}` id reported as a pin. | | `scripts/ci/ui-defect-audit.mjs` | do any live sites ship an interactive label below its WCAG AA floor, or a stack whose rows start at different x? Renders each site; no repo checkout involved. Self-tested by `scripts/ci/test-ui-defect-audit.mjs`, which pins BOTH sides — the real defect is still caught, correct markup stays silent. | Both report into a weekly workflow's job summary rather than only a log. @@ -86,7 +89,7 @@ Ranked by (copies × how identical the logic is). Counts from |---|---|---| | `auto-merge-sweep.sh` | ~~22~~ **6** | **EXTRACTED 2026-08-16/20.** Sixteen repos call the canonical as a reusable workflow, each verified to actually *run* it (a sweep that fails to start looks exactly like one with nothing to do). The six remaining are deliberate: dotfiles is the canonical home and runs it directly; ai-forms, datacat, petvity, solon had dirty working trees owned by other sessions when swept — convert when clear. The two repos that had ever *tested* their copies (evig, orangecat) had that coverage ported into the canonical suite **before** deletion: 17 cases, mutation-proven. | | rate limiting | **14 → adopting** | **Extracted 2026-08-20 as [`limitkit`](https://github.com/maonakamoto/limitkit)** (see registry above). fleetcrown converted as the proving consumer; 13 files remain across 8 repos, orangecat first in line (its ADR-0002 becomes a 12-line `Store` adapter + three deletions). The ratchet holds the count until each adoption lands. | -| AI provider client | **16** | evig 7, orangecat 5. `ai-ration` already owns the hard part (chain, 429, budget); these are the callers. | +| AI provider client | **16** | evig 7, orangecat 5. `ai-ration` already owns the hard part (chain, 429, budget); these are the callers. **Priced 2026-08-26:** Groq retired the llama-3.x family and six pins across botsmann, evig, kivvi, orangecat and truthseeker died at once, with three deployed apps failing live and AOZ reporting a missing key it already had. Every one of those repos hand-rolls this layer; fleetcrown, which adopted the package, was unaffected. | | logger | **10** | sbb-lost-found alone has 4. | | health route | **8** | Identical shape in 8 repos; a 20-line contract. | | ~~`@ai-native-cms/core`~~ | — | **Withdrawn — measurement error.** `maonakamoto/revampit` *redirects* to `maonakamoto/evig` (renamed in the pivot); the "two repos" with byte-identical trees were two clones of ONE repo. Nothing to extract. Two directories are not two repos: check `git remote -v` before reporting cross-repo duplication. | diff --git a/scripts/ci/model-pin-audit.mjs b/scripts/ci/model-pin-audit.mjs new file mode 100755 index 0000000..ca9d310 --- /dev/null +++ b/scripts/ci/model-pin-audit.mjs @@ -0,0 +1,565 @@ +#!/usr/bin/env node +/** + * Fleet audit: is any model id this fleet pins no longer served? + * + * WHY THIS EXISTS + * --------------- + * On 2026-08-26 the AOZ assistant answered "KI-Assistent nicht konfiguriert. + * Bitte GROQ_API_KEY setzen" on a deployment whose key was valid. Groq had + * retired the whole llama-3.x family; the pinned `llama-3.3-70b-versatile` + * returned 404 model_not_found. Production had been broken as long as the demo + * and nobody knew, because the only thing the app can say is "not configured". + * + * That is not an AOZ bug. Every Groq id pinned anywhere in the fleet was dead + * the same morning — llama-3.1-8b-instant, llama3-8b-8192, gemma2-9b-it, + * mixtral-8x7b-32768, llama-3.2-3b-instruct — across eleven repos, with three + * deployed apps failing live. + * + * A pinned free model is not a configuration, it is a scheduled outage. The + * schedule is set by the vendor and nobody here is told. So this asks the only + * authority that knows: the vendor's own catalogue. + * + * WHY A CENTRAL SCRIPT AND NOT A CHECK PER REPO + * --------------------------------------------- + * Same doctrine as verify-floor-audit.sh and ui-defect-audit.mjs. A check + * copied into twenty repos drifts into twenty versions — SHARED.md counts the + * bill for exactly that habit. More to the point, a per-repo check only ever + * runs in repos somebody still touches, and the repos that rot quietly are + * precisely the ones nobody touches. This one needs no adoption at all. + * + * WHY IT REUSES ai-ration + * ----------------------- + * `checkCatalog` already answers this, already distinguishes the three states + * that matter, and already carries the scars — its own docstring records four + * of nine default pins gone and a consumer silently failing for eight days. + * Writing a second vendor query here would be this repo committing the sin it + * exists to police. It takes the chain as an ARGUMENT, so it generalises to + * arbitrary ids with no change to the package. + * + * ZERO TOKENS, WHICH IS THE WHOLE POINT + * ------------------------------------- + * One GET /models per vendor. No completion, no spend. That is the difference + * between a check that runs on a timer and a command somebody is supposed to + * remember — and "supposed to remember" is what failed for eight days. + * + * WHAT IT DOES NOT PROVE + * ---------------------- + * That a listed model WORKS. Existence is cheap; capability is not. A model + * can be listed and still refuse tool calls — of nine free models probed for + * ai-ration's default chain, five answered only via a text protocol. If the + * surface is a tool loop, probe with a real tool call before pinning. This + * audit catches the retirement, not the mismatch. + * + * It also judges only ids it can ATTRIBUTE to a vendor it queried. An + * unattributed id is reported and not judged, because "I could not look" and + * "it is gone" are different answers and collapsing them invents outages. + * + * Usage: + * node scripts/ci/model-pin-audit.mjs # audit, exit 1 on dead pins + * node scripts/ci/model-pin-audit.mjs --warn-only # report, always exit 0 + * node scripts/ci/model-pin-audit.mjs --local # scan ~/dev checkouts + * + * Env: GH_OWNER (default maonakamoto), GH_LIMIT (default 100), + * FLEET_ROOT (default ~/dev, --local only), + * AI_RATION_FROM (path to a repo that installs ai-ration), + * GROQ_API_KEY / OPENROUTER_API_KEY (to read the catalogues). + */ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; + +const exec = promisify(execFile); + +const WARN_ONLY = process.argv.includes("--warn-only"); +const LOCAL = process.argv.includes("--local"); +const OWNER = process.env.GH_OWNER ?? "maonakamoto"; +const GH_LIMIT = process.env.GH_LIMIT ?? "100"; +const FLEET_ROOT = process.env.FLEET_ROOT ?? join(homedir(), "dev"); + +/** + * The vendors whose catalogue we can actually read, and the markers that tie a + * pin to one of them. + * + * Attribution is by MARKER rather than by the shape of the id, because the shape + * lies: `openai/gpt-oss-20b` is a Groq model id AND an OpenRouter routing id, + * and at OpenRouter the missing `:free` suffix is the difference between free + * and billed. Only the surrounding code knows which vendor is meant. + */ +export const VENDORS = [ + { + id: "groq", + queryable: true, + baseUrl: "https://api.groq.com/openai/v1", + keyEnv: "GROQ_API_KEY", + markers: [/groq/i], + }, + { + id: "openrouter", + queryable: true, + baseUrl: "https://openrouter.ai/api/v1", + keyEnv: "OPENROUTER_API_KEY", + markers: [/openrouter/i], + }, + // Not queryable here — no catalogue call is wired for these. They are listed + // so their ids are ATTRIBUTED and reported unchecked, rather than falling to + // whichever queryable vendor happens to sit nearest in the file. Markers are + // deliberately specific: bare /openai/ would match Groq's own + // `api.groq.com/openai/v1` path and every `openai/gpt-oss-*` id it serves. + { id: "xai", queryable: false, markers: [/api\.x\.ai/i, /\bXAI_API_KEY\b/, /\bgrok\b/i] }, + { id: "anthropic", queryable: false, markers: [/api\.anthropic\.com/i, /\bANTHROPIC_API_KEY\b/] }, + { id: "openai", queryable: false, markers: [/api\.openai\.com/i, /\bOPENAI_API_KEY\b/] }, + { id: "google", queryable: false, markers: [/generativelanguage\.googleapis/i, /\bGEMINI_API_KEY\b/] }, +]; + +/** + * Files worth opening, in two tiers. + * + * LIKELY names an AI module outright. POSSIBLE is the long tail that a + * name-based filter misses: botsmann keeps its model id in `lib/constants.ts`, + * which mentions no vendor in its path and was invisible to the first version + * of this filter. A pin does not have to live in a file called `provider.ts`. + */ +const LIKELY_PATH = + /(^|\/)\.env\.example$|(^|\/)env\.(ts|js|mjs)$|(^|\/)(lib|src|app|apps|packages|config)\/.*(provider|model|llm|chat|ai|openai|anthropic)[^/]*\.(ts|js|mjs)$/i; + +const POSSIBLE_PATH = + /(^|\/)(lib|src|app|apps|packages|config)\/.*(constants?|config|settings|defaults)[^/]*\.(ts|js|mjs)$/i; + +const CANDIDATE_PATH = new RegExp(`(${LIKELY_PATH.source})|(${POSSIBLE_PATH.source})`, "i"); + +/** Never open these, whatever they are named. */ +const SKIP_PATH = + /(^|\/)(node_modules|dist|build|\.next|coverage|__tests__|__fixtures__)\/|(^|\/)\.claude\/worktrees\//; + +const MAX_FILES_PER_REPO = 90; + +/** A vendor named 40+ lines from a pin is not describing that pin. */ +const MAX_ATTRIBUTION_DISTANCE = 40; + +// ── Extraction ─────────────────────────────────────────────────────────────── + +/** + * Is this string plausibly a model id rather than any other quoted thing? + * + * Deliberately permissive on shape and strict on the obvious negatives. A false + * POSITIVE costs one line in a report that says "not judged"; a false NEGATIVE + * is the outage this whole file exists to prevent. + */ +export function looksLikeModelId(s) { + if (typeof s !== "string") return false; + if (s.length < 3 || s.length > 80) return false; + if (/\s/.test(s)) return false; + if (s.includes("${")) return false; // interpolated: resolved at runtime, not pinned + if (/^https?:/i.test(s)) return false; + if (/^[./~@]/.test(s)) return false; + if (/\.(ts|tsx|js|mjs|cjs|json|css|scss|md|png|jpe?g|svg|ico|txt|ya?ml)$/i.test(s)) return false; + if (/^[A-Z][A-Z0-9_]*$/.test(s)) return false; // SCREAMING_CASE is an env name + // Model ids essentially always carry a version digit or a vendor/ prefix. + return /\d/.test(s) || s.includes("/"); +} + +/** + * Pull candidate model ids out of one file's text, with the line each sits on. + * + * Three shapes cover how this fleet writes them: + * 1. a `model:` / `models:` assignment, single value or array + * 2. a *_MODEL constant or Zod `.default(...)` + * 3. a bare `GROQ_MODEL=...` line in a .env file + */ +export function extractPins(text) { + const found = new Map(); // id -> line number (first sighting) + const lineOf = (index) => text.slice(0, index).split("\n").length; + + const remember = (id, index) => { + if (!looksLikeModelId(id)) return; + if (!found.has(id)) found.set(id, lineOf(index)); + }; + + // 1. model: 'x' | models: ['a', 'b'] + for (const m of text.matchAll(/\bmodels?\s*[:=]\s*(\[[\s\S]{0,400}?\]|['"`][^'"`\n]{0,120}['"`])/gi)) { + for (const q of m[1].matchAll(/['"`]([^'"`\n]+)['"`]/g)) remember(q[1], m.index); + } + + // 2. GROQ_MODEL: z.string().default('x') | const DEFAULT_MODEL = 'x' + for (const m of text.matchAll( + /\b([A-Za-z][A-Za-z0-9_]*MODEL[A-Za-z0-9_]*)\s*[:=][^\n]{0,80}?['"`]([^'"`\n]+)['"`]/g, + )) { + remember(m[2], m.index); + } + + // 3. .env style, unquoted or quoted, no code around it + for (const m of text.matchAll(/^[ \t]*(?:export[ \t]+)?[A-Z][A-Z0-9_]*MODEL[A-Z0-9_]*\s*=\s*["']?([^"'\s#]+)/gm)) { + remember(m[1], m.index); + } + + return [...found].map(([id, line]) => ({ id, line })); +} + +/** + * Which vendor does this pin belong to? + * + * Nearest-marker first: a file can legitimately name both vendors — AOZ's + * provider.ts resolves Groq AND OpenRouter in one module — so a file-wide vote + * would attribute both vendors' pins to whichever appeared more. The pin's own + * neighbourhood is what actually says which branch it is in. Only when the + * window is silent do we fall back to the file, and only when the file names + * exactly one vendor. + * + * Returns a vendor id, or null for "cannot tell" — which is reported, not judged. + */ +export function attribute(text, line, vendors = VENDORS, maxDistance = MAX_ATTRIBUTION_DISTANCE) { + const lines = text.split("\n"); + + /** + * Distance to this vendor's nearest mention ABOVE the pin, and below it. + * + * Above is what decides. Every provider module in this fleet is written as + * `if (provider === "groq") { url = ...; body = { model: "..." } }` — the + * branch that owns a model literal always opens above it. Ranking by raw + * proximity instead put `grok-3-mini` (xAI, line 89) with Groq, whose URL sat + * at line 71, and tied kivvi's real Groq pin exactly between two vendors. + */ + const distances = (vendor) => { + let above = Infinity; + let below = Infinity; + for (let i = 0; i < lines.length; i++) { + if (!vendor.markers.some((re) => re.test(lines[i]))) continue; + const d = i + 1 - line; + if (d <= 0) above = Math.min(above, -d); + else below = Math.min(below, d); + } + return { above, below }; + }; + + const scored = vendors.map((v) => ({ id: v.id, ...distances(v) })); + + const fromAbove = scored.filter((s) => s.above <= maxDistance).sort((a, b) => a.above - b.above); + if (fromAbove.length === 1) return fromAbove[0].id; + if (fromAbove.length > 1 && fromAbove[0].above < fromAbove[1].above) return fromAbove[0].id; + + // Nothing above governs it — a pin at the top of a file, or a config block + // whose vendor is named afterwards. Fall back to the nearest mention below. + if (fromAbove.length === 0) { + const fromBelow = scored.filter((s) => s.below <= maxDistance).sort((a, b) => a.below - b.below); + if (fromBelow.length === 1) return fromBelow[0].id; + if (fromBelow.length > 1 && fromBelow[0].below < fromBelow[1].below) return fromBelow[0].id; + } + + const named = vendors.filter((v) => v.markers.some((re) => re.test(text))); + return named.length === 1 ? named[0].id : null; +} + +/** + * Turn per-file findings into per-vendor id lists plus the unattributable rest. + * Pure, so the self-test can drive it without a network or a checkout. + */ +export function collate(findings) { + const byVendor = new Map(); + const unattributed = []; + + for (const f of findings) { + if (!f.vendor) { + unattributed.push(f); + continue; + } + if (!byVendor.has(f.vendor)) byVendor.set(f.vendor, []); + byVendor.get(f.vendor).push(f); + } + return { byVendor, unattributed }; +} + +/** + * Apply catalogue verdicts to the findings. + * + * `live` is a Map of vendorId -> Set of ids, or null for a vendor whose + * catalogue could not be read. Null is NOT an empty set: treating "I could not + * look" as "nothing is there" reports every pin as retired and invents an + * outage, which is worse than silence because someone acts on it. + */ +export function judge(findings, live) { + return findings.map((f) => { + if (!f.vendor) return { ...f, state: "unattributed" }; + const set = live.get(f.vendor); + if (!set) return { ...f, state: "unchecked" }; + return { ...f, state: set.has(f.id) ? "ok" : "gone" }; + }); +} + +// ── Reading the fleet ──────────────────────────────────────────────────────── + +/** + * Which files get opened when a repo has more candidates than the cap allows. + * + * Ranked, never arbitrary: a file that names a vendor or an AI concern outranks + * a generic `config.ts`, so the cap sheds the least likely candidates first. + * And every truncation is RECORDED — a bounded sweep that stays quiet about + * what it skipped reads exactly like a sweep that found nothing. + */ +const truncated = []; + +function rank(paths, repoName) { + const sorted = [...paths].sort((a, b) => { + const ta = LIKELY_PATH.test(a) ? 0 : 1; + const tb = LIKELY_PATH.test(b) ? 0 : 1; + return ta - tb || a.length - b.length; + }); + if (sorted.length > MAX_FILES_PER_REPO) { + // What was dropped matters more than how many. Everything in the LIKELY + // tier is opened first, so a truncation that sheds only generic config + // files has not touched the audit's real coverage — and saying which it was + // is the difference between a caveat and an alarm. + const likelyDropped = sorted.slice(MAX_FILES_PER_REPO).filter((p) => LIKELY_PATH.test(p)).length; + truncated.push({ repo: repoName, seen: sorted.length, opened: MAX_FILES_PER_REPO, likelyDropped }); + } + return sorted.slice(0, MAX_FILES_PER_REPO); +} + +async function gh(args) { + const { stdout } = await exec("gh", args, { maxBuffer: 64 * 1024 * 1024 }); + return stdout; +} + +/** Remote, default-branch view. Clones drift; this repo has been bitten by that. */ +async function remoteRepos() { + const raw = await gh([ + "repo", "list", OWNER, + "--limit", String(GH_LIMIT), + "--no-archived", + "--json", "name,defaultBranchRef,isFork", + ]); + return JSON.parse(raw) + .filter((r) => !r.isFork && r.defaultBranchRef?.name) + .map((r) => ({ name: r.name, branch: r.defaultBranchRef.name })); +} + +async function remoteFiles(repo) { + let tree; + try { + tree = JSON.parse( + await gh(["api", `repos/${OWNER}/${repo.name}/git/trees/${repo.branch}?recursive=1`]), + ); + } catch { + return []; // empty repo, or no access — not a finding + } + const all = (tree.tree ?? []) + .filter((n) => n.type === "blob" && !SKIP_PATH.test("/" + n.path) && CANDIDATE_PATH.test(n.path)) + .map((n) => n.path); + const paths = rank(all, repo.name); + + const out = []; + for (const path of paths) { + try { + const body = JSON.parse( + await gh(["api", `repos/${OWNER}/${repo.name}/contents/${path}?ref=${repo.branch}`]), + ); + if (body.encoding !== "base64" || !body.content) continue; + out.push({ path, text: Buffer.from(body.content, "base64").toString("utf8") }); + } catch { + /* a path that vanished between tree and read is not a finding */ + } + } + return out; +} + +function localRepos() { + return readdirSync(FLEET_ROOT) + .filter((n) => !n.startsWith("_") && existsSync(join(FLEET_ROOT, n, ".git"))) + .map((name) => ({ name, branch: "(local)" })); +} + +function walk(dir, depth, acc) { + if (depth < 0 || acc.length >= MAX_FILES_PER_REPO * 6) return acc; + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return acc; + } + for (const e of entries) { + const full = join(dir, e.name); + if (e.isDirectory()) { + if (/^(node_modules|dist|build|\.next|coverage|\.git|\.claude)$/.test(e.name)) continue; + walk(full, depth - 1, acc); + } else if (CANDIDATE_PATH.test(full) && !SKIP_PATH.test(full)) { + try { + if (statSync(full).size < 400_000) acc.push(full); + } catch { /* raced */ } + } + } + return acc; +} + +function localFiles(repo) { + const root = join(FLEET_ROOT, repo.name); + const all = walk(root, 6, []).map((full) => full.slice(root.length + 1)); + return rank(all, repo.name).map((path) => ({ + path, + text: readFileSync(join(root, path), "utf8"), + })); +} + +// ── The catalogue, via ai-ration ───────────────────────────────────────────── + +function loadAiRation() { + const candidates = [ + process.env.AI_RATION_FROM, + join(homedir(), "dev", "fleetcrown"), + join(homedir(), "dev", "ai-ration"), + ].filter(Boolean); + + for (const root of candidates) { + for (const entry of [ + join(root, "node_modules", "ai-ration", "dist", "index.js"), + join(root, "dist", "index.js"), + ]) { + if (existsSync(entry)) return import(entry); + } + } + console.error( + "✗ ai-ration not found. Set AI_RATION_FROM=/path/to/a/repo that installs it,\n" + + " or build it once: (cd ~/dev/ai-ration && npm i && npm run build)", + ); + process.exit(2); +} + +/** + * Ask each vendor what it still lists, for exactly the ids we found. + * Returns Map(vendorId -> Set|null), where null means "could not look". + */ +async function readCatalogues(byVendor, checkCatalog) { + const live = new Map(); + for (const vendor of VENDORS) { + if (!vendor.queryable) continue; + const findings = byVendor.get(vendor.id); + if (!findings?.length) continue; + + const ids = [...new Set(findings.map((f) => f.id))]; + const [verdict] = await checkCatalog([ + { + id: vendor.id, + baseUrl: vendor.baseUrl, + keyEnv: vendor.keyEnv, + models: ids, + dailyTokens: 0, + }, + ]); + live.set(vendor.id, verdict.live ? new Set(verdict.live) : null); + } + return live; +} + +// ── Report ─────────────────────────────────────────────────────────────────── + +function report(judged) { + const gone = judged.filter((j) => j.state === "gone"); + const unchecked = judged.filter((j) => j.state === "unchecked"); + const unattributed = judged.filter((j) => j.state === "unattributed"); + const ok = judged.filter((j) => j.state === "ok"); + + const lines = []; + + if (gone.length) { + lines.push("RETIRED — the vendor no longer lists these, so every call using them fails:"); + const byId = new Map(); + for (const g of gone) { + const key = `${g.vendor}/${g.id}`; + if (!byId.has(key)) byId.set(key, []); + byId.get(key).push(`${g.repo}:${g.path}:${g.line}`); + } + for (const [key, sites] of [...byId].sort()) { + lines.push(` GONE ${key}`); + for (const s of sites.sort()) lines.push(` ${s}`); + } + lines.push(""); + } + + if (unchecked.length) { + const vendors = [...new Set(unchecked.map((u) => u.vendor))].sort(); + lines.push( + `${unchecked.length} pin(s) UNCHECKED — no readable catalogue for: ${vendors.join(", ")}.`, + ); + lines.push(" That is not a pass for them. Set the vendor key to judge these."); + lines.push(""); + } + + if (unattributed.length) { + const ids = [...new Set(unattributed.map((u) => u.id))].sort(); + lines.push(`${unattributed.length} pin(s) not attributable to a vendor we query — listed, not judged:`); + for (const id of ids.slice(0, 20)) lines.push(` ? ${id}`); + if (ids.length > 20) lines.push(` ? … and ${ids.length - 20} more`); + lines.push(""); + } + + lines.push( + `${ok.length} pin(s) confirmed live · ${gone.length} retired · ` + + `${unchecked.length} unchecked · ${unattributed.length} unattributed`, + ); + + if (truncated.length) { + lines.push(""); + const blind = truncated.filter((t) => t.likelyDropped > 0); + lines.push(`COVERAGE — ${truncated.length} repo(s) had more candidate files than the cap of ${MAX_FILES_PER_REPO}:`); + for (const t of truncated) { + const tail = t.likelyDropped > 0 ? `, ${t.likelyDropped} of them likely-AI` : ", none likely-AI"; + lines.push(` ${t.repo}: opened ${t.opened} of ${t.seen}${tail}`); + } + lines.push( + blind.length + ? " Files naming an AI concern were dropped — raise MAX_FILES_PER_REPO." + : " Only generic config files were dropped; every likely-AI file was opened.", + ); + } + + if (gone.length) { + lines.push(""); + lines.push("A pin is a scheduled outage. The durable fix is a chain across VENDORS —"); + lines.push("see SHARED.md → ai-ration. Repinning buys time until the next retirement."); + } + + return lines.join("\n"); +} + +// ── Main ───────────────────────────────────────────────────────────────────── + +export async function main() { + const { checkCatalog } = await loadAiRation(); + + const repos = LOCAL ? localRepos() : await remoteRepos(); + const findings = []; + + for (const repo of repos) { + const files = LOCAL ? localFiles(repo) : await remoteFiles(repo); + for (const file of files) { + for (const pin of extractPins(file.text)) { + findings.push({ + repo: repo.name, + path: file.path, + line: pin.line, + id: pin.id, + vendor: attribute(file.text, pin.line), + }); + } + } + } + + const { byVendor } = collate(findings); + const live = await readCatalogues(byVendor, checkCatalog); + const judged = judge(findings, live); + + console.log(report(judged)); + console.log(`\ninspected ${repos.length} repo(s)${LOCAL ? " (local checkouts)" : " on their default branches"}`); + + const dead = judged.some((j) => j.state === "gone"); + process.exit(dead && !WARN_ONLY ? 1 : 0); +} + +const invokedDirectly = + process.argv[1] && import.meta.url === `file://${process.argv[1]}`; +if (invokedDirectly) { + main().catch((err) => { + console.error(`✗ audit failed: ${err?.message ?? err}`); + process.exit(2); + }); +} diff --git a/scripts/ci/test-model-pin-audit.mjs b/scripts/ci/test-model-pin-audit.mjs new file mode 100755 index 0000000..d050b08 --- /dev/null +++ b/scripts/ci/test-model-pin-audit.mjs @@ -0,0 +1,272 @@ +#!/usr/bin/env node +/** + * Self-test for model-pin-audit.mjs. + * Run: node scripts/ci/test-model-pin-audit.mjs + * + * A detector that has quietly stopped detecting reports a clean fleet, and a + * clean report from a broken detector is worse than no report — it is an absent + * check that prints a ✓. So every fixture asserts a VERDICT, and both sides are + * pinned: + * + * - the real AOZ regression is still caught (positive) + * - the corrected file stays silent (negative) + * - quoted things that are not model ids stay out of the report + * - an unreadable catalogue reports UNCHECKED, never GONE + * + * That last one is the expensive mistake. Treating "I could not look" as + * "nothing is there" marks every pin retired and invents a fleet-wide outage + * that somebody will act on. Silence is recoverable; a false alarm at this + * scale is not. + * + * Fixtures are inline strings — no network, no gh, no checkout, no key. + */ +import { extractPins, attribute, collate, judge, looksLikeModelId } from "./model-pin-audit.mjs"; + +let failures = 0; +function check(name, actual, expected) { + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + if (a === e) { + console.log(` ok ${name}`); + } else { + console.log(` FAIL ${name}\n expected ${e}\n actual ${a}`); + failures++; + } +} + +// ── Fixtures ───────────────────────────────────────────────────────────────── + +/** aoz-housing/src/lib/env.ts as it stood when both AI surfaces went down. */ +const AOZ_BROKEN = ` +import { z } from 'zod' + +const schema = z.object({ + DATABASE_URL: z.string().url(), + CRON_SECRET: z.string().min(16).optional(), + + // AI: whichever key is set decides the provider. + GROQ_API_KEY: z.string().optional(), + GROQ_MODEL: z.string().default('llama-3.3-70b-versatile'), + OPENROUTER_API_KEY: z.string().optional(), + OPENROUTER_MODEL: z.string().default('openai/gpt-oss-20b:free'), +}) +`; + +/** The same file after the fix. Must produce no findings at all. */ +const AOZ_FIXED = AOZ_BROKEN.replace("llama-3.3-70b-versatile", "openai/gpt-oss-120b"); + +/** aoz-housing/src/lib/ai/provider.ts — one module, BOTH vendors named. */ +const TWO_VENDOR_FILE = ` +const GROQ_API_URL = 'https://api.groq.com/openai/v1/chat/completions' +const OPENROUTER_API_URL = 'https://openrouter.ai/api/v1/chat/completions' + +export async function getConfig() { + if (provider === 'groq') { + return { url: GROQ_API_URL, model: 'llama-3.1-8b-instant' } + } + return { url: OPENROUTER_API_URL, model: 'meta-llama/llama-3.3-70b-instruct:free' } +} +`; + +/** Quoted strings that must never be mistaken for model ids. */ +const NOISE = ` +import { BRAND } from '@/lib/config/brand' +const path = './lib/ai/provider.ts' +const url = 'https://api.groq.com/openai/v1/chat/completions' +const key = 'GROQ_API_KEY' +const model = 'llama-3.1-8b-instant' +const style = 'rounded-lg' +`; + +// ── looksLikeModelId ───────────────────────────────────────────────────────── + +console.log("looksLikeModelId"); +check("accepts a versioned id", looksLikeModelId("llama-3.3-70b-versatile"), true); +check("accepts a routed id", looksLikeModelId("openai/gpt-oss-20b:free"), true); +check("rejects a relative path", looksLikeModelId("./lib/ai/provider.ts"), false); +check("rejects a url", looksLikeModelId("https://api.groq.com/openai/v1"), false); +check("rejects an env var name", looksLikeModelId("GROQ_API_KEY"), false); +check("rejects a css class", looksLikeModelId("rounded-lg"), false); +check("rejects a module specifier", looksLikeModelId("@/lib/config/brand"), false); +// fleetcrown builds its model id from ai-ration's chain at call time. A +// computed id is the ABSENCE of a pin, and the first run reported it retired. +check("rejects an interpolated id", looksLikeModelId("${link.provider.id}/${link.model}"), false); + +// ── extraction ─────────────────────────────────────────────────────────────── + +console.log("\nextractPins"); +check( + "finds both pins in the broken AOZ env schema", + extractPins(AOZ_BROKEN).map((p) => p.id).sort(), + ["llama-3.3-70b-versatile", "openai/gpt-oss-20b:free"], +); +check( + "finds the model literals in a two-vendor provider", + extractPins(TWO_VENDOR_FILE).map((p) => p.id).sort(), + ["llama-3.1-8b-instant", "meta-llama/llama-3.3-70b-instruct:free"], +); +check( + "picks the model id out of a file full of other quoted strings", + extractPins(NOISE).map((p) => p.id), + ["llama-3.1-8b-instant"], +); + +// ── attribution ────────────────────────────────────────────────────────────── + +console.log("\nattribute"); +{ + const pins = extractPins(AOZ_BROKEN); + const groq = pins.find((p) => p.id === "llama-3.3-70b-versatile"); + const or = pins.find((p) => p.id === "openai/gpt-oss-20b:free"); + check("GROQ_MODEL line attributes to groq", attribute(AOZ_BROKEN, groq.line), "groq"); + check("OPENROUTER_MODEL line attributes to openrouter", attribute(AOZ_BROKEN, or.line), "openrouter"); +} +{ + // The case a file-wide vote gets wrong: both vendors named in one module, so + // only the pin's own neighbourhood says which branch it belongs to. + const pins = extractPins(TWO_VENDOR_FILE); + const g = pins.find((p) => p.id === "llama-3.1-8b-instant"); + const o = pins.find((p) => p.id === "meta-llama/llama-3.3-70b-instruct:free"); + check("nearest marker wins for the groq branch", attribute(TWO_VENDOR_FILE, g.line), "groq"); + check("nearest marker wins for the openrouter branch", attribute(TWO_VENDOR_FILE, o.line), "openrouter"); +} +check( + "a pin with no vendor anywhere is not attributed", + attribute("const model = 'some-model-9b'", 1), + null, +); + +// ── judging ────────────────────────────────────────────────────────────────── + +console.log("\njudge"); +const findingsFrom = (repo, path, text) => + extractPins(text).map((p) => ({ + repo, + path, + line: p.line, + id: p.id, + vendor: attribute(text, p.line), + })); + +const GROQ_LIVE = new Set(["openai/gpt-oss-120b", "openai/gpt-oss-20b", "qwen/qwen3.8-27b"]); +const OR_LIVE = new Set(["openai/gpt-oss-20b:free"]); + +{ + const findings = findingsFrom("aoz-housing", "src/lib/env.ts", AOZ_BROKEN); + const live = new Map([["groq", GROQ_LIVE], ["openrouter", OR_LIVE]]); + const judged = judge(findings, live); + + check( + "THE REGRESSION: the retired llama pin is caught", + judged.filter((j) => j.state === "gone").map((j) => j.id), + ["llama-3.3-70b-versatile"], + ); + check( + "the still-served openrouter pin beside it stays quiet", + judged.filter((j) => j.state === "ok").map((j) => j.id), + ["openai/gpt-oss-20b:free"], + ); +} + +{ + // The negative that matters most: correct code must produce a silent report, + // or the audit trains people to ignore it. + const findings = findingsFrom("aoz-housing", "src/lib/env.ts", AOZ_FIXED); + const live = new Map([["groq", GROQ_LIVE], ["openrouter", OR_LIVE]]); + const judged = judge(findings, live); + check("the FIXED file reports nothing retired", judged.filter((j) => j.state === "gone"), []); + check("and confirms both pins live", judged.filter((j) => j.state === "ok").length, 2); +} + +{ + // An unreadable catalogue must never read as rot. This is the guard against + // a missing key printing a fleet-wide outage that somebody then acts on. + const findings = findingsFrom("aoz-housing", "src/lib/env.ts", AOZ_BROKEN); + const live = new Map([["groq", null], ["openrouter", null]]); + const judged = judge(findings, live); + check("no key => UNCHECKED, never GONE", judged.filter((j) => j.state === "gone"), []); + check("and every pin is reported unchecked", judged.filter((j) => j.state === "unchecked").length, 2); +} + +{ + const findings = [{ repo: "x", path: "y", line: 1, id: "some-model-9b", vendor: null }]; + const judged = judge(findings, new Map()); + check("an unattributed pin is listed, not judged", judged[0].state, "unattributed"); +} + +// ── collate ────────────────────────────────────────────────────────────────── + +console.log("\ncollate"); +{ + const findings = [ + ...findingsFrom("a", "env.ts", AOZ_BROKEN), + { repo: "b", path: "c.ts", line: 1, id: "mystery-1b", vendor: null }, + ]; + const { byVendor, unattributed } = collate(findings); + check("groups by vendor", [...byVendor.keys()].sort(), ["groq", "openrouter"]); + check("keeps the unattributable aside", unattributed.map((u) => u.id), ["mystery-1b"]); +} + +// ── vendor confusion ───────────────────────────────────────────────────────── +// +// Found by running the audit for the first time, so it is pinned here. +// +// kivvi/apps/web/lib/ai/call-provider.ts dispatches four vendors in one +// function. `grok-3-mini` is xAI's model; Groq is a different company whose +// name differs by one letter, and whose base URL sat 18 lines above the pin +// while xAI's sat 6 above. Ranking by raw proximity called a live xAI model a +// retired Groq one — and the Groq pin six lines up tied EXACTLY between the two +// vendors, so a tie-break by proximity alone lost a true positive as well. +// +// Both halves are asserted: the xAI id must not be judged against Groq's +// catalogue, and the real Groq pin must still be caught. +const KIVVI_MULTI_VENDOR = ` +export async function callProvider(provider, apiKey, systemPrompt, userText, maxTokens) { + let url; + let headers; + let body; + + if (provider === "groq") { + url = "https://api.groq.com/openai/v1/chat/completions"; + headers = { Authorization: \`Bearer \${apiKey}\` }; + body = { + model: "llama-3.1-8b-instant", + messages: openaiMessages, + }; + } else if (provider === "xai") { + url = "https://api.x.ai/v1/chat/completions"; + headers = { Authorization: \`Bearer \${apiKey}\` }; + body = { + model: "grok-3-mini", + messages: openaiMessages, + }; + } +} +`; + +console.log("\nvendor confusion (regression)"); +{ + const pins = extractPins(KIVVI_MULTI_VENDOR); + const groqPin = pins.find((p) => p.id === "llama-3.1-8b-instant"); + const xaiPin = pins.find((p) => p.id === "grok-3-mini"); + + check("both vendor pins are extracted", Boolean(groqPin && xaiPin), true); + check("xAI's grok-3-mini is NOT attributed to groq", attribute(KIVVI_MULTI_VENDOR, xaiPin.line), "xai"); + check("the groq pin above it still attributes to groq", attribute(KIVVI_MULTI_VENDOR, groqPin.line), "groq"); + + // Only groq is queryable, so the xAI id must land in unchecked — never judged + // against a catalogue that was never going to list it. + const findings = pins.map((p) => ({ + repo: "kivvi", + path: "apps/web/lib/ai/call-provider.ts", + line: p.line, + id: p.id, + vendor: attribute(KIVVI_MULTI_VENDOR, p.line), + })); + const judged = judge(findings, new Map([["groq", GROQ_LIVE]])); + check("the real groq pin is still caught as retired", judged.filter((j) => j.state === "gone").map((j) => j.id), ["llama-3.1-8b-instant"]); + check("the xAI pin is unchecked, not retired", judged.filter((j) => j.state === "unchecked").map((j) => j.id), ["grok-3-mini"]); +} + +console.log(failures ? `\n✗ ${failures} failure(s)` : "\n✓ all checks pass"); +process.exit(failures ? 1 : 0);