diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6af7d9..afed102 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,16 @@ jobs: - name: The auto-merge sweep behaves run: bash scripts/ci/test-auto-merge-sweep.sh + # This detector reports on repos nobody is watching, so its own failure + # mode is silence: a clean report from a broken audit is worse than no + # audit, because it prints a ✓. Both sides are pinned — the real AOZ + # regression is still caught, corrected code stays quiet — plus the two + # false positives the first live run produced (xAI's `grok-3-mini` filed + # under Groq, and a computed `${...}` id read as a pin). No network, no + # key, no checkout, so it runs here as well as in the daily sweep. + - name: The model-pin audit still detects, and still stays quiet + run: node scripts/ci/test-model-pin-audit.mjs + # Drift guard. The fleet is on v7; templates handing out v4 is exactly # how this repo fell behind the repos it governs. - name: No stale action versions diff --git a/.github/workflows/model-pins.yml b/.github/workflows/model-pins.yml new file mode 100644 index 0000000..be1a042 --- /dev/null +++ b/.github/workflows/model-pins.yml @@ -0,0 +1,95 @@ +# Is any model id this fleet pins still served by its vendor? +# +# On 2026-08-26 the AOZ assistant reported "KI-Assistent nicht konfiguriert" on +# a deployment whose key was valid. Groq had retired the whole llama-3.x family. +# Production had been failing exactly as long as the demo, and nobody knew, +# because "not configured" is the only thing the app can say. +# +# Six pins across five repos were dead the same morning. None of it was +# detectable from inside a repo: every gate was green, because the code is +# correct and the vendor changed underneath it. +# +# Daily, not weekly. A retirement is decided by someone outside this fleet and +# lands without warning — unlike the UI defects next door, which arrive with a +# design change. The check costs ZERO tokens (one GET /models per vendor), so +# the only argument against running it often is noise, and a silent day is one +# line in a job summary. +name: Model pins + +on: + schedule: + # 07:10 UTC, before the working day — a retirement found at 09:00 is a + # morning's work; one found by a user is an outage of unknown age. + - cron: '10 7 * * *' + workflow_dispatch: + inputs: + strict: + description: 'Fail the run when pins are retired' + type: boolean + default: false + pull_request: + paths: + - 'scripts/ci/model-pin-audit.mjs' + - 'scripts/ci/test-model-pin-audit.mjs' + - '.github/workflows/model-pins.yml' + +permissions: + contents: read + +jobs: + audit: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 22 + + # The detector's own test needs no network, no key and no checkout, so it + # runs on pull_request too — where the sweep below cannot, because a fork + # has no secrets and an unreadable catalogue is deliberately not a pass. + - name: Self-test the detector + run: node scripts/ci/test-model-pin-audit.mjs + + # dotfiles has no package.json on purpose. Install ai-ration into a + # scratch dir and point the audit at it — the same shape ui-defects.yml + # uses for playwright. The vendor query lives in ai-ration precisely so + # this repo does not grow a second copy of it. + - name: Install ai-ration + if: github.event_name != 'pull_request' + run: | + mkdir -p "$RUNNER_TEMP/air" && cd "$RUNNER_TEMP/air" + npm init -y >/dev/null + npm i --no-audit --no-fund github:maonakamoto/ai-ration#v0.2.1 >/dev/null + + - name: Audit the fleet's pins + if: github.event_name != 'pull_request' + env: + AI_RATION_FROM: ${{ runner.temp }}/air + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Absent keys are handled, not fatal: the vendor is reported UNCHECKED + # rather than clean. "I could not look" is not "nothing is wrong". + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: | + set -uo pipefail + flag=--warn-only + if [ "${{ inputs.strict }}" = "true" ]; then flag=""; fi + + # Into the job summary, not just the log. A finding nobody scrolls to + # is the same as no finding — which is how eight days passed. + { + echo '## Model pins' + echo + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + set +e + node scripts/ci/model-pin-audit.mjs $flag 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + status=${PIPESTATUS[0]} + set -e + + echo '```' >> "$GITHUB_STEP_SUMMARY" + exit "$status" diff --git a/SHARED.md b/SHARED.md index 6c602da..c4f43ae 100644 --- a/SHARED.md +++ b/SHARED.md @@ -28,7 +28,9 @@ the inventory underneath it is **generated**, and the number it produces is a | [`limitkit`](https://github.com/maonakamoto/limitkit) | `npm i github:maonakamoto/limitkit#v0.1.0` | the fleet's **12 hand-rolled rate limiters** (this file's own "next extraction" row). Sliding/fixed windows over an injectable two-method `Store`; **bounded** memory default (the unbounded-Map leak is impossible by construction); standard `X-RateLimit-*` + `Retry-After` headers — what orangecat's ADR-0002 specified seven months before anything enforced it; `clientIp()`. Refusals count nothing, so a hammered key recovers. Ships no middleware and **no limit values** — how many attempts a route allows is app semantics, asserted locally. | **Adopted:** `ai-forms` — fleetcrown, evig, aoz-housing, surf-your-life. -`ai-ration` — fleetcrown. `threadkit` — **nobody yet**. +`ai-ration` — fleetcrown, **and this repo** (`model-pin-audit.mjs` calls +`checkCatalog`; the audit needed exactly the vendor query the package owns, so +writing a second one here would have been this file's own sin). `threadkit` — **nobody yet**. `limitkit` — fleetcrown (proving consumer; its old limiter had the unbounded Map). **Next adopter should be orangecat** — it closes ADR-0002 by making its Upstash client a 12-line `Store` adapter and deleting three of its four @@ -73,6 +75,7 @@ fleet-wide checker. | Script | Answers | |---|---| | `scripts/ci/verify-floor-audit.sh` | does every repo's `verify` actually run lint + typecheck + test? | +| `scripts/ci/model-pin-audit.mjs` | is any model id the fleet pins no longer served by its vendor? Zero tokens — one `GET /models` per vendor — so it runs DAILY. Uses `ai-ration`'s `checkCatalog` rather than a second vendor query. Self-tested by `scripts/ci/test-model-pin-audit.mjs`, which pins both sides and both real defects the first run produced: xAI's `grok-3-mini` misfiled under Groq, and a computed `${...}` id reported as a pin. | | `scripts/ci/ui-defect-audit.mjs` | do any live sites ship an interactive label below its WCAG AA floor, or a stack whose rows start at different x? Renders each site; no repo checkout involved. Self-tested by `scripts/ci/test-ui-defect-audit.mjs`, which pins BOTH sides — the real defect is still caught, correct markup stays silent. | Both report into a weekly workflow's job summary rather than only a log. @@ -86,7 +89,7 @@ Ranked by (copies × how identical the logic is). Counts from |---|---|---| | `auto-merge-sweep.sh` | ~~22~~ **6** | **EXTRACTED 2026-08-16/20.** Sixteen repos call the canonical as a reusable workflow, each verified to actually *run* it (a sweep that fails to start looks exactly like one with nothing to do). The six remaining are deliberate: dotfiles is the canonical home and runs it directly; ai-forms, datacat, petvity, solon had dirty working trees owned by other sessions when swept — convert when clear. The two repos that had ever *tested* their copies (evig, orangecat) had that coverage ported into the canonical suite **before** deletion: 17 cases, mutation-proven. | | rate limiting | **14 → adopting** | **Extracted 2026-08-20 as [`limitkit`](https://github.com/maonakamoto/limitkit)** (see registry above). fleetcrown converted as the proving consumer; 13 files remain across 8 repos, orangecat first in line (its ADR-0002 becomes a 12-line `Store` adapter + three deletions). The ratchet holds the count until each adoption lands. | -| AI provider client | **16** | evig 7, orangecat 5. `ai-ration` already owns the hard part (chain, 429, budget); these are the callers. | +| AI provider client | **16** | evig 7, orangecat 5. `ai-ration` already owns the hard part (chain, 429, budget); these are the callers. **Priced 2026-08-26:** Groq retired the llama-3.x family and six pins across botsmann, evig, kivvi, orangecat and truthseeker died at once, with three deployed apps failing live and AOZ reporting a missing key it already had. Every one of those repos hand-rolls this layer; fleetcrown, which adopted the package, was unaffected. | | logger | **10** | sbb-lost-found alone has 4. | | health route | **8** | Identical shape in 8 repos; a 20-line contract. | | ~~`@ai-native-cms/core`~~ | — | **Withdrawn — measurement error.** `maonakamoto/revampit` *redirects* to `maonakamoto/evig` (renamed in the pivot); the "two repos" with byte-identical trees were two clones of ONE repo. Nothing to extract. Two directories are not two repos: check `git remote -v` before reporting cross-repo duplication. | diff --git a/scripts/ci/model-pin-audit.mjs b/scripts/ci/model-pin-audit.mjs new file mode 100755 index 0000000..ca9d310 --- /dev/null +++ b/scripts/ci/model-pin-audit.mjs @@ -0,0 +1,565 @@ +#!/usr/bin/env node +/** + * Fleet audit: is any model id this fleet pins no longer served? + * + * WHY THIS EXISTS + * --------------- + * On 2026-08-26 the AOZ assistant answered "KI-Assistent nicht konfiguriert. + * Bitte GROQ_API_KEY setzen" on a deployment whose key was valid. Groq had + * retired the whole llama-3.x family; the pinned `llama-3.3-70b-versatile` + * returned 404 model_not_found. Production had been broken as long as the demo + * and nobody knew, because the only thing the app can say is "not configured". + * + * That is not an AOZ bug. Every Groq id pinned anywhere in the fleet was dead + * the same morning — llama-3.1-8b-instant, llama3-8b-8192, gemma2-9b-it, + * mixtral-8x7b-32768, llama-3.2-3b-instruct — across eleven repos, with three + * deployed apps failing live. + * + * A pinned free model is not a configuration, it is a scheduled outage. The + * schedule is set by the vendor and nobody here is told. So this asks the only + * authority that knows: the vendor's own catalogue. + * + * WHY A CENTRAL SCRIPT AND NOT A CHECK PER REPO + * --------------------------------------------- + * Same doctrine as verify-floor-audit.sh and ui-defect-audit.mjs. A check + * copied into twenty repos drifts into twenty versions — SHARED.md counts the + * bill for exactly that habit. More to the point, a per-repo check only ever + * runs in repos somebody still touches, and the repos that rot quietly are + * precisely the ones nobody touches. This one needs no adoption at all. + * + * WHY IT REUSES ai-ration + * ----------------------- + * `checkCatalog` already answers this, already distinguishes the three states + * that matter, and already carries the scars — its own docstring records four + * of nine default pins gone and a consumer silently failing for eight days. + * Writing a second vendor query here would be this repo committing the sin it + * exists to police. It takes the chain as an ARGUMENT, so it generalises to + * arbitrary ids with no change to the package. + * + * ZERO TOKENS, WHICH IS THE WHOLE POINT + * ------------------------------------- + * One GET /models per vendor. No completion, no spend. That is the difference + * between a check that runs on a timer and a command somebody is supposed to + * remember — and "supposed to remember" is what failed for eight days. + * + * WHAT IT DOES NOT PROVE + * ---------------------- + * That a listed model WORKS. Existence is cheap; capability is not. A model + * can be listed and still refuse tool calls — of nine free models probed for + * ai-ration's default chain, five answered only via a text protocol. If the + * surface is a tool loop, probe with a real tool call before pinning. This + * audit catches the retirement, not the mismatch. + * + * It also judges only ids it can ATTRIBUTE to a vendor it queried. An + * unattributed id is reported and not judged, because "I could not look" and + * "it is gone" are different answers and collapsing them invents outages. + * + * Usage: + * node scripts/ci/model-pin-audit.mjs # audit, exit 1 on dead pins + * node scripts/ci/model-pin-audit.mjs --warn-only # report, always exit 0 + * node scripts/ci/model-pin-audit.mjs --local # scan ~/dev checkouts + * + * Env: GH_OWNER (default maonakamoto), GH_LIMIT (default 100), + * FLEET_ROOT (default ~/dev, --local only), + * AI_RATION_FROM (path to a repo that installs ai-ration), + * GROQ_API_KEY / OPENROUTER_API_KEY (to read the catalogues). + */ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; + +const exec = promisify(execFile); + +const WARN_ONLY = process.argv.includes("--warn-only"); +const LOCAL = process.argv.includes("--local"); +const OWNER = process.env.GH_OWNER ?? "maonakamoto"; +const GH_LIMIT = process.env.GH_LIMIT ?? "100"; +const FLEET_ROOT = process.env.FLEET_ROOT ?? join(homedir(), "dev"); + +/** + * The vendors whose catalogue we can actually read, and the markers that tie a + * pin to one of them. + * + * Attribution is by MARKER rather than by the shape of the id, because the shape + * lies: `openai/gpt-oss-20b` is a Groq model id AND an OpenRouter routing id, + * and at OpenRouter the missing `:free` suffix is the difference between free + * and billed. Only the surrounding code knows which vendor is meant. + */ +export const VENDORS = [ + { + id: "groq", + queryable: true, + baseUrl: "https://api.groq.com/openai/v1", + keyEnv: "GROQ_API_KEY", + markers: [/groq/i], + }, + { + id: "openrouter", + queryable: true, + baseUrl: "https://openrouter.ai/api/v1", + keyEnv: "OPENROUTER_API_KEY", + markers: [/openrouter/i], + }, + // Not queryable here — no catalogue call is wired for these. They are listed + // so their ids are ATTRIBUTED and reported unchecked, rather than falling to + // whichever queryable vendor happens to sit nearest in the file. Markers are + // deliberately specific: bare /openai/ would match Groq's own + // `api.groq.com/openai/v1` path and every `openai/gpt-oss-*` id it serves. + { id: "xai", queryable: false, markers: [/api\.x\.ai/i, /\bXAI_API_KEY\b/, /\bgrok\b/i] }, + { id: "anthropic", queryable: false, markers: [/api\.anthropic\.com/i, /\bANTHROPIC_API_KEY\b/] }, + { id: "openai", queryable: false, markers: [/api\.openai\.com/i, /\bOPENAI_API_KEY\b/] }, + { id: "google", queryable: false, markers: [/generativelanguage\.googleapis/i, /\bGEMINI_API_KEY\b/] }, +]; + +/** + * Files worth opening, in two tiers. + * + * LIKELY names an AI module outright. POSSIBLE is the long tail that a + * name-based filter misses: botsmann keeps its model id in `lib/constants.ts`, + * which mentions no vendor in its path and was invisible to the first version + * of this filter. A pin does not have to live in a file called `provider.ts`. + */ +const LIKELY_PATH = + /(^|\/)\.env\.example$|(^|\/)env\.(ts|js|mjs)$|(^|\/)(lib|src|app|apps|packages|config)\/.*(provider|model|llm|chat|ai|openai|anthropic)[^/]*\.(ts|js|mjs)$/i; + +const POSSIBLE_PATH = + /(^|\/)(lib|src|app|apps|packages|config)\/.*(constants?|config|settings|defaults)[^/]*\.(ts|js|mjs)$/i; + +const CANDIDATE_PATH = new RegExp(`(${LIKELY_PATH.source})|(${POSSIBLE_PATH.source})`, "i"); + +/** Never open these, whatever they are named. */ +const SKIP_PATH = + /(^|\/)(node_modules|dist|build|\.next|coverage|__tests__|__fixtures__)\/|(^|\/)\.claude\/worktrees\//; + +const MAX_FILES_PER_REPO = 90; + +/** A vendor named 40+ lines from a pin is not describing that pin. */ +const MAX_ATTRIBUTION_DISTANCE = 40; + +// ── Extraction ─────────────────────────────────────────────────────────────── + +/** + * Is this string plausibly a model id rather than any other quoted thing? + * + * Deliberately permissive on shape and strict on the obvious negatives. A false + * POSITIVE costs one line in a report that says "not judged"; a false NEGATIVE + * is the outage this whole file exists to prevent. + */ +export function looksLikeModelId(s) { + if (typeof s !== "string") return false; + if (s.length < 3 || s.length > 80) return false; + if (/\s/.test(s)) return false; + if (s.includes("${")) return false; // interpolated: resolved at runtime, not pinned + if (/^https?:/i.test(s)) return false; + if (/^[./~@]/.test(s)) return false; + if (/\.(ts|tsx|js|mjs|cjs|json|css|scss|md|png|jpe?g|svg|ico|txt|ya?ml)$/i.test(s)) return false; + if (/^[A-Z][A-Z0-9_]*$/.test(s)) return false; // SCREAMING_CASE is an env name + // Model ids essentially always carry a version digit or a vendor/ prefix. + return /\d/.test(s) || s.includes("/"); +} + +/** + * Pull candidate model ids out of one file's text, with the line each sits on. + * + * Three shapes cover how this fleet writes them: + * 1. a `model:` / `models:` assignment, single value or array + * 2. a *_MODEL constant or Zod `.default(...)` + * 3. a bare `GROQ_MODEL=...` line in a .env file + */ +export function extractPins(text) { + const found = new Map(); // id -> line number (first sighting) + const lineOf = (index) => text.slice(0, index).split("\n").length; + + const remember = (id, index) => { + if (!looksLikeModelId(id)) return; + if (!found.has(id)) found.set(id, lineOf(index)); + }; + + // 1. model: 'x' | models: ['a', 'b'] + for (const m of text.matchAll(/\bmodels?\s*[:=]\s*(\[[\s\S]{0,400}?\]|['"`][^'"`\n]{0,120}['"`])/gi)) { + for (const q of m[1].matchAll(/['"`]([^'"`\n]+)['"`]/g)) remember(q[1], m.index); + } + + // 2. GROQ_MODEL: z.string().default('x') | const DEFAULT_MODEL = 'x' + for (const m of text.matchAll( + /\b([A-Za-z][A-Za-z0-9_]*MODEL[A-Za-z0-9_]*)\s*[:=][^\n]{0,80}?['"`]([^'"`\n]+)['"`]/g, + )) { + remember(m[2], m.index); + } + + // 3. .env style, unquoted or quoted, no code around it + for (const m of text.matchAll(/^[ \t]*(?:export[ \t]+)?[A-Z][A-Z0-9_]*MODEL[A-Z0-9_]*\s*=\s*["']?([^"'\s#]+)/gm)) { + remember(m[1], m.index); + } + + return [...found].map(([id, line]) => ({ id, line })); +} + +/** + * Which vendor does this pin belong to? + * + * Nearest-marker first: a file can legitimately name both vendors — AOZ's + * provider.ts resolves Groq AND OpenRouter in one module — so a file-wide vote + * would attribute both vendors' pins to whichever appeared more. The pin's own + * neighbourhood is what actually says which branch it is in. Only when the + * window is silent do we fall back to the file, and only when the file names + * exactly one vendor. + * + * Returns a vendor id, or null for "cannot tell" — which is reported, not judged. + */ +export function attribute(text, line, vendors = VENDORS, maxDistance = MAX_ATTRIBUTION_DISTANCE) { + const lines = text.split("\n"); + + /** + * Distance to this vendor's nearest mention ABOVE the pin, and below it. + * + * Above is what decides. Every provider module in this fleet is written as + * `if (provider === "groq") { url = ...; body = { model: "..." } }` — the + * branch that owns a model literal always opens above it. Ranking by raw + * proximity instead put `grok-3-mini` (xAI, line 89) with Groq, whose URL sat + * at line 71, and tied kivvi's real Groq pin exactly between two vendors. + */ + const distances = (vendor) => { + let above = Infinity; + let below = Infinity; + for (let i = 0; i < lines.length; i++) { + if (!vendor.markers.some((re) => re.test(lines[i]))) continue; + const d = i + 1 - line; + if (d <= 0) above = Math.min(above, -d); + else below = Math.min(below, d); + } + return { above, below }; + }; + + const scored = vendors.map((v) => ({ id: v.id, ...distances(v) })); + + const fromAbove = scored.filter((s) => s.above <= maxDistance).sort((a, b) => a.above - b.above); + if (fromAbove.length === 1) return fromAbove[0].id; + if (fromAbove.length > 1 && fromAbove[0].above < fromAbove[1].above) return fromAbove[0].id; + + // Nothing above governs it — a pin at the top of a file, or a config block + // whose vendor is named afterwards. Fall back to the nearest mention below. + if (fromAbove.length === 0) { + const fromBelow = scored.filter((s) => s.below <= maxDistance).sort((a, b) => a.below - b.below); + if (fromBelow.length === 1) return fromBelow[0].id; + if (fromBelow.length > 1 && fromBelow[0].below < fromBelow[1].below) return fromBelow[0].id; + } + + const named = vendors.filter((v) => v.markers.some((re) => re.test(text))); + return named.length === 1 ? named[0].id : null; +} + +/** + * Turn per-file findings into per-vendor id lists plus the unattributable rest. + * Pure, so the self-test can drive it without a network or a checkout. + */ +export function collate(findings) { + const byVendor = new Map(); + const unattributed = []; + + for (const f of findings) { + if (!f.vendor) { + unattributed.push(f); + continue; + } + if (!byVendor.has(f.vendor)) byVendor.set(f.vendor, []); + byVendor.get(f.vendor).push(f); + } + return { byVendor, unattributed }; +} + +/** + * Apply catalogue verdicts to the findings. + * + * `live` is a Map of vendorId -> Set of ids, or null for a vendor whose + * catalogue could not be read. Null is NOT an empty set: treating "I could not + * look" as "nothing is there" reports every pin as retired and invents an + * outage, which is worse than silence because someone acts on it. + */ +export function judge(findings, live) { + return findings.map((f) => { + if (!f.vendor) return { ...f, state: "unattributed" }; + const set = live.get(f.vendor); + if (!set) return { ...f, state: "unchecked" }; + return { ...f, state: set.has(f.id) ? "ok" : "gone" }; + }); +} + +// ── Reading the fleet ──────────────────────────────────────────────────────── + +/** + * Which files get opened when a repo has more candidates than the cap allows. + * + * Ranked, never arbitrary: a file that names a vendor or an AI concern outranks + * a generic `config.ts`, so the cap sheds the least likely candidates first. + * And every truncation is RECORDED — a bounded sweep that stays quiet about + * what it skipped reads exactly like a sweep that found nothing. + */ +const truncated = []; + +function rank(paths, repoName) { + const sorted = [...paths].sort((a, b) => { + const ta = LIKELY_PATH.test(a) ? 0 : 1; + const tb = LIKELY_PATH.test(b) ? 0 : 1; + return ta - tb || a.length - b.length; + }); + if (sorted.length > MAX_FILES_PER_REPO) { + // What was dropped matters more than how many. Everything in the LIKELY + // tier is opened first, so a truncation that sheds only generic config + // files has not touched the audit's real coverage — and saying which it was + // is the difference between a caveat and an alarm. + const likelyDropped = sorted.slice(MAX_FILES_PER_REPO).filter((p) => LIKELY_PATH.test(p)).length; + truncated.push({ repo: repoName, seen: sorted.length, opened: MAX_FILES_PER_REPO, likelyDropped }); + } + return sorted.slice(0, MAX_FILES_PER_REPO); +} + +async function gh(args) { + const { stdout } = await exec("gh", args, { maxBuffer: 64 * 1024 * 1024 }); + return stdout; +} + +/** Remote, default-branch view. Clones drift; this repo has been bitten by that. */ +async function remoteRepos() { + const raw = await gh([ + "repo", "list", OWNER, + "--limit", String(GH_LIMIT), + "--no-archived", + "--json", "name,defaultBranchRef,isFork", + ]); + return JSON.parse(raw) + .filter((r) => !r.isFork && r.defaultBranchRef?.name) + .map((r) => ({ name: r.name, branch: r.defaultBranchRef.name })); +} + +async function remoteFiles(repo) { + let tree; + try { + tree = JSON.parse( + await gh(["api", `repos/${OWNER}/${repo.name}/git/trees/${repo.branch}?recursive=1`]), + ); + } catch { + return []; // empty repo, or no access — not a finding + } + const all = (tree.tree ?? []) + .filter((n) => n.type === "blob" && !SKIP_PATH.test("/" + n.path) && CANDIDATE_PATH.test(n.path)) + .map((n) => n.path); + const paths = rank(all, repo.name); + + const out = []; + for (const path of paths) { + try { + const body = JSON.parse( + await gh(["api", `repos/${OWNER}/${repo.name}/contents/${path}?ref=${repo.branch}`]), + ); + if (body.encoding !== "base64" || !body.content) continue; + out.push({ path, text: Buffer.from(body.content, "base64").toString("utf8") }); + } catch { + /* a path that vanished between tree and read is not a finding */ + } + } + return out; +} + +function localRepos() { + return readdirSync(FLEET_ROOT) + .filter((n) => !n.startsWith("_") && existsSync(join(FLEET_ROOT, n, ".git"))) + .map((name) => ({ name, branch: "(local)" })); +} + +function walk(dir, depth, acc) { + if (depth < 0 || acc.length >= MAX_FILES_PER_REPO * 6) return acc; + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return acc; + } + for (const e of entries) { + const full = join(dir, e.name); + if (e.isDirectory()) { + if (/^(node_modules|dist|build|\.next|coverage|\.git|\.claude)$/.test(e.name)) continue; + walk(full, depth - 1, acc); + } else if (CANDIDATE_PATH.test(full) && !SKIP_PATH.test(full)) { + try { + if (statSync(full).size < 400_000) acc.push(full); + } catch { /* raced */ } + } + } + return acc; +} + +function localFiles(repo) { + const root = join(FLEET_ROOT, repo.name); + const all = walk(root, 6, []).map((full) => full.slice(root.length + 1)); + return rank(all, repo.name).map((path) => ({ + path, + text: readFileSync(join(root, path), "utf8"), + })); +} + +// ── The catalogue, via ai-ration ───────────────────────────────────────────── + +function loadAiRation() { + const candidates = [ + process.env.AI_RATION_FROM, + join(homedir(), "dev", "fleetcrown"), + join(homedir(), "dev", "ai-ration"), + ].filter(Boolean); + + for (const root of candidates) { + for (const entry of [ + join(root, "node_modules", "ai-ration", "dist", "index.js"), + join(root, "dist", "index.js"), + ]) { + if (existsSync(entry)) return import(entry); + } + } + console.error( + "✗ ai-ration not found. Set AI_RATION_FROM=/path/to/a/repo that installs it,\n" + + " or build it once: (cd ~/dev/ai-ration && npm i && npm run build)", + ); + process.exit(2); +} + +/** + * Ask each vendor what it still lists, for exactly the ids we found. + * Returns Map(vendorId -> Set|null), where null means "could not look". + */ +async function readCatalogues(byVendor, checkCatalog) { + const live = new Map(); + for (const vendor of VENDORS) { + if (!vendor.queryable) continue; + const findings = byVendor.get(vendor.id); + if (!findings?.length) continue; + + const ids = [...new Set(findings.map((f) => f.id))]; + const [verdict] = await checkCatalog([ + { + id: vendor.id, + baseUrl: vendor.baseUrl, + keyEnv: vendor.keyEnv, + models: ids, + dailyTokens: 0, + }, + ]); + live.set(vendor.id, verdict.live ? new Set(verdict.live) : null); + } + return live; +} + +// ── Report ─────────────────────────────────────────────────────────────────── + +function report(judged) { + const gone = judged.filter((j) => j.state === "gone"); + const unchecked = judged.filter((j) => j.state === "unchecked"); + const unattributed = judged.filter((j) => j.state === "unattributed"); + const ok = judged.filter((j) => j.state === "ok"); + + const lines = []; + + if (gone.length) { + lines.push("RETIRED — the vendor no longer lists these, so every call using them fails:"); + const byId = new Map(); + for (const g of gone) { + const key = `${g.vendor}/${g.id}`; + if (!byId.has(key)) byId.set(key, []); + byId.get(key).push(`${g.repo}:${g.path}:${g.line}`); + } + for (const [key, sites] of [...byId].sort()) { + lines.push(` GONE ${key}`); + for (const s of sites.sort()) lines.push(` ${s}`); + } + lines.push(""); + } + + if (unchecked.length) { + const vendors = [...new Set(unchecked.map((u) => u.vendor))].sort(); + lines.push( + `${unchecked.length} pin(s) UNCHECKED — no readable catalogue for: ${vendors.join(", ")}.`, + ); + lines.push(" That is not a pass for them. Set the vendor key to judge these."); + lines.push(""); + } + + if (unattributed.length) { + const ids = [...new Set(unattributed.map((u) => u.id))].sort(); + lines.push(`${unattributed.length} pin(s) not attributable to a vendor we query — listed, not judged:`); + for (const id of ids.slice(0, 20)) lines.push(` ? ${id}`); + if (ids.length > 20) lines.push(` ? … and ${ids.length - 20} more`); + lines.push(""); + } + + lines.push( + `${ok.length} pin(s) confirmed live · ${gone.length} retired · ` + + `${unchecked.length} unchecked · ${unattributed.length} unattributed`, + ); + + if (truncated.length) { + lines.push(""); + const blind = truncated.filter((t) => t.likelyDropped > 0); + lines.push(`COVERAGE — ${truncated.length} repo(s) had more candidate files than the cap of ${MAX_FILES_PER_REPO}:`); + for (const t of truncated) { + const tail = t.likelyDropped > 0 ? `, ${t.likelyDropped} of them likely-AI` : ", none likely-AI"; + lines.push(` ${t.repo}: opened ${t.opened} of ${t.seen}${tail}`); + } + lines.push( + blind.length + ? " Files naming an AI concern were dropped — raise MAX_FILES_PER_REPO." + : " Only generic config files were dropped; every likely-AI file was opened.", + ); + } + + if (gone.length) { + lines.push(""); + lines.push("A pin is a scheduled outage. The durable fix is a chain across VENDORS —"); + lines.push("see SHARED.md → ai-ration. Repinning buys time until the next retirement."); + } + + return lines.join("\n"); +} + +// ── Main ───────────────────────────────────────────────────────────────────── + +export async function main() { + const { checkCatalog } = await loadAiRation(); + + const repos = LOCAL ? localRepos() : await remoteRepos(); + const findings = []; + + for (const repo of repos) { + const files = LOCAL ? localFiles(repo) : await remoteFiles(repo); + for (const file of files) { + for (const pin of extractPins(file.text)) { + findings.push({ + repo: repo.name, + path: file.path, + line: pin.line, + id: pin.id, + vendor: attribute(file.text, pin.line), + }); + } + } + } + + const { byVendor } = collate(findings); + const live = await readCatalogues(byVendor, checkCatalog); + const judged = judge(findings, live); + + console.log(report(judged)); + console.log(`\ninspected ${repos.length} repo(s)${LOCAL ? " (local checkouts)" : " on their default branches"}`); + + const dead = judged.some((j) => j.state === "gone"); + process.exit(dead && !WARN_ONLY ? 1 : 0); +} + +const invokedDirectly = + process.argv[1] && import.meta.url === `file://${process.argv[1]}`; +if (invokedDirectly) { + main().catch((err) => { + console.error(`✗ audit failed: ${err?.message ?? err}`); + process.exit(2); + }); +} diff --git a/scripts/ci/test-model-pin-audit.mjs b/scripts/ci/test-model-pin-audit.mjs new file mode 100755 index 0000000..d050b08 --- /dev/null +++ b/scripts/ci/test-model-pin-audit.mjs @@ -0,0 +1,272 @@ +#!/usr/bin/env node +/** + * Self-test for model-pin-audit.mjs. + * Run: node scripts/ci/test-model-pin-audit.mjs + * + * A detector that has quietly stopped detecting reports a clean fleet, and a + * clean report from a broken detector is worse than no report — it is an absent + * check that prints a ✓. So every fixture asserts a VERDICT, and both sides are + * pinned: + * + * - the real AOZ regression is still caught (positive) + * - the corrected file stays silent (negative) + * - quoted things that are not model ids stay out of the report + * - an unreadable catalogue reports UNCHECKED, never GONE + * + * That last one is the expensive mistake. Treating "I could not look" as + * "nothing is there" marks every pin retired and invents a fleet-wide outage + * that somebody will act on. Silence is recoverable; a false alarm at this + * scale is not. + * + * Fixtures are inline strings — no network, no gh, no checkout, no key. + */ +import { extractPins, attribute, collate, judge, looksLikeModelId } from "./model-pin-audit.mjs"; + +let failures = 0; +function check(name, actual, expected) { + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + if (a === e) { + console.log(` ok ${name}`); + } else { + console.log(` FAIL ${name}\n expected ${e}\n actual ${a}`); + failures++; + } +} + +// ── Fixtures ───────────────────────────────────────────────────────────────── + +/** aoz-housing/src/lib/env.ts as it stood when both AI surfaces went down. */ +const AOZ_BROKEN = ` +import { z } from 'zod' + +const schema = z.object({ + DATABASE_URL: z.string().url(), + CRON_SECRET: z.string().min(16).optional(), + + // AI: whichever key is set decides the provider. + GROQ_API_KEY: z.string().optional(), + GROQ_MODEL: z.string().default('llama-3.3-70b-versatile'), + OPENROUTER_API_KEY: z.string().optional(), + OPENROUTER_MODEL: z.string().default('openai/gpt-oss-20b:free'), +}) +`; + +/** The same file after the fix. Must produce no findings at all. */ +const AOZ_FIXED = AOZ_BROKEN.replace("llama-3.3-70b-versatile", "openai/gpt-oss-120b"); + +/** aoz-housing/src/lib/ai/provider.ts — one module, BOTH vendors named. */ +const TWO_VENDOR_FILE = ` +const GROQ_API_URL = 'https://api.groq.com/openai/v1/chat/completions' +const OPENROUTER_API_URL = 'https://openrouter.ai/api/v1/chat/completions' + +export async function getConfig() { + if (provider === 'groq') { + return { url: GROQ_API_URL, model: 'llama-3.1-8b-instant' } + } + return { url: OPENROUTER_API_URL, model: 'meta-llama/llama-3.3-70b-instruct:free' } +} +`; + +/** Quoted strings that must never be mistaken for model ids. */ +const NOISE = ` +import { BRAND } from '@/lib/config/brand' +const path = './lib/ai/provider.ts' +const url = 'https://api.groq.com/openai/v1/chat/completions' +const key = 'GROQ_API_KEY' +const model = 'llama-3.1-8b-instant' +const style = 'rounded-lg' +`; + +// ── looksLikeModelId ───────────────────────────────────────────────────────── + +console.log("looksLikeModelId"); +check("accepts a versioned id", looksLikeModelId("llama-3.3-70b-versatile"), true); +check("accepts a routed id", looksLikeModelId("openai/gpt-oss-20b:free"), true); +check("rejects a relative path", looksLikeModelId("./lib/ai/provider.ts"), false); +check("rejects a url", looksLikeModelId("https://api.groq.com/openai/v1"), false); +check("rejects an env var name", looksLikeModelId("GROQ_API_KEY"), false); +check("rejects a css class", looksLikeModelId("rounded-lg"), false); +check("rejects a module specifier", looksLikeModelId("@/lib/config/brand"), false); +// fleetcrown builds its model id from ai-ration's chain at call time. A +// computed id is the ABSENCE of a pin, and the first run reported it retired. +check("rejects an interpolated id", looksLikeModelId("${link.provider.id}/${link.model}"), false); + +// ── extraction ─────────────────────────────────────────────────────────────── + +console.log("\nextractPins"); +check( + "finds both pins in the broken AOZ env schema", + extractPins(AOZ_BROKEN).map((p) => p.id).sort(), + ["llama-3.3-70b-versatile", "openai/gpt-oss-20b:free"], +); +check( + "finds the model literals in a two-vendor provider", + extractPins(TWO_VENDOR_FILE).map((p) => p.id).sort(), + ["llama-3.1-8b-instant", "meta-llama/llama-3.3-70b-instruct:free"], +); +check( + "picks the model id out of a file full of other quoted strings", + extractPins(NOISE).map((p) => p.id), + ["llama-3.1-8b-instant"], +); + +// ── attribution ────────────────────────────────────────────────────────────── + +console.log("\nattribute"); +{ + const pins = extractPins(AOZ_BROKEN); + const groq = pins.find((p) => p.id === "llama-3.3-70b-versatile"); + const or = pins.find((p) => p.id === "openai/gpt-oss-20b:free"); + check("GROQ_MODEL line attributes to groq", attribute(AOZ_BROKEN, groq.line), "groq"); + check("OPENROUTER_MODEL line attributes to openrouter", attribute(AOZ_BROKEN, or.line), "openrouter"); +} +{ + // The case a file-wide vote gets wrong: both vendors named in one module, so + // only the pin's own neighbourhood says which branch it belongs to. + const pins = extractPins(TWO_VENDOR_FILE); + const g = pins.find((p) => p.id === "llama-3.1-8b-instant"); + const o = pins.find((p) => p.id === "meta-llama/llama-3.3-70b-instruct:free"); + check("nearest marker wins for the groq branch", attribute(TWO_VENDOR_FILE, g.line), "groq"); + check("nearest marker wins for the openrouter branch", attribute(TWO_VENDOR_FILE, o.line), "openrouter"); +} +check( + "a pin with no vendor anywhere is not attributed", + attribute("const model = 'some-model-9b'", 1), + null, +); + +// ── judging ────────────────────────────────────────────────────────────────── + +console.log("\njudge"); +const findingsFrom = (repo, path, text) => + extractPins(text).map((p) => ({ + repo, + path, + line: p.line, + id: p.id, + vendor: attribute(text, p.line), + })); + +const GROQ_LIVE = new Set(["openai/gpt-oss-120b", "openai/gpt-oss-20b", "qwen/qwen3.8-27b"]); +const OR_LIVE = new Set(["openai/gpt-oss-20b:free"]); + +{ + const findings = findingsFrom("aoz-housing", "src/lib/env.ts", AOZ_BROKEN); + const live = new Map([["groq", GROQ_LIVE], ["openrouter", OR_LIVE]]); + const judged = judge(findings, live); + + check( + "THE REGRESSION: the retired llama pin is caught", + judged.filter((j) => j.state === "gone").map((j) => j.id), + ["llama-3.3-70b-versatile"], + ); + check( + "the still-served openrouter pin beside it stays quiet", + judged.filter((j) => j.state === "ok").map((j) => j.id), + ["openai/gpt-oss-20b:free"], + ); +} + +{ + // The negative that matters most: correct code must produce a silent report, + // or the audit trains people to ignore it. + const findings = findingsFrom("aoz-housing", "src/lib/env.ts", AOZ_FIXED); + const live = new Map([["groq", GROQ_LIVE], ["openrouter", OR_LIVE]]); + const judged = judge(findings, live); + check("the FIXED file reports nothing retired", judged.filter((j) => j.state === "gone"), []); + check("and confirms both pins live", judged.filter((j) => j.state === "ok").length, 2); +} + +{ + // An unreadable catalogue must never read as rot. This is the guard against + // a missing key printing a fleet-wide outage that somebody then acts on. + const findings = findingsFrom("aoz-housing", "src/lib/env.ts", AOZ_BROKEN); + const live = new Map([["groq", null], ["openrouter", null]]); + const judged = judge(findings, live); + check("no key => UNCHECKED, never GONE", judged.filter((j) => j.state === "gone"), []); + check("and every pin is reported unchecked", judged.filter((j) => j.state === "unchecked").length, 2); +} + +{ + const findings = [{ repo: "x", path: "y", line: 1, id: "some-model-9b", vendor: null }]; + const judged = judge(findings, new Map()); + check("an unattributed pin is listed, not judged", judged[0].state, "unattributed"); +} + +// ── collate ────────────────────────────────────────────────────────────────── + +console.log("\ncollate"); +{ + const findings = [ + ...findingsFrom("a", "env.ts", AOZ_BROKEN), + { repo: "b", path: "c.ts", line: 1, id: "mystery-1b", vendor: null }, + ]; + const { byVendor, unattributed } = collate(findings); + check("groups by vendor", [...byVendor.keys()].sort(), ["groq", "openrouter"]); + check("keeps the unattributable aside", unattributed.map((u) => u.id), ["mystery-1b"]); +} + +// ── vendor confusion ───────────────────────────────────────────────────────── +// +// Found by running the audit for the first time, so it is pinned here. +// +// kivvi/apps/web/lib/ai/call-provider.ts dispatches four vendors in one +// function. `grok-3-mini` is xAI's model; Groq is a different company whose +// name differs by one letter, and whose base URL sat 18 lines above the pin +// while xAI's sat 6 above. Ranking by raw proximity called a live xAI model a +// retired Groq one — and the Groq pin six lines up tied EXACTLY between the two +// vendors, so a tie-break by proximity alone lost a true positive as well. +// +// Both halves are asserted: the xAI id must not be judged against Groq's +// catalogue, and the real Groq pin must still be caught. +const KIVVI_MULTI_VENDOR = ` +export async function callProvider(provider, apiKey, systemPrompt, userText, maxTokens) { + let url; + let headers; + let body; + + if (provider === "groq") { + url = "https://api.groq.com/openai/v1/chat/completions"; + headers = { Authorization: \`Bearer \${apiKey}\` }; + body = { + model: "llama-3.1-8b-instant", + messages: openaiMessages, + }; + } else if (provider === "xai") { + url = "https://api.x.ai/v1/chat/completions"; + headers = { Authorization: \`Bearer \${apiKey}\` }; + body = { + model: "grok-3-mini", + messages: openaiMessages, + }; + } +} +`; + +console.log("\nvendor confusion (regression)"); +{ + const pins = extractPins(KIVVI_MULTI_VENDOR); + const groqPin = pins.find((p) => p.id === "llama-3.1-8b-instant"); + const xaiPin = pins.find((p) => p.id === "grok-3-mini"); + + check("both vendor pins are extracted", Boolean(groqPin && xaiPin), true); + check("xAI's grok-3-mini is NOT attributed to groq", attribute(KIVVI_MULTI_VENDOR, xaiPin.line), "xai"); + check("the groq pin above it still attributes to groq", attribute(KIVVI_MULTI_VENDOR, groqPin.line), "groq"); + + // Only groq is queryable, so the xAI id must land in unchecked — never judged + // against a catalogue that was never going to list it. + const findings = pins.map((p) => ({ + repo: "kivvi", + path: "apps/web/lib/ai/call-provider.ts", + line: p.line, + id: p.id, + vendor: attribute(KIVVI_MULTI_VENDOR, p.line), + })); + const judged = judge(findings, new Map([["groq", GROQ_LIVE]])); + check("the real groq pin is still caught as retired", judged.filter((j) => j.state === "gone").map((j) => j.id), ["llama-3.1-8b-instant"]); + check("the xAI pin is unchecked, not retired", judged.filter((j) => j.state === "unchecked").map((j) => j.id), ["grok-3-mini"]); +} + +console.log(failures ? `\n✗ ${failures} failure(s)` : "\n✓ all checks pass"); +process.exit(failures ? 1 : 0);