From 5669c0e6baa1c7f54e60630474a95cc6424a44ce Mon Sep 17 00:00:00 2001 From: Mao Nakamoto <41178744+maonakamoto@users.noreply.github.com> Date: Thu, 27 Aug 2026 01:06:16 +0200 Subject: [PATCH] fix(ai): repin five dead model ids, and stop needing to do this by hand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five of this repo's model ids were retired by their vendors. Four of them at once, in the same file. `src/lib/ai/providers.ts` had the Groq default, the large-context fallback, the OpenRouter default and the OpenRouter vision model all dead — the entire cascade except the Groq vision model. `src/lib/hirn/providers/groq.ts` and `openrouter.ts` carried two more, so both of this repo's AI layers were down. The history matters more than the repin. This block has now been repinned three times and the previous two are still legible in its own comments: Llama-4-Scout was decommissioned, so it moved to `llama-3.3-70b-versatile`, and Groq then retired the entire llama-3.x family. The comment in `hirn/openrouter.ts` even predicted its own death — "free catalogues rot, so replace it with another `:free` id" — and was right, and nothing acted on it. So the durable part of this change is not the new ids. It is that two checks now run without being asked: - dotfiles/scripts/ci/model-pin-audit.mjs asks Groq and OpenRouter DAILY whether these exact ids are still listed. It is the one that catches NEW rot, because it talks to the vendor rather than to a list. It did not catch these: `src/lib/hirn/providers/` was never opened, because nothing in that path said "ai" loudly enough. That gap is fixed and tested. - a new test here catches the cheaper mistake — a dead id reintroduced by copying an old line, reverting a file or merging a stale branch. It matches FAMILIES rather than individual ids, because what happened twice was a whole lineage withdrawn at once, not one model being deprecated. That test strips comments before matching, deliberately. The comments above these constants name every retired id on purpose — that history is the most useful thing on the page — and a sibling repo already shipped a guard that grepped a function body and failed on a COMMENT, reporting on prose instead of behaviour. A self-test asserts the stripper still finds a genuinely present id, so the sweep cannot pass on an empty haystack. Every replacement was verified against the live catalogue on 2026-08-27. The vision id was chosen on three properties rather than on name: `:free` with `pricing.prompt = 0`, `image` among its input modalities, and tool support. ── One thing fixed that was not a model id ──────────────────────────────────── `npm run verify` could not pass on a clean checkout. ESLint was linting `.claude/worktrees/` — complete copies of this repo living inside it — and reported 641 errors, every single one from those copies. The real tree had zero. CI checks out without the worktrees, so CI was green while every developer's local verify was red. That is the failure mode this fleet has already been bitten by: a gate that fails on good code teaches people to reach for --no-verify, and then the day it means something nobody reads it. Ignored by family, so a new worktree needs no new line. ── Noted, not fixed ─────────────────────────────────────────────────────────── This repo has TWO independent AI provider layers, `src/lib/ai` and `src/lib/hirn/providers`, each with its own Groq and OpenRouter client and its own model ids. That duplication is why five ids needed changing instead of two, and it will cause the same work again. Merging them is a design decision with real consequences for the Hirn feature, so it belongs to a human rather than to this PR. verify: lint (0 errors), typecheck, 538 suites / 7766 tests, build. Co-Authored-By: Claude Opus 5 --- eslint.config.mjs | 13 ++ .../ai/__tests__/no-retired-models.test.ts | 116 ++++++++++++++++++ src/lib/ai/providers.ts | 43 +++++-- src/lib/hirn/providers/groq.ts | 5 +- src/lib/hirn/providers/openrouter.ts | 7 +- 5 files changed, 170 insertions(+), 14 deletions(-) create mode 100644 src/lib/ai/__tests__/no-retired-models.test.ts diff --git a/eslint.config.mjs b/eslint.config.mjs index 591d1c714..2747797c1 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -87,6 +87,19 @@ const eslintConfig = [ "node_modules/**", "examples/**", "packages/**", + // Per-session agent worktrees are complete checkouts of this repo living + // inside it, so without this every file gets linted twice: once here and + // once per worktree. Measured 2026-08-27 — `npx eslint .` reported 641 + // errors and ALL 641 were inside .claude/worktrees. The real tree had + // zero. + // + // That makes `npm run verify` permanently red on a clean repo, while CI — + // which checks out without the worktrees — stays green. A gate that fails + // on good code is worse than no gate: it is what teaches people to reach + // for --no-verify, and then the day it means something, nobody reads it. + // + // Ignored by FAMILY, not by instance, so a new worktree needs no new line. + ".claude/worktrees/**", ], }, ...nextConfig, diff --git a/src/lib/ai/__tests__/no-retired-models.test.ts b/src/lib/ai/__tests__/no-retired-models.test.ts new file mode 100644 index 000000000..12b3816f7 --- /dev/null +++ b/src/lib/ai/__tests__/no-retired-models.test.ts @@ -0,0 +1,116 @@ +/** + * No retired model id may survive anywhere in this repo's AI layers. + * + * This exists because evig has been repinned three times for the same reason, + * and the history is still legible in the comments: Llama-4-Scout was + * decommissioned, so the ids moved to `llama-3.3-70b-versatile`, and Groq then + * retired the entire llama-3.x family. At the low point FOUR ids were dead at + * once — the Groq default, the large-context fallback, the OpenRouter default + * and the OpenRouter vision model — and none of them failed anything until a + * user saw it. + * + * A repin cannot prevent the next retirement; only a check that runs without + * being asked can. Two now do: + * + * - dotfiles/scripts/ci/model-pin-audit.mjs asks Groq and OpenRouter DAILY + * whether these exact ids are still listed. That is the one that catches + * new rot, because it talks to the vendor. + * - this test, which catches the cheaper mistake: someone reintroducing an + * id already known to be dead, by copying an old line, reverting a file, or + * merging a stale branch. + * + * ── Why this reads source text, and why that is normally wrong ────────────── + * + * These ids are module-private constants. Nothing exports them, so there is no + * value to assert against, and the alternative — exporting internals purely to + * make them testable — would be a worse trade. + * + * Reading source has a specific failure mode that has already bitten this fleet + * once: a sibling repo's guard grepped a function body and failed on a COMMENT + * that named the old id, so it reported on prose rather than behaviour. The + * comments above these very constants name every retired id on purpose, because + * that history is the most useful thing on the page. + * + * So comments are stripped before anything is matched. The test then sees only + * code, which is the only thing that can call a vendor. + */ +import { readFileSync, readdirSync, statSync } from 'node:fs' +import { join } from 'node:path' + +/** Source with `//` and block comments removed. Not a parser; sufficient here. */ +function stripComments(source: string): string { + return source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:])\/\/[^\n]*/g, '$1') +} + +function walk(dir: string): string[] { + const out: string[] = [] + for (const entry of readdirSync(dir)) { + const full = join(dir, entry) + if (statSync(full).isDirectory()) { + if (entry === '__tests__' || entry === 'node_modules') continue + out.push(...walk(full)) + } else if (/\.(ts|tsx)$/.test(entry)) { + out.push(full) + } + } + return out +} + +/** + * Families, not individual ids. + * + * The failure here was never one model being deprecated — it was a whole + * lineage withdrawn at once, twice. Listing `llama-3.3-70b-versatile` alone + * would pass happily on `llama-3.1-8b-instant`, which died the same day. + */ +const RETIRED = [ + { pattern: /\bllama-3\.\d[\w.-]*/i, why: 'Groq retired the entire llama-3.x family' }, + { pattern: /\bllama-4-scout[\w.-]*/i, why: 'decommissioned; this repo was already repinned off it once' }, + { pattern: /\bopenai\/gpt-oss-20b:free\b/i, why: 'retired from OpenRouter’s catalogue' }, + { pattern: /\bgoogle\/gemini-2\.0-flash-001\b/i, why: 'retired from OpenRouter’s catalogue' }, +] + +const AI_DIRS = ['src/lib/ai', 'src/lib/hirn'] + +describe('retired model ids', () => { + const files = AI_DIRS.flatMap((dir) => walk(join(process.cwd(), dir))) + + it('finds AI source to check, so an empty sweep cannot pass vacuously', () => { + // Without this, a moved directory turns the whole file below into a + // guarantee about nothing — which reads identically to a clean result. + expect(files.length).toBeGreaterThan(5) + }) + + it.each(RETIRED)('carries no id matching $pattern ($why)', ({ pattern }) => { + const offenders: string[] = [] + + for (const file of files) { + const code = stripComments(readFileSync(file, 'utf8')) + // Only quoted strings can be sent to a vendor. This also keeps a bare + // word in an identifier or type name from tripping the check. + for (const match of code.matchAll(/['"`]([^'"`\n]+)['"`]/g)) { + if (pattern.test(match[1])) { + offenders.push(`${file.replace(process.cwd() + '/', '')}: ${match[1]}`) + } + } + } + + // Named, not counted: the failure should say which file and which id. + expect(offenders).toEqual([]) + }) + + it('still catches an id that is genuinely present', () => { + // Guards the stripper and the matcher together. If `stripComments` ever + // ate real code, or the quoted-string scan stopped finding anything, every + // assertion above would pass on an empty haystack. + const sample = ` + // a comment naming 'llama-3.3-70b-versatile' must NOT count + const model = 'llama-3.1-8b-instant' + ` + const code = stripComments(sample) + const found = [...code.matchAll(/['"`]([^'"`\n]+)['"`]/g)].map((m) => m[1]) + + expect(found).toContain('llama-3.1-8b-instant') + expect(found).not.toContain('llama-3.3-70b-versatile') + }) +}) diff --git a/src/lib/ai/providers.ts b/src/lib/ai/providers.ts index bf242a201..85a6d48d6 100644 --- a/src/lib/ai/providers.ts +++ b/src/lib/ai/providers.ts @@ -19,22 +19,41 @@ import { ORG } from '@/config/org' // CONFIGURATION (SSOT - all AI provider settings in one place) // ============================================================================= +// ── Read the comments below before changing a model id ────────────────────── +// +// This block has now been repinned three times, and the first two repins are +// still visible in its history: Llama-4-Scout was decommissioned, so it moved +// to `llama-3.3-70b-versatile` — and Groq has since retired the entire +// llama-3.x family, so that id died too, along with every OpenRouter llama-3 +// entry beside it. Four of the five ids here were dead simultaneously. +// +// Repinning is what this repo keeps doing and it is not what fixes it. What +// fixes it is that dotfiles/scripts/ci/model-pin-audit.mjs now asks both +// vendors DAILY whether these ids still exist, so the next retirement surfaces +// within a day instead of on a user's screen. Every id below was verified +// present in the live catalogue on 2026-08-27. const GROQ_API_URL = 'https://api.groq.com/openai/v1/chat/completions' -const GROQ_MODEL = 'llama-3.3-70b-versatile' -// Fallback for large prompts. Groq decommissioned Llama-4-Scout (404 -// model_not_found), so fall back to the versatile 128k model — a dead model id -// here silently broke the large-prompt retry path. -const GROQ_LARGE_CONTEXT_MODEL = 'llama-3.3-70b-versatile' +const GROQ_MODEL = 'openai/gpt-oss-120b' +// Fallback for large prompts. Same 131k context as the default above, which is +// the whole Groq free lineup's window now — Groq's catalogue no longer has a +// larger free model to escalate to, so this is a retry rather than an upgrade. +const GROQ_LARGE_CONTEXT_MODEL = 'openai/gpt-oss-120b' const OPENROUTER_API_URL = 'https://openrouter.ai/api/v1/chat/completions' -const OPENROUTER_MODEL = 'meta-llama/llama-3.3-70b-instruct:free' - -// Vision-capable models (multimodal). Groq RETIRED Llama-4-Scout — its whole -// lineup lost vision except Qwen3 (verified: qwen/qwen3.6-27b accepts image_url); -// OpenRouter keeps a free vision model as fallback. Used by callVisionWithFallback -// so photo analysis works on prod (Ollama vision is local-dev only, not deployed). +const OPENROUTER_MODEL = 'nvidia/nemotron-3-super-120b-a12b:free' + +// Vision-capable models (multimodal). Groq's lineup lost vision except Qwen3 +// (verified: qwen/qwen3.6-27b accepts image_url, and it is still listed); +// OpenRouter keeps a free vision model as fallback. Used by +// callVisionWithFallback so photo analysis works on prod (Ollama vision is +// local-dev only, not deployed). +// +// The OpenRouter vision id was `meta-llama/llama-3.2-11b-vision-instruct:free`, +// retired with the rest of that family. Its replacement was chosen against the +// live catalogue on three properties, not on name: `:free` with +// `pricing.prompt = 0`, `image` in its input modalities, and tool support. const GROQ_VISION_MODEL = 'qwen/qwen3.6-27b' -const OPENROUTER_VISION_MODEL = 'meta-llama/llama-3.2-11b-vision-instruct:free' +const OPENROUTER_VISION_MODEL = 'google/gemma-4-26b-a4b-it:free' const DEFAULT_TIMEOUT_MS = 60000 diff --git a/src/lib/hirn/providers/groq.ts b/src/lib/hirn/providers/groq.ts index 1da4da5fa..0e9fbbd0a 100644 --- a/src/lib/hirn/providers/groq.ts +++ b/src/lib/hirn/providers/groq.ts @@ -16,7 +16,10 @@ import type { } from './types' const GROQ_API_URL = 'https://api.groq.com/openai/v1' -const DEFAULT_MODEL = 'llama-3.3-70b-versatile' +// Groq retired the whole llama-3.x family; this read +// `llama-3.3-70b-versatile`, so every Hirn chat answered by Groq returned 404 +// with a valid key. Verified present in the live catalogue 2026-08-27. +const DEFAULT_MODEL = 'openai/gpt-oss-120b' const REQUEST_TIMEOUT_MS = 30_000 const AVAILABILITY_TIMEOUT_MS = 5_000 diff --git a/src/lib/hirn/providers/openrouter.ts b/src/lib/hirn/providers/openrouter.ts index 6dc52ff83..1bb26244c 100644 --- a/src/lib/hirn/providers/openrouter.ts +++ b/src/lib/hirn/providers/openrouter.ts @@ -32,7 +32,12 @@ const OPENROUTER_API_URL = 'https://openrouter.ai/api/v1' * without a deploy — and when it rots, replace it with another `:free` id rather * than dropping the suffix. */ -const DEFAULT_MODEL = process.env.OPENROUTER_MODEL?.trim() || 'openai/gpt-oss-20b:free' +// `openai/gpt-oss-20b:free` has since been retired — the comment above already +// said this would happen and asked for another `:free` id rather than dropping +// the suffix, which is exactly what this is. Verified 2026-08-27: `:free`, and +// `pricing.prompt = 0` in OpenRouter's own catalogue. +const DEFAULT_MODEL = + process.env.OPENROUTER_MODEL?.trim() || 'nvidia/nemotron-3-super-120b-a12b:free' const REQUEST_TIMEOUT_MS = 30_000 const AVAILABILITY_TIMEOUT_MS = 5_000