From 640b931a59b642830d5507c5d8563096a48b542f Mon Sep 17 00:00:00 2001 From: Georgy Butaev <41178744+g-but@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:48:00 +0200 Subject: [PATCH] fix(ai): the platform LLM was calling a model that no longer exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by asking the Cat a real question in production and reading why it answered with its fallback: platform-llm: model call failed status: 404 model: llama-3.3-70b-versatile Groq stopped serving that id. Confirmed against the live API: 14 models available, and it is not among them. This was not the Cat's problem. Every callPlatformJson caller returned null — the offer engine, both writing engines, prompt suggestions, platform feedback, image suggestions, the voice intent router, and the Cat. Each is written to "degrade gracefully", so eight features degraded gracefully into doing nothing, and the log line was a warn. Three changes, and the id is the least important one. FAILOVER. resolveProvider returned the FIRST provider whose key existed and stopped, so a dead Groq id took everything down while OpenRouter sat configured and unused. It now returns a LIST and the call walks it. The ordering is unchanged and is not the point — the point is that there is a second entry at all. A 404 is now an ERROR, not a warn, and says what it means: the pinned id has rotted. A model that no longer exists is a configuration fault, not a hiccup; it will fail identically until a human changes a constant, so it must not share a log level with a timeout. check:ai-models asks each provider whether it still serves what we pinned, reading the ids out of source so the check cannot drift from the constants. Deliberately NOT in `verify`: it needs the network and a provider key, and a gate that goes red when an API hiccups is a gate that gets switched off. It runs where the keys live, and a provider it cannot ask is reported as SKIPPED on its own line — because the failure mode of this entire class of check is a skip being mistaken for a pass. Proven both ways against the live APIs: green on the new id, and red on the old one with "'llama-3.3-70b-versatile' is NOT served (14 models available)". Co-Authored-By: Claude Opus 5 --- package.json | 1 + scripts/check-ai-models.mjs | 117 +++++++++++++++++++++++ src/services/cat/platform-llm.ts | 154 +++++++++++++++++++++---------- 3 files changed, 222 insertions(+), 50 deletions(-) create mode 100644 scripts/check-ai-models.mjs diff --git a/package.json b/package.json index 8b59e54a4..3f1ab2987 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "check:schema-columns": "node scripts/check-schema-columns.mjs", "check:currency-units": "node scripts/check-currency-units.mjs", "check:rpc-exists": "node scripts/check-rpc-exists.mjs", + "check:ai-models": "node scripts/check-ai-models.mjs", "check:mdx": "node scripts/check-mdx.mjs", "verify": "npm run ci:docs && npm run check:accent-ink && npm run type-check && npm run check:sizes && npm run audit:routes && npm run lint && npm run check:duplication && npm run check:dead-fields && npm run check:schema-columns && npm run check:currency-units && npm run check:rpc-exists && npm run check:mdx && npm run test:unit -- --watchAll=false", "audit:schema": "node scripts/db/audit-schema-drift.mjs", diff --git a/scripts/check-ai-models.mjs b/scripts/check-ai-models.mjs new file mode 100644 index 000000000..25661c174 --- /dev/null +++ b/scripts/check-ai-models.mjs @@ -0,0 +1,117 @@ +#!/usr/bin/env node +/* eslint-disable no-console */ +/** + * check-ai-models.mjs — every pinned model id must still be served. + * + * A model id is a string in a constant. Nothing type-checks it, nothing tests + * it, and providers retire models on their own schedule — so the id keeps + * compiling long after it stops existing, and the only symptom is a 404 inside + * a caller written to "degrade gracefully". + * + * That is not hypothetical and it is not the first time. On 2026-08-26 Groq had + * stopped serving `llama-3.3-70b-versatile`, which platform-llm.ts pinned. Every + * callPlatformJson caller returned null: the offer engine, both writing engines, + * prompt suggestions, platform feedback, image suggestions, the voice intent + * router, and the Cat's replies. Eight features degraded gracefully into doing + * nothing at all, and the log line for it was a `warn`. + * + * WHY THIS IS NOT IN `verify` + * It needs the network and a provider key, and a gate that goes red when an API + * hiccups is a gate that gets disabled. Run it where the keys live: + * + * npm run check:ai-models # uses .env.local / the environment + * + * With no key for a provider it SKIPS that provider and says so — loudly enough + * that a skip cannot be mistaken for a pass, which is the failure mode this + * whole class of check keeps falling into. + */ + +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const ROOT = new URL('..', import.meta.url).pathname; + +/** Read an env var, falling back to .env.local so this runs from a worktree. */ +function env(key) { + if (process.env[key]) { + return process.env[key]; + } + try { + const text = readFileSync(join(ROOT, '.env.local'), 'utf8'); + return text.match(new RegExp(`^${key}=(.*)$`, 'm'))?.[1]?.trim(); + } catch { + return undefined; + } +} + +/** Pinned ids, read from source so this file cannot drift from the constant. */ +function pinnedModels() { + const llm = readFileSync(join(ROOT, 'src/services/cat/platform-llm.ts'), 'utf8'); + const groq = llm.match(/const GROQ_MODEL = '([^']+)'/)?.[1]; + + const models = readFileSync(join(ROOT, 'src/config/ai-models.ts'), 'utf8'); + const openRouter = models.match(/export const DEFAULT_FREE_MODEL_ID = '([^']+)'/)?.[1]; + + return [ + { provider: 'groq', id: groq, keyName: 'GROQ_API_KEY', url: 'https://api.groq.com/openai/v1/models' }, + { provider: 'openrouter', id: openRouter, keyName: 'OPENROUTER_API_KEY', url: 'https://openrouter.ai/api/v1/models' }, + ]; +} + +async function servedIds(url, key) { + const res = await fetch(url, { headers: { Authorization: `Bearer ${key}` } }); + if (!res.ok) { + throw new Error(`${res.status} from ${url}`); + } + const json = await res.json(); + return new Set((json.data ?? []).map(m => m.id)); +} + +const pinned = pinnedModels(); +const missing = []; +const skipped = []; + +for (const entry of pinned) { + if (!entry.id) { + missing.push(`${entry.provider}: could not read the pinned id from source`); + continue; + } + const key = env(entry.keyName); + if (!key) { + skipped.push(`${entry.provider} (${entry.id}) — no ${entry.keyName}`); + continue; + } + try { + const served = await servedIds(entry.url, key); + if (served.has(entry.id)) { + console.log(`[check-ai-models] ${entry.provider}: ${entry.id} — served (${served.size} models)`); + } else { + missing.push(`${entry.provider}: '${entry.id}' is NOT served (${served.size} models available)`); + } + } catch (error) { + skipped.push(`${entry.provider} (${entry.id}) — could not ask: ${error.message}`); + } +} + +// A skip is not a pass. Say so on its own line so it cannot be skimmed past. +for (const line of skipped) { + console.log(`[check-ai-models] SKIPPED — ${line}`); +} + +if (missing.length > 0) { + console.error( + `\n[check-ai-models] FAIL: ${missing.length} pinned model(s) no longer exist:\n` + + missing.map(m => ` ${m}`).join('\n') + + '\n\n Every callPlatformJson caller returns null against a retired id, and\n' + + ' each one is written to degrade gracefully — so the symptom is features\n' + + ' quietly doing nothing. Update the constant and redeploy.\n' + ); + process.exit(1); +} + +if (skipped.length === pinned.length) { + console.error('\n[check-ai-models] nothing was checked — no provider keys available.\n'); + process.exit(1); +} + +console.log('[check-ai-models] OK — every pinned model is still served.'); diff --git a/src/services/cat/platform-llm.ts b/src/services/cat/platform-llm.ts index 68520e3f9..b9b47e525 100644 --- a/src/services/cat/platform-llm.ts +++ b/src/services/cat/platform-llm.ts @@ -18,7 +18,24 @@ import { DEFAULT_FREE_MODEL_ID } from '@/config/ai-models'; // registry is now the one place ids live, guarded by the free-model catalog // probe (health-probes.ts), so drift is detected there instead of re-pinned // here. -const GROQ_MODEL = 'llama-3.3-70b-versatile'; +/** + * Groq's general-purpose model. + * + * `llama-3.3-70b-versatile` was pinned here and STOPPED BEING SERVED. Groq + * answered 404 for it, every callPlatformJson caller returned null, and because + * the failure was logged at warn and swallowed by callers that "degrade + * gracefully", eight features degraded gracefully into doing nothing: the offer + * engine, both writing engines, prompt suggestions, platform feedback, image + * suggestions, the voice intent router, and the Cat's replies. Verified against + * the live API on 2026-08-26 — Groq served 14 models and that was not among + * them. + * + * Model ids rot. This is the fifth time in this fleet, and the comment below + * already said so about OpenRouter. The durable answer is not a better id, it + * is the failover underneath and `npm run check:ai-models`, which asks each + * provider whether it still serves what we pinned. + */ +const GROQ_MODEL = 'openai/gpt-oss-120b'; const OPENROUTER_MODEL = DEFAULT_FREE_MODEL_ID; export interface PlatformJsonOpts { @@ -41,26 +58,33 @@ interface Provider { isOpenRouter: boolean; } -function resolveProvider(_longform: boolean): Provider | null { +/** + * Providers to try, in order. + * + * Groq first — fast, and handles long-form JSON inside the free TPM budget. + * OpenRouter after it, and that ORDERING IS NOT THE POINT: what matters is that + * there is a second entry at all. This used to return the FIRST provider whose + * key existed and stop, so when Groq's pinned model stopped being served there + * was no path out — a dead id took every platform-LLM feature down with it and + * OpenRouter sat there configured and unused. + */ +function resolveProviders(): Provider[] { + const providers: Provider[] = []; const groqKey = process.env.GROQ_API_KEY; const openRouterKey = process.env.OPENROUTER_API_KEY; - // Prefer Groq — it's the verified-working provider on the box (fast, handles - // long-form JSON within the free TPM budget). OpenRouter is a fallback only: - // its free model IDs rot and 404 (both gpt-oss-120b:free and llama-4-maverick:free - // returned 404 in prod), so never route to it when Groq is available. if (groqKey) { - return { + providers.push({ url: `${PROVIDER_BASE_URLS.groq}/chat/completions`, model: GROQ_MODEL, apiKey: groqKey, isOpenRouter: false, - }; + }); } if (openRouterKey) { - return openRouter(openRouterKey, OPENROUTER_MODEL); + providers.push(openRouter(openRouterKey, OPENROUTER_MODEL)); } - return null; + return providers; } function openRouter(apiKey: string, model: string): Provider { @@ -81,20 +105,12 @@ export async function callPlatformJson( user: string, opts: PlatformJsonOpts = {} ): Promise { - const provider = resolveProvider(!!opts.longform); - if (!provider) { + const providers = resolveProviders(); + if (providers.length === 0) { logger.warn('platform-llm: no platform AI key configured', {}, 'PlatformLLM'); return null; } - const headers: Record = { - 'Content-Type': 'application/json', - Authorization: `Bearer ${provider.apiKey}`, - }; - if (provider.isOpenRouter) { - headers['HTTP-Referer'] = process.env.NEXT_PUBLIC_APP_URL || 'https://orangecat.ch'; - } - const messages = [ { role: 'system', content: system }, { role: 'user', content: user }, @@ -102,42 +118,80 @@ export async function callPlatformJson( const maxTokens = opts.maxTokens ?? (opts.longform ? 3000 : 1400); const temperature = opts.temperature ?? 0.6; - const call = (jsonMode: boolean) => - fetch(provider.url, { - method: 'POST', - headers, - ...(opts.timeoutMs ? { signal: AbortSignal.timeout(opts.timeoutMs) } : {}), - body: JSON.stringify({ - model: provider.model, - messages, - temperature, - max_tokens: maxTokens, - ...(jsonMode ? { response_format: { type: 'json_object' } } : {}), - }), - }); + let lastStatus: number | null = null; - try { - // Some free models 400 on response_format — retry once without it and lean - // on parseJsonLoose (the system prompt already demands JSON-only output). - let response = await call(true); - if (!response.ok) { + for (const provider of providers) { + const headers: Record = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${provider.apiKey}`, + }; + if (provider.isOpenRouter) { + headers['HTTP-Referer'] = process.env.NEXT_PUBLIC_APP_URL || 'https://orangecat.ch'; + } + + const call = (jsonMode: boolean) => + fetch(provider.url, { + method: 'POST', + headers, + ...(opts.timeoutMs ? { signal: AbortSignal.timeout(opts.timeoutMs) } : {}), + body: JSON.stringify({ + model: provider.model, + messages, + temperature, + max_tokens: maxTokens, + ...(jsonMode ? { response_format: { type: 'json_object' } } : {}), + }), + }); + + try { + // Some free models 400 on response_format — retry once without it and lean + // on parseJsonLoose (the system prompt already demands JSON-only output). + let response = await call(true); + if (!response.ok) { + response = await call(false); + } + + if (response.ok) { + const json = (await response.json()) as { + choices?: { message?: { content?: string } }[]; + }; + return json.choices?.[0]?.message?.content ?? null; + } + + lastStatus = response.status; + // 404 means the model id no longer exists, which is a CONFIGURATION fault + // rather than a hiccup: it will fail identically until someone changes the + // constant. warn was too quiet — it degraded eight features to silence for + // as long as nobody read the logs. + if (response.status === 404) { + logger.error( + 'platform-llm: model no longer served — the pinned id has rotted', + { model: provider.model, provider: provider.isOpenRouter ? 'openrouter' : 'groq' }, + 'PlatformLLM' + ); + } else { + logger.warn( + 'platform-llm: model call failed', + { status: response.status, model: provider.model }, + 'PlatformLLM' + ); + } + } catch (err) { logger.warn( - 'platform-llm: json-mode call failed, retrying without response_format', - { status: response.status, model: provider.model }, + 'platform-llm: model call threw', + { err: String(err), model: provider.model }, 'PlatformLLM' ); - response = await call(false); - } - if (!response.ok) { - logger.warn('platform-llm: model call failed', { status: response.status }, 'PlatformLLM'); - return null; } - const json = (await response.json()) as { choices?: { message?: { content?: string } }[] }; - return json.choices?.[0]?.message?.content ?? null; - } catch (err) { - logger.warn('platform-llm: model call threw', { err: String(err) }, 'PlatformLLM'); - return null; + // Fall through to the next provider. } + + logger.error( + 'platform-llm: every provider failed', + { providers: providers.length, lastStatus }, + 'PlatformLLM' + ); + return null; } /**