diff --git a/package.json b/package.json index 8b59e54a4..3f1ab2987 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "check:schema-columns": "node scripts/check-schema-columns.mjs", "check:currency-units": "node scripts/check-currency-units.mjs", "check:rpc-exists": "node scripts/check-rpc-exists.mjs", + "check:ai-models": "node scripts/check-ai-models.mjs", "check:mdx": "node scripts/check-mdx.mjs", "verify": "npm run ci:docs && npm run check:accent-ink && npm run type-check && npm run check:sizes && npm run audit:routes && npm run lint && npm run check:duplication && npm run check:dead-fields && npm run check:schema-columns && npm run check:currency-units && npm run check:rpc-exists && npm run check:mdx && npm run test:unit -- --watchAll=false", "audit:schema": "node scripts/db/audit-schema-drift.mjs", diff --git a/scripts/check-ai-models.mjs b/scripts/check-ai-models.mjs new file mode 100644 index 000000000..25661c174 --- /dev/null +++ b/scripts/check-ai-models.mjs @@ -0,0 +1,117 @@ +#!/usr/bin/env node +/* eslint-disable no-console */ +/** + * check-ai-models.mjs — every pinned model id must still be served. + * + * A model id is a string in a constant. Nothing type-checks it, nothing tests + * it, and providers retire models on their own schedule — so the id keeps + * compiling long after it stops existing, and the only symptom is a 404 inside + * a caller written to "degrade gracefully". + * + * That is not hypothetical and it is not the first time. On 2026-08-26 Groq had + * stopped serving `llama-3.3-70b-versatile`, which platform-llm.ts pinned. Every + * callPlatformJson caller returned null: the offer engine, both writing engines, + * prompt suggestions, platform feedback, image suggestions, the voice intent + * router, and the Cat's replies. Eight features degraded gracefully into doing + * nothing at all, and the log line for it was a `warn`. + * + * WHY THIS IS NOT IN `verify` + * It needs the network and a provider key, and a gate that goes red when an API + * hiccups is a gate that gets disabled. Run it where the keys live: + * + * npm run check:ai-models # uses .env.local / the environment + * + * With no key for a provider it SKIPS that provider and says so — loudly enough + * that a skip cannot be mistaken for a pass, which is the failure mode this + * whole class of check keeps falling into. + */ + +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const ROOT = new URL('..', import.meta.url).pathname; + +/** Read an env var, falling back to .env.local so this runs from a worktree. */ +function env(key) { + if (process.env[key]) { + return process.env[key]; + } + try { + const text = readFileSync(join(ROOT, '.env.local'), 'utf8'); + return text.match(new RegExp(`^${key}=(.*)$`, 'm'))?.[1]?.trim(); + } catch { + return undefined; + } +} + +/** Pinned ids, read from source so this file cannot drift from the constant. */ +function pinnedModels() { + const llm = readFileSync(join(ROOT, 'src/services/cat/platform-llm.ts'), 'utf8'); + const groq = llm.match(/const GROQ_MODEL = '([^']+)'/)?.[1]; + + const models = readFileSync(join(ROOT, 'src/config/ai-models.ts'), 'utf8'); + const openRouter = models.match(/export const DEFAULT_FREE_MODEL_ID = '([^']+)'/)?.[1]; + + return [ + { provider: 'groq', id: groq, keyName: 'GROQ_API_KEY', url: 'https://api.groq.com/openai/v1/models' }, + { provider: 'openrouter', id: openRouter, keyName: 'OPENROUTER_API_KEY', url: 'https://openrouter.ai/api/v1/models' }, + ]; +} + +async function servedIds(url, key) { + const res = await fetch(url, { headers: { Authorization: `Bearer ${key}` } }); + if (!res.ok) { + throw new Error(`${res.status} from ${url}`); + } + const json = await res.json(); + return new Set((json.data ?? []).map(m => m.id)); +} + +const pinned = pinnedModels(); +const missing = []; +const skipped = []; + +for (const entry of pinned) { + if (!entry.id) { + missing.push(`${entry.provider}: could not read the pinned id from source`); + continue; + } + const key = env(entry.keyName); + if (!key) { + skipped.push(`${entry.provider} (${entry.id}) — no ${entry.keyName}`); + continue; + } + try { + const served = await servedIds(entry.url, key); + if (served.has(entry.id)) { + console.log(`[check-ai-models] ${entry.provider}: ${entry.id} — served (${served.size} models)`); + } else { + missing.push(`${entry.provider}: '${entry.id}' is NOT served (${served.size} models available)`); + } + } catch (error) { + skipped.push(`${entry.provider} (${entry.id}) — could not ask: ${error.message}`); + } +} + +// A skip is not a pass. Say so on its own line so it cannot be skimmed past. +for (const line of skipped) { + console.log(`[check-ai-models] SKIPPED — ${line}`); +} + +if (missing.length > 0) { + console.error( + `\n[check-ai-models] FAIL: ${missing.length} pinned model(s) no longer exist:\n` + + missing.map(m => ` ${m}`).join('\n') + + '\n\n Every callPlatformJson caller returns null against a retired id, and\n' + + ' each one is written to degrade gracefully — so the symptom is features\n' + + ' quietly doing nothing. Update the constant and redeploy.\n' + ); + process.exit(1); +} + +if (skipped.length === pinned.length) { + console.error('\n[check-ai-models] nothing was checked — no provider keys available.\n'); + process.exit(1); +} + +console.log('[check-ai-models] OK — every pinned model is still served.'); diff --git a/src/services/cat/platform-llm.ts b/src/services/cat/platform-llm.ts index 68520e3f9..b9b47e525 100644 --- a/src/services/cat/platform-llm.ts +++ b/src/services/cat/platform-llm.ts @@ -18,7 +18,24 @@ import { DEFAULT_FREE_MODEL_ID } from '@/config/ai-models'; // registry is now the one place ids live, guarded by the free-model catalog // probe (health-probes.ts), so drift is detected there instead of re-pinned // here. -const GROQ_MODEL = 'llama-3.3-70b-versatile'; +/** + * Groq's general-purpose model. + * + * `llama-3.3-70b-versatile` was pinned here and STOPPED BEING SERVED. Groq + * answered 404 for it, every callPlatformJson caller returned null, and because + * the failure was logged at warn and swallowed by callers that "degrade + * gracefully", eight features degraded gracefully into doing nothing: the offer + * engine, both writing engines, prompt suggestions, platform feedback, image + * suggestions, the voice intent router, and the Cat's replies. Verified against + * the live API on 2026-08-26 — Groq served 14 models and that was not among + * them. + * + * Model ids rot. This is the fifth time in this fleet, and the comment below + * already said so about OpenRouter. The durable answer is not a better id, it + * is the failover underneath and `npm run check:ai-models`, which asks each + * provider whether it still serves what we pinned. + */ +const GROQ_MODEL = 'openai/gpt-oss-120b'; const OPENROUTER_MODEL = DEFAULT_FREE_MODEL_ID; export interface PlatformJsonOpts { @@ -41,26 +58,33 @@ interface Provider { isOpenRouter: boolean; } -function resolveProvider(_longform: boolean): Provider | null { +/** + * Providers to try, in order. + * + * Groq first — fast, and handles long-form JSON inside the free TPM budget. + * OpenRouter after it, and that ORDERING IS NOT THE POINT: what matters is that + * there is a second entry at all. This used to return the FIRST provider whose + * key existed and stop, so when Groq's pinned model stopped being served there + * was no path out — a dead id took every platform-LLM feature down with it and + * OpenRouter sat there configured and unused. + */ +function resolveProviders(): Provider[] { + const providers: Provider[] = []; const groqKey = process.env.GROQ_API_KEY; const openRouterKey = process.env.OPENROUTER_API_KEY; - // Prefer Groq — it's the verified-working provider on the box (fast, handles - // long-form JSON within the free TPM budget). OpenRouter is a fallback only: - // its free model IDs rot and 404 (both gpt-oss-120b:free and llama-4-maverick:free - // returned 404 in prod), so never route to it when Groq is available. if (groqKey) { - return { + providers.push({ url: `${PROVIDER_BASE_URLS.groq}/chat/completions`, model: GROQ_MODEL, apiKey: groqKey, isOpenRouter: false, - }; + }); } if (openRouterKey) { - return openRouter(openRouterKey, OPENROUTER_MODEL); + providers.push(openRouter(openRouterKey, OPENROUTER_MODEL)); } - return null; + return providers; } function openRouter(apiKey: string, model: string): Provider { @@ -81,20 +105,12 @@ export async function callPlatformJson( user: string, opts: PlatformJsonOpts = {} ): Promise { - const provider = resolveProvider(!!opts.longform); - if (!provider) { + const providers = resolveProviders(); + if (providers.length === 0) { logger.warn('platform-llm: no platform AI key configured', {}, 'PlatformLLM'); return null; } - const headers: Record = { - 'Content-Type': 'application/json', - Authorization: `Bearer ${provider.apiKey}`, - }; - if (provider.isOpenRouter) { - headers['HTTP-Referer'] = process.env.NEXT_PUBLIC_APP_URL || 'https://orangecat.ch'; - } - const messages = [ { role: 'system', content: system }, { role: 'user', content: user }, @@ -102,42 +118,80 @@ export async function callPlatformJson( const maxTokens = opts.maxTokens ?? (opts.longform ? 3000 : 1400); const temperature = opts.temperature ?? 0.6; - const call = (jsonMode: boolean) => - fetch(provider.url, { - method: 'POST', - headers, - ...(opts.timeoutMs ? { signal: AbortSignal.timeout(opts.timeoutMs) } : {}), - body: JSON.stringify({ - model: provider.model, - messages, - temperature, - max_tokens: maxTokens, - ...(jsonMode ? { response_format: { type: 'json_object' } } : {}), - }), - }); + let lastStatus: number | null = null; - try { - // Some free models 400 on response_format — retry once without it and lean - // on parseJsonLoose (the system prompt already demands JSON-only output). - let response = await call(true); - if (!response.ok) { + for (const provider of providers) { + const headers: Record = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${provider.apiKey}`, + }; + if (provider.isOpenRouter) { + headers['HTTP-Referer'] = process.env.NEXT_PUBLIC_APP_URL || 'https://orangecat.ch'; + } + + const call = (jsonMode: boolean) => + fetch(provider.url, { + method: 'POST', + headers, + ...(opts.timeoutMs ? { signal: AbortSignal.timeout(opts.timeoutMs) } : {}), + body: JSON.stringify({ + model: provider.model, + messages, + temperature, + max_tokens: maxTokens, + ...(jsonMode ? { response_format: { type: 'json_object' } } : {}), + }), + }); + + try { + // Some free models 400 on response_format — retry once without it and lean + // on parseJsonLoose (the system prompt already demands JSON-only output). + let response = await call(true); + if (!response.ok) { + response = await call(false); + } + + if (response.ok) { + const json = (await response.json()) as { + choices?: { message?: { content?: string } }[]; + }; + return json.choices?.[0]?.message?.content ?? null; + } + + lastStatus = response.status; + // 404 means the model id no longer exists, which is a CONFIGURATION fault + // rather than a hiccup: it will fail identically until someone changes the + // constant. warn was too quiet — it degraded eight features to silence for + // as long as nobody read the logs. + if (response.status === 404) { + logger.error( + 'platform-llm: model no longer served — the pinned id has rotted', + { model: provider.model, provider: provider.isOpenRouter ? 'openrouter' : 'groq' }, + 'PlatformLLM' + ); + } else { + logger.warn( + 'platform-llm: model call failed', + { status: response.status, model: provider.model }, + 'PlatformLLM' + ); + } + } catch (err) { logger.warn( - 'platform-llm: json-mode call failed, retrying without response_format', - { status: response.status, model: provider.model }, + 'platform-llm: model call threw', + { err: String(err), model: provider.model }, 'PlatformLLM' ); - response = await call(false); - } - if (!response.ok) { - logger.warn('platform-llm: model call failed', { status: response.status }, 'PlatformLLM'); - return null; } - const json = (await response.json()) as { choices?: { message?: { content?: string } }[] }; - return json.choices?.[0]?.message?.content ?? null; - } catch (err) { - logger.warn('platform-llm: model call threw', { err: String(err) }, 'PlatformLLM'); - return null; + // Fall through to the next provider. } + + logger.error( + 'platform-llm: every provider failed', + { providers: providers.length, lastStatus }, + 'PlatformLLM' + ); + return null; } /**