diff --git a/package.json b/package.json index faa9a07..0ddcb14 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ai-ration", - "version": "0.2.0", + "version": "0.2.1", "description": "Stay on free LLM tiers: a multi-vendor fallback chain, rate-limit classification that tells the three kinds of 429 apart, and fair-share rationing of a shared daily pool across users.", "license": "MIT", "author": "Mao Nakamoto", @@ -47,6 +47,7 @@ "lint": "eslint .", "typecheck": "tsc -p tsconfig.json --noEmit", "test": "node --test test/*.test.js", + "check:catalog": "npm run build && node scripts/check-catalog.mjs", "verify": "npm run lint && npm run typecheck && npm run build && npm test", "prepare": "npm run build" }, diff --git a/scripts/check-catalog.mjs b/scripts/check-catalog.mjs new file mode 100644 index 0000000..0618a49 --- /dev/null +++ b/scripts/check-catalog.mjs @@ -0,0 +1,38 @@ +/** + * Are this package's DEFAULT pins still real? + * + * `freeChain()` ships a list of model ids, and vendors retire ids without + * notice. On 2026-08-25 four of the nine defaults were already gone — both Groq + * models, so the chain led with a vendor that could never answer, plus the + * preferred OpenRouter fallback. Every consumer inherited that. + * + * Zero tokens: one GET /models per provider. Cheap enough to run on a schedule, + * which is the only kind of check that catches rot without someone remembering. + * + * Run: node scripts/check-catalog.mjs (npm run check:catalog) + * Needs GROQ_API_KEY / OPENROUTER_API_KEY for the providers you want checked; + * a provider with no key is reported UNCHECKED, which is not a pass. + * + * Exit 1 only on CONFIRMED rot, so a keyless or offline run does not fail a + * pipeline for something it could not see. + */ +import { freeChain, checkCatalog, catalogReport, hasRot, deadProviders } from "../dist/index.js"; + +const verdicts = await checkCatalog(freeChain()); +console.log(catalogReport(verdicts)); +console.log(""); + +const dead = deadProviders(verdicts); +if (dead.length) { + console.error(`A whole vendor is gone (${dead.join(", ")}) — the chain is back to a single point of failure.`); +} +if (hasRot(verdicts)) { + console.error("Retired model ids are still pinned. Re-probe replacements and update freeChain()."); + process.exit(1); +} +const unchecked = verdicts.reduce((n, v) => n + v.unchecked.length, 0); +if (unchecked) { + console.log(`No rot found among the ids that could be checked; ${unchecked} were not checkable.`); +} else { + console.log("Every default pin still exists at its vendor."); +} diff --git a/src/catalog.ts b/src/catalog.ts new file mode 100644 index 0000000..573d710 --- /dev/null +++ b/src/catalog.ts @@ -0,0 +1,149 @@ +/** + * catalog — has the vendor retired a model this chain still asks for? + * + * The chain exists because a single pinned free model is a scheduled outage. + * That reasoning has a hole: the chain itself is a list of pinned ids, so it + * rots too, and a chain whose first vendor is entirely dead is a slower version + * of the failure it was built to prevent. + * + * Not hypothetical. On 2026-08-25 `freeChain()` was checked against the live + * catalogues and FOUR of its nine ids were gone — both Groq models (the whole + * first vendor) and two OpenRouter ids, one of them the preferred fallback. The + * consumer that also used the Groq id for direct, unchained calls had been + * silently failing for eight days. + * + * Why this lives in the package rather than in each app: the check is the same + * everywhere, and the app that wrote its own first wrote it slightly + * differently. One implementation, shared by name and by value. + * + * Cheap on purpose — one GET /models per provider and ZERO tokens. That is what + * makes it schedulable, which is the whole difference between a check that runs + * nightly and a command someone is supposed to remember. A tool-call probe + * costs real tokens and cannot run on a timer; existence can. + */ + +import { providerModels, type Env, type Provider } from "./chain.js"; + +export type CatalogVerdict = { + provider: string; + /** + * Ids the vendor currently lists, or NULL when the catalogue could not be + * read (no key, network failure, non-200, unparseable body). + * + * Null is not an empty list. Treating "I could not look" as "nothing is + * there" reports every model as retired and invents an outage; treating it + * as "all fine" hides a real one. Callers must handle three states. + */ + live: string[] | null; + /** Pinned ids confirmed present. Empty when `live` is null. */ + present: string[]; + /** Pinned ids the vendor no longer lists. Empty when `live` is null. */ + missing: string[]; + /** Pinned ids whose status is unknown because `live` is null. */ + unchecked: string[]; +}; + +export type CheckCatalogOptions = { + env?: Env; + /** Injectable for tests; defaults to global fetch. */ + fetchImpl?: typeof fetch; + timeoutMs?: number; +}; + +/** Ids listed by one provider, or null when the catalogue could not be read. */ +async function liveIds( + provider: Provider, + key: string, + fetchImpl: typeof fetch, + timeoutMs: number, +): Promise { + try { + const res = await fetchImpl(`${provider.baseUrl.replace(/\/$/, "")}/models`, { + headers: { Authorization: `Bearer ${key}` }, + signal: AbortSignal.timeout(timeoutMs), + }); + if (!res.ok) return null; + const body = (await res.json()) as { data?: Array<{ id?: unknown }> }; + if (!Array.isArray(body?.data)) return null; + const ids = body.data + .map((m) => (typeof m?.id === "string" ? m.id : "")) + .filter((id): id is string => id.length > 0); + // A catalogue that parses but lists nothing is a malformed answer, not a + // vendor with no models. Refusing it keeps a bad response from reading as + // total rot. + return ids.length > 0 ? ids : null; + } catch { + return null; + } +} + +/** + * Check every model a chain would try against what its vendor still lists. + * + * Honours the same env overrides `usableChain` does, so it checks the ids this + * deployment would ACTUALLY call — not the library defaults an operator has + * already routed around. + */ +export async function checkCatalog( + chain: Provider[], + opts: CheckCatalogOptions = {}, +): Promise { + const env = opts.env ?? process.env; + const fetchImpl = opts.fetchImpl ?? fetch; + const timeoutMs = opts.timeoutMs ?? 20_000; + + const out: CatalogVerdict[] = []; + for (const provider of chain) { + const pinned = providerModels(provider, env); + const key = env[provider.keyEnv]?.trim(); + const live = key ? await liveIds(provider, key, fetchImpl, timeoutMs) : null; + + if (!live) { + out.push({ provider: provider.id, live: null, present: [], missing: [], unchecked: pinned }); + continue; + } + const set = new Set(live); + out.push({ + provider: provider.id, + live, + present: pinned.filter((m) => set.has(m)), + missing: pinned.filter((m) => !set.has(m)), + unchecked: [], + }); + } + return out; +} + +/** True when any pinned id is confirmed gone. Unchecked providers do NOT count + * — an unreadable catalogue is not evidence of rot. */ +export function hasRot(verdicts: CatalogVerdict[]): boolean { + return verdicts.some((v) => v.missing.length > 0); +} + +/** True when a whole vendor's models are gone, i.e. the chain has lost a link + * entirely. Worth separating: a chain that still has vendors is degraded, a + * chain that has lost one is back to being a single point of failure. */ +export function deadProviders(verdicts: CatalogVerdict[]): string[] { + return verdicts + .filter((v) => v.live !== null && v.present.length === 0 && v.missing.length > 0) + .map((v) => v.provider); +} + +/** Human-readable report. Keeps could-not-look visibly distinct from a pass. */ +export function catalogReport(verdicts: CatalogVerdict[]): string { + const lines: string[] = []; + for (const v of verdicts) { + if (v.live === null) { + lines.push(`? ${v.provider}: catalogue unreadable (no key, or the request failed) — ${v.unchecked.length} id(s) UNCHECKED`); + for (const m of v.unchecked) lines.push(` ? ${m}`); + continue; + } + for (const m of v.present) lines.push(` ok ${v.provider}/${m}`); + for (const m of v.missing) lines.push(` GONE ${v.provider}/${m}`); + } + const dead = deadProviders(verdicts); + if (dead.length) lines.push(`\nEVERY model is gone at: ${dead.join(", ")} — the chain has lost that vendor entirely.`); + const unchecked = verdicts.reduce((n, v) => n + v.unchecked.length, 0); + if (unchecked) lines.push(`\n${unchecked} id(s) could not be checked. That is not a pass for them.`); + return lines.join("\n"); +} diff --git a/src/chain.ts b/src/chain.ts index ebe93ea..5edcca8 100644 --- a/src/chain.ts +++ b/src/chain.ts @@ -59,6 +59,23 @@ export type Provider = { modelsEnv?: string; /** Env var overriding `dailyTokens` at call time. */ dailyTokensEnv?: string; + /** + * Does this vendor use ROUTED ids, where `vendor/model` names weights it + * resells and a `:free` suffix is the difference between free routing and a + * per-call charge? True for OpenRouter. + * + * It matters because the same STRING means different things at different + * vendors. `openai/gpt-oss-20b` bills at OpenRouter (no `:free`), while at + * Groq it is simply that vendor's name for a model whose cost depends on the + * account tier. Deciding cost from the id alone was safe only while + * non-routed vendors used bare ids like `llama-3.1-8b-instant`; Groq now + * ships vendor-prefixed ids, so the shape no longer identifies the vendor. + * + * Defaults to false: claiming an id is routed when it is not would report a + * free model as paid, and the reverse — assuming free — is the direction + * this module exists to refuse. + */ + routed?: boolean; }; export type Env = Record; @@ -135,7 +152,14 @@ export function freeChain(prefix = "AI"): Provider[] { id: "groq", baseUrl: "https://api.groq.com/openai/v1", keyEnv: "GROQ_API_KEY", - models: ["llama-3.3-70b-versatile", "llama-3.1-8b-instant"], + // Re-probed 2026-08-25 against the live catalog. The previous pins, + // `llama-3.3-70b-versatile` and `llama-3.1-8b-instant`, were BOTH gone — + // so this "fallback chain" led with a fully dead vendor and every caller + // paid two 404s before reaching OpenRouter. FleetCrown, whose direct + // (non-chain) calls used the same id and had no fallback at all, was + // silently down for eight days. Both ids below answered with a correct + // native tool_call when probed, which is the bar this list is held to. + models: ["openai/gpt-oss-120b", "openai/gpt-oss-20b"], // Not a guess: Groq's own TPD refusal names it — "on tokens per day // (TPD): Limit 100000". Org-wide, so every feature sharing the key draws // from this same pool. @@ -145,12 +169,17 @@ export function freeChain(prefix = "AI"): Provider[] { id: "openrouter", baseUrl: "https://openrouter.ai/api/v1", keyEnv: "OPENROUTER_API_KEY", + // Routed ids: `:free` is the whole difference between free routing and a + // per-call charge for the same weights. See Provider.routed. + routed: true, + // Re-checked 2026-08-25 against the 419-model live catalog. Two entries + // were retired and are removed here: `openai/gpt-oss-20b:free` — which + // was FIRST, so the preferred fallback 404'd on every call — and + // `nvidia/nemotron-3-nano-30b-a3b:free`. The five below were present. models: [ - "openai/gpt-oss-20b:free", "nvidia/nemotron-3-super-120b-a12b:free", "nvidia/nemotron-3.5-lightning:free", "google/gemma-4-26b-a4b-it:free", - "nvidia/nemotron-3-nano-30b-a3b:free", "cohere/north-mini-code:free", "openrouter/free", ], @@ -186,6 +215,12 @@ export type CostVerdict = "free" | "paid" | "unknown"; * * Guessing "free" there would be the dangerous direction — it is what let three * of these through code review. + * + * IMPORTANT: this reads the id as a ROUTED (OpenRouter-shape) id, because that + * is the only shape where the string decides. It is therefore wrong to apply to + * an id from a vendor that merely happens to prefix its own models — Groq's + * `openai/gpt-oss-120b` is not a routed OpenAI id, and this function would call + * it paid. When you know the provider, use `modelCostAt`; `paidModelsIn` does. */ export function modelCost(id: string): CostVerdict { const model = id.trim(); @@ -196,15 +231,33 @@ export function modelCost(id: string): CostVerdict { return model.endsWith(":free") ? "free" : "paid"; } +/** + * Cost of a model AT a specific provider — the honest signature, because the + * same id answers differently at different vendors (see `Provider.routed`). + * + * At a non-routed vendor the id carries no cost information at all: what you + * pay is the account's tier there, which no string can report. That is the + * same "unknown" a bare id has always returned, now correct for vendor-prefixed + * ids too. + */ +export function modelCostAt(provider: Provider, model: string): CostVerdict { + return provider.routed ? modelCost(model) : "unknown"; +} + /** * Assert every model in a chain is free, for apps that must never bill. * + * Judges each id AT ITS PROVIDER. Flagging Groq's `openai/gpt-oss-120b` as paid + * because it contains a slash would be a false alarm that pressures someone + * into "fixing" a working free model — and a guard that cries wolf gets + * disabled, taking the three real cases it does catch with it. + * * Returns the offending ids rather than throwing: the caller knows whether a * paid link is a bug or a deliberate, opted-in upgrade, and a library that * throws on the second case forces people to route around it. */ export function paidModelsIn(chain: Provider[]): string[] { - return chain.flatMap((p) => p.models).filter((m) => modelCost(m) === "paid"); + return chain.flatMap((p) => p.models.filter((m) => modelCostAt(p, m) === "paid")); } /** diff --git a/src/index.ts b/src/index.ts index 133d3af..5be19c1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,15 @@ /** * ai-ration — keep an LLM app on free tiers, and share what is free fairly. * - * Three pieces that each solve a distinct failure, and are useful separately: + * Four pieces that each solve a distinct failure, and are useful separately: * * chain — a fallback list ACROSS VENDORS, because a single pinned free * model is a scheduled outage and a smaller model at the same * vendor draws on the same exhausted daily budget. + * catalog — has the vendor retired an id the chain still asks for? The + * chain is itself a list of pins, so it rots too; on 2026-08-25 + * four of nine default ids were already gone, including an + * entire vendor. Zero tokens, so it can run on a schedule. * limits — tell the three kinds of 429 apart, because they need opposite * responses and only the body distinguishes them. * fair-share — divide a fixed daily pool across active users so the person who @@ -26,12 +30,22 @@ export { withEnvPrefix, freeChain, modelCost, + modelCostAt, paidModelsIn, dayCapacityTokens, usableChain, chainFrom, } from "./chain.js"; +export { + type CatalogVerdict, + type CheckCatalogOptions, + checkCatalog, + hasRot, + deadProviders, + catalogReport, +} from "./catalog.js"; + export { type RateLimitKind, classifyRateLimit, diff --git a/test/catalog.test.js b/test/catalog.test.js new file mode 100644 index 0000000..f1f9aab --- /dev/null +++ b/test/catalog.test.js @@ -0,0 +1,100 @@ +/** + * The chain is itself a list of pins, so it rots. These tests pin the part that + * is easy to get subtly wrong: telling "the vendor retired this" apart from + * "I could not read the catalogue". Conflating them either invents an outage or + * hides one. + * + * No network and no keys — fetch is injected. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + checkCatalog, + hasRot, + deadProviders, + catalogReport, + withEnvPrefix, +} from 'ai-ration'; + +const provider = (id, models, keyEnv) => + withEnvPrefix('T', { id, baseUrl: `https://${id}.test/v1`, keyEnv, models, dailyTokens: 1000 }); + +/** A fetch that serves a fixed catalogue, or a status, per host. */ +const fakeFetch = (byHost) => async (url) => { + const host = new URL(url).host; + const entry = byHost[host]; + if (entry === undefined) return { ok: false, status: 404, json: async () => ({}) }; + if (typeof entry === 'number') return { ok: false, status: entry, json: async () => ({}) }; + return { ok: true, status: 200, json: async () => ({ data: entry.map((id) => ({ id })) }) }; +}; + +const ENV = { T_GROQ_API_KEY: 'k', GROQ_API_KEY: 'k', OPENROUTER_API_KEY: 'k' }; + +test('a retired id is reported GONE, a live one is not', async () => { + const chain = [provider('groq', ['alive-1', 'retired-1'], 'GROQ_API_KEY')]; + const v = await checkCatalog(chain, { + env: ENV, + fetchImpl: fakeFetch({ 'groq.test': ['alive-1', 'something-else'] }), + }); + assert.deepEqual(v[0].present, ['alive-1']); + assert.deepEqual(v[0].missing, ['retired-1']); + assert.equal(hasRot(v), true); +}); + +test('an unreadable catalogue is UNCHECKED — never reported as rot', async () => { + // This is the failure that matters most. A 500, a network blip or a missing + // key must not make every pinned model look retired. + const chain = [provider('groq', ['a', 'b'], 'GROQ_API_KEY')]; + const v = await checkCatalog(chain, { env: ENV, fetchImpl: fakeFetch({ 'groq.test': 500 }) }); + assert.equal(v[0].live, null); + assert.deepEqual(v[0].missing, []); + assert.deepEqual(v[0].unchecked, ['a', 'b']); + assert.equal(hasRot(v), false, 'a failed lookup was reported as retired models'); +}); + +test('no API key is UNCHECKED, not a pass and not an outage', async () => { + const chain = [provider('groq', ['a'], 'MISSING_KEY_ENV')]; + const v = await checkCatalog(chain, { env: {}, fetchImpl: fakeFetch({ 'groq.test': ['a'] }) }); + assert.equal(v[0].live, null); + assert.deepEqual(v[0].unchecked, ['a']); + assert.equal(hasRot(v), false); + assert.match(catalogReport(v), /UNCHECKED/); + assert.match(catalogReport(v), /not a pass/); +}); + +test('a catalogue that parses but lists nothing is unreadable, not total rot', async () => { + const chain = [provider('groq', ['a'], 'GROQ_API_KEY')]; + const v = await checkCatalog(chain, { env: ENV, fetchImpl: fakeFetch({ 'groq.test': [] }) }); + assert.equal(v[0].live, null, 'an empty catalogue was believed'); + assert.equal(hasRot(v), false); +}); + +test('a vendor whose every model is gone is named — the chain lost a link', async () => { + // The 2026-08-25 case: both Groq ids retired, so the "fallback chain" led + // with a vendor that could never answer. + const chain = [ + provider('groq', ['gone-1', 'gone-2'], 'GROQ_API_KEY'), + provider('openrouter', ['ok-1', 'gone-3'], 'OPENROUTER_API_KEY'), + ]; + const v = await checkCatalog(chain, { + env: ENV, + fetchImpl: fakeFetch({ 'groq.test': ['other'], 'openrouter.test': ['ok-1'] }), + }); + assert.deepEqual(deadProviders(v), ['groq']); + assert.match(catalogReport(v), /EVERY model is gone at: groq/); + // openrouter is degraded, not dead — it must NOT be listed. + assert.ok(!deadProviders(v).includes('openrouter')); +}); + +test('an env override is what gets checked — not the library default', async () => { + // Checking the defaults would give a clean report on a box the operator has + // already routed around, and miss the ids it actually calls. + const chain = [provider('groq', ['default-model'], 'GROQ_API_KEY')]; + const env = { ...ENV, T_GROQ_MODELS: 'override-model' }; + const v = await checkCatalog(chain, { + env, + fetchImpl: fakeFetch({ 'groq.test': ['default-model'] }), + }); + assert.deepEqual(v[0].missing, ['override-model'], 'checked the default instead of the override'); +}); diff --git a/test/cost.test.js b/test/cost.test.js index 7a6eda1..8a9ab4f 100644 --- a/test/cost.test.js +++ b/test/cost.test.js @@ -6,7 +6,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { modelCost, paidModelsIn, freeChain } from 'ai-ration'; +import { modelCost, modelCostAt, paidModelsIn, freeChain } from 'ai-ration'; test('the three ids that were actually billing are all caught', () => { assert.equal(modelCost('anthropic/claude-sonnet-5'), 'paid'); @@ -35,3 +35,35 @@ test('the shipped free chain contains no paid model', () => { // The package would have no standing to flag anyone else's chain otherwise. assert.deepEqual(paidModelsIn(freeChain('TEST')), []); }); + +const routed = { id: 'openrouter', baseUrl: 'x', keyEnv: 'K', models: [], dailyTokens: 1, routed: true }; +const direct = { id: 'groq', baseUrl: 'x', keyEnv: 'K', models: [], dailyTokens: 1 }; + +test('the same id is PAID at a routed vendor and UNKNOWN at a direct one', () => { + // Cost is not a property of the string. `openai/gpt-oss-20b` bills at + // OpenRouter (routed, no `:free`); at Groq it is that vendor's own name for a + // model whose cost is the account's tier. Judging by shape alone was safe + // only while direct vendors used bare ids like `llama-3.1-8b-instant` — Groq + // now ships vendor-prefixed ids, which is what broke this. + assert.equal(modelCostAt(routed, 'openai/gpt-oss-20b'), 'paid'); + assert.equal(modelCostAt(direct, 'openai/gpt-oss-20b'), 'unknown'); + assert.equal(modelCostAt(routed, 'openai/gpt-oss-20b:free'), 'free'); +}); + +test('a direct vendor never yields "free" from the id alone', () => { + // "unknown" is the honest answer AND the safe direction. Returning "free" + // here would reopen the exact hole modelCost was written to close. + for (const id of ['openai/gpt-oss-120b', 'llama-3.1-8b-instant', 'anything/at-all:free']) { + assert.notEqual(modelCostAt(direct, id), 'free', `${id} was assumed free at a direct vendor`); + } +}); + +test('provider-awareness does NOT weaken the guard where the real incidents were', () => { + // All three production incidents were routed ids missing `:free`. Those must + // still be caught, or this change traded a false alarm for a real miss. + const chain = [ + { ...direct, models: ['openai/gpt-oss-120b'] }, + { ...routed, models: ['anthropic/claude-sonnet-5', 'google/gemini-2.0-flash-001'] }, + ]; + assert.deepEqual(paidModelsIn(chain), ['anthropic/claude-sonnet-5', 'google/gemini-2.0-flash-001']); +});