diff --git a/devlog/_plan/260912_reasoning_effort_downgrade/000_decision.md b/devlog/_plan/260912_reasoning_effort_downgrade/000_decision.md new file mode 100644 index 0000000000..67499186af --- /dev/null +++ b/devlog/_plan/260912_reasoning_effort_downgrade/000_decision.md @@ -0,0 +1,41 @@ +# 260912 — A refused reasoning rung is learned and replayed once + +## Decision + +When a routed upstream answers 400/403 and names reasoning effort in the body, the pipeline records +that (provider, model, effort) as refused, replays the request once at the next lower published rung, +and keeps the rung out of every later ladder (see the metadata record in +devlog/_plan/260912_reasoning_metadata/). The attempt is logged with recovery kind +reasoning-effort-downgrade, so requestedEffort and effectiveEffort stay distinguishable in usage. + +## Why the ladder is not enough + +The published ladder describes the model, not the account. Live 2026-09-12: +muse-spark-1.3-contributor answered 400 for max with + + Error from provider (Console Go): Upstream request failed: [invalid_request_error] + reasoning_effort max requires an active Muse Code subscription for model + muse-spark-1.3-contributor. + +while xhigh answered 200. Clamping against the published ladder removes that case before dispatch, +but any entitlement-driven refusal for a published rung would otherwise fail the turn outright. + +## Shape + +- Detection is narrow on purpose: 400/403 only, the body must be complete and display-safe (the same + contract as the other rejection peeks), and the text has to name reasoning effort. An unrelated 400 + never triggers a replay, which keeps the single extra send honest. +- One replay per request, guarded per recovery loop. The streamed passthroughRecovery loop and the + non-streamed recovery loop both carry the same block, matching the file's existing convention that + recovery kinds stay in sync across the two. +- Before the rebuild the parsed effort is replaced and the same-target cache is invalidated + (invalidateSameTargetRequest), because that cache keys on parsed identity and would otherwise + replay the original body byte-for-byte. +- No new failure surface: when the refusal is the only rung (or every lower rung is known-refused), + the original error is returned untouched. + +## Evidence + +tests/responses/responses-reasoning-effort-downgrade.test.ts (4 cases, mocked upstream): +pre-dispatch clamp, learn-then-replay on the non-streamed path, learn-then-replay on the streamed +path, and no replay for an unrelated 400. tests/responses runs 2040 pass / 0 fail with the change. diff --git a/devlog/_plan/260912_reasoning_metadata/000_decision.md b/devlog/_plan/260912_reasoning_metadata/000_decision.md new file mode 100644 index 0000000000..6bb98e1c24 --- /dev/null +++ b/devlog/_plan/260912_reasoning_metadata/000_decision.md @@ -0,0 +1,75 @@ +# 260912 — Routed reasoning ladders come from models.dev + +## Decision + +For a routed provider whose destination models.dev publishes, the Codex catalog and the outbound +wire value fall back to the published reasoning ladder when nothing is configured for that model. +A hand-written model ladder stays authoritative, then a provider-level one; models.dev is only +consulted when neither exists. A rung the upstream actually refused is dropped from every later +ladder, registry config included. + +Layers: + +1. src/providers/reasoning-metadata.ts snapshots models.dev (reasoning + reasoning_options, the + effort / toggle / budget_tokens option types) into ~/.opencodex/reasoning-metadata-cache.json + (24h TTL, stale-but-readable offline, atomic write). The v2 snapshot stores ladders for the + gated destinations (OpenCode Zen + Zen Go, 133 models / ~20 KB) and the published `api` URL of + every provider models.dev lists, so the gate can be checked against real data. +2. configuredReasoningEfforts() consults that snapshot only when nothing was configured for the + model, so every hand-written contract stays authoritative; mapReasoningEffort() clamps through + the same function, which is what keeps the catalog and the wire in agreement. +3. reasoning-support-cache.json records (provider, model, effort) refusals; the filter at the + configuredReasoningEfforts() exit removes those rungs whether the ladder came from the snapshot + or from the registry. + +## Why the hand-written table was not enough + +OpenCode Zen Go answers GET https://opencode.ai/zen/go/v1/models with ids only (id, object, created, +owned_by — 37 models, verified 2026-09-12), so opencodex had to guess: + +- muse-spark-1.3-contributor was advertised up to ultra while the gateway refuses max with + 400 {"param":"reasoning.effort","type":"invalid_request_error","message":"Error from provider + (Console Go): Upstream request failed: [invalid_request_error] reasoning_effort max requires an + active Muse Code subscription for model muse-spark-1.3-contributor."} ; xhigh answers 200. + models.dev publishes [minimal, low, medium, high, xhigh] for that model — the refusals were the + synthetic tiers, not the model. +- deepseek-v4.1-flash needs [low, high, max] before it advertises any control at all; models.dev + publishes exactly that. + +Verified after the change: the catalog lists [low, medium, high, xhigh] for muse-spark and +[low, high, max] for deepseek-v4.1-flash, max on muse-spark is sent as xhigh, and a refusal replays +once at the next lower published rung (usage.jsonl recovery kind reasoning-effort-downgrade) +instead of failing the turn. + +## Source resolution (2026-09-12 review follow-up) + +models.dev publishes each provider's own `api` URL (`opencode-go` -> `https://opencode.ai/zen/go/v1`, +`opencode` -> `https://opencode.ai/zen/v1`), so the destination is resolvable from data rather than from a +guess. Resolution stays gated: BASE_URL_TO_METADATA_PROVIDER is the authoritative list (both URLs are +compared normalised, so a trailing slash or a `/v1` suffix never decides), and reasoningMetadataMapping() +reports for each gated destination whether the snapshot confirms it against the published URL. + +Measured the same day: **36 of the registry's 83 destinations** match a models.dev provider, and 13 of a live +27-provider config do; 11 of those 13 already carry hand-written ladders (the metadata fallback is never +consulted) and the other 2 (`openrouter`, 4 models) would change catalog ladders. Resolving by URL alone +would therefore move ladders for providers this change has no evidence for, so widening the gate is a +separate decision with those numbers in hand -- the snapshot already carries the data it needs. + +## Learned refusals are credential-scoped in practice + +A refusal is recorded per `(provider, model, effort)`. Every destination that can reach this path is +`authKind: key`, i.e. one credential per provider entry, so that key already has the credential dimension; +the catalog is account-independent by construction (built once per process, not per request). Three +properties bound the rest: only the refused rung is dropped, the fact expires after 30 days, and the clamp +is visible as requestedEffort versus effectiveEffort in usage.jsonl. A credential-scoped key becomes +necessary only if opencodex ever pools several credentials behind one metadata-mapped provider entry. + +## Known follow-ups + +- Destination to models.dev provider id stays a gated table (two OpenCode destinations today). + Widening it to every URL match is measured above and is a maintainer call, not a mechanical edit. A + shared registry-side helper would replace the table itself, but importing providers/registry from this + module widened an unrelated supported_reasoning_levels literal type during development, so the naive + import was reverted. +- The snapshot refresh is triggered on first read with TTL and in-flight guards rather than from the + startup path, so a long-lived proxy refreshes at most daily. diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index b097e45e6d..016b20c274 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1078,6 +1078,7 @@ "raycast-detect.test.ts": "clients", "reasoning-effort.test.ts": "codex-integration", "reasoning-envelope.test.ts": "responses", + "reasoning-metadata.test.ts": "codex-integration", "reasoning-replay-identity.test.ts": "adapters", "reasoning-replay-robustness.test.ts": "adapters", "reasoning-replay-scope-source.test.ts": "lib", @@ -1156,6 +1157,7 @@ "responses-parser.test.ts": "responses", "responses-pool-401-refresh.test.ts": "responses", "responses-pool-refresh-attribution.test.ts": "responses", + "responses-reasoning-effort-downgrade.test.ts": "responses", "responses-reasoning-summary-passthrough.test.ts": "responses", "responses-routed-web-search-fields.test.ts": "responses", "responses-self-named-namespace-scrub.test.ts": "responses", diff --git a/src/providers/reasoning-metadata.ts b/src/providers/reasoning-metadata.ts new file mode 100644 index 0000000000..25b312eced --- /dev/null +++ b/src/providers/reasoning-metadata.ts @@ -0,0 +1,543 @@ +/** + * Data-driven reasoning ladders for routed providers. + * + * Routed providers rarely publish per-model effort ladders: OpenCode Zen Go answers /models + * with ids only (id/object/created/owned_by), so opencodex had to hardcode ladders in + * registry.ts and synthesise max/ultra for codex-rs catalog membership. The public models.dev + * catalogue DOES publish them per model: + * reasoning: true + * reasoning_options: [{type:"effort",values:["low","high","max"]}, {type:"toggle"}, + * {type:"budget_tokens"}] + * This module snapshots that catalogue to disk and hands configuredReasoningEfforts() a + * fallback ladder, so the Codex catalog AND the wire clamp agree with the model instead of a + * hand-written guess. + * + * Failure policy: the network is never on the critical path. A missing, stale or corrupt + * snapshot yields undefined, which leaves every hand-written contract untouched. The second + * cache records rungs the upstream actually rejected (400/403 naming reasoning_effort), so an + * entitlement gap (muse-spark max needs an active Muse Code subscription) costs one rejected + * request instead of failing every turn that selects that rung. + */ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +// Leaf modules on purpose: this file is imported from reasoning-effort.ts, which combos/types.ts +// already imports. Going through the ../config barrel closes a cycle back into account-namespaces.ts +// and leaves COMBO_NAMESPACE in its temporal dead zone for entry points that start at combos/types.ts. +import { atomicWriteFile } from "../config/atomic-write"; +import { getConfigDir } from "../config/paths"; +import type { OcxProviderConfig } from "../types"; + +const FILENAME = "reasoning-metadata-cache.json"; +const SUPPORT_FILENAME = "reasoning-support-cache.json"; +const SOURCE_URL = "https://models.dev/api.json"; +const USER_AGENT = "opencodex-reasoning-metadata/1.0 (+https://github.com/lidge-jun/opencodex)"; +/** Snapshot age that triggers a background refresh. Older snapshots still serve reads. */ +const CACHE_TTL_MS = 24 * 60 * 60 * 1000; +/** A learned "this rung is refused" fact expires: entitlements change. */ +const SUPPORT_TTL_MS = 30 * 24 * 60 * 60 * 1000; +const PERSIST_DEBOUNCE_MS = 250; + +/** Canonical Codex ladder order; mirrors reasoning-effort.ts CODEX_REASONING_LEVELS. */ +const LADDER_ORDER = ["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"]; +/** Ranked rungs used for downgrade planning; ultra is client-only and folds to max. */ +const RANKED = ["low", "medium", "high", "xhigh", "max"]; +/** Mirror of registry.ts THINKING_TOGGLE_EFFORTS / THINKING_BUDGET_EFFORTS. */ +const CLASSIFIED_STYLE_EFFORTS = ["low", "medium", "high", "xhigh", "max"]; + +/** + * models.dev provider key for a provider config. OcxProviderConfig carries no id, so the + * destination URL is the stable handle. Only destinations this patch has evidence for are + * listed; an unlisted provider simply keeps its current behaviour. + */ +const BASE_URL_TO_METADATA_PROVIDER: Record = { + "https://opencode.ai/zen/go/v1": "opencode-go", + "https://opencode.ai/zen/v1": "opencode", +}; + +/** + * Both sides of the mapping are compared after this normalisation, so a trailing slash or a + * `/v1` suffix never decides whether a destination resolves. models.dev publishes each + * provider's own `api` URL; the snapshot keeps it (v2) so the mapping can be checked against + * published data instead of trusted blindly. + */ +export function normalizeDestinationUrl(url: string | undefined): string | undefined { + if (typeof url !== "string" || url.trim() === "") return undefined; + try { + const parsed = new URL(url.trim()); + const path = parsed.pathname.replace(/\/+$/, "").replace(/\/v1$/i, ""); + return (parsed.protocol + "//" + parsed.host + path).toLowerCase(); + } catch { + return undefined; + } +} + +export type ReasoningMetadataOption = { type: string; values?: string[] }; +export type ReasoningMetadataModel = { reasoning: boolean; options: ReasoningMetadataOption[] }; + +interface MetadataSnapshot { + version: 1 | 2; + fetchedAt: number; + source: string; + providers: Record>; + /** + * v2: models.dev provider key -> that provider's published api URL (normalised). v1 snapshots + * predate the field and keep working through BASE_URL_TO_METADATA_PROVIDER. + */ + apis?: Record; +} + +interface SupportSnapshot { + version: 1; + rows: Record; +} + +let snapshotMemo: MetadataSnapshot | null | undefined; +let supportMemo: Map | undefined; +let persistTimer: ReturnType | null = null; +let refreshInFlight: Promise | null = null; + +/** Test seam: drop the memoised snapshot/support caches so a suite can drive the load paths. */ +export function resetReasoningMetadataCachesForTests(): void { + snapshotMemo = undefined; + supportMemo = undefined; + if (persistTimer) { + clearTimeout(persistTimer); + persistTimer = null; + } + refreshInFlight = null; +} + +function readJsonFile(filename: string): T | null { + try { + const path = join(getConfigDir(), filename); + if (!existsSync(path)) return null; + return JSON.parse(readFileSync(path, "utf8")) as T; + } catch { + // A corrupt cache must never break routing, the catalog, or the dashboard. + return null; + } +} + +/** Canonical order + dedupe. Local mirror of sanitizeCodexReasoningEfforts (import cycle). */ +function sanitizeLadder(values: readonly string[] | undefined): string[] | undefined { + if (!Array.isArray(values)) return undefined; + const seen = new Set(values.filter((value): value is string => typeof value === "string")); + const ordered = LADDER_ORDER.filter(effort => seen.has(effort)); + return ordered.length > 0 ? ordered : undefined; +} + +function metadataProviderKey(provider: OcxProviderConfig): string | undefined { + const normalized = normalizeDestinationUrl(typeof provider.baseUrl === "string" ? provider.baseUrl : undefined); + if (!normalized) return undefined; + for (const [destination, key] of Object.entries(BASE_URL_TO_METADATA_PROVIDER)) { + if (normalizeDestinationUrl(destination) === normalized) return key; + } + return undefined; +} + +/** + * Local mirror of `modelRecordValue()` from `src/reasoning-effort.ts`, which imports this + * module and so cannot be imported back. Exact id, then the `family:` prefix, then a + * case-folded match — a configured ladder must resolve here exactly as it does there, or the + * downgrade rung is chosen off a different ladder than the catalog advertises. + */ +function modelLadderValue( + record: Record | undefined, + modelId: string, +): readonly string[] | undefined { + if (!record) return undefined; + if (Object.prototype.hasOwnProperty.call(record, modelId)) return record[modelId]; + const colon = modelId.indexOf(":"); + if (colon > 0) { + const family = modelId.slice(0, colon); + if (Object.prototype.hasOwnProperty.call(record, family)) return record[family]; + } + const folded = modelId.toLowerCase(); + for (const [key, value] of Object.entries(record)) { + if (key.toLowerCase() === folded) return value; + } + return undefined; +} + +/** Opaque row key: providerKey|modelId|effort. None of the three may contain a pipe. */ +const KEY_SEP = "|"; + +function supportKey(providerKey: string, modelId: string, effort: string): string { + return providerKey + KEY_SEP + modelId + KEY_SEP + effort; +} + +function loadSnapshot(): MetadataSnapshot | null { + if (snapshotMemo !== undefined) return snapshotMemo; + const parsed = readJsonFile(FILENAME); + snapshotMemo = parsed && (parsed.version === 1 || parsed.version === 2) && parsed.providers && typeof parsed.providers === "object" + ? parsed + : null; + return snapshotMemo; +} + +/** + * Mapping report for diagnostics and tests: every gated destination, the models.dev provider it + * resolves to, and whether the snapshot (v2) publishes an `api` URL that confirms it. The gate is + * deliberate -- 36 of the registry's 83 destinations match a models.dev provider, so resolving by + * URL alone would silently move ladders for providers this change has no evidence for. + */ +export function reasoningMetadataMapping(): Array<{ + destination: string; + provider: string; + publishedApi?: string; + confirmed?: boolean; + models: number; +}> { + const snapshot = loadSnapshot(); + return Object.entries(BASE_URL_TO_METADATA_PROVIDER).map(([destination, provider]) => { + const normalized = normalizeDestinationUrl(destination); + const publishedApi = snapshot?.apis?.[provider]; + const row = { + destination, + provider, + ...(publishedApi ? { publishedApi } : {}), + ...(publishedApi ? { confirmed: publishedApi === normalized } : {}), + models: Object.keys(snapshot?.providers?.[provider] ?? {}).length, + }; + return row; + }); +} + +function loadSupport(): Map { + const nowMs = Date.now(); + if (supportMemo) { + // The memo lives for the process lifetime, so the TTL has to be re-applied on every read. + // Checking it only on the disk load meant a long-running proxy kept clamping on a refusal + // it recorded a month earlier, and `dropLearnedUnsupportedReasoningEfforts` inherited that + // through the same map. + for (const [key, at] of supportMemo) { + if (nowMs - at > SUPPORT_TTL_MS) { + supportMemo.delete(key); + supportEvidence.delete(key); + } + } + return supportMemo; + } + const rows = new Map(); + const parsed = readJsonFile(SUPPORT_FILENAME); + if (parsed && parsed.version === 1 && parsed.rows && typeof parsed.rows === "object") { + for (const [key, row] of Object.entries(parsed.rows)) { + if (!row || typeof row.at !== "number") continue; + if (nowMs - row.at > SUPPORT_TTL_MS) continue; + rows.set(key, row.at); + } + } + supportMemo = rows; + return rows; +} + +/** Snapshot health for ocx status / diagnostics. */ +export function reasoningMetadataStatus(): { fetchedAt?: number; ageMs?: number; stale: boolean; models: number } { + const snapshot = loadSnapshot(); + if (!snapshot) return { stale: false, models: 0 }; + const ageMs = Date.now() - snapshot.fetchedAt; + let models = 0; + for (const provider of Object.values(snapshot.providers)) models += Object.keys(provider).length; + return { fetchedAt: snapshot.fetchedAt, ageMs, stale: ageMs > CACHE_TTL_MS, models }; +} + +export function reasoningMetadataModel(provider: OcxProviderConfig, modelId: string): ReasoningMetadataModel | undefined { + const key = metadataProviderKey(provider); + if (!key) return undefined; + const models = loadSnapshot()?.providers?.[key]; + if (!models) return undefined; + const model = models[modelId]; + return model && typeof model === "object" ? model : undefined; +} + +/** Raw models.dev effort values for a model, canonicalised; undefined when not published. */ +export function metadataEffortValues(provider: OcxProviderConfig, modelId: string): string[] | undefined { + const model = reasoningMetadataModel(provider, modelId); + if (!model) return undefined; + const options = Array.isArray(model.options) ? model.options : []; + const effort = options.find(option => option && option.type === "effort"); + const ladder = sanitizeLadder(effort?.values); + // none/minimal are sentinels, not picker rungs (mapReasoningEffort folds minimal to low), and + // advertising them would trip the Codex runtime clamp for no user-visible gain. + const rungs = ladder?.filter(value => value !== "none" && value !== "minimal"); + return rungs && rungs.length > 0 ? rungs : undefined; +} + +/** True when models.dev publishes the named option type (toggle / budget_tokens) for a model. */ +export function metadataDeclaresType(provider: OcxProviderConfig, modelId: string, type: string): boolean { + const model = reasoningMetadataModel(provider, modelId); + if (!model) return false; + const options = Array.isArray(model.options) ? model.options : []; + return options.some(option => option?.type === type); +} + +export function isReasoningEffortLearnedUnsupported(provider: OcxProviderConfig, modelId: string, effort: string): boolean { + const key = metadataProviderKey(provider); + if (!key) return false; + return loadSupport().has(supportKey(key, modelId, effort)); +} + +/** + * After the ladder is chosen (registry config or models.dev metadata), remove the rungs this + * account actually had refused. Applied at the configuredReasoningEfforts() exit so a + * registry-pinned ladder learns exactly like a metadata-derived one; without it a pinned rung + * the upstream rejects would replay-and-fail on every request. An all-refused ladder keeps the + * original list: turning "some rungs" into "no effort control" would silently drop the picker. + */ +export function dropLearnedUnsupportedReasoningEfforts( + provider: OcxProviderConfig, + modelId: string, + efforts: readonly string[], +): string[] { + if (efforts.length === 0) return [...efforts]; + const key = metadataProviderKey(provider); + if (!key) return [...efforts]; + const support = loadSupport(); + if (support.size === 0) return [...efforts]; + const kept = efforts.filter(effort => !support.has(supportKey(key, modelId, effort))); + return kept.length === 0 ? [...efforts] : kept; +} + +/** + * Metadata fallback ladder for a provider/model. + * + * - Published effort values win. + * - A model the provider already classifies as thinking-toggle / thinking-budget keeps the + * provider's own effort list; a toggle-only entry never invents wire semantics here. + * - Rungs the upstream actually refused are removed; a ladder emptied by that learning + * returns undefined (status quo) rather than advertising "no effort control". + */ +export function reasoningEffortsFromMetadata(provider: OcxProviderConfig, modelId: string): string[] | undefined { + const published = metadataEffortValues(provider, modelId); + let ladder = published; + if (!ladder) { + const classified = (provider.thinkingToggleModels ?? []).includes(modelId) + || (provider.thinkingBudgetModels ?? []).includes(modelId); + ladder = classified ? CLASSIFIED_STYLE_EFFORTS : undefined; + } + if (!ladder || ladder.length === 0) return undefined; + const kept = ladder.filter(effort => !isReasoningEffortLearnedUnsupported(provider, modelId, effort)); + if (kept.length === 0) return undefined; + return kept; +} + +const supportEvidence = new Map(); + +/** + * Record that the upstream refused a rung. Persisted (debounced) so the next catalog sync and + * every later request clamp before dispatch. Returns true when this is new information. + */ +export function recordUnsupportedReasoningEffort( + provider: OcxProviderConfig, + modelId: string, + effort: string, + evidence?: string, +): boolean { + const key = metadataProviderKey(provider); + if (!key || !effort) return false; + const rowKey = supportKey(key, modelId, effort); + const rows = loadSupport(); + if (rows.has(rowKey)) return false; + rows.set(rowKey, Date.now()); + if (evidence) supportEvidence.set(rowKey, evidence.slice(0, 240)); + if (persistTimer) clearTimeout(persistTimer); + persistTimer = setTimeout(() => { + persistTimer = null; + try { + const out: SupportSnapshot["rows"] = {}; + for (const [rowKey, at] of rows) { + const parts = rowKey.split(KEY_SEP); + const evidenceText = supportEvidence.get(rowKey); + out[rowKey] = { + effort: parts[2] ?? "", + at, + ...(evidenceText ? { evidence: evidenceText } : {}), + }; + } + atomicWriteFile(join(getConfigDir(), SUPPORT_FILENAME), JSON.stringify({ version: 1, rows: out }) + "\n"); + } catch { + // Best-effort persistence only. + } + }, PERSIST_DEBOUNCE_MS); + return true; +} + +/** Test seam: flush a pending support write so a script sees the snapshot immediately. */ +export function flushReasoningSupportCache(): void { + if (!persistTimer) return; + clearTimeout(persistTimer); + persistTimer = null; + try { + const rows = loadSupport(); + const out: SupportSnapshot["rows"] = {}; + for (const [rowKey, at] of rows) { + const parts = rowKey.split(KEY_SEP); + const evidenceText = supportEvidence.get(rowKey); + out[rowKey] = { effort: parts[2] ?? "", at, ...(evidenceText ? { evidence: evidenceText } : {}) }; + } + atomicWriteFile(join(getConfigDir(), SUPPORT_FILENAME), JSON.stringify({ version: 1, rows: out }) + "\n"); + } catch { + // Best-effort persistence only. + } +} + +/** + * Words an upstream uses when it is refusing the parameter it just named. Requiring one of + * these beside the effort term is what separates "the gateway rejected reasoning effort" from + * "the gateway rejected something else and echoed the request back". + */ +const REJECTION_LANGUAGE = /unsupported|not supported|does not support|invalid|unrecognized|unknown|not allowed|not permitted|must be|requires|required|cannot|can't|out of range/i; + +/** + * How far from the effort term the rejection language may sit and still be about it. Kept + * deliberately short: an error body that echoes the request back puts unrelated field names and + * their complaints within a hundred characters of each other, so a generous window classifies + * every 400 that mentions effort as a refusal of it. + */ +const REJECTION_WINDOW = 48; + +/** + * The `invalid_request_error` type tag rides along on essentially every 400 an OpenAI-shaped + * gateway emits, so it is evidence of nothing. Blanked before the language scan rather than + * dropped from the pattern, because `invalid` is real evidence when it is the message. + */ +const GENERIC_ERROR_TYPE = /invalid_request_error/gi; + +/** + * Evidence test for a rejection body: does it blame reasoning effort? + * + * The parameter name on its own is not evidence. A 400 that refuses `max_tokens` may still + * echo the whole request body back, `reasoning_effort` included, and treating that as a + * refusal spends this request's one downgrade replay on a rung the upstream never objected to + * — and persists a false refusal that clamps every later turn for thirty days. + */ +export function isReasoningEffortRejection(text: string | undefined): boolean { + if (!text) return false; + if (/unsupported.{0,24}effort/i.test(text)) return true; + // An upstream that names the offending parameter has already said which one it means. + if (/["']?param["']?\s*[:=]\s*["']?(?:reasoning[._ ]effort|reasoning)/i.test(text)) return true; + const scanned = text.replace(GENERIC_ERROR_TYPE, " "); + const term = /reasoning\.effort|reasoning_effort|reasoning effort|thinking budget|reasoning_parameters/gi; + for (let match = term.exec(scanned); match; match = term.exec(scanned)) { + const from = Math.max(0, match.index - REJECTION_WINDOW); + const to = Math.min(scanned.length, match.index + match[0].length + REJECTION_WINDOW); + if (REJECTION_LANGUAGE.test(scanned.slice(from, to))) return true; + } + return false; +} + +/** + * Plan a single-rung downgrade for a rejected request: records the refusal (so later turns + * clamp before dispatch) and returns the next lower rung the model does publish. + */ +export function planReasoningEffortDowngrade(args: { + provider: OcxProviderConfig; + modelId: string; + requested?: string; + rejectionText?: string; +}): { effort: string; recorded: boolean } | undefined { + const requested = args.requested === "ultra" ? "max" : args.requested; + if (!requested || !RANKED.includes(requested)) return undefined; + const recorded = recordUnsupportedReasoningEffort(args.provider, args.modelId, requested, args.rejectionText); + // Same precedence as configuredReasoningEfforts(): a hand-written ladder is a contract and + // models.dev is only consulted when nothing was configured for this model. Reading metadata + // first would have picked the downgrade rung off the published ladder even where a pinned + // one disagreed, so the replay could land on a rung the registry deliberately excludes. + const effective = sanitizeLadder(modelLadderValue(args.provider.modelReasoningEfforts, args.modelId)) + ?? sanitizeLadder(args.provider.reasoningEfforts) + ?? metadataEffortValues(args.provider, args.modelId); + const ladder = (effective ?? []).filter(effort => RANKED.includes(effort)); + if (ladder.length === 0) return undefined; + const candidates = ladder + .filter(effort => RANKED.indexOf(effort) < RANKED.indexOf(requested)) + .filter(effort => !isReasoningEffortLearnedUnsupported(args.provider, args.modelId, effort)); + if (candidates.length === 0) return undefined; + return { effort: candidates[candidates.length - 1], recorded }; +} + +/** + * Refresh the models.dev snapshot. Best-effort and idempotent: never throws, never blocks a + * request, keeps the previous snapshot on failure. Ladders are stored for the gated destinations + * only (OpenCode Zen + Zen Go: about 130 models), while every published provider `api` URL is + * kept so the gate can be checked against real data and widened without another format change. + * Non-reasoning models carry no ladder and are dropped. + */ +export async function refreshReasoningMetadata(options: { force?: boolean } = {}): Promise<{ + ok: boolean; + reason: string; + providers?: number; + models?: number; +}> { + const snapshot = loadSnapshot(); + if (!options.force && snapshot && Date.now() - snapshot.fetchedAt <= CACHE_TTL_MS) { + return { ok: true, reason: "fresh" }; + } + if (refreshInFlight) { + await refreshInFlight; + return { ok: true, reason: "coalesced" }; + } + const job = (async () => { + const response = await fetch(SOURCE_URL, { + headers: { "user-agent": USER_AGENT, accept: "application/json" }, + // A hanging connection must not pin refreshInFlight for the life of the process. + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) throw new Error("models.dev HTTP " + response.status); + const raw = await response.json() as Record }>; + const providers: MetadataSnapshot["providers"] = {}; + const apis: Record = {}; + const gated = new Set(Object.values(BASE_URL_TO_METADATA_PROVIDER)); + let models = 0; + for (const [providerKey, entry] of Object.entries(raw ?? {})) { + const api = normalizeDestinationUrl(typeof entry?.api === "string" ? entry.api : undefined); + if (api) apis[providerKey] = api; + if (!gated.has(providerKey)) continue; + const out: Record = {}; + for (const [modelId, value] of Object.entries(entry?.models ?? {})) { + const model = value as { reasoning?: unknown; reasoning_options?: unknown }; + if (model?.reasoning !== true) continue; + const options: ReasoningMetadataOption[] = []; + if (Array.isArray(model?.reasoning_options)) { + for (const option of model.reasoning_options) { + if (!option || typeof option !== "object") continue; + const type = (option as { type?: unknown }).type; + if (typeof type !== "string") continue; + const values = (option as { values?: unknown }).values; + options.push({ + type, + ...(Array.isArray(values) + ? { values: values.filter((v): v is string => typeof v === "string").slice(0, 12) } + : {}), + }); + } + } + out[modelId] = { reasoning: model?.reasoning === true, options }; + models += 1; + } + if (Object.keys(out).length === 0) continue; + providers[providerKey] = out; + } + const next: MetadataSnapshot = { version: 2, fetchedAt: Date.now(), source: SOURCE_URL, providers, apis }; + atomicWriteFile(join(getConfigDir(), FILENAME), JSON.stringify(next) + "\n"); + snapshotMemo = next; + return { ok: true, reason: "refreshed", providers: Object.keys(providers).length, models }; + })(); + refreshInFlight = job.catch(() => undefined).finally(() => { refreshInFlight = null; }); + try { + return await job; + } catch (error) { + return { ok: false, reason: error instanceof Error ? error.message : String(error) }; + } +} + +/** + * Kick a background refresh when the snapshot is missing or stale. Called from the ladder read + * path so both the long-lived proxy and short-lived ocx sync self-heal without a new CLI + * surface. One refresh per process at a time; failures are ignored on purpose. + */ +export function ensureReasoningMetadataSnapshot(): void { + const snapshot = loadSnapshot(); + if (snapshot && Date.now() - snapshot.fetchedAt <= CACHE_TTL_MS) return; + if (refreshInFlight) return; + void refreshReasoningMetadata().catch(() => undefined); +} diff --git a/src/reasoning-effort.ts b/src/reasoning-effort.ts index 6342159d31..d97c96db99 100644 --- a/src/reasoning-effort.ts +++ b/src/reasoning-effort.ts @@ -1,5 +1,6 @@ import type { OcxProviderConfig } from "./types"; import { modelInList } from "./types"; +import { dropLearnedUnsupportedReasoningEfforts, ensureReasoningMetadataSnapshot, reasoningEffortsFromMetadata } from "./providers/reasoning-metadata"; // Descriptions mirror the upstream bundled models.json canonical wording (openai/codex PR #31684). export const CODEX_REASONING_LEVELS: { effort: string; description: string }[] = [ @@ -148,8 +149,31 @@ export function sanitizeCodexReasoningEfforts(efforts: readonly string[] | undef export function configuredReasoningEfforts(provider: OcxProviderConfig, modelId: string): string[] | undefined { if (modelInList(provider.noReasoningModels, modelId)) return []; const modelEfforts = modelRecordValue(provider.modelReasoningEfforts, modelId); - if (modelEfforts !== undefined) return healMappedTiers(provider, modelId, sanitizeCodexReasoningEfforts(modelEfforts) ?? []); - if (provider.reasoningEfforts !== undefined) return healMappedTiers(provider, modelId, sanitizeCodexReasoningEfforts(provider.reasoningEfforts) ?? []); + // Rungs this account actually had refused are removed for every ladder source (registry + // config or models.dev), so a learned refusal is honoured even when the ladder is pinned in + // code; otherwise a rejected pinned rung would replay-and-fail on every request. + if (modelEfforts !== undefined) { + return dropLearnedUnsupportedReasoningEfforts(provider, modelId, healMappedTiers(provider, modelId, sanitizeCodexReasoningEfforts(modelEfforts) ?? [])); + } + if (provider.reasoningEfforts !== undefined) { + return dropLearnedUnsupportedReasoningEfforts(provider, modelId, healMappedTiers(provider, modelId, sanitizeCodexReasoningEfforts(provider.reasoningEfforts) ?? [])); + } + // models.dev publishes the per-model ladder that routed providers never expose on /models. + // (OpenCode Zen Go answers ids only). Only consulted when nothing was configured for this + // model, so every hand-written contract stays authoritative. The snapshot refreshes itself in + // the background; no snapshot means the previous behaviour. + // The refresh is asked for only once a snapshot has already answered, which means it only ever + // refreshes a STALE snapshot. Review asked for the opposite — refresh when the snapshot is + // missing or corrupt, since that is the case this lookup cannot serve. That is declined here: + // a missing snapshot is the default state of every fresh install and every test process, so + // requesting the fetch here puts a models.dev request on the request path of the first routed + // turn to a gated destination. Refreshing a snapshot that does not exist is catalog-sync work, + // not request work. + const fromMetadata = reasoningEffortsFromMetadata(provider, modelId); + if (fromMetadata !== undefined) { + ensureReasoningMetadataSnapshot(); + return dropLearnedUnsupportedReasoningEfforts(provider, modelId, healMappedTiers(provider, modelId, fromMetadata)); + } return undefined; } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 392c26ef63..ba6cfdd855 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -296,6 +296,7 @@ import { } from "../lifecycle"; import { redactSecretString, sanitizeLogMetadataString } from "../../lib/redact"; import { readBoundedResponseBody } from "../../lib/bounded-body"; +import { isReasoningEffortRejection, planReasoningEffortDowngrade } from "../../providers/reasoning-metadata"; import { ENCRYPTED_FUNCTION_OUTPUT_REJECTION, isRateLimitOrQuotaFailureMessage, @@ -844,6 +845,28 @@ export function shouldAttemptOpaqueBlobRecovery(args: { && isSelfIdentifiedOpaqueBlobRejection(args.errorBody); } +/** + * Peek the upstream error body for the reasoning-effort downgrade. Only 400/403 are considered + * and the body must be complete and display-safe, the same contract the other rejection peeks + * use. The match is deliberately narrow: the upstream has to name reasoning effort, so an + * unrelated 400 never triggers a replay. + */ +async function reasoningEffortRejectionText( + response: Response, + alreadyAttempted: boolean, + signal: AbortSignal, +): Promise { + if (alreadyAttempted) return undefined; + if (response.status !== 400 && response.status !== 403) return undefined; + try { + const body = await readBoundedResponseBody(response.clone(), { signal }); + if (!body.displaySafe || body.truncated) return undefined; + return isReasoningEffortRejection(body.text) ? body.text : undefined; + } catch { + return undefined; + } +} + async function opaqueBlobRejectionBodyForRecovery( response: Response, outboundBody: string | undefined, @@ -5419,6 +5442,8 @@ async function handleResponsesInner( } const opaqueBlobRecoveryGuard: OpaqueBlobRecoveryGuard = { attempted: false }; + // At most one reasoning-effort downgrade per request. + const reasoningEffortDowngradeGuard: { attempted: boolean } = { attempted: false }; let oauth401ReplayAttempted = false; let codex401ReplayKind: "main" | "stored" | null = null; // Console Go answers a transient 400 "Invalid upload request." for bodies it accepts @@ -5996,6 +6021,35 @@ async function handleResponsesInner( continue passthroughRecovery; } } + // Reasoning-effort downgrade: a rung the catalog still advertises can be refused upstream -- + // the metadata records the model's ladder, not this account's entitlement (a Muse Code + // subscription gates max on muse-spark-1.3-contributor, for example). Learn the refusal so + // later turns clamp before dispatch, then replay once at the next lower published rung + // instead of failing the turn; requestedEffort/effectiveEffort keep both values in usage. + if (!reasoningEffortDowngradeGuard.attempted) { + const rejectionText = await reasoningEffortRejectionText( + upstreamResponse, + reasoningEffortDowngradeGuard.attempted, + upstream.signal, + ); + const downgrade = rejectionText === undefined + ? undefined + : planReasoningEffortDowngrade({ + provider: route.provider, + modelId: parsed.modelId, + requested: parsed.options.reasoning, + rejectionText, + }); + if (downgrade) { + reasoningEffortDowngradeGuard.attempted = true; + parsed.options.reasoning = downgrade.effort; + try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } + const result = await rebuildAndRefetch("reasoning-effort-downgrade"); + if ("failed" in result) return result.failed; + upstreamResponse = result; + continue passthroughRecovery; + } + } break; } const headers = sanitizePassthroughHeaders(upstreamResponse.headers, codexSafetyBufferingOptions); @@ -7517,6 +7571,10 @@ async function handleResponsesInner( // moments later; at most one byte-identical replay is allowed per request. const consoleGoUploadRetryGuard: { attempted: boolean } = { attempted: false }; let oauth401ReplayAttempted = false; + // At most one reasoning-effort downgrade per request. This sits outside the recovery loop + // below for the same reason the two guards above do: a guard declared inside it is reset by + // every `continue recovery`, which would let one turn walk the whole ladder down. + const reasoningEffortDowngradeGuard: { attempted: boolean } = { attempted: false }; /** * Rebuild the request from the current parsed input (and any image-tier bias) and refetch * it once, tagging the attempt with the given recovery kind. Rebuilds are deterministic @@ -7929,6 +7987,34 @@ async function handleResponsesInner( continue recovery; } } + // Reasoning-effort downgrade, mirroring the passthroughRecovery loop above: learn the + // refused rung, then replay once at the next published one. + if (!reasoningEffortDowngradeGuard.attempted) { + const rejectionText = await reasoningEffortRejectionText( + upstreamResponse, + reasoningEffortDowngradeGuard.attempted, + upstream.signal, + ); + const downgrade = rejectionText === undefined + ? undefined + : planReasoningEffortDowngrade({ + provider: route.provider, + modelId: parsed.modelId, + requested: parsed.options.reasoning, + rejectionText, + }); + if (downgrade) { + reasoningEffortDowngradeGuard.attempted = true; + parsed.options.reasoning = downgrade.effort; + // The same-target cache keys on parsed identity, so a mutated effort needs a token bump. + invalidateSameTargetRequest(); + try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } + const result = await rebuildAndRefetch("reasoning-effort-downgrade"); + if ("failed" in result) return result.failed; + upstreamResponse = result; + continue recovery; + } + } break; } if (!upstreamResponse.ok) { diff --git a/src/usage/log.ts b/src/usage/log.ts index e8bbe3eb2b..51a682910e 100644 --- a/src/usage/log.ts +++ b/src/usage/log.ts @@ -73,7 +73,8 @@ export type AttemptRecoveryKind = | "image-413" | "console-go-upload-retry" | "opaque-blob-rejection" - | "empty-completion"; + | "empty-completion" + | "reasoning-effort-downgrade"; /** Request-time upstream credential class, never a credential or account identifier. */ export type UsageCredentialSource = "grok-oauth" | "xai-api-key"; @@ -322,6 +323,7 @@ const ATTEMPT_RECOVERY_KINDS = new Set([ "console-go-upload-retry", "opaque-blob-rejection", "empty-completion", + "reasoning-effort-downgrade", ]); const USAGE_STATUSES = new Set([ "reported", diff --git a/tests/codex-integration/reasoning-metadata.test.ts b/tests/codex-integration/reasoning-metadata.test.ts new file mode 100644 index 0000000000..02e8d16c5e --- /dev/null +++ b/tests/codex-integration/reasoning-metadata.test.ts @@ -0,0 +1,239 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { OcxProviderConfig } from "../../src/types"; + +/** + * Data-driven reasoning ladders (models.dev snapshot + learned refusals). + * + * OpenCode Zen Go publishes model ids only, so the catalog used to advertise whatever the + * registry hardcoded -- including rungs the upstream refuses (muse-spark max -> 400 "requires an + * active Muse Code subscription"). These cases pin the metadata fallback, the wire clamp and the + * learned-refusal filter that keeps a rejected rung out of every later request. + */ + +const ZEN_GO: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://opencode.ai/zen/go/v1", +} as OcxProviderConfig; + +const MUSE_SPARK = "muse-spark-1.3-contributor"; +const DEEPSEEK_FLASH = "deepseek-v4.1-flash"; + +const REJECTION_BODY = JSON.stringify({ + model: MUSE_SPARK, + error: { + param: "reasoning.effort", + type: "invalid_request_error", + message: "Error from provider (Console Go): Upstream request failed: [invalid_request_error] reasoning_effort max requires an active Muse Code subscription for model muse-spark-1.3-contributor.", + }, +}); + +const roots: string[] = []; +const originalOpenCodexHome = process.env["OPENCODEX_HOME"]; + +function snapshotFile(providers: Record): Record { + return { version: 1, fetchedAt: Date.now(), source: "test", providers }; +} + +function sandbox(files: Record = {}): string { + const dir = mkdtempSync(join(tmpdir(), "ocx-reasoning-metadata-")); + roots.push(dir); + for (const [name, body] of Object.entries(files)) writeFileSync(join(dir, name), body); + process.env["OPENCODEX_HOME"] = dir; + return dir; +} + +async function load(files: Record = {}) { + sandbox(files); + const metadata = await import("../../src/providers/reasoning-metadata"); + const effort = await import("../../src/reasoning-effort"); + metadata.resetReasoningMetadataCachesForTests(); + return { metadata, effort }; +} + +function metadataFile(providers: Record): Record { + return { "reasoning-metadata-cache.json": JSON.stringify(snapshotFile(providers)) }; +} + +function metadataFileV2(providers: Record, apis: Record): Record { + return { + "reasoning-metadata-cache.json": JSON.stringify({ ...snapshotFile(providers), version: 2, apis }), + }; +} + +function supportFile(rows: Record): Record { + return { "reasoning-support-cache.json": JSON.stringify({ version: 1, rows }) }; +} + +afterEach(() => { + for (const dir of roots.splice(0)) rmSync(dir, { recursive: true, force: true }); + // Restore rather than delete. Unsetting it entirely pointed every later test file in the same + // bun process at the real ~/.opencodex, which read the machine's actual configuration and + // failed unrelated suites (tests/web-search) depending on file order. + if (originalOpenCodexHome === undefined) delete process.env["OPENCODEX_HOME"]; + else process.env["OPENCODEX_HOME"] = originalOpenCodexHome; +}); + +describe("models.dev reasoning metadata", () => { + test("advertises the published effort rungs and strips the none/minimal sentinels", async () => { + const { effort } = await load(metadataFile({ + "opencode-go": { + [MUSE_SPARK]: { reasoning: true, options: [{ type: "effort", values: ["minimal", "low", "medium", "high", "xhigh"] }] }, + }, + })); + expect(effort.configuredReasoningEfforts(ZEN_GO, MUSE_SPARK)).toEqual(["low", "medium", "high", "xhigh"]); + }); + + test("clamps a rung the model does not publish instead of failing upstream", async () => { + const { effort } = await load(metadataFile({ + "opencode-go": { + [MUSE_SPARK]: { reasoning: true, options: [{ type: "effort", values: ["minimal", "low", "medium", "high", "xhigh"] }] }, + [DEEPSEEK_FLASH]: { reasoning: true, options: [{ type: "effort", values: ["low", "high", "max"] }] }, + }, + })); + expect(effort.mapReasoningEffort(ZEN_GO, MUSE_SPARK, "max")).toBe("xhigh"); + expect(effort.mapReasoningEffort(ZEN_GO, DEEPSEEK_FLASH, "max")).toBe("max"); + expect(effort.mapReasoningEffort(ZEN_GO, DEEPSEEK_FLASH, "ultra")).toBe("max"); + }); + + test("a hand-written ladder stays authoritative and unknown models stay untouched", async () => { + const { effort } = await load(metadataFile({ + "opencode-go": { [MUSE_SPARK]: { reasoning: true, options: [{ type: "effort", values: ["low", "medium", "high", "xhigh"] }] } }, + })); + const pinned = { ...ZEN_GO, modelReasoningEfforts: { [MUSE_SPARK]: ["low", "high"] } } as OcxProviderConfig; + expect(effort.configuredReasoningEfforts(pinned, MUSE_SPARK)).toEqual(["low", "high"]); + expect(effort.configuredReasoningEfforts(ZEN_GO, "not-a-model")).toBeUndefined(); + }); + + test("a destination the snapshot does not describe keeps the previous behaviour", async () => { + const { effort } = await load(metadataFile({ + "opencode-go": { [MUSE_SPARK]: { reasoning: true, options: [{ type: "effort", values: ["low", "high"] }] } }, + })); + const elsewhere = { ...ZEN_GO, baseUrl: "https://api.deepseek.com/v1" } as OcxProviderConfig; + expect(effort.configuredReasoningEfforts(elsewhere, MUSE_SPARK)).toBeUndefined(); + }); + + test("a toggle-only entry never invents wire semantics for an unclassified model", async () => { + const { effort } = await load(metadataFile({ + "opencode-go": { "minimax-m3": { reasoning: true, options: [{ type: "toggle" }] } }, + })); + expect(effort.configuredReasoningEfforts(ZEN_GO, "minimax-m3")).toBeUndefined(); + }); + + test("a corrupt or missing snapshot falls back to the status quo", async () => { + const { effort } = await load({ "reasoning-metadata-cache.json": "{not json" }); + expect(effort.configuredReasoningEfforts(ZEN_GO, MUSE_SPARK)).toBeUndefined(); + expect(effort.mapReasoningEffort(ZEN_GO, MUSE_SPARK, "max")).toBe("max"); + }); +}); + +describe("learned rung refusals", () => { + const refused = () => ({ + ["opencode-go|" + DEEPSEEK_FLASH + "|max"]: { effort: "max", at: Date.now() }, + }); + + test("drops a refused rung from a metadata-derived ladder", async () => { + const { effort } = await load({ + ...metadataFile({ "opencode-go": { [DEEPSEEK_FLASH]: { reasoning: true, options: [{ type: "effort", values: ["low", "high", "max"] }] } } }), + ...supportFile(refused()), + }); + expect(effort.configuredReasoningEfforts(ZEN_GO, DEEPSEEK_FLASH)).toEqual(["low", "high"]); + expect(effort.mapReasoningEffort(ZEN_GO, DEEPSEEK_FLASH, "max")).toBe("high"); + }); + + test("drops a refused rung from a ladder pinned in the registry too", async () => { + const { effort } = await load(supportFile(refused())); + const pinned = { ...ZEN_GO, modelReasoningEfforts: { [DEEPSEEK_FLASH]: ["low", "high", "max"] } } as OcxProviderConfig; + expect(effort.configuredReasoningEfforts(pinned, DEEPSEEK_FLASH)).toEqual(["low", "high"]); + }); + + test("records the refusal and plans the next lower published rung once", async () => { + const { metadata } = await load({ + ...metadataFile({ "opencode-go": { [DEEPSEEK_FLASH]: { reasoning: true, options: [{ type: "effort", values: ["low", "high", "max"] }] } } }), + }); + const first = metadata.planReasoningEffortDowngrade({ + provider: ZEN_GO, modelId: DEEPSEEK_FLASH, requested: "max", rejectionText: REJECTION_BODY, + }); + expect(first).toEqual({ effort: "high", recorded: true }); + metadata.flushReasoningSupportCache(); + const second = metadata.planReasoningEffortDowngrade({ + provider: ZEN_GO, modelId: DEEPSEEK_FLASH, requested: "max", rejectionText: REJECTION_BODY, + }); + expect(second).toEqual({ effort: "high", recorded: false }); + }); + + test("classifies only reasoning-effort refusals", async () => { + const { metadata } = await load(); + expect(metadata.isReasoningEffortRejection(REJECTION_BODY)).toBe(true); + expect(metadata.isReasoningEffortRejection(JSON.stringify({ error: { message: "Invalid upload request." } }))).toBe(false); + expect(metadata.isReasoningEffortRejection(undefined)).toBe(false); + }); + + // The near miss the parameter name alone cannot tell apart: the upstream is refusing + // `max_tokens` and merely echoing the request it received, `reasoning_effort` included. + // Reading that as a refusal spends the turn's one downgrade replay and persists a refusal + // that clamps the ladder for thirty days. + test("an unrelated refusal that echoes reasoning_effort is not a reasoning-effort refusal", async () => { + const { metadata } = await load(); + const echoed = JSON.stringify({ + error: { + param: "max_tokens", + type: "invalid_request_error", + message: "max_tokens must be a positive integer.", + }, + request: { model: "muse-spark-1.3-contributor", reasoning_effort: "max", max_tokens: -1 }, + }); + expect(metadata.isReasoningEffortRejection(echoed)).toBe(false); + }); + + test("still classifies a refusal that names the effort parameter without the word effort", async () => { + const { metadata } = await load(); + expect(metadata.isReasoningEffortRejection(JSON.stringify({ + error: { param: "reasoning.effort", message: "Unsupported value for this model." }, + }))).toBe(true); + }); +}); + +describe("destination resolution", () => { + const ZEN = { ...ZEN_GO, baseUrl: "https://opencode.ai/zen/v1" } as OcxProviderConfig; + const MODEL = { [MUSE_SPARK]: { reasoning: true, options: [{ type: "effort", values: ["low", "high"] }] } }; + + test("resolves the OpenCode family and tolerates a trailing slash", async () => { + const { effort } = await load(metadataFile({ "opencode": MODEL, "opencode-go": MODEL })); + expect(effort.configuredReasoningEfforts(ZEN, MUSE_SPARK)).toEqual(["low", "high"]); + const trailingSlash = { ...ZEN_GO, baseUrl: "https://opencode.ai/zen/go/v1/" } as OcxProviderConfig; + expect(effort.configuredReasoningEfforts(trailingSlash, MUSE_SPARK)).toEqual(["low", "high"]); + }); + + // 36 of the registry's 83 destinations match a models.dev provider, so resolving by URL alone + // would move ladders for providers this change has no evidence for. The gate stays explicit. + test("a destination that only matches by URL stays gated out", async () => { + const { effort } = await load(metadataFileV2( + { "some-upstream": MODEL }, + { "some-upstream": "https://api.some-upstream.example/v1" }, + )); + const provider = { ...ZEN_GO, baseUrl: "https://api.some-upstream.example/v1" } as OcxProviderConfig; + expect(effort.configuredReasoningEfforts(provider, MUSE_SPARK)).toBeUndefined(); + }); + + test("the v2 snapshot confirms the gate against each provider's published api url", async () => { + const { metadata } = await load(metadataFileV2( + { "opencode-go": MODEL }, + { "opencode-go": "https://opencode.ai/zen/go" }, + )); + expect(metadata.reasoningMetadataMapping()).toEqual([ + { destination: "https://opencode.ai/zen/go/v1", provider: "opencode-go", publishedApi: "https://opencode.ai/zen/go", confirmed: true, models: 1 }, + { destination: "https://opencode.ai/zen/v1", provider: "opencode", models: 0 }, + ]); + }); + + test("a v1 snapshot without published api urls still resolves through the table", async () => { + const { metadata } = await load(metadataFile({ "opencode-go": MODEL })); + const [zenGo] = metadata.reasoningMetadataMapping(); + expect(zenGo.publishedApi).toBeUndefined(); + expect(zenGo.confirmed).toBeUndefined(); + expect(zenGo.models).toBe(1); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 0e311dfb36..be611619d0 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -907,6 +907,7 @@ "raycast-detect.test.ts": "clients", "reasoning-effort.test.ts": "codex-integration", "reasoning-envelope.test.ts": "responses", + "reasoning-metadata.test.ts": "codex-integration", "reasoning-replay-identity.test.ts": "adapters", "reasoning-replay-robustness.test.ts": "adapters", "reasoning-replay-scope-source.test.ts": "lib", @@ -985,6 +986,7 @@ "responses-parser.test.ts": "responses", "responses-pool-401-refresh.test.ts": "responses", "responses-pool-refresh-attribution.test.ts": "responses", + "responses-reasoning-effort-downgrade.test.ts": "responses", "responses-reasoning-summary-passthrough.test.ts": "responses", "responses-routed-web-search-fields.test.ts": "responses", "responses-self-named-namespace-scrub.test.ts": "responses", diff --git a/tests/responses/responses-reasoning-effort-downgrade.test.ts b/tests/responses/responses-reasoning-effort-downgrade.test.ts new file mode 100644 index 0000000000..ee18288fce --- /dev/null +++ b/tests/responses/responses-reasoning-effort-downgrade.test.ts @@ -0,0 +1,227 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { handleResponses } from "../../src/server/responses/core"; +import { resetReasoningMetadataCachesForTests } from "../../src/providers/reasoning-metadata"; +import type { RequestLogContext } from "../../src/server/request-log"; +import type { OcxConfig } from "../../src/types"; + +/** + * Rejected-rung learning on the request path: a rung the catalog advertises can still be refused + * upstream because the ladder describes the model, not this account's entitlement (max on + * muse-spark-1.3-contributor needs an active Muse Code subscription). The pipeline must learn the + * refusal, replay once at the next published rung, and never replay an unrelated 400. + */ + +const originalFetch = globalThis.fetch; +const originalOpenCodexHome = process.env.OPENCODEX_HOME; +const MODEL = "muse-spark-1.3-contributor"; +const REFUSAL = JSON.stringify({ + error: { + param: "reasoning.effort", + type: "invalid_request_error", + message: "Error from provider (Console Go): Upstream request failed: [invalid_request_error] reasoning_effort max requires an active Muse Code subscription for model muse-spark-1.3-contributor.", + }, +}); +const UNRELATED = JSON.stringify({ error: { type: "invalid_request_error", message: "Invalid upload request." } }); + +let testDir = ""; + +function writeSnapshot(values: string[]): void { + writeFileSync(join(testDir, "reasoning-metadata-cache.json"), JSON.stringify({ + version: 1, + fetchedAt: Date.now(), + source: "test", + providers: { "opencode-go": { [MODEL]: { reasoning: true, options: [{ type: "effort", values }] } } }, + })); +} + +function config(): OcxConfig { + return { + defaultProvider: "first", + providers: { + first: { + adapter: "openai-chat", + baseUrl: "https://opencode.ai/zen/go/v1", + authMode: "key", + apiKey: "test-key", + }, + }, + } as OcxConfig; +} + +/** + * The Chat config above routes through the generic `recovery:` loop. muse-spark is an + * `openai-responses` destination in the registry, and that wire takes the separate + * `passthroughRecovery:` loop, which carries its own copy of the downgrade block. Covering only + * the Chat config would have left that copy untested while the test name claimed otherwise. + */ +function passthroughConfig(): OcxConfig { + return { + defaultProvider: "first", + providers: { + first: { + adapter: "openai-responses", + baseUrl: "https://opencode.ai/zen/go/v1", + authMode: "key", + apiKey: "test-key", + }, + }, + } as OcxConfig; +} + +function effortOf(body: Record | undefined): unknown { + if (!body) return undefined; + const reasoning = body.reasoning; + if (reasoning && typeof reasoning === "object" && "effort" in reasoning) { + return (reasoning as { effort?: unknown }).effort; + } + return body.reasoning_effort; +} + +function request(stream = false): Request { + return new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "first/" + MODEL, + stream, + store: false, + reasoning: { effort: "max" }, + input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "go" }] }], + }), + }); +} + +function success(): Response { + return Response.json({ id: "resp-ok", object: "response", status: "completed", model: MODEL, output: [] }); +} + +beforeEach(() => { + testDir = mkdtempSync(join(tmpdir(), "ocx-reasoning-downgrade-")); + process.env.OPENCODEX_HOME = testDir; + resetReasoningMetadataCachesForTests(); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + resetReasoningMetadataCachesForTests(); + if (originalOpenCodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalOpenCodexHome; + rmSync(testDir, { recursive: true, force: true }); +}); + +describe("rejected reasoning rungs", () => { + test("clamps a rung the model does not publish before dispatch", async () => { + writeSnapshot(["minimal", "low", "medium", "high", "xhigh"]); + const outbound: Array> = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + return success(); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(request(), config(), logCtx); + await response.text(); + + expect(response.status).toBe(200); + expect(outbound).toHaveLength(1); + expect(outbound[0]?.reasoning_effort).toBe("xhigh"); + }); + + test("learns the refusal and replays once at the next published rung", async () => { + writeSnapshot(["low", "high", "max"]); + const outbound: Array> = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + return outbound.length === 1 + ? new Response(REFUSAL, { status: 400, headers: { "content-type": "application/json" } }) + : success(); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(request(), config(), logCtx); + await response.text(); + + expect(outbound).toHaveLength(2); + expect(outbound[0]?.reasoning_effort).toBe("max"); + expect(outbound[1]?.reasoning_effort).toBe("high"); + expect(logCtx.activeAttempt?.recoveryKinds).toEqual(["reasoning-effort-downgrade"]); + }); + + test("does not replay an unrelated 400", async () => { + writeSnapshot(["low", "high", "max"]); + const outbound: Array> = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + return new Response(UNRELATED, { status: 400, headers: { "content-type": "application/json" } }); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(request(), config(), logCtx); + await response.text(); + + expect(outbound).toHaveLength(1); + expect(response.ok).toBe(false); + expect(logCtx.activeAttempt?.recoveryKinds ?? []).toEqual([]); + }); + test("replays once on the streamed generic-recovery path too", async () => { + writeSnapshot(["low", "high", "max"]); + const outbound: Array> = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + return outbound.length === 1 + ? new Response(REFUSAL, { status: 400, headers: { "content-type": "application/json" } }) + : success(); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(request(true), config(), logCtx); + await response.text(); + + expect(outbound).toHaveLength(2); + expect(outbound[1]?.reasoning_effort).toBe("high"); + expect(logCtx.activeAttempt?.recoveryKinds).toEqual(["reasoning-effort-downgrade"]); + }); + + test("replays once on the Responses passthrough path", async () => { + writeSnapshot(["low", "high", "max"]); + const outbound: Array> = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + return outbound.length === 1 + ? new Response(REFUSAL, { status: 400, headers: { "content-type": "application/json" } }) + : success(); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(request(true), passthroughConfig(), logCtx); + await response.text(); + + expect(outbound).toHaveLength(2); + expect(effortOf(outbound[0])).toBe("max"); + expect(effortOf(outbound[1])).toBe("high"); + expect(logCtx.activeAttempt?.recoveryKinds).toEqual(["reasoning-effort-downgrade"]); + }); + + // The guard used to live inside the generic `recovery:` loop, so every `continue recovery` + // handed the turn a fresh downgrade budget and one request could walk the ladder down. + test("downgrades at most once even when the replay is refused again", async () => { + writeSnapshot(["low", "medium", "high", "max"]); + const outbound: Array> = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + return new Response(REFUSAL, { status: 400, headers: { "content-type": "application/json" } }); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(request(), config(), logCtx); + await response.text(); + + expect(outbound).toHaveLength(2); + expect(effortOf(outbound[0])).toBe("max"); + expect(effortOf(outbound[1])).toBe("high"); + expect(response.ok).toBe(false); + }); +});