diff --git a/CHANGELOG.md b/CHANGELOG.md index a487e0b3..57dd4253 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project are recorded here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the versioning follows [Semantic Versioning](https://semver.org/). +## [Unreleased] + +### ✨ Added + +- A review that fails because the model is unavailable now says so and tells you what to do — with a local CLI provider (claude / codex) the model comes from that CLI's own configuration, so it has to be changed there. + +### 🔧 Fixed + +- A failed review now shows the provider's actual error instead of only "all fallback models failed" — the real cause (an unavailable model, an expired login, an exhausted quota) was previously swallowed and never reached the run card. +- A local CLI provider that exits successfully but returns an empty reply is now reported as a failure naming that cause, rather than as an unexplained LLM failure. + ## [0.11.2] - 2026-07-28 > Highlights of this release: diff --git a/CHANGELOG.zh-CN.md b/CHANGELOG.zh-CN.md index 7b3a085e..7af6d2b5 100644 --- a/CHANGELOG.zh-CN.md +++ b/CHANGELOG.zh-CN.md @@ -5,6 +5,17 @@ 本项目所有重要变更记录于此。格式参考 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/), 版本号遵循 [语义化版本](https://semver.org/lang/zh-CN/)。 +## [Unreleased] + +### ✨ 新增 + +- 因模型不可用而失败的评审现在会明确说明,并给出处理方式——使用本地 CLI 供应商(claude / codex)时,模型来自该 CLI 自身的配置,需要在那里更换。 + +### 🔧 修复 + +- 评审失败时现在会展示供应商返回的真实错误,而不再只有一句「所有备选模型均调用失败」——真正的原因(模型不可用、登录过期、额度耗尽)此前被吞掉,从未出现在运行卡片上。 +- 本地 CLI 供应商正常退出却返回空回复时,现在会作为失败上报并指明该原因,而不再表现为一次无从解释的 LLM 调用失败。 + ## [0.11.2] - 2026-07-28 > 本次发布要点: diff --git a/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/cli/install.py b/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/cli/install.py index 8d4c3e09..caa0e91b 100644 --- a/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/cli/install.py +++ b/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/cli/install.py @@ -11,7 +11,7 @@ import sys from ..runtime import _debug, strip_cache_break -from ..usage import _emit_usage_tokens +from ..usage import _emit_llm_error, _emit_usage_tokens from .specs import _CLI_SPECS @@ -88,12 +88,18 @@ async def run_cli_chat(bin_name, system, user) -> str: except Exception as exc: # noqa: BLE001 raise RuntimeError(f"failed to start CLI '{bin_name}': {exc}") from exc out, err = await proc.communicate(prompt.encode("utf-8")) + out_text = (out or b"").decode("utf-8", "replace") if proc.returncode != 0: - raise RuntimeError( - f"CLI '{bin_name}' exit code {proc.returncode}: " - f"{(err or b'').decode('utf-8', 'replace')[:500]}" - ) - text, usage = spec["parser"]((out or b"").decode("utf-8", "replace")) + # Error source: some CLIs report the failure only on **stdout** (codex writes turn.failed / error events into its + # JSONL stream and leaves stderr empty), which would otherwise surface as an empty reason and lose the real cause + # (auth / model unavailable / quota). When the spec provides an extractor, take stdout's verdict first and fall + # back to stderr; either way keep the raise contract (pr-agent turns it into "all fallback models failed"). + extractor = spec.get("error_extractor") + detail = (extractor(out_text) if extractor else None) or (err or b"").decode("utf-8", "replace") + detail = (detail or "")[:500] + _emit_llm_error(bin_name, detail) + raise RuntimeError(f"CLI '{bin_name}' exit code {proc.returncode}: {detail}") + text, usage = spec["parser"](out_text) if usage: # prompt_tokens ≈ total input-side size, output_tokens ≈ completion (input/output_tokens share the same names across both). # The cache fields differ in convention between the two: @@ -115,6 +121,18 @@ async def run_cli_chat(bin_name, system, user) -> str: cache_read_tokens=cache_read if isinstance(cache_read, int) else None, turns=turns if isinstance(turns, int) else None, ) + # Empty reply on a zero exit code: the CLI ran to completion but produced no assistant message (codex does this when a turn + # dies on a tool/feature failure, e.g. Code Mode failing closed because its host binary is missing). Downstream this became + # an empty prompt into pr-agent's load_yaml → the opaque "all fallback models failed"; raise here instead, carrying any error + # event the stream did report, so the run card names the actual cause. Usage is emitted first, keeping the token count truthful. + if not (text or "").strip(): + extractor = spec.get("error_extractor") + detail = (extractor(out_text) if extractor else None) or "" + msg = f"CLI '{bin_name}' returned an empty reply" + ( + f": {detail[:500]}" if detail else " (no error reported in its output)" + ) + _emit_llm_error(bin_name, msg) + raise RuntimeError(msg) return text diff --git a/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/cli/parsers.py b/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/cli/parsers.py index 5dc7b27c..7e504720 100644 --- a/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/cli/parsers.py +++ b/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/cli/parsers.py @@ -71,3 +71,45 @@ def _parse_codex_output(stdout): if isinstance(usage, dict) and turns: usage["num_turns"] = turns return (text if isinstance(text, str) else ""), usage + + +def _extract_codex_error(stdout): + """Pull the failure cause out of the `codex exec --json` event stream, returning a message string or None. + + codex reports failures on **stdout** (the JSONL stream) and leaves stderr empty, so without this the caller would + raise with an empty detail and the real cause (auth, model 404, quota ...) would never reach the logs. Precedence: + - `turn.failed` → error.message is the terminal verdict, most accurate; + - otherwise the last `type=="error"` event (the intermediate retry notices carry the same upstream message); + - otherwise the last `item.completed` whose item.type=="error" (startup-stage errors, e.g. an unavailable feature host). + """ + import json + + turn_failed = None + last_error = None + last_item_error = None + for line in (stdout or "").splitlines(): + line = line.strip() + if not line: + continue + try: + ev = json.loads(line) + except Exception: # noqa: BLE001 - skip non-JSON lines (logs, etc.) + continue + if not isinstance(ev, dict): + continue + etype = ev.get("type") + if etype == "turn.failed": + err = ev.get("error") + if isinstance(err, dict) and isinstance(err.get("message"), str): + turn_failed = err["message"] + elif etype == "error": + msg = ev.get("message") + if isinstance(msg, str): + last_error = msg + elif etype == "item.completed": + item = ev.get("item") + if isinstance(item, dict) and item.get("type") == "error": + msg = item.get("message") + if isinstance(msg, str): + last_item_error = msg + return turn_failed or last_error or last_item_error diff --git a/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/cli/specs.py b/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/cli/specs.py index 8145fd61..9e0ef3af 100644 --- a/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/cli/specs.py +++ b/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/cli/specs.py @@ -1,6 +1,6 @@ """Spec table for adapted local CLI commands: argv flags (the prompt always goes via stdin) + output parser + billing env to strip. Registering one entry here is enough for a new command; the renderer-side whitelist validation must stay in sync (see LlmProfileForm.validateProfile).""" -from .parsers import _parse_claude_output, _parse_codex_output +from .parsers import _extract_codex_error, _parse_claude_output, _parse_codex_output # `low_effort_flags`: argv to append for the low-effort tier (only enabled by the Agent orchestration channel via MEEBOX_CLI_REASONING, # see install.py). Commands with a trailing `-` (stdin) insert these flags before the `-`, keeping `-` last. @@ -30,6 +30,9 @@ ], "low_effort_flags": ["-c", "model_reasoning_effort=low"], "parser": _parse_codex_output, + # codex writes its failures into the stdout event stream and leaves stderr empty, so a non-zero exit needs + # the cause extracted from stdout (see install.py); commands without this key just fall back to stderr. + "error_extractor": _extract_codex_error, "strip_env": ("OPENAI_API_KEY", "CODEX_API_KEY"), }, } diff --git a/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/usage.py b/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/usage.py index bccf7e6b..74b45105 100644 --- a/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/usage.py +++ b/apps/desktop/scripts/pragent-shim/meebox_pragent_shim/usage.py @@ -1,5 +1,10 @@ """Real token usage collection: emitted to stderr as a sentinel line `@@MEEBOX_USAGE@@ {json}`, which the main process onLine -accumulates from (see apps/desktop/src/main/ipc.ts). Takes only tokens, not cost. Fault-tolerant throughout.""" +accumulates from (see apps/desktop/src/main/ipc.ts). Takes only tokens, not cost. Fault-tolerant throughout. + +The same stderr sentinel channel also carries `@@MEEBOX_LLM_ERROR@@ {json}` (see _emit_llm_error): pr-agent's +retry_with_fallback_models logs the underlying exception into loguru's `artifact=` field, which the default format never +prints, so a failing LLM call reaches stdout as nothing but "Failed to generate prediction with any model". The sentinel +carries the real cause out of band, past that lossy log line.""" import sys from .runtime import _debug @@ -67,3 +72,22 @@ def _emit_usage_tokens( print(f"@@MEEBOX_USAGE@@ {json.dumps(rec)}", file=sys.stderr, flush=True) except Exception as exc: # noqa: BLE001 _debug(f"emit cli usage failed (ignored): {exc}") + + +def _emit_llm_error(cli, message) -> None: + """Emit the real cause of a failed LLM call to stderr as `@@MEEBOX_LLM_ERROR@@ {json}`, right before raising. + + Emitting is purely additive — the exception is still raised and pr-agent still runs its fallback retry; this only + keeps the cause from being swallowed by that retry's log line, so the run card can show what actually went wrong + (and classify it, e.g. an unavailable model) instead of the generic "all fallback models failed". + """ + try: + import json + + print( + f"@@MEEBOX_LLM_ERROR@@ {json.dumps({'cli': cli, 'message': message})}", + file=sys.stderr, + flush=True, + ) + except Exception as exc: # noqa: BLE001 + _debug(f"emit llm error failed (ignored): {exc}") diff --git a/apps/desktop/src/main/services/pr-agent/run-executor.ts b/apps/desktop/src/main/services/pr-agent/run-executor.ts index 398e0234..7701eb8f 100644 --- a/apps/desktop/src/main/services/pr-agent/run-executor.ts +++ b/apps/desktop/src/main/services/pr-agent/run-executor.ts @@ -15,6 +15,7 @@ import { } from '@meebox/pr-agent-bridge'; import { addFindingClosure, + classifyLlmFailure, dropPendingFindingDrafts, finishReviewRun, parseReviewOutput, @@ -37,7 +38,8 @@ import { accumulateUsageSentinel, finalizeUsage, newUsageAcc, - stripUsageSentinels, + parseLlmErrorSentinel, + stripShimSentinels, } from './usage.js'; import { neutralizeWorktreeInstructions } from './worktree-sanitize.js'; @@ -171,12 +173,18 @@ export class RunExecutor { durationMs: Date.now() - t0, exitCode: result.exitCode, stdout, - stderr: stripUsageSentinels(result.stderr), + stderr: stripShimSentinels(result.stderr), tokenUsage, }; if (parsed.llmFailure) { + // pr-agent's fallback retry only logs "Failed to generate prediction with any model" to stdout and hides the real + // exception in loguru's artifact field, so prefer the shim's stderr sentinel when present — it carries the actual + // provider/CLI error, which is also the only text worth classifying into an actionable hint. + const sentinel = parseLlmErrorSentinel(result.stderr); + const message = sentinel?.message ?? parsed.llmFailure.message; + const errorHint = classifyLlmFailure(message); this.ctx.logger.warn( - { runId, reason: parsed.llmFailure.message }, + { runId, reason: message, cli: sentinel?.cli, hint: errorHint }, 'pragent exit 0 but LLM call failed; marking run as failed', ); // Failed runs get no structured collection — findings set empty, UI shows only raw output (no chatpane finding card). @@ -184,7 +192,8 @@ export class RunExecutor { ...base, status: 'failed', errorReason: 'llm-error', - errorMessage: parsed.llmFailure.message, + errorMessage: message, + ...(errorHint ? { errorHint } : {}), findings: [], }; } @@ -225,7 +234,7 @@ export class RunExecutor { errorReason: err.reason, errorMessage: err.message, stdout: err.result.stdout, - stderr: stripUsageSentinels(err.result.stderr), + stderr: stripShimSentinels(err.result.stderr), findings: [], tokenUsage, }; diff --git a/apps/desktop/src/main/services/pr-agent/usage.ts b/apps/desktop/src/main/services/pr-agent/usage.ts index 33ba4076..922fac06 100644 --- a/apps/desktop/src/main/services/pr-agent/usage.ts +++ b/apps/desktop/src/main/services/pr-agent/usage.ts @@ -73,13 +73,51 @@ export function finalizeUsage(acc: UsageAcc): TokenUsage | undefined { } /** - * Strip usage sentinel lines from stderr before persistence: onLine already intercepts them in real time without forwarding, but exec internally + * Strip shim sentinel lines from stderr before persistence: onLine already intercepts them in real time without forwarding, but exec internally * accumulates all stderr into result.stderr (including sentinels), so clear these noise lines before persisting. */ -export function stripUsageSentinels(stderr: string | undefined): string | undefined { +export function stripShimSentinels(stderr: string | undefined): string | undefined { if (!stderr) return stderr; return stderr .split('\n') - .filter((l) => !l.includes(USAGE_SENTINEL)) + .filter((l) => !l.includes(USAGE_SENTINEL) && !l.includes(LLM_ERROR_SENTINEL)) .join('\n'); } + +/** + * LLM-error sentinel-line prefix (kept consistent with the shim's usage._emit_llm_error). Shares the stderr sentinel + * channel with usage: pr-agent's fallback retry logs the underlying exception into loguru's `artifact=` field, which the + * default format drops, so the real cause never reaches stdout — the shim routes it out of band through this line. + */ +export const LLM_ERROR_SENTINEL = '@@MEEBOX_LLM_ERROR@@'; + +/** + * Pull the LLM-call cause out of stderr sentinel lines, returning the **last** one (a run may retry several times; the last + * is what actually sank the run). Bad JSON / a missing message is ignored — this is a diagnostic enrichment, never a + * reason to fail differently. + */ +export function parseLlmErrorSentinel( + stderr: string | undefined, +): { cli?: string; message: string } | undefined { + if (!stderr) return undefined; + let found: { cli?: string; message: string } | undefined; + for (const line of stderr.split('\n')) { + const i = line.indexOf(LLM_ERROR_SENTINEL); + if (i < 0) continue; + try { + const rec = JSON.parse(line.slice(i + LLM_ERROR_SENTINEL.length).trim()) as { + cli?: unknown; + message?: unknown; + }; + if (typeof rec.message === 'string' && rec.message.trim()) { + found = { + cli: typeof rec.cli === 'string' ? rec.cli : undefined, + message: rec.message.trim(), + }; + } + } catch { + // Malformed sentinel → skip, keep any earlier one. + } + } + return found; +} diff --git a/apps/desktop/src/renderer/src/components/features/chat/components/RunResultView.tsx b/apps/desktop/src/renderer/src/components/features/chat/components/RunResultView.tsx index 2387d75c..610122f7 100644 --- a/apps/desktop/src/renderer/src/components/features/chat/components/RunResultView.tsx +++ b/apps/desktop/src/renderer/src/components/features/chat/components/RunResultView.tsx @@ -229,6 +229,12 @@ export function RunResultView({ {run.errorMessage && !isCancelled && (
{run.errorMessage}
)} + {/* Actionable hint on top of the raw cause: the message above is the provider's own wording (English, technical), + which doesn't tell the user what to do. A recognized kind adds one localized line of remedy — currently + 「the model is unavailable, change it」, whose fix lives outside the app for a local CLI provider. */} + {run.errorHint === 'model-unavailable' && !isCancelled && ( +

{t('chatPane.llmErrorHint.modelUnavailable')}

+ )} {/* Failed / cancelled no longer show a separate output block: the pr-agent log is already in the collapsible「raw output」above (stdout contains the [pr-agent stdout log] segment, same source as stderr), avoiding duplicating the same log into two blocks. */} diff --git a/apps/desktop/src/renderer/src/i18n/locales/de-DE.json b/apps/desktop/src/renderer/src/i18n/locales/de-DE.json index f4083b55..47f5e205 100644 --- a/apps/desktop/src/renderer/src/i18n/locales/de-DE.json +++ b/apps/desktop/src/renderer/src/i18n/locales/de-DE.json @@ -98,6 +98,9 @@ "goToSettings": "Zu den Einstellungen", "inputAria": "Chat-Eingabe", "llmCallFailed": "LLM-Aufruf fehlgeschlagen", + "llmErrorHint": { + "modelUnavailable": "Das angeforderte Modell ist nicht verfügbar. Bei einem lokalen CLI-Anbieter stammt das Modell aus der Konfiguration dieser CLI — stellen Sie dort auf ein unterstütztes Modell um und führen Sie den Lauf erneut aus." + }, "mergeConfirmLabel": "Mergen", "mergeConfirmMessage": "„{{title}}“ in den Zielbranch mergen? Dies kann nicht rückgängig gemacht werden.", "mergeConfirmTitle": "PR mergen", diff --git a/apps/desktop/src/renderer/src/i18n/locales/en-US.json b/apps/desktop/src/renderer/src/i18n/locales/en-US.json index def9c738..45fbae10 100644 --- a/apps/desktop/src/renderer/src/i18n/locales/en-US.json +++ b/apps/desktop/src/renderer/src/i18n/locales/en-US.json @@ -98,6 +98,9 @@ "goToSettings": "Go to Settings", "inputAria": "Chat input", "llmCallFailed": "LLM call failed", + "llmErrorHint": { + "modelUnavailable": "The requested model is unavailable. With a local CLI provider the model comes from that CLI's own configuration — switch it to a supported model there, then run again." + }, "mergeConfirmLabel": "Merge", "mergeConfirmMessage": "Merge \"{{title}}\" into its target branch? This can't be undone.", "mergeConfirmTitle": "Merge PR", diff --git a/apps/desktop/src/renderer/src/i18n/locales/ja-JP.json b/apps/desktop/src/renderer/src/i18n/locales/ja-JP.json index 5f1542f7..2929031a 100644 --- a/apps/desktop/src/renderer/src/i18n/locales/ja-JP.json +++ b/apps/desktop/src/renderer/src/i18n/locales/ja-JP.json @@ -98,6 +98,9 @@ "goToSettings": "設定へ移動", "inputAria": "チャット入力", "llmCallFailed": "LLM の呼び出しに失敗しました", + "llmErrorHint": { + "modelUnavailable": "要求されたモデルは利用できません。ローカル CLI プロバイダーの場合、モデルはその CLI 自身の設定で決まります。CLI 側でサポートされているモデルに変更してから再実行してください。" + }, "mergeConfirmLabel": "マージ", "mergeConfirmMessage": "「{{title}}」をターゲットブランチにマージしますか?この操作は取り消せません。", "mergeConfirmTitle": "PR をマージ", diff --git a/apps/desktop/src/renderer/src/i18n/locales/zh-CN.json b/apps/desktop/src/renderer/src/i18n/locales/zh-CN.json index e220dc1b..ad5bec2d 100644 --- a/apps/desktop/src/renderer/src/i18n/locales/zh-CN.json +++ b/apps/desktop/src/renderer/src/i18n/locales/zh-CN.json @@ -98,6 +98,9 @@ "goToSettings": "去设置", "inputAria": "聊天输入框", "llmCallFailed": "LLM 调用失败", + "llmErrorHint": { + "modelUnavailable": "所请求的模型不可用。使用本地 CLI 供应商时,模型由该 CLI 自身的配置决定,请在 CLI 中改用受支持的模型后重新运行。" + }, "mergeConfirmLabel": "合并", "mergeConfirmMessage": "确定将「{{title}}」合并到目标分支?此操作不可撤销。", "mergeConfirmTitle": "合并 PR", diff --git a/apps/desktop/src/renderer/src/styles/features/chat/run.scss b/apps/desktop/src/renderer/src/styles/features/chat/run.scss index 020a84a3..4a861667 100644 --- a/apps/desktop/src/renderer/src/styles/features/chat/run.scss +++ b/apps/desktop/src/renderer/src/styles/features/chat/run.scss @@ -41,6 +41,14 @@ color: $text-muted; } +// Actionable remedy line under the raw cause: prose (not mono like .chat-error-detail), so it reads as guidance +// from the app rather than as more provider output. +.chat-error-hint { + margin: $space-2 0 0; + font-size: $fs-xs; + line-height: 1.5; +} + // stderr / stdout collapsible area embedded in a failed run: one extra level of indentation inside the chat-error red banner .chat-error-stderr, .chat-error-stdout { diff --git a/docs/arch/02-agent/05-pragent-runtime.md b/docs/arch/02-agent/05-pragent-runtime.md index b0b904e2..f253597a 100644 --- a/docs/arch/02-agent/05-pragent-runtime.md +++ b/docs/arch/02-agent/05-pragent-runtime.md @@ -104,6 +104,11 @@ litellm**. - **Reuse the CLI's own login state**: the subprocess inherits `HOME`/`USERPROFILE`, and the CLI reads its own login credentials (e.g. `~/.claude`) to run. To avoid a stray API key in the local environment leaking in and overriding the CLI's own login method, the shim explicitly strips `ANTHROPIC_API_KEY` / `ANTHROPIC_AUTH_TOKEN` from the subprocess env. The model used, the quota and compliance are all decided by that CLI's account and the user's authorization. +- **Error source on a non-zero exit**: not every CLI reports its failure on stderr — `codex exec --json` writes `turn.failed` / `error` + events into the **stdout** JSONL stream and leaves stderr empty, so taking stderr alone would raise with an empty reason and the real + cause (auth expired / model unavailable / quota) would never reach the log, leaving only pr-agent's generic "all fallback models failed". + A spec may therefore declare an `error_extractor` that pulls the verdict out of stdout (precedence `turn.failed` → the last `error` + event → an `item.completed` error item), with stderr as the fallback for commands without one. - **Proxy auto pass-through**: the subprocess env is copied from `os.environ` (only the two API keys above removed), and `HTTP(S)_PROXY` / `NO_PROXY` are kept as-is → `claude`'s egress automatically goes through the user-configured proxy (see [Networking & proxy](../99-core/03-networking-proxy.md)), no extra setup needed. - **token usage**: from the claude JSON's `usage`, construct the same `@@MEEBOX_USAGE@@` sentinel, accumulated by the same main-process path. The ↑ total input takes diff --git a/packages/poller/src/parse-output.ts b/packages/poller/src/parse-output.ts index 0a3bf21d..265119ea 100644 --- a/packages/poller/src/parse-output.ts +++ b/packages/poller/src/parse-output.ts @@ -22,6 +22,35 @@ export interface ParsedReviewOutput { llmFailure?: { message: string }; } +/** + * Actionable classification of an LLM failure, used by the UI to add a "what to do about it" hint on top of the raw + * technical message. Only cases with a concrete user action get a kind; everything else stays unclassified (undefined) + * and renders the message alone. + * + * - `model-unavailable`: the model the provider was asked for does not exist / is not accessible for this account. + * Typical with a local CLI provider, whose model is pinned in the CLI's own config (e.g. `~/.codex/config.toml`) + * and can be retired upstream at any time — the app never chose it, so the fix is to change it there. + */ +export type LlmFailureKind = 'model-unavailable'; + +/** + * Classify an LLM failure message into an actionable kind, or undefined when nothing actionable is recognized. + * Matches the wording of both the direct-API path (litellm `NotFoundError` / `model_not_found`) and the local CLI path + * (codex's `The model \`x\` does not exist or you do not have access to it` / `is not supported when using ...`). + */ +export function classifyLlmFailure(message: string): LlmFailureKind | undefined { + const m = stripAnsi(message); + if ( + /model_not_found/i.test(m) || + /\bmodel\b[^\n]{0,80}\bdoes not exist\b/i.test(m) || + /\bmodel\b[^\n]{0,80}\bis not supported\b/i.test(m) || + /\bdo(?:es)? not have access to (?:it|this model)\b/i.test(m) + ) { + return 'model-unavailable'; + } + return undefined; +} + /** * Scan stdout for a marker of all LLM calls failing. When pr-agent's fallback retry exhausts all alternate * models and still fails, it only logger.error's one line "Failed to PR: Failed to generate diff --git a/packages/poller/tests/parse-output.test.ts b/packages/poller/tests/parse-output.test.ts index 0e26e369..0de269ab 100644 --- a/packages/poller/tests/parse-output.test.ts +++ b/packages/poller/tests/parse-output.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { + classifyLlmFailure, parseReviewOutput, parseStructuredAsk, sectionToFinding, @@ -698,3 +699,33 @@ describe('parseStructuredAsk', () => { expect(parseReviewOutput('x', 'ask').askVerdict).toBeUndefined(); }); }); + +describe('classifyLlmFailure', () => { + it('codex: model retired upstream (404 wording) → model-unavailable', () => { + expect( + classifyLlmFailure( + "CLI 'codex' exit code 1: unexpected status 404 Not Found: The model `gpt-5.5` does not exist or you do not have access to it., url: https://chatgpt.com/backend-api/codex/responses", + ), + ).toBe('model-unavailable'); + }); + + it('codex: model not entitled for the account → model-unavailable', () => { + expect( + classifyLlmFailure( + "The 'gpt-5.1-codex' model is not supported when using Codex with a ChatGPT account.", + ), + ).toBe('model-unavailable'); + }); + + it('direct API: litellm model_not_found → model-unavailable', () => { + expect( + classifyLlmFailure('litellm.NotFoundError: The model `x` was not found (model_not_found)'), + ).toBe('model-unavailable'); + }); + + it('unrelated failures stay unclassified (no misleading remedy)', () => { + expect(classifyLlmFailure('litellm.AuthenticationError: invalid api key')).toBeUndefined(); + expect(classifyLlmFailure('Read timed out after 600s')).toBeUndefined(); + expect(classifyLlmFailure("CLI 'codex' returned an empty reply")).toBeUndefined(); + }); +}); diff --git a/packages/shared/src/poller-contract.ts b/packages/shared/src/poller-contract.ts index 5b6ecd5b..be2e9380 100644 --- a/packages/shared/src/poller-contract.ts +++ b/packages/shared/src/poller-contract.ts @@ -345,6 +345,13 @@ export interface ReviewRun { exitCode?: number; errorReason?: ReviewRunFailureReason; errorMessage?: string; + /** + * Actionable classification of the failure, when one is recognized (see poller's classifyLlmFailure). errorMessage stays + * the raw technical cause; this drives an extra "how to fix it" line in the UI, localized by the frontend from the kind. + * Currently only `model-unavailable` (the provider rejected the requested model — for a local CLI provider the model + * lives in that CLI's own config, so the user has to change it there). Historical runs don't have it. + */ + errorHint?: 'model-unavailable'; /** Raw stdout text; still kept after M3-B2 parses it into findings, for "see original" debugging */ stdout?: string; /** Raw stderr text */