From d5cef3b1eb6fc875408e342793beed13f3e534d9 Mon Sep 17 00:00:00 2001 From: Devil Date: Wed, 30 Sep 2026 23:06:50 -0400 Subject: [PATCH] feat: model registry, doctor v2 and CLODEX.md migration - clodex/models.py: built-in Codex catalog with retirement dates and successors, the live catalog (`codex debug models`, cached 24h) for supported reasoning efforts, Claude aliases/efforts, and validation of a loaded config: retired/retiring models, unsupported efforts, reviewer and quorum problems, sandbox/approval/workspace values. - clodex/capabilities.py: probe `claude --help` / `codex exec --help` per CLI version and cache it; command builders drop optional flags (--json-schema, --output-schema, -o, --fallback-model, ...) a CLI is known to lack instead of failing. - doctor: login state (`claude auth status`, `codex login status`), capability probe, model/config diagnostics with fixes; errors fail, `--strict` fails on warnings. - Builds, audits and `task start` refuse to run on a retired Codex model (clear message, exit 2, CLODEX_ALLOW_RETIRED_MODEL=1 overrides); a worker for an already-queued run records the failure. - `clodex init --migrate [--split-claude] [--dry-run]`: rewrites retired models and unsupported efforts (and optionally the flat claude layout) in the front matter only, preserving comments, quotes and CRLF. - Replaces the hardcoded gpt-5.5 guard in config.py with the registry. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 20 +- clodex/capabilities.py | 113 ++++++++++ clodex/cli.py | 36 +++- clodex/commands.py | 48 +++-- clodex/config.py | 25 +-- clodex/doctor.py | 82 +++++++- clodex/migrate.py | 137 +++++++++++++ clodex/models.py | 265 ++++++++++++++++++++++++ clodex/tasks.py | 3 + clodex/workflow.py | 10 + tests/support.py | 45 ++++ tests/test_model_registry.py | 385 +++++++++++++++++++++++++++++++++++ 12 files changed, 1123 insertions(+), 46 deletions(-) create mode 100644 clodex/capabilities.py create mode 100644 clodex/migrate.py create mode 100644 clodex/models.py create mode 100644 tests/test_model_registry.py diff --git a/README.md b/README.md index 1ec8f7b..9b31ae4 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,8 @@ PowerShell: | `clodex init` | Install native Claude/Codex instructions and MCP config | | `clodex native status` | Show native instruction and MCP config state | | `clodex native doctor` | Run native setup checks, CLI readiness, and launcher checks | -| `clodex doctor` | Check Python, git, Claude Code, Codex, and `CLODEX.md` | +| `clodex doctor [--strict]` | Check Python, git, both CLIs (login, supported flags), the live Codex model catalog, and `CLODEX.md` settings; `--strict` fails on warnings | +| `clodex init --migrate [--split-claude] [--dry-run]` | Update `CLODEX.md` settings that no longer work (retired models, unsupported efforts) | | `clodex plan ""` | Run Claude planning only | | `clodex build ""` | Run plan, implementation, and dual audit loop in an isolated worktree | | `clodex audit --diff` | Audit current uncommitted changes | @@ -175,6 +176,23 @@ approved diff hash; a local-workspace run is reported as already in the working that fails unexpectedly is marked `failed` (traceback in `error.txt`) and its worktree is removed; approved and blocked runs keep theirs until `clodex clean `. +### Models and `clodex doctor` + +`clodex doctor` validates the configuration against what is actually installed: + +- **Models:** retired Codex models are errors, ones retiring soon are warnings (both name the + replacement). Efforts are checked against the live catalog (`codex debug models`, cached for + 24h in `.clodex/models-cache.json`), so a stale `reasoning_effort` is caught before a run. +- **CLI flags:** the installed `claude` and `codex` are probed once per version + (`.clodex/capabilities.json`). Missing optional flags (`--json-schema`, `--output-schema`, ...) + are warnings and Clodex simply runs without them; missing required flags are errors. +- **Login:** `claude auth status` and `codex login status`. +- **Settings:** reviewers, quorum, sandbox, approval profile and workspace backend. + +`clodex build`, `audit` and `task start` refuse to start on a retired Codex model +(`clodex init --migrate` fixes the file; `CLODEX_ALLOW_RETIRED_MODEL=1` overrides). Migration edits +only the YAML front matter, keeps comments and line endings, and is safe to re-run. + ### Audit quorum `audit.quorum` decides when the reviewers agree: `unanimous` (default), `majority`, or a number diff --git a/clodex/capabilities.py b/clodex/capabilities.py new file mode 100644 index 0000000..2ebf27b --- /dev/null +++ b/clodex/capabilities.py @@ -0,0 +1,113 @@ +"""Detect which flags the installed Claude / Codex CLIs support by reading their `--help`. + +Version numbers say little about which flags exist, so this probes the real help text and +caches the result per CLI version in `.clodex/capabilities.json`. Command builders consult the +cache and drop optional flags a CLI is known to lack; an empty cache means "assume supported". +""" + +from __future__ import annotations + +import json +import re +import shutil +import subprocess +from pathlib import Path +from typing import Any + +CACHE_NAME = "capabilities.json" + +# Without these a run cannot work at all. +REQUIRED_FLAGS: dict[str, tuple[str, ...]] = { + "claude": ("--effort", "--permission-mode", "--output-format", "--model"), + "codex": ("--sandbox", "--model", "--cd"), +} +# Clodex degrades gracefully without these (prompt + schema validation + stdout parsing). +OPTIONAL_FLAGS: dict[str, tuple[str, ...]] = { + "claude": ("--json-schema", "--fallback-model", "--max-budget-usd"), + "codex": ("--output-schema", "--output-last-message", "--approve-for-me", "--ephemeral"), +} +HELP_COMMANDS: dict[str, list[str]] = { + "claude": ["claude", "--help"], + "codex": ["codex", "exec", "--help"], +} +VERSION_COMMANDS: dict[str, list[str]] = { + "claude": ["claude", "--version"], + "codex": ["codex", "--version"], +} + + +def flags_in_help(text: str, flags: tuple[str, ...]) -> list[str]: + return [flag for flag in flags if re.search(rf"(? str | None: + exe = shutil.which(argv[0]) + if not exe: + return None + try: + result = subprocess.run([exe, *argv[1:]], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, stdin=subprocess.DEVNULL, check=False) + except (OSError, subprocess.TimeoutExpired): + return None + return (result.stdout or "") + (result.stderr or "") if result.returncode == 0 else None + + +def _version(cli: str) -> str | None: + output = _run(VERSION_COMMANDS[cli]) + return output.strip().splitlines()[0] if output and output.strip() else None + + +def probe_cli(cli: str) -> dict[str, Any]: + """`{version, probed, supported, missing_required, missing_optional}`; probed is False if help was unreadable.""" + version = _version(cli) + help_text = _run(HELP_COMMANDS[cli]) + if help_text is None: + return {"version": version, "probed": False, "supported": [], "missing_required": [], "missing_optional": []} + required, optional = REQUIRED_FLAGS[cli], OPTIONAL_FLAGS[cli] + found = flags_in_help(help_text, required + optional) + return { + "version": version, + "probed": True, + "supported": found, + "missing_required": [flag for flag in required if flag not in found], + "missing_optional": [flag for flag in optional if flag not in found], + } + + +def cache_path(repo_root: Path) -> Path: + return repo_root / ".clodex" / CACHE_NAME + + +def load(repo_root: Path) -> dict[str, dict[str, Any]]: + try: + data = json.loads(cache_path(repo_root).read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + return data if isinstance(data, dict) else {} + + +def probe_all(repo_root: Path, clis: tuple[str, ...] = ("claude", "codex"), use_cache: bool = True) -> dict[str, dict[str, Any]]: + """Probe each CLI, reusing the cached result while its version is unchanged.""" + cached = load(repo_root) if use_cache else {} + result: dict[str, dict[str, Any]] = {} + for cli in clis: + version = _version(cli) + previous = cached.get(cli) + if previous and previous.get("probed") and version and previous.get("version") == version: + result[cli] = previous + else: + result[cli] = probe_cli(cli) + try: + path = cache_path(repo_root) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + except OSError: + pass + return result + + +def supports(repo_root: Path, cli: str, flag: str) -> bool | None: + """True/False if the cache knows, None if unknown (then callers assume supported).""" + info = load(repo_root).get(cli) + if not info or not info.get("probed"): + return None + return flag in info.get("supported", []) diff --git a/clodex/cli.py b/clodex/cli.py index b97a0df..d99b352 100644 --- a/clodex/cli.py +++ b/clodex/cli.py @@ -6,10 +6,13 @@ from pathlib import Path from typing import Any +from .config import ConfigError from .doctor import run_doctor from .evals import run_local_evals from .hooks import hook_config, ingest_hook_event from .mcp_server import main as mcp_main +from .migrate import migrate_contract +from .models import ModelRetiredError from .native import ( ManagedBlockError, apply_native_install, @@ -22,17 +25,44 @@ def main(argv: list[str] | None = None) -> int: + try: + return _main(argv) + except (ConfigError, ModelRetiredError) as exc: + print(f"clodex: error: {exc}", file=sys.stderr) + return 2 + + +def handle_migrate(args: argparse.Namespace) -> int: + contract = Path.cwd() / "CLODEX.md" + if not contract.is_file(): + print_output({"ok": False, "error": f"{contract} not found"}, args.json) + return 1 + with contract.open(encoding="utf-8", newline="") as handle: # keep CRLF endings as they are + text = handle.read() + migrated, changes = migrate_contract(text, split_claude=args.split_claude) + written = bool(changes) and not args.dry_run + if written: + with contract.open("w", encoding="utf-8", newline="") as handle: + handle.write(migrated) + print_output({"ok": True, "file": str(contract), "changes": changes, "dry_run": args.dry_run, "written": written}, args.json) + return 0 + + +def _main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="clodex", description="Claude Code + Codex CLI workflow orchestrator") parser.add_argument("--json", action="store_true", help="Print machine-readable JSON where supported") sub = parser.add_subparsers(dest="command", required=True) - sub.add_parser("doctor", help="Check local Clodex, Claude Code, Codex, and git readiness") + doctor = sub.add_parser("doctor", help="Check local Clodex, Claude Code, Codex, and git readiness") + doctor.add_argument("--strict", action="store_true", help="Treat warnings (e.g. a model retiring soon) as failures") init = sub.add_parser("init", help="Install native Claude Code and Codex collaboration instructions") init.add_argument("--global", action="store_true", dest="global_mode") init.add_argument("--dry-run", action="store_true") init.add_argument("--no-mcp-config", action="store_true") init.add_argument("--force", action="store_true") + init.add_argument("--migrate", action="store_true", help="Update CLODEX.md settings that no longer work (retired models, unsupported efforts)") + init.add_argument("--split-claude", action="store_true", help="With --migrate: split flat claude.model/effort into plan and audit roles") native = sub.add_parser("native", help="Inspect native Clodex setup") native_sub = native.add_subparsers(dest="native_command", required=True) @@ -120,9 +150,11 @@ def main(argv: list[str] | None = None) -> int: if args.command == "mcp-server": return mcp_main() if args.command == "doctor": - exit_code, data = run_doctor() + exit_code, data = run_doctor(strict=args.strict) print_output(data, args.json) return exit_code + if args.command == "init" and args.migrate: + return handle_migrate(args) if args.command == "init": try: exit_code = 0 diff --git a/clodex/commands.py b/clodex/commands.py index c95e0b0..9fee70b 100644 --- a/clodex/commands.py +++ b/clodex/commands.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from pathlib import Path +from . import capabilities from .config import ClodexConfig from .schemas import compact_schema, schema_path @@ -65,11 +66,11 @@ def _claude_command(name: str, config: ClodexConfig, role: str, schema_name: str "json", ] fallback = settings.get("fallback_model") - if fallback: + if fallback and capabilities.supports(config.repo_root, "claude", "--fallback-model") is not False: argv.extend(["--fallback-model", ",".join(fallback) if isinstance(fallback, (list, tuple)) else str(fallback)]) - if settings.get("max_budget_usd") not in (None, ""): + if settings.get("max_budget_usd") not in (None, "") and capabilities.supports(config.repo_root, "claude", "--max-budget-usd") is not False: argv.extend(["--max-budget-usd", str(settings["max_budget_usd"])]) - if inline_schema_supported(settings.get("json_schema", "auto")): + if inline_schema_supported(settings.get("json_schema", "auto")) and capabilities.supports(config.repo_root, "claude", "--json-schema") is not False: argv.extend(["--json-schema", compact_schema(schema_name)]) return AgentCommand(name=name, argv=argv, schema_name=schema_name) @@ -107,26 +108,29 @@ def codex_review_command(config: ClodexConfig, repo_root: Path) -> AgentCommand: # --output-schema, so audits run as a read-only `codex exec`; the audit prompt # already embeds the diff. codex = config.codex_role("audit") + argv = [ + "codex", + "exec", + "-m", + str(codex["model"]), + "-c", + f'model_reasoning_effort="{codex["reasoning_effort"]}"', + "-c", + 'approval_policy="never"', + "--sandbox", + "read-only", + "--ephemeral", + "-C", + str(repo_root), + ] + # An older Codex without these flags still works: the prompt asks for JSON, stdout is + # parsed, and the reply is validated against the schema either way. + if capabilities.supports(config.repo_root, "codex", "--output-schema") is not False: + argv.extend(["--output-schema", str(schema_path("audit_verdict"))]) + argv.append("-") return AgentCommand( name="codex-audit", - argv=[ - "codex", - "exec", - "-m", - str(codex["model"]), - "-c", - f'model_reasoning_effort="{codex["reasoning_effort"]}"', - "-c", - 'approval_policy="never"', - "--sandbox", - "read-only", - "--ephemeral", - "-C", - str(repo_root), - "--output-schema", - str(schema_path("audit_verdict")), - "-", - ], + argv=argv, schema_name="audit_verdict", - capture_last_message=True, + capture_last_message=capabilities.supports(config.repo_root, "codex", "--output-last-message") is not False, ) diff --git a/clodex/config.py b/clodex/config.py index ea1accb..0275b33 100644 --- a/clodex/config.py +++ b/clodex/config.py @@ -1,11 +1,12 @@ from __future__ import annotations import sys -from datetime import date from dataclasses import dataclass, field from pathlib import Path from typing import Any +from .models import retirement + try: import yaml except ImportError: # npm installs have no site-packages PyYAML; use the vendored copy @@ -64,26 +65,17 @@ class ConfigError(ValueError): """CLODEX.md could not be loaded.""" -# Codex models being retired: model -> (retire date, successor). A fuller -# registry replaces this once model validation lands. -RETIRING_CODEX_MODELS: dict[str, tuple[str, str]] = { - "gpt-5.5": ("2026-10-14", "gpt-6.1-sol"), -} _warned_models: set[str] = set() def warn_if_model_retiring(model: str) -> None: - entry = RETIRING_CODEX_MODELS.get(model) - if entry is None or model in _warned_models: + info = retirement(model) + if info is None or model in _warned_models: return _warned_models.add(model) - retire_on, successor = entry - verb = "has retired" if date.today() >= date.fromisoformat(retire_on) else "retires" - print( - f"clodex: warning: Codex model '{model}' {verb} on {retire_on}. " - f"Set `model: {successor}` under `codex:` in the CLODEX.md front matter.", - file=sys.stderr, - ) + verb = "has retired" if info.retired else "retires" + fix = f" Set `model: {info.successor}` under `codex:` in the CLODEX.md front matter (or run `clodex init --migrate`)." if info.successor else "" + print(f"clodex: warning: Codex model '{model}' {verb} on {info.on.isoformat()}.{fix}", file=sys.stderr) @dataclass(frozen=True) @@ -180,6 +172,9 @@ def load_config(repo_root: Path | None = None) -> ClodexConfig: parsed = parse_front_matter(front_matter, str(contract)) merged = deep_merge(DEFAULT_CONFIG, parsed) warn_if_model_retiring(str(merged["codex"].get("model", ""))) + audit_override = merged["codex"].get("audit") + if isinstance(audit_override, dict) and audit_override.get("model"): + warn_if_model_retiring(str(audit_override["model"])) return ClodexConfig(repo_root=root, raw=merged, prompt_body=body.strip(), user=parsed) diff --git a/clodex/doctor.py b/clodex/doctor.py index 6837389..0aca5b6 100644 --- a/clodex/doctor.py +++ b/clodex/doctor.py @@ -1,15 +1,19 @@ from __future__ import annotations +import json import shutil import subprocess import sys from pathlib import Path from typing import Any -from .config import load_config +from . import capabilities +from .config import ConfigError, load_config +from .models import Diagnostic, refresh_codex_catalog, validate -def run_doctor(repo_root: Path | None = None) -> tuple[int, dict[str, Any]]: +def run_doctor(repo_root: Path | None = None, strict: bool = False, probe: bool = True) -> tuple[int, dict[str, Any]]: + """Check the local setup. Errors always fail; with `strict`, warnings fail too.""" config = load_config(repo_root) checks: dict[str, Any] = { "python": { @@ -28,15 +32,81 @@ def run_doctor(repo_root: Path | None = None) -> tuple[int, dict[str, Any]]: "state_path": str(config.state_path), "runs_root": str(config.runs_root), } - ok = ( + base_ok = ( checks["python"]["ok"] and checks["contract"]["ok"] and checks["git"]["ok"] and checks["claude"]["ok"] and checks["codex"]["ok"] ) - checks["ok"] = ok - return (0 if ok else 1), checks + + diagnostics: list[Diagnostic] = [] + catalog = None + if probe and checks["codex"]["ok"]: + catalog = refresh_codex_catalog(config.repo_root / ".clodex" / "models-cache.json") + if probe: + installed = tuple(cli for cli in ("claude", "codex") if checks[cli]["ok"]) + found = capabilities.probe_all(config.repo_root, installed) if installed else {} + checks["capabilities"] = found + for cli, info in found.items(): + for flag in info.get("missing_required", []): + diagnostics.append(Diagnostic("error", f"{cli} CLI", f"`{cli}` does not support {flag}, which Clodex requires", f"upgrade {cli}")) + for flag in info.get("missing_optional", []): + diagnostics.append(Diagnostic("warning", f"{cli} CLI", f"`{cli}` does not support {flag}; Clodex will run without it", f"upgrade {cli} for full functionality")) + checks["auth"] = check_auth(checks) + for cli, status in checks["auth"].items(): + if status.get("status") == "logged-out": + diagnostics.append(Diagnostic("error", f"{cli} auth", f"{cli} is not logged in", f"run `{'claude auth login' if cli == 'claude' else 'codex login'}`")) + try: + diagnostics.extend(validate(config, catalog)) + except ConfigError as exc: # pragma: no cover - load_config already raised for broken front matter + diagnostics.append(Diagnostic("error", "CLODEX.md", str(exc))) + + errors = [item for item in diagnostics if item.level == "error"] + warnings = [item for item in diagnostics if item.level == "warning"] + checks["diagnostics"] = [item.as_dict() for item in diagnostics] + checks["ok"] = bool(base_ok and not errors and not (strict and warnings)) + return (0 if checks["ok"] else 1), checks + + +def check_auth(checks: dict[str, Any]) -> dict[str, dict[str, str]]: + """Whether each installed CLI is logged in. `unknown` when the CLI cannot say.""" + status: dict[str, dict[str, str]] = {} + if checks["claude"]["ok"]: + output = _run_text(["claude", "auth", "status"]) + state = "unknown" + if output is not None: + try: + state = "logged-in" if json.loads(output).get("loggedIn") else "logged-out" + except ValueError: + state = "unknown" + status["claude"] = {"status": state} + if checks["codex"]["ok"]: + exe = shutil.which("codex") + state = "unknown" + if exe: + try: + result = subprocess.run([exe, "login", "status"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, stdin=subprocess.DEVNULL, check=False) + text = (result.stdout + result.stderr).lower() + if result.returncode == 0: + state = "logged-out" if "not logged in" in text else "logged-in" + elif "not logged in" in text: + state = "logged-out" + except (OSError, subprocess.TimeoutExpired): + pass + status["codex"] = {"status": state} + return status + + +def _run_text(argv: list[str]) -> str | None: + exe = shutil.which(argv[0]) + if not exe: + return None + try: + result = subprocess.run([exe, *argv[1:]], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, stdin=subprocess.DEVNULL, check=False) + except (OSError, subprocess.TimeoutExpired): + return None + return result.stdout if result.returncode == 0 else None def check_command(argv: list[str]) -> dict[str, Any]: @@ -44,7 +114,7 @@ def check_command(argv: list[str]) -> dict[str, Any]: if not exe: return {"ok": False, "path": None, "version": None} try: - result = subprocess.run([exe, *argv[1:]], capture_output=True, text=True, encoding="utf-8", errors="replace", check=False) + result = subprocess.run([exe, *argv[1:]], capture_output=True, text=True, encoding="utf-8", errors="replace", stdin=subprocess.DEVNULL, check=False) except OSError as exc: return {"ok": False, "path": exe, "version": None, "error": str(exc)} output = (result.stdout or result.stderr).strip().splitlines() diff --git a/clodex/migrate.py b/clodex/migrate.py new file mode 100644 index 0000000..2f22f5e --- /dev/null +++ b/clodex/migrate.py @@ -0,0 +1,137 @@ +"""`clodex init --migrate`: fix CLODEX.md settings that no longer work. + +Edits only the YAML front matter, line by line, so comments, ordering and line endings +survive. It is idempotent: a migrated file yields no further changes. +""" + +from __future__ import annotations + +import re +from datetime import date + +from .models import nearest_effort, retirement, supported_efforts + +_KEY = re.compile(r"^(?P *)(?P[A-Za-z_][\w-]*) *:(?P.*?)(?P\r?\n?)$") +_VALUE = re.compile(r"^(?P *)(?P[\"']?)(?P[^\s#\"']*)(?P[\"']?)(?P\s*(?:#.*)?)$") + + +def _is_content(line: str) -> bool: + stripped = line.strip() + return bool(stripped) and not stripped.startswith("#") + + +def _indent(line: str) -> int: + return len(line) - len(line.lstrip(" ")) + + +def _front_matter(lines: list[str]) -> tuple[int, int] | None: + if not lines or lines[0].strip() != "---": + return None + for index in range(1, len(lines)): + if lines[index].strip() == "---": + return 1, index + return None + + +def _find_key(lines: list[str], span: tuple[int, int], key: str) -> tuple[int, tuple[int, int]] | None: + """Locate `key:` among the direct children of `span`; returns (line index, span of its own children).""" + start, end = span + child_indent = next((_indent(lines[i]) for i in range(start, end) if _is_content(lines[i])), None) + if child_indent is None: + return None + for index in range(start, end): + match = _KEY.match(lines[index]) + if match and _indent(lines[index]) == child_indent and match.group("key") == key: + stop = index + 1 + while stop < end and (not _is_content(lines[stop]) or _indent(lines[stop]) > child_indent): + stop += 1 + return index, (index + 1, stop) + return None + + +def _get_scalar(line: str) -> str | None: + match = _KEY.match(line) + value = _VALUE.match(match.group("rest")) if match else None + return value.group("value") if value and value.group("value") else None + + +def _set_scalar(lines: list[str], index: int, new_value: str) -> None: + match = _KEY.match(lines[index]) + assert match is not None + value = _VALUE.match(match.group("rest")) + assert value is not None + lines[index] = ( + f"{match.group('indent')}{match.group('key')}:{value.group('lead') or ' '}" + f"{value.group('open')}{new_value}{value.group('close')}{value.group('tail')}{match.group('eol')}" + ) + + +def migrate_contract(text: str, split_claude: bool = False, today: date | None = None) -> tuple[str, list[str]]: + """Return (new text, human-readable list of changes).""" + lines = text.splitlines(keepends=True) + front = _front_matter(lines) + if front is None: + return text, [] + changes: list[str] = [] + + codex = _find_key(lines, front, "codex") + if codex is not None: + scopes = [("codex", codex[1])] + audit = _find_key(lines, codex[1], "audit") + if audit is not None: + scopes.append(("codex.audit", audit[1])) + for name, span in scopes: + model_at = _find_key(lines, span, "model") + effort_at = _find_key(lines, span, "reasoning_effort") + model = _get_scalar(lines[model_at[0]]) if model_at else None + if model and model_at: + info = retirement(model, today) + if info is not None and info.successor: + _set_scalar(lines, model_at[0], info.successor) + state = "retired" if info.retired else "retires" + changes.append(f"{name}.model: {model} -> {info.successor} ({state} {info.on.isoformat()})") + model = info.successor + effort = _get_scalar(lines[effort_at[0]]) if effort_at else None + efforts = supported_efforts(model) if model else None + if effort and effort_at and efforts and effort not in efforts: + fixed = nearest_effort(effort, efforts) + _set_scalar(lines, effort_at[0], fixed) + changes.append(f"{name}.reasoning_effort: {effort} -> {fixed} (not supported by {model})") + + if split_claude: + changes.extend(_split_claude(lines, front)) + + return "".join(lines), changes + + +def _split_claude(lines: list[str], front: tuple[int, int]) -> list[str]: + claude = _find_key(lines, front, "claude") + if claude is None: + return [] + header, span = claude + if _find_key(lines, span, "plan") or _find_key(lines, span, "audit"): + return [] # already split + model_at = _find_key(lines, span, "model") + effort_at = _find_key(lines, span, "effort") + if not model_at and not effort_at: + return [] + model = (_get_scalar(lines[model_at[0]]) if model_at else None) or "opus" + effort = (_get_scalar(lines[effort_at[0]]) if effort_at else None) or "max" + audit_effort = "high" if effort == "max" else effort + child_indent = next((_indent(lines[i]) for i in range(span[0], span[1]) if _is_content(lines[i])), _indent(lines[header]) + 2) + pad = " " * child_indent + eol = "\r\n" if lines[header].endswith("\r\n") else "\n" + block = [ + f"{pad}plan:{eol}", + f"{pad} model: {model}{eol}", + f"{pad} effort: {effort}{eol}", + f"{pad}audit:{eol}", + f"{pad} model: {model}{eol}", + f"{pad} effort: {audit_effort}{eol}", + ] + flat = sorted(item[0] for item in (model_at, effort_at) if item) + insert_at = flat[0] + for index in reversed(flat): + del lines[index] + lines[insert_at:insert_at] = block + return [f"claude: split flat model/effort into plan ({model}/{effort}) and audit ({model}/{audit_effort})"] diff --git a/clodex/models.py b/clodex/models.py new file mode 100644 index 0000000..dc992bc --- /dev/null +++ b/clodex/models.py @@ -0,0 +1,265 @@ +"""Known Claude / Codex models: effort levels, retirement dates and config validation.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +from dataclasses import dataclass +from datetime import UTC, date, datetime, timedelta +from pathlib import Path +from typing import Any + +EFFORT_ORDER = ("low", "medium", "high", "xhigh", "max", "ultra") +CLAUDE_EFFORTS = ("low", "medium", "high", "xhigh", "max") +CLAUDE_ALIASES = frozenset({"opus", "sonnet", "haiku", "fable", "best", "opusplan", "default"}) +_CLAUDE_MODEL = re.compile(r"^(claude-[a-z0-9][a-z0-9._-]*|(opus|sonnet|haiku|fable)(\[[0-9a-z]+\])?)$") +CATALOG_TTL = timedelta(hours=24) + +_UP_TO_ULTRA = ("low", "medium", "high", "xhigh", "max", "ultra") +_UP_TO_MAX = ("low", "medium", "high", "xhigh", "max") +_UP_TO_XHIGH = ("low", "medium", "high", "xhigh") + + +@dataclass(frozen=True) +class CodexModel: + slug: str + efforts: tuple[str, ...] + retires: str | None = None # ISO date the model stops working + successor: str | None = None + + +# Snapshot of the Codex catalog (`codex debug models`) as of 2026-09. The live catalog, +# when available, takes precedence for efforts; this table carries the retirement dates. +BUILTIN_CODEX: dict[str, CodexModel] = { + model.slug: model + for model in ( + CodexModel("gpt-6.1-sol", _UP_TO_ULTRA), + CodexModel("gpt-6-sol", _UP_TO_ULTRA), + CodexModel("gpt-6-astra", _UP_TO_ULTRA), + CodexModel("gpt-6-luna", _UP_TO_MAX), + CodexModel("gpt-5.6-sol", _UP_TO_ULTRA), + CodexModel("gpt-5.6-terra", _UP_TO_ULTRA), + CodexModel("gpt-5.6-luna", _UP_TO_MAX), + CodexModel("gpt-5.5", _UP_TO_XHIGH, retires="2026-10-14", successor="gpt-6.1-sol"), + CodexModel("gpt-5.4", _UP_TO_XHIGH, retires="2026-08-31", successor="gpt-6.1-sol"), + CodexModel("gpt-5.4-mini", _UP_TO_XHIGH, retires="2026-08-31", successor="gpt-6.1-sol"), + CodexModel("gpt-5.3-codex-spark", _UP_TO_XHIGH, retires="2026-09-14", successor="gpt-6.1-sol"), + ) +} + + +def today() -> date: + return date.today() + + +@dataclass(frozen=True) +class Retirement: + model: str + on: date + successor: str | None + retired: bool + + +def retirement(model: str, on: date | None = None) -> Retirement | None: + """Retirement info for a model that is retired or has a retirement date; None otherwise.""" + entry = BUILTIN_CODEX.get(model) + if entry is None or not entry.retires: + return None + retire_date = date.fromisoformat(entry.retires) + return Retirement(model, retire_date, entry.successor, (on or today()) >= retire_date) + + +class ModelRetiredError(ValueError): + """The configured Codex model no longer exists, so a run would fail immediately.""" + + +def ensure_usable(config: Any, on: date | None = None) -> None: + """Refuse to start a run on a retired Codex model. CLODEX_ALLOW_RETIRED_MODEL=1 overrides.""" + if os.environ.get("CLODEX_ALLOW_RETIRED_MODEL"): + return + for where, settings in (("codex.model", config.codex_role("build")), ("codex.audit.model", config.codex_role("audit"))): + info = retirement(str(settings["model"]), on) + if info is not None and info.retired: + fix = f" Run `clodex init --migrate` (or set `{where}: {info.successor}`)." if info.successor else "" + raise ModelRetiredError( + f"Codex model '{info.model}' ({where}) retired on {info.on.isoformat()}.{fix} " + "Set CLODEX_ALLOW_RETIRED_MODEL=1 to try anyway." + ) + + +def supported_efforts(model: str, catalog: dict[str, dict[str, Any]] | None = None) -> tuple[str, ...] | None: + """Reasoning efforts a Codex model accepts, from the live catalog if present, else the built-in table.""" + if catalog and model in catalog and catalog[model].get("efforts"): + return tuple(catalog[model]["efforts"]) + entry = BUILTIN_CODEX.get(model) + return entry.efforts if entry else None + + +def nearest_effort(effort: str, supported: tuple[str, ...]) -> str: + """The highest supported effort not above `effort`; the lowest supported one if none is.""" + if effort in supported: + return effort + rank = EFFORT_ORDER.index(effort) if effort in EFFORT_ORDER else len(EFFORT_ORDER) + at_or_below = [level for level in supported if level in EFFORT_ORDER and EFFORT_ORDER.index(level) <= rank] + return at_or_below[-1] if at_or_below else supported[0] + + +# ---------------------------------------------------------------- live catalog + + +def parse_catalog(raw: str) -> dict[str, dict[str, Any]]: + """Parse `codex debug models` JSON into {slug: {efforts, visibility, default_effort}}.""" + data = json.loads(raw) + models = data.get("models", []) if isinstance(data, dict) else data + catalog: dict[str, dict[str, Any]] = {} + for item in models: + slug = item.get("slug") or item.get("id") + if not slug: + continue + levels = item.get("supported_reasoning_levels") or item.get("supported_reasoning_efforts") or [] + efforts = [level.get("effort") if isinstance(level, dict) else level for level in levels] + catalog[str(slug)] = { + "efforts": [str(effort) for effort in efforts if effort], + "visibility": item.get("visibility"), + "default_effort": item.get("default_reasoning_level") or item.get("default_reasoning_effort"), + } + return catalog + + +def refresh_codex_catalog(cache_path: Path | None = None, now: datetime | None = None, force: bool = False) -> dict[str, dict[str, Any]] | None: + """The live Codex model catalog, cached for 24h. None when Codex is unavailable and nothing is cached.""" + now = now or datetime.now(UTC) + cached = _read_cache(cache_path) + if cached and not force and now - cached[0] < CATALOG_TTL: + return cached[1] + exe = shutil.which("codex") + if exe: + try: + result = subprocess.run([exe, "debug", "models"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=60, check=False) + if result.returncode == 0: + catalog = parse_catalog(result.stdout) + if catalog: + _write_cache(cache_path, now, catalog) + return catalog + except (OSError, ValueError, subprocess.TimeoutExpired): + pass + return cached[1] if cached else None # stale beats nothing + + +def _read_cache(path: Path | None) -> tuple[datetime, dict[str, dict[str, Any]]] | None: + if path is None or not path.is_file(): + return None + try: + data = json.loads(path.read_text(encoding="utf-8")) + return datetime.fromisoformat(data["fetched_at"]), data["models"] + except (OSError, ValueError, KeyError, TypeError): + return None + + +def _write_cache(path: Path | None, now: datetime, catalog: dict[str, dict[str, Any]]) -> None: + if path is None: + return + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"fetched_at": now.isoformat(), "models": catalog}, indent=1) + "\n", encoding="utf-8") + except OSError: + pass + + +# ---------------------------------------------------------------- validation + + +@dataclass(frozen=True) +class Diagnostic: + level: str # "error" | "warning" + where: str + message: str + fix: str | None = None + + def as_dict(self) -> dict[str, str]: + data = {"level": self.level, "where": self.where, "message": self.message} + if self.fix: + data["fix"] = self.fix + return data + + +def codex_model_diagnostics(where: str, model: str, effort: str, catalog: dict[str, dict[str, Any]] | None, on: date | None = None) -> list[Diagnostic]: + found: list[Diagnostic] = [] + info = retirement(model, on) + if info is not None: + fix = f"set `{where.rsplit('.', 1)[0]}.model: {info.successor}` (or run `clodex init --migrate`)" if info.successor else None + if info.retired: + found.append(Diagnostic("error", where, f"Codex model '{model}' retired on {info.on.isoformat()}", fix)) + else: + found.append(Diagnostic("warning", where, f"Codex model '{model}' retires on {info.on.isoformat()}", fix)) + elif catalog and model not in catalog and model not in BUILTIN_CODEX: + found.append(Diagnostic("warning", where, f"'{model}' is not in the Codex model catalog (`codex debug models`)")) + efforts = supported_efforts(model, catalog) + if efforts is not None and effort not in efforts and not (info and info.retired): + found.append( + Diagnostic( + "error", + where.rsplit(".", 1)[0] + ".reasoning_effort", + f"effort '{effort}' is not supported by '{model}' (supported: {', '.join(efforts)})", + f"use one of: {', '.join(efforts)}", + ) + ) + elif efforts is None and effort not in EFFORT_ORDER: + found.append(Diagnostic("warning", where.rsplit(".", 1)[0] + ".reasoning_effort", f"unknown reasoning effort '{effort}'")) + return found + + +def claude_model_diagnostics(where: str, model: str, effort: str) -> list[Diagnostic]: + found: list[Diagnostic] = [] + if not _CLAUDE_MODEL.match(model): + found.append(Diagnostic("warning", f"{where}.model", f"'{model}' is not a known Claude alias or model id", f"aliases: {', '.join(sorted(CLAUDE_ALIASES - {'default'}))}")) + if effort not in CLAUDE_EFFORTS: + found.append(Diagnostic("error", f"{where}.effort", f"effort '{effort}' is not valid for claude (use {', '.join(CLAUDE_EFFORTS)})")) + return found + + +def validate(config: Any, catalog: dict[str, dict[str, Any]] | None = None, on: date | None = None) -> list[Diagnostic]: + """Check a loaded CLODEX.md config (duck-typed ClodexConfig) for problems that would break a run.""" + from .quorum import parse_quorum + + found: list[Diagnostic] = [] + for role in ("plan", "audit"): + settings = config.claude_role(role) + found.extend(claude_model_diagnostics(f"claude.{role}", str(settings["model"]), str(settings["effort"]))) + build = config.codex_role("build") + found.extend(codex_model_diagnostics("codex.model", str(build["model"]), str(build["reasoning_effort"]), catalog, on)) + audit = config.codex_role("audit") + if (audit["model"], audit["reasoning_effort"]) != (build["model"], build["reasoning_effort"]): + found.extend(codex_model_diagnostics("codex.audit.model", str(audit["model"]), str(audit["reasoning_effort"]), catalog, on)) + + codex = config.codex + if str(codex.get("sandbox")) not in {"read-only", "workspace-write", "danger-full-access"}: + found.append(Diagnostic("error", "codex.sandbox", f"unknown sandbox '{codex.get('sandbox')}'", "use read-only, workspace-write or danger-full-access")) + if str(codex.get("approval_profile")) not in {"ci", "local", "auto_review"}: + found.append(Diagnostic("error", "codex.approval_profile", f"unknown approval profile '{codex.get('approval_profile')}'", "use ci, local or auto_review")) + if str(config.workspace.get("backend")) not in {"git-worktree", "local"}: + found.append(Diagnostic("error", "workspace.backend", f"unknown workspace backend '{config.workspace.get('backend')}'", "use git-worktree or local")) + + reviewers = config.reviewers + seen: set[str] = set() + for index, reviewer in enumerate(reviewers): + where = f"audit.reviewers[{index}]" + reviewer_id = str(reviewer.get("id", "")) + if not reviewer_id: + found.append(Diagnostic("error", where, "reviewer has no id")) + elif reviewer_id in seen: + found.append(Diagnostic("error", where, f"duplicate reviewer id '{reviewer_id}'")) + seen.add(reviewer_id) + if str(reviewer.get("backend")) not in {"claude", "codex"}: + found.append(Diagnostic("error", where, f"reviewer '{reviewer_id}' has unknown backend '{reviewer.get('backend')}'", "use claude or codex")) + required = [reviewer for reviewer in reviewers if reviewer.get("required", True)] + if not required: + found.append(Diagnostic("error", "audit.reviewers", "no required reviewers: every run would end blocked", "mark at least one reviewer required: true")) + try: + parse_quorum(config.audit.get("quorum", "unanimous")) + except ValueError as exc: + found.append(Diagnostic("error", "audit.quorum", str(exc), "use unanimous, majority or a number")) + return found diff --git a/clodex/tasks.py b/clodex/tasks.py index bd97762..2a6ccf1 100644 --- a/clodex/tasks.py +++ b/clodex/tasks.py @@ -11,6 +11,7 @@ from .artifacts import ArtifactStore, make_run_id, make_task_id from .config import ClodexConfig, load_config +from .models import ensure_usable from .procs import kill_tree, pid_alive, popen_isolation_kwargs from .state import TERMINAL_STATUSES, StateStore from .workflow import WorkflowResult @@ -80,6 +81,8 @@ def start( approval_profile: str | None = None, dry_run: bool = False, ) -> WorkflowResult: + if not dry_run: + ensure_usable(self.config) task_id = make_task_id(task) run_id = make_run_id(task_id) selected_workspace = workspace_backend or self.config.workspace["backend"] diff --git a/clodex/workflow.py b/clodex/workflow.py index ae94d2e..049812b 100644 --- a/clodex/workflow.py +++ b/clodex/workflow.py @@ -13,6 +13,7 @@ from .commands import claude_audit_command, claude_plan_command, codex_exec_command, codex_review_command from .config import ClodexConfig, load_config from .jsonutil import AgentEnvelopeError, extract_json_object +from .models import ensure_usable from .prompts import audit_diff_excerpt, audit_prompt, fix_prompt, implementation_prompt, plan_prompt from .quorum import evaluate as evaluate_quorum, required_fixes, verdict_approved from .schemas import SchemaValidationError, check as check_schema @@ -97,6 +98,7 @@ def build( }, ) + ensure_usable(self.config) task_id = make_task_id(task) run_id = make_run_id(task_id) artifacts = ArtifactStore(self.config, run_id, self.state) @@ -115,11 +117,19 @@ def run_existing( if run is None: raise ValueError(f"Unknown run: {run_id}") artifacts = ArtifactStore(self.config, run_id, self.state) + try: + ensure_usable(self.config) + except ValueError as exc: + # The worker was already started; record why instead of leaving the run queued. + self.state.update_run(run_id, "failed", error=str(exc)) + self.state.update_task(str(run["task_id"]), "failed") + return WorkflowResult("failed", run_id, str(run["task_id"]), str(artifacts.path), {"error": str(exc)}) return self._execute_build(str(run["prompt"]), str(run["task_id"]), run_id, artifacts, workspace_backend, approval_profile, apply_changes) def audit(self, dry_run: bool = False) -> WorkflowResult: if dry_run: return WorkflowResult("dry-run", None, None, None, {"commands": self.dry_run_commands()}) + ensure_usable(self.config) task = "Audit current uncommitted changes" task_id = make_task_id(task) run_id = make_run_id(task_id) diff --git a/tests/support.py b/tests/support.py index 4c5f4d8..bc0fbe0 100644 --- a/tests/support.py +++ b/tests/support.py @@ -25,6 +25,18 @@ def cli_test_python() -> str | None: return None +FAKE_CODEX_CATALOG = { + "models": [ + {"slug": "gpt-6.1-sol", "visibility": "list", "default_reasoning_level": "low", + "supported_reasoning_levels": [{"effort": e} for e in ("low", "medium", "high", "xhigh", "max", "ultra")]}, + {"slug": "gpt-6-luna", "visibility": "list", "default_reasoning_level": "medium", + "supported_reasoning_levels": [{"effort": e} for e in ("low", "medium", "high", "xhigh", "max")]}, + {"slug": "gpt-5.5", "visibility": "list", "default_reasoning_level": "medium", + "supported_reasoning_levels": [{"effort": e} for e in ("low", "medium", "high", "xhigh")]}, + ] +} + + def cleanup_tempdir(tmp: tempfile.TemporaryDirectory, attempts: int = 50) -> None: # Orphaned fake CLI children can briefly hold the directory on Windows. for attempt in range(attempts): @@ -67,6 +79,10 @@ def __init__( fail_reviewers: tuple[str, ...] = (), slow_reviewers: tuple[str, ...] = (), reject_reviewers: tuple[str, ...] = (), + help_lacks: tuple[str, ...] = (), + claude_logged_in: bool = True, + codex_logged_in: bool = True, + codex_catalog: dict | None = None, ): self.reject_once = reject_once self.malformed_once = malformed_once @@ -76,6 +92,10 @@ def __init__( self.fail_reviewers = tuple(fail_reviewers) self.slow_reviewers = tuple(slow_reviewers) self.reject_reviewers = tuple(reject_reviewers) + self.help_lacks = tuple(help_lacks) + self.claude_logged_in = claude_logged_in + self.codex_logged_in = codex_logged_in + self.codex_catalog = codex_catalog self.sleep_seconds = sleep_seconds self.include_clodex = include_clodex @@ -114,6 +134,31 @@ def _write_fake(self): # Answer before reading stdin: callers like `doctor` don't close it. print(name + ' fake 1.0.0') raise SystemExit(0) +CLAUDE_HELP_FLAGS = ['--model', '--effort', '--permission-mode', '--output-format', '--json-schema', '--fallback-model', '--max-budget-usd'] +CODEX_HELP_FLAGS = ['--sandbox', '--model', '--cd', '--output-schema', '--output-last-message', '--approve-for-me', '--ephemeral'] +LACKS = {self.help_lacks!r} +if '--help' in args: + flags = CLAUDE_HELP_FLAGS if name == 'claude' else CODEX_HELP_FLAGS + print('Usage: ' + name + ' [options]') + for flag in flags: + if flag not in LACKS: + print(' ' + flag + ' fake option') + raise SystemExit(0) +if name == 'claude' and args[:2] == ['auth', 'status']: + print(json.dumps({{'loggedIn': {self.claude_logged_in!r}, 'authMethod': 'fake'}})) + raise SystemExit(0) +if name == 'codex' and args[:2] == ['login', 'status']: + if {self.codex_logged_in!r}: + print('Logged in using fake') + raise SystemExit(0) + print('Not logged in') + raise SystemExit(1) +if name == 'codex' and args[:2] == ['debug', 'models']: + if {self.codex_catalog!r} is None: + sys.stderr.write('error: unrecognized subcommand\\n') + raise SystemExit(2) + print(json.dumps({self.codex_catalog!r})) + raise SystemExit(0) stdin = sys.stdin.read() if {self.sleep_seconds!r}: diff --git a/tests/test_model_registry.py b/tests/test_model_registry.py new file mode 100644 index 0000000..8a146d5 --- /dev/null +++ b/tests/test_model_registry.py @@ -0,0 +1,385 @@ +"""Tests: model registry, doctor v2, capability probing, retired-model refusal and migration.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import unittest +from datetime import UTC, date, datetime, timedelta +from pathlib import Path +from unittest import mock + +from clodex.capabilities import flags_in_help, supports +from clodex.commands import claude_plan_command, codex_review_command +from clodex.config import load_config +from clodex.doctor import run_doctor +from clodex.migrate import migrate_contract +from clodex.models import ( + ModelRetiredError, + nearest_effort, + parse_catalog, + refresh_codex_catalog, + retirement, + supported_efforts, + validate, +) +from clodex.tasks import TaskManager +from clodex.workflow import ClodexWorkflow +from tests.support import FAKE_CODEX_CATALOG, ROOT, FakeCliPath, TempRepo + +BEFORE = date(2026, 9, 30) +AFTER = date(2026, 10, 15) + + +def write_contract(repo: Path, front_matter: str) -> None: + (repo / "CLODEX.md").write_text(f"---\n{front_matter}---\nbody\n", encoding="utf-8") + + +def messages(diagnostics, level: str) -> list[str]: + return [item.message for item in diagnostics if item.level == level] + + +class RegistryTests(unittest.TestCase): + def test_retirement_dates(self): + self.assertIsNone(retirement("gpt-6.1-sol")) + self.assertIsNone(retirement("something-custom")) + soon = retirement("gpt-5.5", BEFORE) + self.assertFalse(soon.retired) + self.assertEqual((soon.on, soon.successor), (date(2026, 10, 14), "gpt-6.1-sol")) + self.assertTrue(retirement("gpt-5.5", date(2026, 10, 14)).retired) + self.assertTrue(retirement("gpt-5.5", AFTER).retired) + self.assertTrue(retirement("gpt-5.4", BEFORE).retired) + + def test_efforts_prefer_the_live_catalog_over_the_builtin_table(self): + self.assertEqual(supported_efforts("gpt-6-luna"), ("low", "medium", "high", "xhigh", "max")) + self.assertIsNone(supported_efforts("unknown-model")) + live = {"gpt-6-luna": {"efforts": ["low", "high"]}} + self.assertEqual(supported_efforts("gpt-6-luna", live), ("low", "high")) + self.assertEqual(supported_efforts("gpt-6.1-sol", live), supported_efforts("gpt-6.1-sol"), "models missing from the catalog fall back") + + def test_nearest_effort(self): + luna = supported_efforts("gpt-6-luna") + self.assertEqual(nearest_effort("ultra", luna), "max") + self.assertEqual(nearest_effort("high", luna), "high") + self.assertEqual(nearest_effort("xhigh", ("low", "high")), "high") + self.assertEqual(nearest_effort("low", ("medium", "high")), "medium", "nothing below: use the lowest supported") + + def test_parse_catalog(self): + catalog = parse_catalog(json.dumps(FAKE_CODEX_CATALOG)) + self.assertEqual(catalog["gpt-6-luna"]["efforts"][-1], "max") + self.assertEqual(catalog["gpt-6.1-sol"]["default_effort"], "low") + plain = parse_catalog(json.dumps([{"id": "x", "supported_reasoning_efforts": ["low", "high"]}])) + self.assertEqual(plain["x"]["efforts"], ["low", "high"]) + + +class CatalogCacheTests(unittest.TestCase): + def test_catalog_is_fetched_cached_refreshed_and_survives_failure(self): + now = datetime(2026, 9, 30, tzinfo=UTC) + with TempRepo() as repo: + cache = repo / ".clodex" / "models-cache.json" + with FakeCliPath(codex_catalog=FAKE_CODEX_CATALOG): + first = refresh_codex_catalog(cache, now=now) + self.assertIn("gpt-6.1-sol", first) + self.assertTrue(cache.is_file()) + with mock.patch("clodex.models.subprocess.run", side_effect=AssertionError("fresh cache must not refetch")): + self.assertEqual(refresh_codex_catalog(cache, now=now + timedelta(hours=1)), first) + changed = {"models": [{"slug": "gpt-7", "supported_reasoning_levels": [{"effort": "low"}]}]} + with FakeCliPath(codex_catalog=changed): + refreshed = refresh_codex_catalog(cache, now=now + timedelta(hours=25)) + self.assertEqual(list(refreshed), ["gpt-7"]) + with FakeCliPath(codex_catalog=None): # `codex debug models` fails: keep the stale copy + self.assertEqual(list(refresh_codex_catalog(cache, now=now + timedelta(hours=60))), ["gpt-7"]) + with mock.patch("clodex.models.shutil.which", return_value=None): # no codex installed, nothing cached + self.assertIsNone(refresh_codex_catalog(repo / "no-such-cache.json")) + + +class ValidateTests(unittest.TestCase): + def check(self, front_matter: str, catalog=None, on=BEFORE): + with TempRepo() as repo: + write_contract(repo, front_matter) + return validate(load_config(repo), catalog, on) + + def test_default_config_is_clean(self): + with TempRepo() as repo: + self.assertEqual(validate(load_config(repo), None, BEFORE), []) + + def test_retiring_and_retired_codex_models(self): + soon = self.check("codex:\n model: gpt-5.5\n", on=BEFORE) + self.assertEqual([d.level for d in soon], ["warning"]) + self.assertIn("retires on 2026-10-14", soon[0].message) + self.assertIn("gpt-6.1-sol", soon[0].fix) + gone = self.check("codex:\n model: gpt-5.5\n", on=AFTER) + self.assertEqual([d.level for d in gone], ["error"]) + self.assertIn("retired on 2026-10-14", gone[0].message) + + def test_unsupported_effort_for_model(self): + errors = messages(self.check("codex:\n model: gpt-6-luna\n reasoning_effort: ultra\n"), "error") + self.assertEqual(len(errors), 1) + self.assertIn("not supported by 'gpt-6-luna'", errors[0]) + self.assertEqual(self.check("codex:\n model: gpt-6.1-sol\n reasoning_effort: ultra\n"), []) + + def test_live_catalog_overrides_builtin_and_flags_unknown_models(self): + catalog = parse_catalog(json.dumps({"models": [{"slug": "gpt-6-luna", "supported_reasoning_levels": [{"effort": "low"}, {"effort": "high"}]}]})) + errors = messages(self.check("codex:\n model: gpt-6-luna\n reasoning_effort: max\n", catalog), "error") + self.assertTrue(errors and "supported: low, high" in errors[0]) + warnings = messages(self.check("codex:\n model: gpt-9-mystery\n reasoning_effort: high\n", catalog), "warning") + self.assertTrue(any("not in the Codex model catalog" in w for w in warnings)) + self.assertEqual(self.check("codex:\n model: gpt-9-mystery\n reasoning_effort: high\n"), [], "no catalog: unknown models are allowed") + + def test_audit_override_is_checked_separately(self): + found = self.check("codex:\n audit:\n model: gpt-5.5\n", on=AFTER) + self.assertEqual([d.where for d in found if d.level == "error"], ["codex.audit.model"]) + + def test_claude_checks(self): + found = self.check("claude:\n plan: {model: opus-ish, effort: turbo}\n") + self.assertTrue(any("not a known Claude alias" in m for m in messages(found, "warning"))) + self.assertTrue(any("'turbo'" in m for m in messages(found, "error"))) + self.assertEqual(self.check("claude:\n plan: {model: claude-opus-5-5, effort: xhigh}\n audit: {model: 'opus[1m]', effort: low}\n"), []) + + def test_reviewer_and_misc_settings(self): + reviewers = json.dumps([ + {"id": "a", "backend": "claude", "required": False}, + {"id": "a", "backend": "gemini", "required": False}, + ]) + found = self.check(f"audit:\n quorum: most\n reviewers: {reviewers}\ncodex:\n sandbox: yolo\n approval_profile: never\nworkspace:\n backend: docker\n") + text = " | ".join(messages(found, "error")) + for expected in ("duplicate reviewer id 'a'", "unknown backend 'gemini'", "no required reviewers", "Unsupported audit.quorum", "unknown sandbox", "unknown approval profile", "unknown workspace backend"): + self.assertIn(expected, text) + + +class CapabilityTests(unittest.TestCase): + def test_flags_in_help_matches_whole_flags_only(self): + text = " --json-schema \n --effort-level foo\n --max-budget-usd \n" + self.assertEqual(flags_in_help(text, ("--json-schema", "--effort", "--max-budget-usd")), ["--json-schema", "--max-budget-usd"]) + + +class DoctorTests(unittest.TestCase): + def test_healthy_setup_has_no_diagnostics_and_writes_caches(self): + with TempRepo() as repo, FakeCliPath(codex_catalog=FAKE_CODEX_CATALOG): + code, data = run_doctor(repo) + self.assertEqual(code, 0, data["diagnostics"]) + self.assertEqual(data["diagnostics"], []) + self.assertEqual(data["auth"], {"claude": {"status": "logged-in"}, "codex": {"status": "logged-in"}}) + self.assertEqual(data["capabilities"]["claude"]["missing_required"], []) + self.assertTrue((repo / ".clodex" / "capabilities.json").is_file()) + self.assertTrue((repo / ".clodex" / "models-cache.json").is_file()) + + def test_missing_optional_flag_warns_and_commands_degrade(self): + with TempRepo() as repo, FakeCliPath(help_lacks=("--json-schema", "--output-schema", "--output-last-message")): + write_contract(repo, "claude:\n json_schema: true\n") + code, data = run_doctor(repo) + self.assertEqual(code, 0) + self.assertEqual(len(messages_from(data, "warning")), 3) + self.assertEqual(run_doctor(repo, strict=True)[0], 1, "--strict fails on warnings") + config = load_config(repo) + self.assertIs(supports(repo, "claude", "--json-schema"), False) + self.assertIs(supports(repo, "codex", "--output-schema"), False) + self.assertNotIn("--json-schema", claude_plan_command(config).argv) + audit = codex_review_command(config, repo) + self.assertNotIn("--output-schema", audit.argv) + self.assertFalse(audit.capture_last_message) + self.assertEqual(audit.schema_name, "audit_verdict", "output is still validated against the schema") + + def test_full_support_keeps_all_flags(self): + with TempRepo() as repo, FakeCliPath(): + write_contract(repo, "claude:\n json_schema: true\n") + run_doctor(repo) + config = load_config(repo) + self.assertIn("--json-schema", claude_plan_command(config).argv) + self.assertIn("--output-schema", codex_review_command(config, repo).argv) + self.assertTrue(codex_review_command(config, repo).capture_last_message) + + def test_missing_required_flag_is_an_error(self): + with TempRepo() as repo, FakeCliPath(help_lacks=("--effort",)): + code, data = run_doctor(repo) + self.assertEqual(code, 1) + self.assertTrue(any("--effort" in m for m in messages_from(data, "error"))) + self.assertFalse(data["ok"]) + + def test_logged_out_clis_are_errors(self): + for kwargs, who in (({"claude_logged_in": False}, "claude"), ({"codex_logged_in": False}, "codex")): + with TempRepo() as repo, FakeCliPath(**kwargs): + code, data = run_doctor(repo) + self.assertEqual(code, 1, who) + self.assertEqual(data["auth"][who]["status"], "logged-out") + self.assertTrue(any(f"{who} is not logged in" in m for m in messages_from(data, "error"))) + + def test_config_problems_from_the_registry_show_up(self): + with TempRepo() as repo, FakeCliPath(codex_catalog=FAKE_CODEX_CATALOG): + write_contract(repo, "codex:\n model: gpt-6-luna\n reasoning_effort: ultra\n") + code, data = run_doctor(repo) + self.assertEqual(code, 1) + self.assertTrue(any("not supported by 'gpt-6-luna'" in m for m in messages_from(data, "error"))) + + def test_probe_can_be_skipped(self): + with TempRepo() as repo, FakeCliPath(help_lacks=("--effort",)): + code, data = run_doctor(repo, probe=False) + self.assertEqual(code, 0) + self.assertNotIn("capabilities", data) + + +def messages_from(data: dict, level: str) -> list[str]: + return [item["message"] for item in data["diagnostics"] if item["level"] == level] + + +class RetiredModelRefusalTests(unittest.TestCase): + def retired_contract(self, repo: Path, scope: str = "codex:\n model: gpt-5.4\n") -> None: + write_contract(repo, scope) + + def test_build_and_audit_refuse_but_plan_and_dry_runs_work(self): + with TempRepo() as repo, FakeCliPath(): + self.retired_contract(repo) + workflow = ClodexWorkflow(repo) + with self.assertRaisesRegex(ModelRetiredError, "gpt-5.4.*retired on 2026-08-31.*init --migrate"): + workflow.build("implement fixture", workspace_backend="local") + with self.assertRaises(ModelRetiredError): + workflow.audit() + self.assertEqual(workflow.build("implement fixture", dry_run=True).status, "dry-run") + self.assertEqual(workflow.plan("plan fixture").status, "planned", "planning only uses Claude") + + def test_audit_override_model_is_checked_too(self): + with TempRepo() as repo, FakeCliPath(): + self.retired_contract(repo, "codex:\n audit:\n model: gpt-5.4\n") + with self.assertRaisesRegex(ModelRetiredError, "codex.audit.model"): + ClodexWorkflow(repo).build("implement fixture", workspace_backend="local") + + def test_environment_override_allows_the_run(self): + with TempRepo() as repo, FakeCliPath(), mock.patch.dict(os.environ, {"CLODEX_ALLOW_RETIRED_MODEL": "1"}): + self.retired_contract(repo) + self.assertEqual(ClodexWorkflow(repo).build("implement fixture", workspace_backend="local").status, "approved") + + def test_task_start_refuses_before_creating_a_run(self): + with TempRepo() as repo: + self.retired_contract(repo) + manager = TaskManager(repo) + with self.assertRaises(ModelRetiredError): + manager.start("slow fixture") + self.assertEqual(manager.state.list_runs(), []) + self.assertEqual(manager.start("slow fixture", dry_run=True).status, "dry-run") + + def test_worker_for_an_already_queued_run_records_the_failure(self): + with TempRepo() as repo: + self.retired_contract(repo) + workflow = ClodexWorkflow(repo) + workflow.state.upsert_task("t", "task", "queued") + workflow.state.create_run("r1", "t", "do it", "queued") + result = workflow.run_existing("r1") + self.assertEqual(result.status, "failed") + run = workflow.state.get_run("r1") + self.assertEqual(run["status"], "failed") + self.assertIn("retired", run["error"]) + + def test_cli_exits_2_with_a_clear_message(self): + with TempRepo() as repo: + self.retired_contract(repo) + result = subprocess.run( + [sys.executable, "-m", "clodex", "build", "something"], + cwd=repo, + env={**os.environ, "PYTHONPATH": str(ROOT)}, + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + ) + self.assertEqual(result.returncode, 2) + self.assertIn("retired on 2026-08-31", result.stderr) + self.assertNotIn("Traceback", result.stderr) + + +class MigrateTests(unittest.TestCase): + def test_retiring_model_is_replaced_keeping_quotes_comments_and_order(self): + text = '---\nversion: 1\ncodex:\n model: "gpt-5.5" # pinned by us\n reasoning_effort: xhigh\n sandbox: workspace-write\n---\nbody stays\n' + new, changes = migrate_contract(text, today=BEFORE) + self.assertEqual(new, text.replace('"gpt-5.5"', '"gpt-6.1-sol"')) + self.assertEqual(changes, ["codex.model: gpt-5.5 -> gpt-6.1-sol (retires 2026-10-14)"]) + self.assertEqual(migrate_contract(new, today=BEFORE), (new, []), "idempotent") + + def test_crlf_endings_are_preserved(self): + text = "---\r\ncodex:\r\n model: gpt-5.4\r\n reasoning_effort: high\r\n---\r\nbody\r\n" + new, changes = migrate_contract(text) + self.assertEqual(new, text.replace("gpt-5.4", "gpt-6.1-sol")) + self.assertNotIn("\n", new.replace("\r\n", "")) + self.assertEqual(len(changes), 1) + + def test_unsupported_effort_is_clamped_to_the_nearest_supported_level(self): + text = "---\ncodex:\n model: gpt-6-luna\n reasoning_effort: ultra # too high\n---\n" + new, changes = migrate_contract(text) + self.assertIn("reasoning_effort: max # too high", new) + self.assertIn("ultra -> max", changes[0]) + sol, none = migrate_contract("---\ncodex:\n model: gpt-5.5\n reasoning_effort: ultra\n---\n", today=BEFORE) + self.assertIn("model: gpt-6.1-sol", sol) + self.assertIn("reasoning_effort: ultra", sol, "ultra is valid on the successor, so it stays") + + def test_audit_override_is_migrated_too(self): + text = "---\ncodex:\n model: gpt-6.1-sol\n audit:\n model: gpt-5.5\n reasoning_effort: xhigh\n---\n" + new, changes = migrate_contract(text, today=BEFORE) + self.assertIn(" model: gpt-6.1-sol\n reasoning_effort: xhigh", new) + self.assertTrue(changes[0].startswith("codex.audit.model")) + + def test_files_without_front_matter_or_codex_section_are_untouched(self): + for text in ("no front matter\nmodel: gpt-5.5\n", "---\nclaude:\n model: opus\n---\n", ""): + self.assertEqual(migrate_contract(text), (text, [])) + + def test_split_claude_creates_plan_and_audit_roles(self): + text = "---\nclaude:\n model: sonnet\n effort: max\n permission_mode: plan\ncodex:\n model: gpt-6.1-sol\n---\nbody\n" + new, changes = migrate_contract(text, split_claude=True) + self.assertEqual( + new, + "---\nclaude:\n plan:\n model: sonnet\n effort: max\n audit:\n model: sonnet\n effort: high\n permission_mode: plan\n" + "codex:\n model: gpt-6.1-sol\n---\nbody\n", + ) + self.assertEqual(len(changes), 1) + self.assertEqual(migrate_contract(new, split_claude=True), (new, []), "already split: nothing to do") + with TempRepo() as repo: + (repo / "CLODEX.md").write_text(new, encoding="utf-8") + config = load_config(repo) + self.assertEqual((config.claude_role("plan")["model"], config.claude_role("plan")["effort"]), ("sonnet", "max")) + self.assertEqual((config.claude_role("audit")["model"], config.claude_role("audit")["effort"]), ("sonnet", "high")) + + def test_split_claude_is_opt_in_and_handles_a_lone_effort(self): + text = "---\nclaude:\n effort: high\n---\n" + self.assertEqual(migrate_contract(text), (text, [])) + new, _ = migrate_contract(text, split_claude=True) + self.assertIn(" plan:\n model: opus\n effort: high\n audit:\n model: opus\n effort: high\n", new) + + def test_shipped_contract_needs_no_migration(self): + text = (ROOT / "CLODEX.md").read_text(encoding="utf-8") + self.assertEqual(migrate_contract(text, split_claude=True, today=AFTER), (text, [])) + + +class MigrateCliTests(unittest.TestCase): + def run_cli(self, repo: Path, *args: str) -> tuple[int, dict]: + result = subprocess.run( + [sys.executable, "-m", "clodex", "--json", "init", "--migrate", *args], + cwd=repo, + env={**os.environ, "PYTHONPATH": str(ROOT)}, + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + ) + return result.returncode, json.loads(result.stdout) + + def test_dry_run_reports_without_writing_then_real_run_writes(self): + with TempRepo() as repo: + original = "---\ncodex:\n model: gpt-5.4\n---\nbody\n" + (repo / "CLODEX.md").write_text(original, encoding="utf-8", newline="") + code, data = self.run_cli(repo, "--dry-run") + self.assertEqual((code, data["written"]), (0, False)) + self.assertEqual(len(data["changes"]), 1) + self.assertEqual((repo / "CLODEX.md").read_text(encoding="utf-8"), original) + code, data = self.run_cli(repo) + self.assertTrue(data["written"]) + self.assertIn("model: gpt-6.1-sol", (repo / "CLODEX.md").read_text(encoding="utf-8")) + self.assertEqual(self.run_cli(repo)[1], {**self.run_cli(repo)[1], "changes": [], "written": False}) + + def test_missing_contract_is_an_error(self): + with TempRepo() as repo: + (repo / "CLODEX.md").unlink() + code, data = self.run_cli(repo) + self.assertEqual(code, 1) + self.assertFalse(data["ok"]) + + +if __name__ == "__main__": + unittest.main()