feat: canonical model registry with CI consistency check

docs/analysis.md

@@ -249,7 +249,28 @@ them in `pyproject.toml` is what keeps historical numbers stable.
 
 ---
 
-## 8. Related documentation
+## 8. Reported-numbers dump
+
+A companion entry point,
+
+```bash
+python -m maestro.analysis.reported_numbers
+```
+
+emits `output/analysis/reported_numbers.json`: the headline totals docs
+and slide decks cite (total cell count, success / failure split,
+aggregate cost). The file is the machine-readable source that
+transcribed numbers in tracked prose must match, and the model-registry
+consistency test in CI enforces that. Regenerate it after any run that
+changes those totals, then update the docs from the file rather than
+from a screenshot of the runner output.
+
+Empty database produces a valid `status: "empty"` payload so a fresh
+checkout can still write the file and pass the consistency test.
+
+---
+
+## 9. Related documentation
 
 - `docs/schema.md`: full database schema reference.
 - `docs/running.md`: how to produce the database in the first place.

src/maestro/analysis/reported_numbers.py

@@ -0,0 +1,138 @@
+"""
+Canonical reported-numbers dump: the values docs quote and slide decks cite
+are computed from the results database here and written to a
+machine-readable JSON file. Downstream text (README, CHANGELOG, docs
+markdown) is checked against this file by the consistency test, so a
+transcribed number cannot silently disagree with the database.
+
+Scope is deliberately narrow: only headline totals that already appear in
+the docs today (total cell count, success / failure split, aggregate
+cost). Statistical results have their own richer artefacts under
+``maestro.analysis.__main__``; this module is the join point where prose
+meets truth.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sqlite3
+from pathlib import Path
+
+from maestro.db.client import get_readonly_connection
+from maestro.experiment_config import DB_PATH
+from maestro.schemas import ReportedNumbers
+
+# Emitted schema version. Bump when a field is renamed or its meaning
+# changes; consumers (docs, the consistency check) can then pin against a
+# specific shape rather than a moving target.
+SCHEMA_VERSION = "1.0"
+
+# Fixed relative location under the analysis output tree. Kept stable so
+# the consistency check does not need a discovery step.
+DEFAULT_OUTPUT_PATH = (
+    Path(__file__).resolve().parents[3]
+    / "output"
+    / "analysis"
+    / "reported_numbers.json"
+)
+
+
+def compute_reported_numbers(conn: sqlite3.Connection) -> ReportedNumbers:
+    """
+    Aggregate the docs-referenced totals from a results DB, read-only.
+
+    Returns ``status="empty"`` when no runs have been recorded yet: every
+    numeric field is still present but set to 0 so a downstream diff still
+    works against a fresh (v2.0.0) checkout.
+    """
+    totals_row = conn.execute(
+        """
+        SELECT
+            COUNT(*) AS total_runs,
+            COALESCE(SUM(CASE WHEN error IS NULL AND output_diagram_code IS NOT NULL
+                              AND TRIM(output_diagram_code) <> ''
+                         THEN 1 ELSE 0 END), 0) AS successes,
+            COALESCE(SUM(CASE WHEN error IS NOT NULL OR output_diagram_code IS NULL
+                              OR TRIM(COALESCE(output_diagram_code, '')) = ''
+                         THEN 1 ELSE 0 END), 0) AS failures,
+            COALESCE(SUM(cost_usd), 0.0) AS total_cost_usd
+        FROM run_results
+        """
+    ).fetchone()
+
+    total_runs = int(totals_row["total_runs"])
+    if total_runs == 0:
+        return ReportedNumbers(
+            schema_version=SCHEMA_VERSION,
+            status="empty",
+            total_runs=0,
+            successes=0,
+            failures=0,
+            total_cost_usd=0.0,
+        )
+
+    return ReportedNumbers(
+        schema_version=SCHEMA_VERSION,
+        status="ok",
+        total_runs=total_runs,
+        successes=int(totals_row["successes"]),
+        failures=int(totals_row["failures"]),
+        # Rounded to cents: docs quote e.g. USD 171.62, so writing extra
+        # digits would create a mismatch on the last decimal that is
+        # cosmetic, not real. Two-place rounding matches the reporting
+        # convention and keeps the diff meaningful.
+        total_cost_usd=round(float(totals_row["total_cost_usd"]), 2),
+    )
+
+
+def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="python -m maestro.analysis.reported_numbers",
+        description=(
+            "Dump the docs-referenced headline totals to a machine-readable "
+            "JSON file. Consumed by the model-registry consistency check."
+        ),
+    )
+    parser.add_argument(
+        "--db",
+        type=Path,
+        default=DB_PATH,
+        help=f"Path to the experiment SQLite database (default: {DB_PATH}).",
+    )
+    parser.add_argument(
+        "--out",
+        type=Path,
+        default=DEFAULT_OUTPUT_PATH,
+        help=(
+            "Destination file for the reported-numbers JSON "
+            f"(default: {DEFAULT_OUTPUT_PATH})."
+        ),
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    """
+    Write the reported-numbers JSON file. Exit code 1 if the DB path does
+    not exist; 0 otherwise (including an empty DB, which still emits a
+    valid file so the docs pipeline can run against a fresh checkout).
+    """
+    args = _parse_args(argv)
+    if not args.db.exists():
+        print(f"ERROR: database not found: {args.db}")
+        return 1
+
+    with get_readonly_connection(args.db) as conn:
+        payload = compute_reported_numbers(conn)
+
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(
+        payload.model_dump_json(indent=2) + "\n",
+        encoding="utf-8",
+    )
+    print(f"Wrote {args.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/maestro/experiment_config.py

@@ -297,6 +297,12 @@
 # April 2026 for the frozen main run. IDs are pinned to dated snapshots where
 # the provider offers one, so the run stays reproducible.
 #
+# The model id on each row is the canonical internal_id from
+# maestro.models.MODEL_REGISTRY (the single source of truth for model
+# naming). A consistency test asserts every non-control entry here is
+# registered, so a typo cannot land pricing under a name no other layer
+# recognises.
+#
 # Note: provider dispatch (run.py) is by substring (claude / gpt / mistral /
 # gemini / deepseek), so any new model id must contain its provider's needle.
 # tests/providers/test_provider_dispatch.py enforces this for every entry here.

src/maestro/models.py

@@ -0,0 +1,194 @@
+"""
+Canonical model registry: the single source of truth for every model name
+MAESTRO uses.
+
+Every place that names a model (pricing table, provider dispatch, viz
+palettes, docs, tests) resolves through ``get_model`` or one of the
+membership helpers. That eliminates the class of drift where the same
+model appears under two spellings in two places, and it gives future
+issues (pricing config, model-specific behaviour) a stable join key.
+
+A dated snapshot is a distinct registry entry, not a mutable attribute
+(the ``ModelSpec`` docstring in ``schemas.py`` records why): swapping
+``claude-haiku-4-5-20251001`` for a newer snapshot means adding a new
+row here, not editing this one.
+"""
+
+from __future__ import annotations
+
+import re
+
+from maestro.schemas import ModelSpec
+
+# Frontier == flagship; efficiency == smaller/cheaper. Two tiers per provider
+# is the shape MAESTRO's paired matrix depends on; if that changes, this
+# constant and viz theme's frontier/efficiency slot mapping move together.
+TIER_FRONTIER = "frontier"
+TIER_EFFICIENCY = "efficiency"
+
+# Regex over an internal id's terminal segment. Two vendor shapes are
+# accepted: 8 consecutive digits (``20251001``) and the hyphenated form
+# ``YYYY-MM-DD`` (``2026-04-23``). The captured group is normalised to the
+# compact 8-digit form on return so downstream comparators do not need to
+# know which shape the vendor picked. 4 consecutive digits do not count
+# (that is a version number, e.g. ``mistral-small-2603``). Loose match
+# is deliberate: a vendor's convention is not something the registry can
+# normalise beyond this compact-form step, only expose.
+_SNAPSHOT_DATE_PATTERN = re.compile(r"-(\d{4}-\d{2}-\d{2}|\d{8})$")
+
+
+def _snapshot_of(internal_id: str) -> str | None:
+    """Extract a snapshot date from the tail of an internal id, compact form."""
+    match = _SNAPSHOT_DATE_PATTERN.search(internal_id)
+    if match is None:
+        return None
+    return match.group(1).replace("-", "")
+
+
+def _spec(
+    internal_id: str,
+    *,
+    provider_id: str,
+    display_name: str,
+    provider_display_name: str,
+    tier: str,
+) -> ModelSpec:
+    """Build a ``ModelSpec`` and auto-derive its snapshot date if present."""
+    return ModelSpec(
+        internal_id=internal_id,
+        provider_id=provider_id,
+        display_name=display_name,
+        provider_display_name=provider_display_name,
+        tier=tier,
+        snapshot_date=_snapshot_of(internal_id),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Canonical registry
+# ---------------------------------------------------------------------------
+#
+# Order in this list is the order every downstream consumer displays models
+# in (viz legends, printed tables, docs generation). Alphabetical by
+# provider, frontier-then-efficiency within each provider: the same shape
+# ``experiment_config.MODELS`` uses, kept in step so the two layers can be
+# diffed at a glance.
+
+_REGISTRY_ENTRIES: tuple[ModelSpec, ...] = (
+    _spec(
+        "claude-opus-4-8",
+        provider_id="anthropic",
+        display_name="Claude Opus 4.8",
+        provider_display_name="Claude",
+        tier=TIER_FRONTIER,
+    ),
+    _spec(
+        "claude-haiku-4-5-20251001",
+        provider_id="anthropic",
+        display_name="Claude Haiku 4.5",
+        provider_display_name="Claude",
+        tier=TIER_EFFICIENCY,
+    ),
+    _spec(
+        "gpt-5.5-2026-04-23",
+        provider_id="openai",
+        display_name="GPT-5.5",
+        provider_display_name="ChatGPT",
+        tier=TIER_FRONTIER,
+    ),
+    _spec(
+        "gpt-5.4-mini-2026-03-17",
+        provider_id="openai",
+        display_name="GPT-5.4 mini",
+        provider_display_name="ChatGPT",
+        tier=TIER_EFFICIENCY,
+    ),
+    _spec(
+        "mistral-medium-3-5",
+        provider_id="mistral",
+        display_name="Mistral Medium 3.5",
+        provider_display_name="Mistral",
+        tier=TIER_FRONTIER,
+    ),
+    _spec(
+        "mistral-small-2603",
+        provider_id="mistral",
+        display_name="Mistral Small",
+        provider_display_name="Mistral",
+        tier=TIER_EFFICIENCY,
+    ),
+    _spec(
+        "gemini-3.5-flash",
+        provider_id="gemini",
+        display_name="Gemini 3.5 Flash",
+        provider_display_name="Gemini",
+        tier=TIER_FRONTIER,
+    ),
+    _spec(
+        "gemini-3.1-flash-lite",
+        provider_id="gemini",
+        display_name="Gemini 3.1 Flash Lite",
+        provider_display_name="Gemini",
+        tier=TIER_EFFICIENCY,
+    ),
+    _spec(
+        "deepseek-v4-pro",
+        provider_id="deepseek",
+        display_name="DeepSeek V4 Pro",
+        provider_display_name="DeepSeek",
+        tier=TIER_FRONTIER,
+    ),
+    _spec(
+        "deepseek-v4-flash",
+        provider_id="deepseek",
+        display_name="DeepSeek V4 Flash",
+        provider_display_name="DeepSeek",
+        tier=TIER_EFFICIENCY,
+    ),
+)
+
+
+MODEL_REGISTRY: dict[str, ModelSpec] = {
+    spec.internal_id: spec for spec in _REGISTRY_ENTRIES
+}
+
+
+# ---------------------------------------------------------------------------
+# Public accessors
+# ---------------------------------------------------------------------------
+
+
+def get_model(internal_id: str) -> ModelSpec:
+    """
+    Resolve an internal id to its ``ModelSpec``. Raises ``KeyError`` with a
+    clear message on an unknown id: silently returning ``None`` would let a
+    typo propagate into a chart or a pricing lookup, which is exactly the
+    drift this registry exists to prevent.
+    """
+    spec = MODEL_REGISTRY.get(internal_id)
+    if spec is None:
+        raise KeyError(
+            f"Unknown model internal_id: {internal_id!r}. "
+            f"Known: {', '.join(sorted(MODEL_REGISTRY))}"
+        )
+    return spec
+
+
+def all_internal_ids() -> set[str]:
+    """Set of every registered internal id (used by consistency scans)."""
+    return set(MODEL_REGISTRY)
+
+
+def registered_models() -> list[ModelSpec]:
+    """Ordered list of every registered ``ModelSpec`` (registry order)."""
+    return list(_REGISTRY_ENTRIES)
+
+
+def models_by_provider(provider_id: str) -> list[ModelSpec]:
+    """
+    Every ``ModelSpec`` whose ``provider_id`` matches, in registry order.
+    Empty list on an unknown provider id: the caller decides whether that
+    is an error (a viz palette missing a provider) or a natural no-op (a
+    key-filtered run that excluded the whole vendor).
+    """
+    return [spec for spec in _REGISTRY_ENTRIES if spec.provider_id == provider_id]