diff --git a/docs/extending.md b/docs/extending.md index d3af9a8..543d832 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -169,7 +169,9 @@ supply-chain incident, a required new symbol, a moved import path). ### 1.6 Register a model -Add an entry to `MODELS` in `src/maestro/experiment_config.py`: +Add an entry to the active pricing snapshot in +`src/maestro/pricing/snapshot_YYYY_MM.py` (the one `DEFAULT_VERSION` in +`src/maestro/pricing/__init__.py` points at): ```python ModelPricing( @@ -180,8 +182,20 @@ ModelPricing( ``` Pricing is USD per 1M tokens, sourced from the provider's public pricing -page on the date of the run. `cost_usd` is computed at write time from -this rate, so a later pricing change does not alter historical rows. +page on the date the snapshot represents. `cost_usd` is computed at write +time from this rate, so a later repricing does not alter historical rows. + +**Repricing an existing model.** Do not edit the existing snapshot. Copy +it to a new dated file (`snapshot_2026_10.py`, `snapshot_2027_01.py`, ...), +update its top-level `VERSION` to the new `YYYY-MM` string and its +`PRICING` list to the new rates, register it in `_VERSIONS`, and bump +`DEFAULT_VERSION` to point at the new file. Historical runs stay pinned +to whichever snapshot was default when they ran; every new run records +its snapshot id in `run_environments.pricing_version` when pricing +capture succeeds, and a soft-failed capture leaves the column NULL +rather than aborting the run. Cross-snapshot analysis is therefore an +explicit research choice (either the column matches a known snapshot or +is NULL), never a silent side effect. ### 1.7 Smoke test @@ -365,9 +379,12 @@ pre-change and post-change runs are never mixed. In particular: - A new provider or strategy is additive and does not require a bump on its own; the new rows are recognisably from the new configuration. -A pricing change on an existing model is not a bump: `cost_usd` is -computed at write time and stored, so historical rows retain their -original cost even after a repricing. +A pricing change on an existing model is not a scoring bump: `cost_usd` +is computed at write time and stored, so historical rows retain their +original cost. It is, however, a pricing-snapshot bump: land the new +rates in a fresh `snapshot_YYYY_MM.py`, bump `DEFAULT_VERSION`, and every +subsequent run's `run_environments.pricing_version` records the new +snapshot id so a cross-snapshot analysis is an explicit choice. See `CHANGELOG.md` and `.github/CONTRIBUTING.md` for the release-line conventions. \ No newline at end of file diff --git a/docs/schema.md b/docs/schema.md index 778d373..9615f4f 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -65,6 +65,7 @@ result traces back to the exact software stack that produced it. | `git_dirty` | INTEGER | Tri-state: 1 = dirty, 0 = clean, NULL = probe failed. Never conflate NULL with clean. | | `lib_versions` | TEXT | JSON blob: `{"anthropic": "0.34.2", "openai": "...", ...}`. Whitelisted runtime deps only (see `db/environment.py`). | | `docker_image_digest` | TEXT | Value of `MAESTRO_IMAGE_DIGEST` at build time. NULL if the build did not pass it. | +| `pricing_version` | TEXT | Active pricing snapshot id (`YYYY-MM`, see `maestro.pricing.DEFAULT_VERSION`) at run time. NULL on pre-migration rows and if pricing capture failed. | | `captured_at` | TEXT NOT NULL | UTC ISO 8601. | ### 2.2 `run_configs` @@ -96,7 +97,7 @@ guarantee. | `prompt_tokens` | INTEGER NOT NULL | Prompt/input token count. 0 for controls and for cells that failed before any call. | | `completion_tokens` | INTEGER NOT NULL | Completion/output token count. | | `duration_ms` | INTEGER NOT NULL | Wall-clock latency of the cell. | -| `cost_usd` | REAL NOT NULL | Computed at write time from token counts and the `ModelPricing` rate captured in `experiment_config.py`. Never recomputed at read time, so a later pricing change does not alter historical rows. | +| `cost_usd` | REAL NOT NULL | Computed at write time from token counts and the `ModelPricing` rate from the active snapshot in `maestro.pricing`. Never recomputed at read time, so a later repricing does not alter historical rows; join to `run_environments.pricing_version` via `run_configs.environment_id` to recover which snapshot produced a row; the join yields a snapshot id when both `environment_id` and `pricing_version` are present, and NULL if capture failed. | | `error` | TEXT | Human-readable error string. NULL means success (the sole flag for `is_success`). | | `retry_count` | INTEGER NOT NULL DEFAULT 0 | Number of retries the provider's retry policy consumed for this cell. | diff --git a/src/maestro/db/client.py b/src/maestro/db/client.py index bbbd161..88d5e17 100644 --- a/src/maestro/db/client.py +++ b/src/maestro/db/client.py @@ -24,6 +24,10 @@ git_dirty INTEGER, lib_versions TEXT, docker_image_digest TEXT, + -- Pricing snapshot the cost columns were computed against (YYYY-MM + -- form; see maestro.pricing). Nullable: pre-migration rows keep NULL, + -- and the environment probe records NULL if pricing capture fails. + pricing_version TEXT, captured_at TEXT NOT NULL ); @@ -147,6 +151,7 @@ def init_db(db_path: Path) -> None: _migrate_add_environment_id_column(conn) _migrate_add_retry_count_column(conn) _migrate_add_container_attachment_columns(conn) + _migrate_add_pricing_version_column(conn) conn.commit() @@ -180,6 +185,18 @@ def _migrate_add_retry_count_column(conn: sqlite3.Connection) -> None: ) +def _migrate_add_pricing_version_column(conn: sqlite3.Connection) -> None: + """ + Add ``run_environments.pricing_version`` to databases that predate the + column. Nullable TEXT: pre-migration rows stay NULL (they were recorded + before pricing was versioned, so the value is genuinely unknown and + fabricating a snapshot label would misrepresent the archive). + """ + cols = {row[1] for row in conn.execute("PRAGMA table_info(run_environments)")} + if "pricing_version" not in cols: + conn.execute("ALTER TABLE run_environments ADD COLUMN pricing_version TEXT") + + def _migrate_add_container_attachment_columns(conn: sqlite3.Connection) -> None: """ Add the container + attachment metric columns to databases that predate diff --git a/src/maestro/db/environment.py b/src/maestro/db/environment.py index 3bc502d..ea419e6 100644 --- a/src/maestro/db/environment.py +++ b/src/maestro/db/environment.py @@ -129,6 +129,21 @@ def _lib_versions() -> dict[str, str | None]: return resolved +def _pricing_version() -> str | None: + """Best-effort probe: environment capture must never crash the run it + describes, so a broken or missing pricing package returns ``None`` rather + than propagating.""" + try: + from maestro.pricing import DEFAULT_VERSION + + return DEFAULT_VERSION + except Exception: + # Env capture must never crash the run it describes; a broken pricing + # package would be caught at import time by the runner long before this. + # Recording None preserves the failure signal instead of guessing. + return None + + def capture_environment( image_digest_env: str = "MAESTRO_IMAGE_DIGEST", ) -> RunEnvironment: @@ -150,5 +165,6 @@ def capture_environment( git_dirty=_git_dirty(), lib_versions=json.dumps(_lib_versions(), sort_keys=True), docker_image_digest=os.environ.get(image_digest_env) or None, + pricing_version=_pricing_version(), captured_at=datetime.now(timezone.utc), ) diff --git a/src/maestro/db/queries.py b/src/maestro/db/queries.py index 3492813..c785d46 100644 --- a/src/maestro/db/queries.py +++ b/src/maestro/db/queries.py @@ -23,9 +23,9 @@ def insert_run_environment(conn: sqlite3.Connection, env: RunEnvironment) -> Non INSERT INTO run_environments (environment_id, os, arch, python, hostname, git_commit, git_dirty, lib_versions, docker_image_digest, - captured_at) + pricing_version, captured_at) VALUES - (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( str(env.environment_id), @@ -38,6 +38,7 @@ def insert_run_environment(conn: sqlite3.Connection, env: RunEnvironment) -> Non None if env.git_dirty is None else int(env.git_dirty), env.lib_versions, env.docker_image_digest, + env.pricing_version, env.captured_at.isoformat(), ), ) diff --git a/src/maestro/experiment_config.py b/src/maestro/experiment_config.py index 41c3b83..180d3f8 100644 --- a/src/maestro/experiment_config.py +++ b/src/maestro/experiment_config.py @@ -4,7 +4,8 @@ Single source of truth for the experiment matrix. To add a new input: append to INPUTS -To add a new model: append to MODELS +To add a new model: add a row to the active pricing snapshot in + ``maestro.pricing`` (never hardcode a rate here) To enable a strategy: add to STRATEGIES (once implemented) """ @@ -13,7 +14,8 @@ import os from pathlib import Path -from maestro.schemas import InputFile, ModelPricing, Strategy, Tier +from maestro.pricing import DEFAULT_VERSION, load_pricing +from maestro.schemas import InputFile, Strategy, Tier # --------------------------------------------------------------------------- # Base path for all data files (relative to project root) @@ -280,98 +282,22 @@ # Model registry - pricing per model for cost calculation # --------------------------------------------------------------------------- -# Synthetic "model" used only for control-strategy rows. Controls bypass the -# LLM entirely so no real model is involved; this entry exists so the -# ``RunConfig.model`` column has an honest value ("control") rather than -# borrowing the name of a real model and lying about what produced the row. -# Zero pricing means control rows never affect cost rollups. -CONTROL_MODEL = ModelPricing( - model="control", - input_price_per_1m=0.0, - output_price_per_1m=0.0, -) - -# Two models per provider: a frontier ("best") model and an efficiency model, -# so the experiment can compare quality against cost within and across vendors. -# Prices are USD per 1M tokens, verified against each provider's pricing page in -# April 2026 for the frozen main run. IDs are pinned to dated snapshots where -# the provider offers one, so the run stays reproducible. -# -# The model id on each row is the canonical internal_id from -# maestro.models.MODEL_REGISTRY (the single source of truth for model -# naming). A consistency test asserts every non-control entry here is -# registered, so a typo cannot land pricing under a name no other layer -# recognises. +# MODELS is derived from the active pricing snapshot in ``maestro.pricing`` +# so rates live in one place: a dated ``snapshot_YYYY_MM.py`` file whose +# version string is recorded on every run (run_environments.pricing_version). +# Every model id here is a canonical internal_id from +# maestro.models.MODEL_REGISTRY; the model-registry consistency test asserts +# the pricing/registry one-to-one invariant, and a typo would fail loudly at +# import via _validate() in the pricing package. # # Note: provider dispatch (run.py) is by substring (claude / gpt / mistral / # gemini / deepseek), so any new model id must contain its provider's needle. # tests/providers/test_provider_dispatch.py enforces this for every entry here. -MODELS: list[ModelPricing] = [ - # Anthropic - ModelPricing( - model="claude-opus-4-8", # frontier - input_price_per_1m=5.00, - output_price_per_1m=25.00, - # Opus 4.7+ removed sampling params; sending temperature returns 400. - supports_temperature=False, - ), - ModelPricing( - model="claude-haiku-4-5-20251001", # efficiency - input_price_per_1m=1.00, - output_price_per_1m=5.00, - ), - # OpenAI (GPT-5 family: max_completion_tokens, no custom temperature) - ModelPricing( - model="gpt-5.5-2026-04-23", # frontier - input_price_per_1m=5.00, - output_price_per_1m=30.00, - supports_temperature=False, - ), - ModelPricing( - model="gpt-5.4-mini-2026-03-17", # efficiency - input_price_per_1m=0.75, - output_price_per_1m=4.50, - supports_temperature=False, - ), - # Mistral - ModelPricing( - model="mistral-medium-3-5", # frontier - input_price_per_1m=1.50, - output_price_per_1m=7.50, - ), - ModelPricing( - model="mistral-small-2603", # efficiency - input_price_per_1m=0.15, - output_price_per_1m=0.60, - ), - # Gemini - ModelPricing( - model="gemini-3.5-flash", # frontier - input_price_per_1m=1.50, - output_price_per_1m=9.00, - ), - ModelPricing( - model="gemini-3.1-flash-lite", # efficiency - input_price_per_1m=0.25, - output_price_per_1m=1.50, - ), - # DeepSeek: the cross-provider replication dimension's emerging-Chinese - # entry (proposal section 3.2), consumed via the OpenAI-compatible endpoint - # (see providers/deepseek.py). Pricing is the cache-MISS (standard) rate; - # DeepSeek also offers a cheaper cache-hit input price, but ModelPricing has - # a single input rate, so cache-miss makes the tracked cost an upper bound - # on actual spend (never an under-count). - ModelPricing( - model="deepseek-v4-pro", # frontier - input_price_per_1m=0.435, - output_price_per_1m=0.87, - ), - ModelPricing( - model="deepseek-v4-flash", # efficiency - input_price_per_1m=0.14, - output_price_per_1m=0.28, - ), -] +PRICING_VERSION, MODELS = load_pricing() + +# Re-export the pricing version constant for consumers (run.py's environment +# capture, tests) that want to name it without importing the pricing package. +DEFAULT_PRICING_VERSION = DEFAULT_VERSION # --------------------------------------------------------------------------- @@ -394,7 +320,8 @@ # Set used by ``build_matrix`` and analysis code to special-case controls: -# - matrix builder uses CONTROL_MODEL and run_number=1 for these strategies +# - matrix builder uses ``maestro.pricing.CONTROL_MODEL`` and run_number=1 +# for these strategies # - analysis can exclude them from ANOVA / cost rollups with # ``WHERE strategy NOT IN (SELECT value FROM control_strategies)`` or the # in-Python equivalent ``s not in CONTROL_STRATEGIES``. diff --git a/src/maestro/pricing/__init__.py b/src/maestro/pricing/__init__.py new file mode 100644 index 0000000..c2e04f4 --- /dev/null +++ b/src/maestro/pricing/__init__.py @@ -0,0 +1,187 @@ +""" +Versioned, date-stamped pricing snapshots. + +Prices decay the moment a vendor updates a page, and cost is one of the +selection axes MAESTRO reports on, so every recorded run must be pinned to +the exact rates that produced its ``cost_usd``. This package holds the +snapshots and exposes two things: + +- ``load_pricing(version)`` returns the ``ModelPricing`` list frozen into that + snapshot, plus its version string. ``experiment_config.MODELS`` is derived + from the default snapshot's pricing so a cost table change is a config bump, + never a stray literal in application code. +- ``get_pricing(model_id, version)`` looks up a single model's rates. It + raises loudly on an unknown model so a typo becomes a run-time failure with + a message, not a silent zero-cost row. + +**Adding a new snapshot.** Copy the newest ``snapshot_YYYY_MM.py`` into a new +dated file (for example ``snapshot_2026_10.py``), update its top-level +``VERSION`` string to the new ``YYYY-MM`` identifier and the ``PRICING`` list to +the current rates, add it to ``_VERSIONS`` below (keyed by the same string), +and bump ``DEFAULT_VERSION`` to point at it. Historical runs stay pinned to +the snapshot that was default when they ran; cross-snapshot cost comparisons +are a research decision the analyst opts into, never a silent side effect. +""" + +from __future__ import annotations + +import re + +from maestro.pricing import snapshot_2026_04 +from maestro.schemas import ModelPricing + +# The version string every new run records under until a fresh snapshot is +# introduced. Deliberately the raw string rather than a re-export of the +# snapshot module's ``VERSION`` so a typo in either would fail loudly at +# import via ``_validate``, and so it is grep-able as a literal identifier +# in commit diffs and archived docs. +DEFAULT_VERSION = "2026-04" + + +# Strict shape a snapshot key must satisfy: four-digit year, hyphen, two-digit +# month 01 to 12. A malformed key sorts oddly in ``available_versions`` and +# reads confusingly in the DB, so ``_validate`` rejects it at import time +# before any run is written under it. +_KEY_PATTERN = re.compile(r"^(\d{4})-(0[1-9]|1[0-2])$") + + +# All snapshots the package knows about, keyed by their ISO-year-month +# version string. Adding a new snapshot means adding a row here in addition +# to bumping ``DEFAULT_VERSION``; the import-time ``_validate`` below refuses +# to load an inconsistent state. +_VERSIONS: dict[str, tuple[str, list[ModelPricing]]] = { + snapshot_2026_04.VERSION: (snapshot_2026_04.VERSION, snapshot_2026_04.PRICING), +} + + +# Synthetic pricing row for control strategies (null / copy / ground-truth): +# they never invoke an LLM, so no real model applies. Kept out of the dated +# snapshots because it is not vendor pricing: rebasing every snapshot on the +# same zero-cost stub would just duplicate the row. ``run.py`` reaches for it +# directly via ``CONTROL_MODEL``. +CONTROL_MODEL = ModelPricing( + model="control", + input_price_per_1m=0.0, + output_price_per_1m=0.0, +) + + +def available_versions() -> tuple[str, ...]: + """Sorted tuple of every registered snapshot version, oldest first.""" + return tuple(sorted(_VERSIONS)) + + +def load_pricing(version: str | None = None) -> tuple[str, list[ModelPricing]]: + """ + Return ``(version, pricing)`` for the requested snapshot. + + ``version`` is a date-stamped identifier of the form ``YYYY-MM`` (for + example ``"2026-04"``); each snapshot file publishes its own such string. + ``None`` resolves to ``DEFAULT_VERSION`` so callers that just want "the + current rates" (the experiment runner, most tests) do not need to name a + version. Unknown versions raise ``KeyError`` with the list of known ones, + so a typo fails immediately rather than silently loading the default. + + Returns the version string alongside the list so callers that persist + provenance (``run_environments.pricing_version``, the CLI banner) never + have to synthesise it and cannot disagree about what "current" means. + The returned list is a shared reference, but each ``ModelPricing`` row + is a frozen Pydantic model, so callers cannot mutate a snapshot's rates + in place and poison another caller's read. + """ + key = version if version is not None else DEFAULT_VERSION + try: + return _VERSIONS[key] + except KeyError as exc: + raise KeyError( + f"Unknown pricing version {key!r}. Known: {', '.join(available_versions())}" + ) from exc + + +def get_pricing(model_id: str, version: str | None = None) -> ModelPricing: + """ + Return the ``ModelPricing`` row for one model under one snapshot. + + ``version`` follows the same ``YYYY-MM`` date-stamped convention as + ``load_pricing``; ``None`` uses ``DEFAULT_VERSION``. ``model_id`` matches + ``ModelPricing.model`` (the same string the canonical registry keys on). + The synthetic ``"control"`` id resolves to ``CONTROL_MODEL`` regardless of + snapshot: controls do not use an LLM, and pretending otherwise would put + zero-cost rows under a vendor's namespace. + + Raises ``KeyError`` with the list of known ids on a miss. Silent zero was + the old behaviour and the reason this module exists: a typo in a run + config would produce a plausible ``$0.000000`` row with no complaint, + which is exactly the drift versioned pricing has to prevent. + """ + if model_id == CONTROL_MODEL.model: + return CONTROL_MODEL + _, pricing = load_pricing(version) + for row in pricing: + if row.model == model_id: + return row + known = ", ".join(sorted(row.model for row in pricing)) + resolved = version if version is not None else DEFAULT_VERSION + raise KeyError( + f"Unknown model {model_id!r} in pricing snapshot {resolved!r}. Known: {known}" + ) + + +def _validate() -> None: + """ + Import-time invariants for the whole package. + + Enforcing them at import time (not lazily) means a broken snapshot fails + the process before any run starts, not halfway through the matrix when + the missing row is finally reached. Four shapes of drift are caught: + + - a snapshot registered under a key that is not a valid ``YYYY-MM`` + identifier (year plus month 01 to 12), which would sort oddly and + read confusingly wherever the id is joined against; + - a snapshot module whose ``VERSION`` disagrees with its ``_VERSIONS`` + key (rename typo); + - a snapshot with duplicate ``ModelPricing.model`` rows (would silently + shadow a rate); + - a ``DEFAULT_VERSION`` that names a snapshot no longer registered + (typo in the bump). + """ + for key in _VERSIONS: + if not _KEY_PATTERN.fullmatch(key): + raise RuntimeError( + f"pricing snapshot key {key!r} is not a valid YYYY-MM identifier " + f"(four-digit year, hyphen, two-digit month 01 to 12)" + ) + if DEFAULT_VERSION not in _VERSIONS: + raise RuntimeError( + f"pricing DEFAULT_VERSION {DEFAULT_VERSION!r} not in _VERSIONS: " + f"{sorted(_VERSIONS)}" + ) + for key, (declared_version, rows) in _VERSIONS.items(): + if key != declared_version: + raise RuntimeError( + f"pricing snapshot key {key!r} disagrees with its VERSION " + f"{declared_version!r}; rename one so they match" + ) + seen: set[str] = set() + duplicates: set[str] = set() + for row in rows: + if row.model in seen: + duplicates.add(row.model) + seen.add(row.model) + if duplicates: + raise RuntimeError( + f"pricing snapshot {key!r} has duplicate model rows: " + f"{sorted(duplicates)}" + ) + + +_validate() + + +__all__ = [ + "CONTROL_MODEL", + "DEFAULT_VERSION", + "available_versions", + "get_pricing", + "load_pricing", +] diff --git a/src/maestro/pricing/snapshot_2026_04.py b/src/maestro/pricing/snapshot_2026_04.py new file mode 100644 index 0000000..c000a31 --- /dev/null +++ b/src/maestro/pricing/snapshot_2026_04.py @@ -0,0 +1,99 @@ +""" +Frozen pricing snapshot for the April 2026 main experiment run. + +The ``VERSION`` string is stored verbatim in ``run_environments.pricing_version`` +for every cell recorded under this snapshot, so historical rows stay +interpretable even after the vendor pages have moved on. A repricing is never +an edit to this file: a new dated snapshot (``snapshot_2026_10.py``, +``snapshot_2027_01.py``, ...) lands beside it with its own ``VERSION`` string, +and ``DEFAULT_VERSION`` in ``maestro.pricing`` bumps to point at the new file. +That way pre-change and post-change runs are pinned to whichever snapshot was +default the day they ran, and cross-snapshot mixing is a research decision +rather than a silent side effect. + +Prices are USD per 1M tokens, verified against each provider's public pricing +page in April 2026. IDs match the canonical registry (``maestro.models``); the +one-to-one invariant is enforced by ``test_pricing_and_registry_are_one_to_one`` +in the model-registry consistency suite. +""" + +from __future__ import annotations + +from maestro.schemas import ModelPricing + +# ISO year-month, stored on every run captured under this snapshot. String is +# deliberately not a date: it identifies the file, not a specific day, and a +# stray ``date`` parse in a downstream consumer would silently drop context. +VERSION = "2026-04" + + +# One entry per active benchmarked model. Order mirrors +# ``maestro.models._REGISTRY_ENTRIES`` (alphabetical by provider, frontier +# then efficiency within each provider); the model-registry consistency test +# blocks silent drift on the id side. +PRICING: list[ModelPricing] = [ + # Anthropic + ModelPricing( + model="claude-opus-4-8", # frontier + input_price_per_1m=5.00, + output_price_per_1m=25.00, + # Opus 4.7+ removed sampling params; sending temperature returns 400. + supports_temperature=False, + ), + ModelPricing( + model="claude-haiku-4-5-20251001", # efficiency + input_price_per_1m=1.00, + output_price_per_1m=5.00, + ), + # OpenAI (GPT-5 family: max_completion_tokens, no custom temperature) + ModelPricing( + model="gpt-5.5-2026-04-23", # frontier + input_price_per_1m=5.00, + output_price_per_1m=30.00, + supports_temperature=False, + ), + ModelPricing( + model="gpt-5.4-mini-2026-03-17", # efficiency + input_price_per_1m=0.75, + output_price_per_1m=4.50, + supports_temperature=False, + ), + # Mistral + ModelPricing( + model="mistral-medium-3-5", # frontier + input_price_per_1m=1.50, + output_price_per_1m=7.50, + ), + ModelPricing( + model="mistral-small-2603", # efficiency + input_price_per_1m=0.15, + output_price_per_1m=0.60, + ), + # Gemini + ModelPricing( + model="gemini-3.5-flash", # frontier + input_price_per_1m=1.50, + output_price_per_1m=9.00, + ), + ModelPricing( + model="gemini-3.1-flash-lite", # efficiency + input_price_per_1m=0.25, + output_price_per_1m=1.50, + ), + # DeepSeek: the cross-provider replication dimension's emerging-Chinese + # entry (proposal section 3.2), consumed via the OpenAI-compatible endpoint + # (see providers/deepseek.py). Pricing is the cache-MISS (standard) rate; + # DeepSeek also offers a cheaper cache-hit input price, but ModelPricing has + # a single input rate, so cache-miss makes the tracked cost an upper bound + # on actual spend (never an under-count). + ModelPricing( + model="deepseek-v4-pro", # frontier + input_price_per_1m=0.435, + output_price_per_1m=0.87, + ), + ModelPricing( + model="deepseek-v4-flash", # efficiency + input_price_per_1m=0.14, + output_price_per_1m=0.28, + ), +] diff --git a/src/maestro/run.py b/src/maestro/run.py index 2488af4..0f7c915 100644 --- a/src/maestro/run.py +++ b/src/maestro/run.py @@ -84,7 +84,6 @@ insert_sub_result, ) from maestro.experiment_config import ( - CONTROL_MODEL, CONTROL_STRATEGIES, DB_PATH, DEFAULT_REPEATS, @@ -92,6 +91,7 @@ MODELS, STRATEGIES, ) +from maestro.pricing import CONTROL_MODEL from maestro.providers.anthropic import AnthropicProvider from maestro.providers.deepseek import DeepSeekProvider from maestro.providers.gemini import GeminiProvider diff --git a/src/maestro/schemas.py b/src/maestro/schemas.py index 600b167..12171ab 100644 --- a/src/maestro/schemas.py +++ b/src/maestro/schemas.py @@ -137,6 +137,13 @@ class RunEnvironment(BaseModel): # Container provenance: set by CI/CD via env var, NULL when running locally docker_image_digest: str | None = None + # Pricing snapshot the cost columns were computed against, in + # ``YYYY-MM`` form (see ``maestro.pricing.DEFAULT_VERSION``). Nullable + # because pre-migration rows predate the column and because the pricing + # capture is best effort like every other environment probe: a bad pricing + # package must never abort the run it is meant to describe. + pricing_version: str | None = None + captured_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) @@ -223,6 +230,8 @@ class ModelPricing(BaseModel): Used to compute cost_usd at write time. """ + model_config = ConfigDict(frozen=True) + model: str input_price_per_1m: float # USD per 1M prompt tokens output_price_per_1m: float # USD per 1M completion tokens diff --git a/tests/pricing/__init__.py b/tests/pricing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pricing/test_pricing.py b/tests/pricing/test_pricing.py new file mode 100644 index 0000000..abe8d17 --- /dev/null +++ b/tests/pricing/test_pricing.py @@ -0,0 +1,227 @@ +""" +Tests for the versioned pricing package. + +Pricing is the join key between token counts and reported cost, so drift +here shows up as wrong dollar figures on published charts. Three shapes +are pinned: + +- **Shape of the public API.** ``load_pricing`` / ``get_pricing`` accept + a ``YYYY-MM`` version string, return a stable structure, and reject + unknown ids with a message that names the offender. +- **Snapshot integrity.** Each dated snapshot's ``VERSION`` matches its + ``_VERSIONS`` key, its ``PRICING`` list has no duplicate model rows, + and ``DEFAULT_VERSION`` points at a registered snapshot. +- **Config wiring.** ``experiment_config.MODELS`` is the default + snapshot's pricing list (not a hardcoded copy), and the environment + probe records the same version string on every captured run. +""" + +from __future__ import annotations + +import pytest + +from maestro import pricing +from maestro.db.environment import capture_environment +from maestro.experiment_config import DEFAULT_PRICING_VERSION, MODELS, PRICING_VERSION +from maestro.pricing import ( + CONTROL_MODEL, + DEFAULT_VERSION, + available_versions, + get_pricing, + load_pricing, + snapshot_2026_04, +) +from maestro.schemas import ModelPricing + +# --------------------------------------------------------------------------- +# Version string format +# --------------------------------------------------------------------------- + + +def test_default_version_is_iso_year_month(): + """The version string is a date stamp, not a semver: enforce the shape.""" + assert DEFAULT_VERSION == "2026-04" + parts = DEFAULT_VERSION.split("-") + assert len(parts) == 2 + year, month = parts + assert year.isdigit() and len(year) == 4 + assert month.isdigit() and len(month) == 2 + assert 1 <= int(month) <= 12 + + +def test_snapshot_module_version_matches_default(): + """The snapshot file declares the same string DEFAULT_VERSION points at.""" + assert snapshot_2026_04.VERSION == DEFAULT_VERSION + + +def test_available_versions_lists_the_default(): + versions = available_versions() + assert DEFAULT_VERSION in versions + # available_versions is sorted so a future addition slots in + # deterministically and this test does not need updating on that path. + assert list(versions) == sorted(versions) + + +# --------------------------------------------------------------------------- +# load_pricing +# --------------------------------------------------------------------------- + + +def test_load_pricing_default_returns_version_and_rows(): + version, rows = load_pricing() + assert version == DEFAULT_VERSION + assert rows, "default snapshot has no pricing rows" + assert all(isinstance(row, ModelPricing) for row in rows) + + +def test_load_pricing_explicit_version_matches_default(): + v_default, rows_default = load_pricing() + v_explicit, rows_explicit = load_pricing(DEFAULT_VERSION) + assert v_default == v_explicit + assert rows_default == rows_explicit + + +def test_load_pricing_unknown_version_raises_with_message(): + with pytest.raises(KeyError) as excinfo: + load_pricing("1999-13") + assert "1999-13" in str(excinfo.value) + # The message names the known versions so the user does not have to grep. + assert DEFAULT_VERSION in str(excinfo.value) + + +def test_snapshot_rows_have_no_duplicate_model_ids(): + """A duplicate row would silently shadow one rate; catch it up front.""" + _, rows = load_pricing() + ids = [row.model for row in rows] + assert len(ids) == len(set(ids)), ( + f"duplicate model rows: {sorted({m for m in ids if ids.count(m) > 1})}" + ) + + +# --------------------------------------------------------------------------- +# get_pricing +# --------------------------------------------------------------------------- + + +def test_get_pricing_returns_row_for_known_model(): + row = get_pricing("claude-opus-4-8") + assert isinstance(row, ModelPricing) + assert row.model == "claude-opus-4-8" + # Sanity: the rate matches the snapshot value; a silent typo in the + # snapshot would be caught by the model-registry consistency suite, + # but this test also fails if get_pricing accidentally normalises the + # rate (rounding, currency conversion). + assert row.input_price_per_1m == 5.00 + assert row.output_price_per_1m == 25.00 + + +def test_get_pricing_control_model_bypasses_snapshot(): + """ + The ``"control"`` id has no vendor pricing; it resolves to CONTROL_MODEL + regardless of the requested version so control rows keep zero cost even + if a snapshot forgot to include it (which it should never do). + """ + row = get_pricing(CONTROL_MODEL.model) + assert row is CONTROL_MODEL + assert row.input_price_per_1m == 0.0 + assert row.output_price_per_1m == 0.0 + + +def test_get_pricing_unknown_model_raises_loudly(): + """The old silent-zero-cost behaviour is what this feature exists to remove.""" + with pytest.raises(KeyError) as excinfo: + get_pricing("no-such-model-xyz") + assert "no-such-model-xyz" in str(excinfo.value) + # Version and known ids are named so the error is actionable. + assert DEFAULT_VERSION in str(excinfo.value) + + +def test_get_pricing_unknown_version_raises(): + with pytest.raises(KeyError): + get_pricing("claude-opus-4-8", version="1999-13") + + +# --------------------------------------------------------------------------- +# Wiring into experiment_config and environment capture +# --------------------------------------------------------------------------- + + +def test_experiment_config_models_is_default_snapshot_pricing(): + """ + ``experiment_config.MODELS`` is derived, not hardcoded: it must equal the + default snapshot's pricing list row-for-row so a snapshot bump propagates + without a stray literal being left behind. + """ + _, expected = load_pricing() + assert MODELS == expected + + +def test_experiment_config_pricing_version_matches_default(): + assert PRICING_VERSION == DEFAULT_VERSION + assert DEFAULT_PRICING_VERSION == DEFAULT_VERSION + + +def test_capture_environment_records_pricing_version(): + """ + ``run_environments.pricing_version`` is the join key from a cost figure + back to the rates that produced it, so the probe must record the active + snapshot id (not None) on a healthy install. + """ + env = capture_environment() + assert env.pricing_version == DEFAULT_VERSION + + +# --------------------------------------------------------------------------- +# _validate: import-time invariants +# --------------------------------------------------------------------------- + + +def test_validate_detects_key_version_mismatch(monkeypatch): + """A snapshot registered under the wrong key would silently return the + wrong rates on a lookup; _validate must catch that at import time.""" + bad = { + "1999-01": ("1999-02", []), # key and VERSION disagree + } + monkeypatch.setattr(pricing, "_VERSIONS", bad) + with pytest.raises(RuntimeError) as excinfo: + pricing._validate() + assert "1999-01" in str(excinfo.value) or "1999-02" in str(excinfo.value) + + +def test_validate_detects_duplicate_rows(monkeypatch): + row = ModelPricing(model="dup", input_price_per_1m=1.0, output_price_per_1m=2.0) + bad = { + DEFAULT_VERSION: (DEFAULT_VERSION, [row, row]), + } + monkeypatch.setattr(pricing, "_VERSIONS", bad) + with pytest.raises(RuntimeError) as excinfo: + pricing._validate() + assert "dup" in str(excinfo.value) + + +def test_validate_detects_orphan_default(monkeypatch): + monkeypatch.setattr(pricing, "DEFAULT_VERSION", "9999-99") + with pytest.raises(RuntimeError) as excinfo: + pricing._validate() + assert "9999-99" in str(excinfo.value) + + +@pytest.mark.parametrize("bad_key", ["04-2026", "2026-13", "2026-1", "2026-00"]) +def test_validate_rejects_malformed_key(monkeypatch, bad_key): + """ + A snapshot key that is not a strict ``YYYY-MM`` identifier would sort + oddly against real ones and read confusingly wherever the id is joined, + so ``_validate`` must reject it at import time. + """ + monkeypatch.setattr(pricing, "_VERSIONS", {bad_key: (bad_key, [])}) + with pytest.raises(RuntimeError) as excinfo: + pricing._validate() + assert bad_key in str(excinfo.value) + assert "YYYY-MM" in str(excinfo.value) + + +def test_validate_accepts_current_snapshot_key(): + """Positive case: the shipped ``2026-04`` key satisfies the regex.""" + # No monkeypatching: we exercise the real _VERSIONS to confirm the + # pattern does not over-reject the snapshot the package ships with. + pricing._validate() diff --git a/tests/test_model_registry_consistency.py b/tests/test_model_registry_consistency.py index 00f15e2..a01ccf8 100644 --- a/tests/test_model_registry_consistency.py +++ b/tests/test_model_registry_consistency.py @@ -27,8 +27,9 @@ import pytest -from maestro.experiment_config import CONTROL_MODEL, MODELS +from maestro.experiment_config import MODELS from maestro.models import MODEL_REGISTRY, all_internal_ids, get_model +from maestro.pricing import CONTROL_MODEL from maestro.schemas import ReportedNumbers _REPO_ROOT = Path(__file__).resolve().parents[1]