From dcb4e5991fc6ac21862fd34e54cd1e4305b48f96 Mon Sep 17 00:00:00 2001 From: Yali and Rona Date: Sun, 30 Aug 2026 14:21:39 +0100 Subject: [PATCH] fix(pricing): resolve model names canonically so priced models stop falling through A model named by a prediction file is priced by an exact lookup in model_cost.json. The name is written by the submitter, so a model that IS in the price table can still fail to resolve on letter case or on the vendor separator, and an unpriceable row is then scored around rather than charged. Two such names are live on the board today: lynkr openai/gpt-oss-120b 8096 rows (96.4%) priced only as openai_gpt-oss-120b glm-4-air-router glm-4-air 8400 rows (100%) no price entry under any spelling Add canonical_key()/resolve_universal_name() to universal_model_names.py, folding case and the `_` vs `/` vendor separator and nothing else -- vendor prefixes are deliberately not stripped, since `vendor/foo` and a bare `foo` are not reliably the same model and `mapping` already records the cases where they are. Both ModelNameManager lookups now go through it; the static variant still raises on an unresolved name, and the non-static variant still records it in missing_models rather than raising, so no existing caller changes behaviour. Verified: all 51 distinct model names across every prediction file resolve exactly as before. Add tools/check_model_names.py, following tools/audit_token_accounting.py, to report the three failure modes and exit 1 under --strict when a named model has no price entry at all. Refs #193 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BydrpgVJTHVLVkCZT9wMXq --- tools/check_model_names.py | 228 +++++++++++++++++++++++++++++++++++++ universal_model_names.py | 104 ++++++++++++++--- 2 files changed, 317 insertions(+), 15 deletions(-) create mode 100644 tools/check_model_names.py diff --git a/tools/check_model_names.py b/tools/check_model_names.py new file mode 100644 index 00000000..86186448 --- /dev/null +++ b/tools/check_model_names.py @@ -0,0 +1,228 @@ +# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project +# SPDX-License-Identifier: Apache-2.0 + +""" +Check that every model a submission names can be priced. + +Addresses issue #193. A model named by a prediction file is priced by looking +its name up in ``model_cost/model_cost.json``. That lookup is exact, while the +name is written by the submitter -- so a model that IS in the price table can +still fail to resolve on letter case or on the vendor separator, and an +unpriceable row is then scored around rather than charged. + +This audit reports three things: + + Problem 1 (registry hygiene): price-table keys that differ only by case or by + ``_`` vs ``/``. Two spellings of one model are two entries that can drift to + two different prices, which is the relabeling hole issue #193 is about. + + Problem 2 (ambiguous universal names): entries in ``universal_names`` / + ``mapping`` that fold onto the same key while naming different models. These + make the canonical fallback ambiguous and should be resolved by hand. + + Problem 3 (unpriceable rows): models named by a prediction file that have no + price entry. Split into names the canonical fold repairs (a spelling + mismatch -- the fix is to normalise the lookup) and names nothing repairs + (genuinely absent from the price table -- the fix is to register a price, + per the pricing policy in issue #193). + +Usage: + python tools/check_model_names.py # audit all routers + python tools/check_model_names.py llm-router lynkr # audit named routers + python tools/check_model_names.py --strict # exit 1 on Problem 3 + +Only "main" rows (for_optimality == False, the rows that feed the RouterArena +score) are counted, matching the leaderboard. +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os +from collections import Counter +from typing import Any, Dict, List, Optional, Tuple + +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +from universal_model_names import ( # noqa: E402 + canonical_collisions, + canonical_key, + resolve_universal_name, +) + +PREDICTIONS_DIR = os.path.join(REPO_ROOT, "router_inference", "predictions") +COST_TABLE = os.path.join(REPO_ROOT, "model_cost", "model_cost.json") + + +def _load_cost_table() -> Dict[str, Any]: + with open(COST_TABLE, "r", encoding="utf-8") as f: + return json.load(f) + + +def _price_of(entry: Dict[str, Any]) -> Tuple[Any, Any]: + return ( + entry.get("input_token_price_per_million"), + entry.get("output_token_price_per_million"), + ) + + +def audit_cost_table(cost: Dict[str, Any]) -> List[Dict[str, Any]]: + """Problem 1: price-table keys that fold together.""" + folded: Dict[str, List[str]] = {} + for key in cost: + folded.setdefault(canonical_key(key), []).append(key) + + duplicates = [] + for fold, keys in sorted(folded.items()): + if len(keys) < 2: + continue + prices = {_price_of(cost[k]) for k in keys} + duplicates.append( + { + "canonical": fold, + "keys": sorted(keys), + "prices": sorted(prices), + "conflict": len(prices) > 1, + } + ) + return duplicates + + +def resolve_price_key( + model_name: str, cost: Dict[str, Any] +) -> Tuple[str, Optional[str]]: + """Resolve a model name to a price-table key. + + Returns (status, key) where status is one of: + "ok" -- the name as written has a price entry + "fold-repairs" -- no exact entry, but a case/separator fold finds one + "missing" -- nothing in the price table names this model + """ + if model_name in cost: + return "ok", model_name + + # A submission may name a model by an alias the pipeline already knows. + universal = resolve_universal_name(model_name) + if universal is not None and universal in cost: + return "ok", universal + + folded = {canonical_key(k): k for k in cost} + for candidate in (model_name, universal): + if candidate is None: + continue + hit = folded.get(canonical_key(candidate)) + if hit is not None: + return "fold-repairs", hit + + return "missing", None + + +def audit_predictions(path: str, cost: Dict[str, Any]) -> Dict[str, Any]: + """Problem 3: models named by one prediction file that cannot be priced.""" + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + + rows = [ + r for r in data if isinstance(r, dict) and not r.get("for_optimality", False) + ] + named: Counter = Counter(r.get("prediction") for r in rows) + + unpriceable = [] + for model, count in named.items(): + if model is None: + continue + status, key = resolve_price_key(model, cost) + if status != "ok": + unpriceable.append( + { + "model": model, + "rows": count, + "share": count / len(rows) if rows else 0.0, + "status": status, + "resolves_to": key, + } + ) + + return { + "router": os.path.basename(path)[: -len(".json")], + "main_rows": len(rows), + "unpriceable": sorted(unpriceable, key=lambda u: -u["rows"]), + } + + +def _resolve_paths(routers: List[str]) -> List[str]: + if not routers: + return sorted( + p + for p in glob.glob(os.path.join(PREDICTIONS_DIR, "*.json")) + if not p.endswith("-robustness.json") + ) + return [os.path.join(PREDICTIONS_DIR, f"{name}.json") for name in routers] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "routers", + nargs="*", + help="Router names to audit (default: every non-robustness prediction file).", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Exit 1 if any named model has no price entry at all (Problem 3).", + ) + args = parser.parse_args() + + cost = _load_cost_table() + + print("Problem 1 - price-table keys that differ only by case or separator") + duplicates = audit_cost_table(cost) + if not duplicates: + print(" none") + for dup in duplicates: + mark = "PRICE CONFLICT" if dup["conflict"] else "duplicate" + print(f" [{mark}] {dup['canonical']}") + print(f" keys: {dup['keys']}") + print(f" prices: {dup['prices']}") + + print("\nProblem 2 - universal names that fold together ambiguously") + if not canonical_collisions: + print(" none") + for key, names in sorted(canonical_collisions.items()): + print(f" {key} <- {names}") + + print("\nProblem 3 - models named by a submission that cannot be priced") + summaries = [audit_predictions(p, cost) for p in _resolve_paths(args.routers)] + header = f"{'router':26} {'model named':40} {'rows':>6} {'share':>7} resolution" + print(header) + print("-" * len(header)) + + any_missing = False + any_row = False + for summary in summaries: + for item in summary["unpriceable"]: + any_row = True + if item["status"] == "missing": + any_missing = True + resolution = "NO PRICE ENTRY - register one (issue #193 policy)" + else: + resolution = f"spelling only - folds to {item['resolves_to']}" + print( + f"{summary['router']:26} {item['model']:40} " + f"{item['rows']:>6} {item['share'] * 100:>6.1f}% {resolution}" + ) + if not any_row: + print(" every model named by every audited submission has a price entry") + + if args.strict and any_missing: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/universal_model_names.py b/universal_model_names.py index 884554a2..4852468a 100644 --- a/universal_model_names.py +++ b/universal_model_names.py @@ -8,6 +8,8 @@ files in ./router_evaluation/llm_inference/outputs/ """ +from __future__ import annotations + universal_names = [ "gpt-3.5-turbo", "gpt-3.5-turbo-1106", @@ -160,6 +162,72 @@ } +def canonical_key(model_name: str) -> str: + """Fold a model name to a key that ignores cosmetic spelling differences. + + The registries in this repo were written by many hands and disagree on two + purely cosmetic points: letter case (``Qwen/Qwen3-Coder-Next`` vs + ``qwen/qwen3-coder-next``) and the vendor separator (``qwen_qwen3-coder`` + vs ``qwen/qwen3-coder``). Both spellings name the same model, but an exact + dict lookup treats them as different models -- and a model that fails to + resolve cannot be priced. See issue #193. + + Folding is deliberately conservative: it normalises case and the separator + and nothing else. It does NOT strip vendor prefixes, because ``vendor/foo`` + and a bare ``foo`` are not reliably the same model and the explicit + ``mapping`` table below already records the cases where they are. + """ + + return model_name.strip().lower().replace("_", "/") + + +def _build_canonical_index() -> tuple[dict[str, str], dict[str, list[str]]]: + """Map canonical_key -> universal name, plus any keys that are ambiguous. + + Two different universal names that fold to the same key, or an alias whose + key shadows a universal name and points somewhere else, make the fold + ambiguous. That is a registry bug rather than a submission bug, so it is + reported (see ``tools/check_model_names.py``) rather than raised at import + time -- importing this module must never be the thing that breaks a run. + """ + + index: dict[str, str] = {} + collisions: dict[str, set[str]] = {} + + def add(key: str, target: str) -> None: + if key in index and index[key] != target: + collisions.setdefault(key, {index[key]}).add(target) + return # first writer wins, matching lookup precedence below + index[key] = target + + for name in universal_names: + add(canonical_key(name), name) + for alias, target in mapping.items(): + add(canonical_key(alias), target) + + return index, {key: sorted(names) for key, names in collisions.items()} + + +canonical_index: dict[str, str] +canonical_collisions: dict[str, list[str]] +canonical_index, canonical_collisions = _build_canonical_index() + + +def resolve_universal_name(model_name: str) -> str | None: + """Resolve a model name to its universal name, or None if unknown. + + Tries, in order: the name as written, the explicit ``mapping`` table, then + the canonical fold. Returning None (rather than the input unchanged) lets + callers decide whether an unresolvable name is fatal. + """ + + if model_name in universal_names: + return model_name + if model_name in mapping: + return mapping[model_name] + return canonical_index.get(canonical_key(model_name)) + + class ModelNameManager: """ Manager for model names. @@ -172,28 +240,34 @@ def __init__(self): self.missing_models = set() def get_universal_name_non_static(self, model_name: str) -> str: - """Convert a model name to its universal equivalent.""" + """Convert a model name to its universal equivalent. - if model_name in universal_names: - return model_name - elif model_name in mapping: - return mapping[model_name] - else: + Non-fatal by design: unknown names are recorded in ``missing_models`` + and returned unchanged, so a caller can survey a whole run before + deciding. Callers that must not proceed on an unknown model should use + the static :meth:`get_universal_name`, or check ``missing_models`` + afterwards. See ``tools/check_model_names.py``. + """ + + resolved = resolve_universal_name(model_name) + if resolved is None: self.missing_models.add(model_name) - # raise ValueError(f"Model name {model_name} not found in universal_names or mapping") + return model_name - return model_name + return resolved @staticmethod def get_universal_name(model_name: str) -> str: - """Convert a model name to its universal equivalent.""" + """Convert a model name to its universal equivalent. - if model_name in universal_names: - return model_name - elif model_name in mapping: - return mapping[model_name] - else: - # self.missing_models.add(model_name) + Raises ``ValueError`` if the name cannot be resolved, so that an + unpriceable model fails loudly rather than being scored around. + """ + + resolved = resolve_universal_name(model_name) + if resolved is None: raise ValueError( f"Model name {model_name} not found in universal_names or mapping" ) + + return resolved