Fix Grok 4.1 Fast → Grok 4.3 mispricing on the leaderboard (#166)

README.md

@@ -37,11 +37,11 @@ For more details, please see our [website](https://routeworks.github.io/leaderbo
 
 | Rank | Router | Affiliation | Acc-Cost Arena | Accuracy | Cost/1K Queries | Optimal Selection | Optimal Cost | Optimal Accuracy | Latency | Robustness |
 |------|--------------------|-----------------------------|--------|----------|---------|-----------------|--------------|----------------|---------|------------|
-| 🥇 | [Cross-Router]() | 👤 [@JiaHg](https://github.com/JiaHg) | 76.12 | 78.14 | $0.30 | 17.66 | 45.49 | 90.31 | — | 67.14 |
-| 🥈 | [vLLM‑SR](https://vllm-semantic-router.com/) [[Code]](https://github.com/vllm-project/semantic-router) [[HF]](https://huggingface.co/llm-semantic-router) | 🎓 vLLM SR Team | 75.30 | 77.18 | $0.30 | 16.81 | 25.10 | 89.37 | — | 67.62 |
-| 🥉 | [Sqwish Router](https://www.sqwish.ai/) | 👤 [@namitha-sqwish](https://github.com/namitha-sqwish) | 75.27 | 76.40 | $0.18 | 7.41 | 25.10 | 90.47 | — | 100.00 |
-| 4 | [Nadir-Tumbler]() | 👤 [@doramirdor](https://github.com/doramirdor) | 75.17 | 75.34 | $0.08 | — | — | — | — | 66.43 |
-| 5 | [AgentForge Router]() | 👤 [@YangY-Z](https://github.com/YangY-Z) | 74.13 | 74.72 | $0.13 | 17.84 | 52.47 | 98.68 | — | 40.48 |
+| 🥇 | [Cross-Router]() | 👤 [@JiaHg](https://github.com/JiaHg) | 75.75 | 78.14 | $0.40 | 17.66 | 45.49 | 90.31 | — | 67.14 |
+| 🥈 | [Sqwish Router](https://www.sqwish.ai/) | 👤 [@namitha-sqwish](https://github.com/namitha-sqwish) | 75.27 | 76.40 | $0.18 | 7.41 | 25.10 | 90.47 | — | 100.00 |
+| 🥉 | [vLLM‑SR](https://vllm-semantic-router.com/) [[Code]](https://github.com/vllm-project/semantic-router) [[HF]](https://huggingface.co/llm-semantic-router) | 🎓 vLLM SR Team | 74.86 | 77.18 | $0.42 | 16.81 | 25.10 | 89.37 | — | 67.62 |
+| 4 | [AgentForge Router]() | 👤 [@YangY-Z](https://github.com/YangY-Z) | 74.13 | 74.72 | $0.13 | 17.84 | 52.47 | 98.68 | — | 40.48 |
+| 5 | [Nadir-Tumbler]() | 👤 [@doramirdor](https://github.com/doramirdor) | 73.44 | 75.34 | $0.37 | — | — | — | — | 66.43 |
 | 6 | [Weave Router](https://workweave.ai) | 🎓 Weave | 72.82 | 76.32 | $0.94 | — | — | — | — | 100.00 |
 | 7 | [Nadir Router](https://github.com/NadirRouter/NadirClaw) | 🎓 NadirRouter | 72.29 | 75.01 | $0.68 | — | — | — | — | 25.48 |
 | 8 | [OrcaRouter‑Adaptive](https://www.orcarouter.ai/) [[Code]](https://github.com/Continuum-AI-Corp/OrcaRouter-Lite) [[Paper]](https://arxiv.org/abs/2605.30736) [[X]](https://x.com/orcarouter) | 🎓 [Continuum AI](https://www.continuum01.ai/) | 72.08 | 75.54 | $1.00 | — | — | — | — | 22.62 |

llm_evaluation/evaluate_models.py

@@ -206,6 +206,29 @@ def load_cost_config(self):
             print(f"Warning: Could not load cost configuration from {cost_file}: {e}")
             self.cost_config = {}
 
+    def _lookup_cost_info(self, model_name: str):
+        """Find the pricing entry for a model name, trying an exact match first
+        and then a substring fallback (historical behaviour). Returns the cost
+        dict, or None if no price is known."""
+        if not self.cost_config or not model_name:
+            return None
+        # Remove _batch suffix if present for cost lookup
+        cost_lookup_name = (
+            model_name[:-6] if model_name.endswith("_batch") else model_name
+        )
+        if cost_lookup_name in self.cost_config:
+            return self.cost_config[cost_lookup_name]
+        for config_name in self.cost_config.keys():
+            if config_name in cost_lookup_name or cost_lookup_name in config_name:
+                return self.cost_config[config_name]
+        return None
+
+    def has_price(self, model_name: str) -> bool:
+        """Whether a price is known for this model name. Used to decide whether a
+        provider-reported actual model (generated_result.model_used) can be
+        billed directly instead of the router's selected slug. See issue #166."""
+        return self._lookup_cost_info(model_name) is not None
+
     def calculate_inference_cost(
         self, model_name: str, token_usage: Dict[str, int]
     ) -> float:
@@ -222,17 +245,7 @@ def calculate_inference_cost(
         if model_name.endswith("_batch"):
             cost_lookup_name = model_name[:-6]  # Remove '_batch' suffix
 
-        # Use model name directly - assume model_cost.json keys match model names exactly
-        # Try to find exact match first
-        if cost_lookup_name in self.cost_config:
-            cost_info = self.cost_config[cost_lookup_name]
-        else:
-            # Try to find partial matches as fallback
-            cost_info = None
-            for config_name in self.cost_config.keys():
-                if config_name in cost_lookup_name or cost_lookup_name in config_name:
-                    cost_info = self.cost_config[config_name]
-                    break
+        cost_info = self._lookup_cost_info(model_name)
 
         if not cost_info:
             print(

llm_evaluation/run.py

@@ -425,12 +425,23 @@ def evaluate_single_prediction(
                 generated_answer, ground_truth, scorer_func, dataset_name
             )
 
-        # Calculate inference cost
-        # Use universal model name for cost lookup to respect user-defined mappings
+        # Calculate inference cost.
+        # Prefer the model the provider actually served (generated_result.model_used)
+        # when it has a known price, so retired/redirected slugs are billed at the
+        # model that truly answered rather than the requested alias. For example the
+        # retired `grok-4-1-fast-reasoning` slug is redirected by xAI to grok-4.3,
+        # which is both stronger and ~5-6x more expensive; billing it at the alias's
+        # old price understates cost. Fall back to the router's selected model
+        # (universal name, to respect universal_model_names.py mappings) when the
+        # provider did not report an actual model or its price is unknown. See #166.
         token_usage = generated_result.get("token_usage", {})
+        cost_model_name = universal_model_name
+        actual_model = generated_result.get("model_used")
+        if actual_model and evaluator.has_price(actual_model):
+            cost_model_name = actual_model
         inference_cost = evaluator.calculate_inference_cost(
-            universal_model_name,
-            token_usage,  # Use universal_model_name to respect mapping in universal_model_names.py
+            cost_model_name,
+            token_usage,
         )
 
         # Update the prediction with evaluation results

model_cost/model_cost.json

@@ -330,5 +330,13 @@
   "z-ai/glm-4.7": {
     "input_token_price_per_million": 0.4,
     "output_token_price_per_million": 1.5
+  },
+  "grok-4.3": {
+    "input_token_price_per_million": 1.25,
+    "output_token_price_per_million": 2.5
+  },
+  "x-ai/grok-4.3": {
+    "input_token_price_per_million": 1.25,
+    "output_token_price_per_million": 2.5
   }
 }

router_inference/check_config_prediction_files.py

@@ -221,6 +221,57 @@ def check_model_costs(
     return len(missing_costs) == 0, errors
 
 
+# Model slugs that upstream providers have retired and silently redirect to a
+# different (often stronger and pricier) model. A submission may still
+# legitimately select these via a provider that keeps hosting the original model
+# (e.g. Azure), so this is a warning rather than a hard failure: it reminds
+# submitters to record the actually-served model (generated_result.model_used)
+# so the evaluator prices the model that truly answered. See issue #166.
+RETIRED_SLUGS = {
+    "grok-4-1-fast-reasoning": "x-ai/grok-4.3 (xAI redirect after 2026-05-15)",
+    "grok-4-1-fast-non-reasoning": "x-ai/grok-4.3 (xAI redirect after 2026-05-15)",
+}
+
+
+def check_retired_slugs(predictions: List[Dict[str, Any]]) -> List[str]:
+    """
+    Warn when predictions select a provider-retired slug, especially when the
+    recorded actual model (generated_result.model_used) differs from the selected
+    slug. Returns a list of warning strings; it never fails the run. See #166.
+    """
+    warnings: List[str] = []
+    counts: Dict[str, int] = {}
+    redirected: Dict[str, int] = {}
+
+    for prediction in predictions:
+        slug = prediction.get("prediction")
+        if slug in RETIRED_SLUGS:
+            counts[slug] = counts.get(slug, 0) + 1
+            generated = prediction.get("generated_result") or {}
+            model_used = generated.get("model_used")
+            if model_used and model_used.split("/")[-1].lower() not in slug.lower():
+                redirected[slug] = redirected.get(slug, 0) + 1
+
+    for slug, count in counts.items():
+        msg = (
+            f"'{slug}' is a retired slug that providers redirect to "
+            f"{RETIRED_SLUGS[slug]}; {count} prediction(s) select it."
+        )
+        if redirected.get(slug):
+            msg += (
+                f" {redirected[slug]} row(s) recorded a different model_used, "
+                "confirming the redirect — these are priced at the actually-served model."
+            )
+        else:
+            msg += (
+                " Record generated_result.model_used with the actually-served model so "
+                "it is priced correctly, or select the resolved model explicitly."
+            )
+        warnings.append(msg)
+
+    return warnings
+
+
 def check_config_models(config: Dict[str, Any]) -> Tuple[bool, List[str]]:
     """
     Check that all model names in config can be found in ModelNameManager.
@@ -627,6 +678,21 @@ def main():
         all_valid = False
         errors_summary.append(f"Cost check error: {str(e)}")
 
+    # Check 5: Warn about retired/redirected model slugs (informational only)
+    print("\n[5] Checking for retired/redirected model slugs...")
+    try:
+        if predictions is not None:
+            slug_warnings = check_retired_slugs(predictions)
+            if slug_warnings:
+                for warning in slug_warnings:
+                    print(f"  ⚠ {warning}")
+            else:
+                print("✓ No retired/redirected model slugs detected")
+        else:
+            print("⚠ Skipping retired-slug check (predictions not loaded)")
+    except Exception as e:
+        print(f"⚠ Error checking retired slugs: {e}")
+
     # Final summary
     print("\n" + "=" * 80)
     if all_valid: