[Setup] Fix white space

.gitignore

@@ -10,6 +10,7 @@ results/
 # Logs
 *.log
 logs/
+/cached_results2/
 
 # Environment files
 .venv

.pre-commit-config.yaml

@@ -81,26 +81,30 @@ repos:
     entry: tools/mypy.sh 1 "3.9"
     language: python
     types: [python]
+    pass_filenames: false
     additional_dependencies: *mypy_deps
     stages: [manual] # Only run in CI
   - id: mypy-3.10
     name: Run mypy for Python 3.10
     entry: tools/mypy.sh 1 "3.10"
     language: python
     types: [python]
+    pass_filenames: false
     additional_dependencies: *mypy_deps
     stages: [manual] # Only run in CI
   - id: mypy-3.11
     name: Run mypy for Python 3.11
     entry: tools/mypy.sh 1 "3.11"
     language: python
     types: [python]
+    pass_filenames: false
     additional_dependencies: *mypy_deps
     stages: [manual] # Only run in CI
   - id: mypy-3.12
     name: Run mypy for Python 3.12
     entry: tools/mypy.sh 1 "3.12"
     language: python
     types: [python]
+    pass_filenames: false
     additional_dependencies: *mypy_deps
     stages: [manual] # Only run in CI

README.md

@@ -7,11 +7,13 @@
 - Internet connection
 
 ### Step 1: Install uv (if you don't have it)
+
 ```bash
 curl -LsSf https://astral.sh/uv/install.sh | sh
 ```
 
 ### Step 2: Install RouterArena
+
 ```bash
 cd RouterArena
 uv sync

llm_evaluation/enhanced_extractor.py

@@ -79,7 +79,7 @@ def has_boxed_pattern(self, text: str) -> bool:
             return False
         return bool(re.search(r"\\boxed\{", text))
 
-    def extract_boxed_answer(self, text: str, dataset: str = None) -> str:
+    def extract_boxed_answer(self, text: str, dataset: Optional[str] = None) -> str:
         """
         Enhanced version of extract_boxed_answer that handles multiple patterns.
 
@@ -221,7 +221,9 @@ def _extract_standard_math_boxed(self, text: str) -> Optional[str]:
 
         return None
 
-    def _extract_enhanced_answer(self, text: str, dataset: str = None) -> Optional[str]:
+    def _extract_enhanced_answer(
+        self, text: str, dataset: Optional[str] = None
+    ) -> Optional[str]:
         """Extract answer using enhanced patterns"""
         # Special handling for LiveCodeBench
         if dataset == "LiveCodeBench":
@@ -430,7 +432,7 @@ def _normalize_chess_move(self, move: str) -> str:
 
 
 # Drop-in replacement functions that maintain compatibility
-def extract_boxed_answer(text: str, dataset: str = None) -> str:
+def extract_boxed_answer(text: str, dataset: Optional[str] = None) -> str:
     """
     Enhanced extract_boxed_answer function that automatically uses improved extraction
     for models with low \boxed{} usage.

llm_evaluation/eval_reasoning.py

@@ -3,6 +3,7 @@
 
 import os
 import json
+from typing import Any, Dict
 from metrics import (
     mcq_exact_match,
     mcq_accuracy,
@@ -201,8 +202,8 @@ def eval(pred_dir, eval_params, pipeline_config, all_data):
         scores: Dictionary of evaluation scores
         raw_results: Detailed results for each prediction
     """
-    scores = dict()
-    all_raw_results = dict()
+    scores: Dict[str, float] = {}
+    all_raw_results: Dict[str, Dict[str, Any]] = {}
 
     # Get the appropriate scorers for this dataset and metrics
     dataset_name = eval_params["dataset"]

llm_evaluation/evaluate_models.py

@@ -31,6 +31,7 @@
 try:
     from universal_model_names import ModelNameManager
 
+    model_name_manager: Optional[ModelNameManager]
     model_name_manager = ModelNameManager()
 except ImportError:
     print("Warning: Could not import ModelNameManager. Model name validation disabled.")
@@ -79,10 +80,12 @@ class ModelEvaluator:
 
     def __init__(self, cached_results_dir: str = "../cached_results/"):
         self.cached_results_dir = cached_results_dir
-        self.all_data = None
-        self.dataset_configs = {}
-        self.existing_results = {}  # Store existing results for incremental evaluation
-        self.cost_config = {}  # Store cost configuration
+        self.all_data: Optional[List[Dict[str, Any]]] = None
+        self.dataset_configs: Dict[str, Dict[str, Any]] = {}
+        self.existing_results: Dict[
+            str, Any
+        ] = {}  # Store existing results for incremental evaluation
+        self.cost_config: Dict[str, Any] = {}  # Store cost configuration
 
         # Load dataset configurations
         self.load_dataset_configs()
@@ -238,7 +241,7 @@ def group_cached_results_by_dataset(
         self, cached_results: List[Dict]
     ) -> Dict[str, List[Dict]]:
         """Group cached results by dataset based on global_index."""
-        dataset_groups = {}
+        dataset_groups: Dict[str, List[Dict[str, Any]]] = {}
 
         for entry in cached_results:
             global_index = entry.get("global_index", "")
@@ -318,7 +321,7 @@ def evaluate_model(self, model_name: str, rerun=False) -> Dict[str, Any]:
 
         # Evaluate each dataset group
         evaluated_count = 0
-        dataset_scores = {}
+        dataset_scores: Dict[str, int] = {}
 
         # Create progress bar for datasets
         dataset_progress = tqdm(
@@ -344,14 +347,21 @@ def evaluate_model(self, model_name: str, rerun=False) -> Dict[str, Any]:
 
             # Evaluate each entry in this dataset
             for entry in dataset_entries:
-                global_index = entry.get("global_index")
+                global_index_val = entry.get("global_index")
                 generated_answer = entry.get("generated_answer", "")
 
                 try:
                     # Get ground truth for this entry
-                    ground_truth = self._get_ground_truth(global_index, dataset_name)
+                    if not isinstance(global_index_val, str):
+                        print(
+                            f"Warning: Invalid global_index {global_index_val} for dataset {dataset_name}"
+                        )
+                        continue
+                    ground_truth = self._get_ground_truth(
+                        global_index_val, dataset_name
+                    )
                     if ground_truth is None:
-                        print(f"Warning: No ground truth found for {global_index}")
+                        print(f"Warning: No ground truth found for {global_index_val}")
                         continue
 
                     # Evaluate using the appropriate scorer
@@ -391,7 +401,7 @@ def evaluate_model(self, model_name: str, rerun=False) -> Dict[str, Any]:
                         "metric": "error",
                         "inference_cost": 0.0,
                     }
-                    print(f"Error evaluating {global_index}: {e}")
+                    print(f"Error evaluating {global_index_val}: {e}")
                     continue
 
             dataset_scores[dataset_name] = len(dataset_entries)
@@ -439,6 +449,8 @@ def _get_ground_truth(self, global_index: str, dataset_name: str) -> Optional[st
                 return None
 
         # For other datasets, find the entry with matching global_index
+        if self.all_data is None:
+            return None
         for item in self.all_data:
             if (
                 item.get("global index") == global_index
@@ -496,7 +508,7 @@ def _compile_final_results(
         avg_cost = total_cost / cost_count if cost_count > 0 else 0.0
 
         # Group results by dataset for detailed reporting
-        dataset_results = {}
+        dataset_results: Dict[str, List[Dict[str, Any]]] = {}
         for entry in cached_results:
             if not entry.get("evaluation_result"):
                 continue
@@ -567,7 +579,10 @@ def main():
 
     args = parser.parse_args()
 
-    universal_name = model_name_manager.get_universal_name(args.model_name)
+    if model_name_manager is not None:
+        universal_name = model_name_manager.get_universal_name(args.model_name)
+    else:
+        universal_name = args.model_name
     print(f"Input model name: {args.model_name}")
     print(f"Universal model name: {universal_name}")