diff --git a/.gemini /config.yaml b/.gemini/config.yaml similarity index 100% rename from .gemini /config.yaml rename to .gemini/config.yaml diff --git a/.github/workflows /pre-commit.yml b/.github/workflows/pre-commit.yml similarity index 100% rename from .github/workflows /pre-commit.yml rename to .github/workflows/pre-commit.yml diff --git a/.gitignore b/.gitignore index c148ca3a..77b8d481 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ results/ # Logs *.log logs/ +/cached_results2/ # Environment files .venv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7de68a15..ae2c1891 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -81,6 +81,7 @@ repos: entry: tools/mypy.sh 1 "3.9" language: python types: [python] + pass_filenames: false additional_dependencies: *mypy_deps stages: [manual] # Only run in CI - id: mypy-3.10 @@ -88,6 +89,7 @@ repos: entry: tools/mypy.sh 1 "3.10" language: python types: [python] + pass_filenames: false additional_dependencies: *mypy_deps stages: [manual] # Only run in CI - id: mypy-3.11 @@ -95,6 +97,7 @@ repos: entry: tools/mypy.sh 1 "3.11" language: python types: [python] + pass_filenames: false additional_dependencies: *mypy_deps stages: [manual] # Only run in CI - id: mypy-3.12 @@ -102,5 +105,6 @@ repos: entry: tools/mypy.sh 1 "3.12" language: python types: [python] + pass_filenames: false additional_dependencies: *mypy_deps stages: [manual] # Only run in CI diff --git a/README.md b/README.md index c512cd21..7b8e39f6 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,13 @@ - Internet connection ### Step 1: Install uv (if you don't have it) + ```bash curl -LsSf https://astral.sh/uv/install.sh | sh ``` ### Step 2: Install RouterArena + ```bash cd RouterArena uv sync diff --git a/llm_evaluation/enhanced_extractor.py b/llm_evaluation/enhanced_extractor.py index 0c292c1a..d7439159 100644 --- a/llm_evaluation/enhanced_extractor.py +++ b/llm_evaluation/enhanced_extractor.py @@ -79,7 +79,7 @@ def has_boxed_pattern(self, text: str) -> bool: return False return bool(re.search(r"\\boxed\{", text)) - def extract_boxed_answer(self, text: str, dataset: str = None) -> str: + def extract_boxed_answer(self, text: str, dataset: Optional[str] = None) -> str: """ Enhanced version of extract_boxed_answer that handles multiple patterns. @@ -221,7 +221,9 @@ def _extract_standard_math_boxed(self, text: str) -> Optional[str]: return None - def _extract_enhanced_answer(self, text: str, dataset: str = None) -> Optional[str]: + def _extract_enhanced_answer( + self, text: str, dataset: Optional[str] = None + ) -> Optional[str]: """Extract answer using enhanced patterns""" # Special handling for LiveCodeBench if dataset == "LiveCodeBench": @@ -430,7 +432,7 @@ def _normalize_chess_move(self, move: str) -> str: # Drop-in replacement functions that maintain compatibility -def extract_boxed_answer(text: str, dataset: str = None) -> str: +def extract_boxed_answer(text: str, dataset: Optional[str] = None) -> str: """ Enhanced extract_boxed_answer function that automatically uses improved extraction for models with low \boxed{} usage. diff --git a/llm_evaluation/eval_reasoning.py b/llm_evaluation/eval_reasoning.py index e6fc7b82..73f3b312 100644 --- a/llm_evaluation/eval_reasoning.py +++ b/llm_evaluation/eval_reasoning.py @@ -3,6 +3,7 @@ import os import json +from typing import Any, Dict from metrics import ( mcq_exact_match, mcq_accuracy, @@ -201,8 +202,8 @@ def eval(pred_dir, eval_params, pipeline_config, all_data): scores: Dictionary of evaluation scores raw_results: Detailed results for each prediction """ - scores = dict() - all_raw_results = dict() + scores: Dict[str, float] = {} + all_raw_results: Dict[str, Dict[str, Any]] = {} # Get the appropriate scorers for this dataset and metrics dataset_name = eval_params["dataset"] diff --git a/llm_evaluation/evaluate_models.py b/llm_evaluation/evaluate_models.py index 4f13ed64..c132e0c2 100644 --- a/llm_evaluation/evaluate_models.py +++ b/llm_evaluation/evaluate_models.py @@ -31,6 +31,7 @@ try: from universal_model_names import ModelNameManager + model_name_manager: Optional[ModelNameManager] model_name_manager = ModelNameManager() except ImportError: print("Warning: Could not import ModelNameManager. Model name validation disabled.") @@ -79,10 +80,12 @@ class ModelEvaluator: def __init__(self, cached_results_dir: str = "../cached_results/"): self.cached_results_dir = cached_results_dir - self.all_data = None - self.dataset_configs = {} - self.existing_results = {} # Store existing results for incremental evaluation - self.cost_config = {} # Store cost configuration + self.all_data: Optional[List[Dict[str, Any]]] = None + self.dataset_configs: Dict[str, Dict[str, Any]] = {} + self.existing_results: Dict[ + str, Any + ] = {} # Store existing results for incremental evaluation + self.cost_config: Dict[str, Any] = {} # Store cost configuration # Load dataset configurations self.load_dataset_configs() @@ -238,7 +241,7 @@ def group_cached_results_by_dataset( self, cached_results: List[Dict] ) -> Dict[str, List[Dict]]: """Group cached results by dataset based on global_index.""" - dataset_groups = {} + dataset_groups: Dict[str, List[Dict[str, Any]]] = {} for entry in cached_results: global_index = entry.get("global_index", "") @@ -318,7 +321,7 @@ def evaluate_model(self, model_name: str, rerun=False) -> Dict[str, Any]: # Evaluate each dataset group evaluated_count = 0 - dataset_scores = {} + dataset_scores: Dict[str, int] = {} # Create progress bar for datasets dataset_progress = tqdm( @@ -344,14 +347,21 @@ def evaluate_model(self, model_name: str, rerun=False) -> Dict[str, Any]: # Evaluate each entry in this dataset for entry in dataset_entries: - global_index = entry.get("global_index") + global_index_val = entry.get("global_index") generated_answer = entry.get("generated_answer", "") try: # Get ground truth for this entry - ground_truth = self._get_ground_truth(global_index, dataset_name) + if not isinstance(global_index_val, str): + print( + f"Warning: Invalid global_index {global_index_val} for dataset {dataset_name}" + ) + continue + ground_truth = self._get_ground_truth( + global_index_val, dataset_name + ) if ground_truth is None: - print(f"Warning: No ground truth found for {global_index}") + print(f"Warning: No ground truth found for {global_index_val}") continue # Evaluate using the appropriate scorer @@ -391,7 +401,7 @@ def evaluate_model(self, model_name: str, rerun=False) -> Dict[str, Any]: "metric": "error", "inference_cost": 0.0, } - print(f"Error evaluating {global_index}: {e}") + print(f"Error evaluating {global_index_val}: {e}") continue dataset_scores[dataset_name] = len(dataset_entries) @@ -439,6 +449,8 @@ def _get_ground_truth(self, global_index: str, dataset_name: str) -> Optional[st return None # For other datasets, find the entry with matching global_index + if self.all_data is None: + return None for item in self.all_data: if ( item.get("global index") == global_index @@ -496,7 +508,7 @@ def _compile_final_results( avg_cost = total_cost / cost_count if cost_count > 0 else 0.0 # Group results by dataset for detailed reporting - dataset_results = {} + dataset_results: Dict[str, List[Dict[str, Any]]] = {} for entry in cached_results: if not entry.get("evaluation_result"): continue @@ -567,7 +579,10 @@ def main(): args = parser.parse_args() - universal_name = model_name_manager.get_universal_name(args.model_name) + if model_name_manager is not None: + universal_name = model_name_manager.get_universal_name(args.model_name) + else: + universal_name = args.model_name print(f"Input model name: {args.model_name}") print(f"Universal model name: {universal_name}") diff --git a/llm_evaluation/livecodebench_util.py b/llm_evaluation/livecodebench_util.py index 3408e0e3..304db062 100644 --- a/llm_evaluation/livecodebench_util.py +++ b/llm_evaluation/livecodebench_util.py @@ -20,7 +20,10 @@ import time import zlib from io import StringIO -from typing import Optional +from typing import Optional, Any, Dict + +# Global storage for original references used by reliability_guard +originals: Dict[str, Any] = {} def has_code(response): @@ -126,8 +129,7 @@ def post_process_tests_inputs(raw_text, is_stdin): # If no matches are found, fall back to line-by-line parsing cleaned_lines = cleaned_string.split("\n") - if test_cases is None: - test_cases = [] + test_cases = [] for line in cleaned_lines: try: test_case = json.loads(line) @@ -229,10 +231,9 @@ def prepare_test_input_output_std(test_case): def run_test_func(completion, is_extracted, test_input, test_output): # print(f"inside: {completion}") + # Define the namespace in which to execute the completion code + namespace: Dict[str, Any] = {} if not is_extracted: - # Define the namespace in which to execute the completion code - namespace = {} - # Execute the generated code in the namespace exec(completion, namespace) @@ -273,8 +274,6 @@ def run_test_func(completion, is_extracted, test_input, test_output): return True, result_output else: - namespace = {} - # Execute the generated code in the namespace exec(completion, namespace) @@ -313,7 +312,7 @@ def run_test_std(completion, test_input, test_output): # Simulate that the code is being run as the main script completion = '__name__ = "__main__"\n' + completion - namespace = {} + namespace: Dict[str, Any] = {} exec(completion, namespace) output_value = output.getvalue().strip() @@ -409,7 +408,7 @@ def swallow_io(redirect_input=True): with contextlib.redirect_stdout(stream), contextlib.redirect_stderr(stream): if redirect_input: - with contextlib.redirect_stdin(StringIO()): # Redirect stdin if enabled + with redirect_stdin(StringIO()): # Redirect stdin if enabled yield stream else: yield stream # Do not redirect stdin @@ -443,8 +442,18 @@ def readable(self, *args, **kwargs): return False -class redirect_stdin(contextlib._RedirectStream): # type: ignore - _stream = "stdin" +class redirect_stdin: + def __init__(self, new_target: Any): + self._new_target = new_target + self._old_target: Any = None + + def __enter__(self) -> Any: + self._old_target = sys.stdin + sys.stdin = self._new_target + return self._new_target + + def __exit__(self, exc_type, exc, tb) -> None: + sys.stdin = self._old_target @contextlib.contextmanager @@ -479,60 +488,68 @@ def reliability_guard(maximum_memory_bytes: Optional[int] = None): import builtins - builtins.exit = None - builtins.quit = None + from typing import Any - import os + # Prepare Any-typed aliases to avoid mypy assignment errors + builtins_mod: Any = builtins + os_mod: Any = os + shutil_mod: Any = shutil + subprocess_mod: Any = subprocess + modules_any: Any = sys.modules os.environ["OMP_NUM_THREADS"] = "1" - os.kill = None - os.system = None - os.putenv = None - os.remove = None - os.removedirs = None - os.rmdir = None - os.fchdir = None - os.setuid = None - os.fork = None - os.forkpty = None - os.killpg = None - os.rename = None - os.renames = None - os.truncate = None - os.replace = None - os.unlink = None - os.fchmod = None - os.fchown = None - os.chmod = None - os.chown = None - os.chroot = None - os.fchdir = None - os.lchflags = None - os.lchmod = None - os.lchown = None - os.getcwd = None - os.chdir = None - - import shutil - - shutil.rmtree = None - shutil.move = None - shutil.chown = None - - import subprocess - - subprocess.Popen = None # type: ignore - - # __builtins__["help"] = None # this line is commented out as it results into error - - import sys - - sys.modules["ipdb"] = None - sys.modules["joblib"] = None - sys.modules["resource"] = None - sys.modules["psutil"] = None - sys.modules["tkinter"] = None + # Disable selected builtins + setattr(builtins_mod, "exit", None) + setattr(builtins_mod, "quit", None) + + # Disable destructive os functions (guard where platform-specific) + for name in [ + "kill", + "system", + "putenv", + "remove", + "removedirs", + "rmdir", + "fchdir", + "setuid", + "fork", + "forkpty", + "killpg", + "rename", + "renames", + "truncate", + "replace", + "unlink", + "fchmod", + "fchown", + "chmod", + "chown", + "chroot", + "getcwd", + "chdir", + "lchflags", + "lchmod", + "lchown", + ]: + try: + setattr(os_mod, name, None) + except Exception: + pass + + # Disable dangerous shutil functions + for name in ["rmtree", "move", "chown"]: + try: + setattr(shutil_mod, name, None) + except Exception: + pass + + # Disable subprocess.Popen + setattr(subprocess_mod, "Popen", None) + + # Hide selected modules + for name in ["ipdb", "joblib", "resource", "psutil", "tkinter"]: + modules_any[name] = None def save_original_references(): @@ -598,7 +615,7 @@ def restore_original_references(): setattr(shutil, func_name, original_func) # Restore 'subprocess' functions - subprocess.Popen = originals["subprocess"]["Popen"] + setattr(subprocess, "Popen", originals["subprocess"]["Popen"]) # type: ignore[misc] # Restore sys modules for module_name, original_module in originals["sys_modules"].items(): diff --git a/llm_evaluation/metric_utils.py b/llm_evaluation/metric_utils.py index ca48f309..22e0784e 100644 --- a/llm_evaluation/metric_utils.py +++ b/llm_evaluation/metric_utils.py @@ -5,67 +5,66 @@ import regex from math import isclose +from typing import Any, Optional, List + from latex2sympy2 import latex2sympy from sympy import N, simplify from sympy.parsing.latex import parse_latex from sympy.parsing.sympy_parser import parse_expr -def choice_answer_clean(pred: str): +def choice_answer_clean(pred: str) -> str: """Helper function for standardizing multiple choice answers""" - pred = pred.strip("\n").rstrip(".").rstrip("/").strip(" ").lstrip(":") - # Clean the answer based on the dataset - tmp = re.findall(r"\b(A|B|C|D|E)\b", pred.upper()) - if tmp: - pred = tmp + cleaned = pred.strip("\n").rstrip(".").rstrip("/").strip(" ").lstrip(":") + matches: List[str] = re.findall(r"\b(A|B|C|D|E)\b", cleaned.upper()) + if matches: + result = matches[-1] else: - pred = [pred.strip().strip(".")] - pred = pred[-1] - # Remove the period at the end, again! - pred = pred.rstrip(".").rstrip("/") - return pred + result = cleaned.strip().strip(".") + result = result.rstrip(".").rstrip("/") + return result -def parse_digits(num): - num = regex.sub(",", "", str(num)) +def parse_digits(num: Any) -> Optional[float]: + normalized = regex.sub(",", "", str(num)) try: - return float(num) + return float(normalized) except Exception: - if num.endswith("%"): - num = num[:-1] - if num.endswith("\\"): - num = num[:-1] + if normalized.endswith("%"): + normalized = normalized[:-1] + if normalized.endswith("\\"): + normalized = normalized[:-1] try: - return float(num) / 100 + return float(normalized) / 100 except Exception: pass return None -def is_digit(num): +def is_digit(num: Any) -> bool: # paired with parse_digits return parse_digits(num) is not None -def numeric_equal(prediction: float, reference: float): +def numeric_equal(prediction: float, reference: float) -> bool: return isclose(reference, prediction, rel_tol=1e-4) -def str_to_pmatrix(input_str): - input_str = input_str.strip() - matrix_str = re.findall(r"\{.*,.*\}", input_str) - pmatrix_list = [] +def str_to_pmatrix(input_str: str) -> str: + stripped = input_str.strip() + matrix_str = re.findall(r"\{.*,.*\}", stripped) + pmatrix_list: List[str] = [] for m in matrix_str: - m = m.strip("{}") - pmatrix = r"\begin{pmatrix}" + m.replace(",", "\\") + r"\end{pmatrix}" + content = m.strip("{}") + pmatrix = r"\begin{pmatrix}" + content.replace(",", "\\") + r"\end{pmatrix}" pmatrix_list.append(pmatrix) return ", ".join(pmatrix_list) -def symbolic_equal(a, b): - def _parse(s): +def symbolic_equal(a: Any, b: Any) -> bool: + def _parse(s: Any) -> Any: for f in [parse_latex, parse_expr, latex2sympy]: try: return f(s.replace("\\\\", "\\")) diff --git a/llm_evaluation/metrics.py b/llm_evaluation/metrics.py index 82867f6b..014ddc76 100644 --- a/llm_evaluation/metrics.py +++ b/llm_evaluation/metrics.py @@ -130,7 +130,7 @@ def classification_score(prediction, ground_truth, **kwargs): score = 0.0 else: best_match = None - highest_similarity = 0 + highest_similarity = 0.0 for string in all_classes: similarity = difflib.SequenceMatcher(None, string, prediction).ratio() if similarity > highest_similarity: @@ -528,24 +528,25 @@ def math_equal( try: # 1. numerical equal if is_digit(prediction) and is_digit(reference): - prediction = parse_digits(prediction) - reference = parse_digits(reference) - # number questions - if include_percentage: - gt_result = [reference / 100, reference, reference * 100] - else: - gt_result = [reference] - for item in gt_result: - try: - if is_close: - if numeric_equal(prediction, item): - return True - else: - if item == prediction: - return True - except Exception: - continue - return False + pred_val = parse_digits(prediction) + ref_val = parse_digits(reference) + if pred_val is not None and ref_val is not None: + # number questions + if include_percentage: + gt_result: list[float] = [ref_val / 100, ref_val, ref_val * 100] + else: + gt_result = [ref_val] + for item in gt_result: + try: + if is_close: + if numeric_equal(pred_val, item): + return True + else: + if item == pred_val: + return True + except Exception: + continue + return False except Exception: pass diff --git a/llm_inference/model_inference.py b/llm_inference/model_inference.py index d0be9d2d..435529ed 100644 --- a/llm_inference/model_inference.py +++ b/llm_inference/model_inference.py @@ -104,6 +104,16 @@ def infer( backoff_time = 2**attempt time.sleep(backoff_time) # Exponential backoff + # Fallback failure result if all retries somehow did not return + return { + "response": "", + "error": "Inference did not return a result", + "success": False, + "token_usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + "provider": provider if "provider" in locals() else "unknown", + "model_used": model_name, + } + def _get_provider(self, model_name: str) -> str: """Determine the API provider based on model name.""" @@ -298,13 +308,24 @@ def _call_openrouter(self, model_name: str, prompt: str) -> Dict[str, Any]: model=model_name, messages=[{"role": "user", "content": prompt}] ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "openrouter", @@ -321,13 +342,24 @@ def _call_openai(self, model_name: str, prompt: str) -> Dict[str, Any]: messages=[{"role": "user", "content": prompt}], ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "openai", @@ -348,13 +380,24 @@ def _call_together(self, model_name: str, prompt: str) -> Dict[str, Any]: model=clean_model_name, messages=[{"role": "user", "content": prompt}] ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "together", @@ -374,14 +417,20 @@ def _call_anthropic(self, model_name: str, prompt: str) -> Dict[str, Any]: messages=[{"role": "user", "content": prompt}], ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "input_tokens", 0) if usage is not None else 0 + output_tokens = getattr(usage, "output_tokens", 0) if usage is not None else 0 + total_tokens = input_tokens + output_tokens + + content0 = response.content[0] + text = getattr(content0, "text", str(content0)) return { - "response": response.content[0].text, + "response": text, "success": True, "token_usage": { - "input_tokens": response.usage.input_tokens, - "output_tokens": response.usage.output_tokens, - "total_tokens": response.usage.input_tokens - + response.usage.output_tokens, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "anthropic", @@ -423,25 +472,41 @@ def _call_mistral(self, model_name: str, prompt: str) -> Dict[str, Any]: clean_model_name = model_name.replace("mistral/", "") + from typing import Any, cast + response = client.chat.complete( model=clean_model_name, - messages=[ - { - "role": "user", - "content": prompt, - } - ], + messages=cast( + Any, + [ + { + "role": "user", + "content": prompt, + } + ], + ), max_tokens=2048, temperature=0.7, ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "mistral", @@ -467,13 +532,24 @@ def _call_azure(self, model_name: str, prompt: str) -> Dict[str, Any]: temperature=0.7, ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "azure", @@ -497,13 +573,24 @@ def _call_deepseek(self, model_name: str, prompt: str) -> Dict[str, Any]: temperature=0.7, ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "deepseek", @@ -527,13 +614,24 @@ def _call_perplexity(self, model_name: str, prompt: str) -> Dict[str, Any]: temperature=0.7, ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "perplexity", diff --git a/llm_inference/pipeline.py b/llm_inference/pipeline.py index 1faab28e..4abbfc62 100644 --- a/llm_inference/pipeline.py +++ b/llm_inference/pipeline.py @@ -23,7 +23,7 @@ def load_jsonl_file(file_path: str) -> List[Dict[str, Any]]: Returns: List of dictionaries parsed from the JSONL file """ - results = [] + results: List[Dict[str, Any]] = [] if not os.path.exists(file_path): return results diff --git a/llm_inference/shared_utils.py b/llm_inference/shared_utils.py index aa138f27..3c74569b 100644 --- a/llm_inference/shared_utils.py +++ b/llm_inference/shared_utils.py @@ -12,6 +12,7 @@ from zoneinfo import ZoneInfo import datetime import logging +from typing import Any, Dict, List, Optional from main_utils import set_logger, lock_seed # Suppress the specific langchain warning @@ -20,7 +21,7 @@ ) -def setup_environment(): +def setup_environment() -> tuple[str, str]: """Set up the environment for LLM inference.""" current_dir = os.path.dirname(os.path.abspath(__file__)) base_dir = os.path.abspath(os.path.join(current_dir, "../")) @@ -39,7 +40,9 @@ def setup_environment(): return current_dir, base_dir -def setup_logging_and_seed(current_dir: str, model_name: str): +def setup_logging_and_seed( + current_dir: str, model_name: str +) -> tuple[logging.Logger, int]: """Set up logging and seed for reproducible results.""" SEED = 42 lock_seed(SEED) @@ -50,7 +53,7 @@ def setup_logging_and_seed(current_dir: str, model_name: str): return logger, SEED -def get_timezone_info(): +def get_timezone_info() -> tuple[ZoneInfo, datetime.datetime]: """Get timezone information for logging.""" ct_timezone = ZoneInfo("America/Chicago") start_time = datetime.datetime.now(ct_timezone) @@ -58,7 +61,7 @@ def get_timezone_info(): def log_pipeline_start( - logger, + logger: logging.Logger, pipeline_type: str, model_name: str, start_time: datetime.datetime, @@ -74,12 +77,12 @@ def log_pipeline_start( def log_pipeline_completion( - logger, + logger: logging.Logger, pipeline_type: str, model_name: str, start_time: datetime.datetime, end_time: datetime.datetime, - results: list = None, + results: Optional[List[Dict[str, Any]]] = None, ): """Log pipeline completion information.""" duration_minutes = (end_time - start_time).total_seconds() / 60 @@ -115,7 +118,7 @@ def create_result_entry( question: str, model_name: str, inference_result: dict, - additional_fields: dict = None, + additional_fields: Optional[Dict[str, Any]] = None, ) -> dict: """Create a standardized result entry.""" result_entry = { diff --git a/pyproject.toml b/pyproject.toml index 3de16e82..2806f0d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,37 +14,37 @@ dependencies = [ "google-generativeai>=0.8.5", "mistralai>=1.9.10", "together>=1.5.25", - + # Cloud Providers "boto3>=1.40.31", "botocore>=1.40.31", - + # Additional LLM Providers "xai-sdk>=1.2.0", "zhipuai>=2.1.5", "openrouter>=1.0", - + # Data Processing "datasets>=4.0.0", "pandas>=2.3.2", "numpy>=2.2.6", "pyarrow>=21.0.0", - + # Utilities "tqdm>=4.67.1", "tiktoken>=0.8.0", "requests>=2.32.5", "python-dotenv>=1.1.1", - + # Text Processing "regex>=2025.9.1", "jieba>=0.42.1", "fuzzywuzzy>=0.18.0", "python-Levenshtein>=0.21.0", - + # Evaluation Metrics "rouge>=1.0.1", - + # Hugging Face "huggingface-hub>=0.34.4", "transformers>=4.56.1", @@ -52,17 +52,17 @@ dependencies = [ "accelerate>=1.10.1", "safetensors>=0.6.2", "peft>=0.17.1", - + # Math and Symbolic Computation "sympy>=1.13.1", "latex2sympy2>=1.9.1", "mpmath>=1.3.0", - + # PyTorch (with CUDA support) "torch>=2.5.1", "torchvision>=0.19.1", "torchaudio>=2.4.1", - + # Other Dependencies "aiohttp>=3.12.4", "pydantic>=2.11.7", @@ -74,18 +74,18 @@ dependencies = [ "fsspec>=2025.3.0", "packaging>=25.0", "typing-extensions>=4.15.0", - + # Additional Utilities "rich>=14.1.0", "click>=8.1.8", "typer>=0.15.4", "python-multipart>=0.0.18", "sortedcontainers>=2.4.0", - + # Optimizations "optimum>=1.27.0", "sentencepiece>=0.2.1", - + # Special Dependencies "notdiamond>=0.4.5", "lxml>=6.0.1", @@ -99,3 +99,9 @@ packages = ["."] [tool.uv] dev-dependencies = [] + +[tool.mypy] +plugins = ['pydantic.mypy'] +ignore_missing_imports = true +check_untyped_defs = true +follow_imports = "silent" diff --git a/router_inference/compare_router_accuracy.py b/router_inference/compare_router_accuracy.py index 3924f06f..bf05f311 100644 --- a/router_inference/compare_router_accuracy.py +++ b/router_inference/compare_router_accuracy.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the RouterArena project # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, Tuple +from typing import Dict, Tuple, DefaultDict, Any import json import os from collections import defaultdict @@ -31,10 +31,12 @@ def build_complete_evaluation_dictionary() -> Dict[str, Dict[str, Tuple[float, f """ print("Building complete evaluation dictionary...") - results_dir = "./cached_results" - # cost_data = load_cost_data() + results_dir = "./cached_results2" + cost_data = load_cost_data() - evaluation_dict = defaultdict(dict) + evaluation_dict: DefaultDict[str, Dict[Any, Tuple[float, float]]] = defaultdict( + dict + ) # Get all evaluation result files if not os.path.exists(results_dir): @@ -45,6 +47,7 @@ def build_complete_evaluation_dictionary() -> Dict[str, Dict[str, Tuple[float, f for file_name in result_files: file_path = os.path.join(results_dir, file_name) + print(f"Processing file: {file_path}") # Load JSONL file (JSON Lines format) results = [] @@ -75,6 +78,33 @@ def build_complete_evaluation_dictionary() -> Dict[str, Dict[str, Tuple[float, f # Get cost for this inference inference_cost = result["evaluation_result"]["inference_cost"] + if inference_cost == 0.0: + # Handle different token naming conventions + # Token info is in result["token_usage"], not in evaluation_result + token_usage = result["token_usage"] + if token_usage is None: + continue + input_tokens = token_usage.get( + "input_tokens", token_usage.get("prompt_tokens", 0) + ) + output_tokens = token_usage.get( + "output_tokens", token_usage.get("completion_tokens", 0) + ) + + if input_tokens and output_tokens: + input_cost_per_million = cost_data[model_name][ + "input_token_price_per_million" + ] + output_cost_per_million = cost_data[model_name][ + "output_token_price_per_million" + ] + + inference_cost = ( + input_tokens / 1_000_000 + ) * input_cost_per_million + ( + output_tokens / 1_000_000 + ) * output_cost_per_million + # Get accuracy (score) accuracy = result["evaluation_result"]["score"] @@ -87,16 +117,16 @@ def build_complete_evaluation_dictionary() -> Dict[str, Dict[str, Tuple[float, f evaluation_dict[model_name][global_index] = (accuracy, inference_cost) # print(f" Added {len(extracted_results)} results for {model_name}") - print( - f" Added {len(extracted_results)} results for {model_name} (unvalidated: {unvalidated_count})" - ) + # print( + # f" Added {len(extracted_results)} results for {model_name} (unvalidated: {unvalidated_count})" + # ) total_pairs = sum(len(queries) for queries in evaluation_dict.values()) print( f"\nCompleted building evaluation dictionary with {total_pairs} total (model, global_index) pairs across {len(evaluation_dict)} models" ) - with open("./llm_evaluation_dict.json", "w") as f: + with open("./router_inference/llm_evaluation_dict.json", "w") as f: json.dump(dict(evaluation_dict), f) return dict(evaluation_dict) @@ -106,7 +136,7 @@ def load_predictions(router_name: str, config: Dict): print(f"Loading predictions for {router_name}.......") model_name_manager = ModelNameManager() - predictions_path = f"./router_inference/predictions/{router_name}.json" + predictions_path = f"./router_inference/predictions2/{router_name}.json" if not os.path.exists(predictions_path): print(f"Warning: Predictions file {predictions_path} not found") return {} @@ -187,7 +217,20 @@ def main(): print(f"Created mapping for {len(global_index_to_bloom_level)} entries") print(f"Sample mapping: {dict(list(global_index_to_bloom_level.items())[:5])}") - router_names = ["carrot"] + router_names = [ + "carrot", + "graphrouter", + "notdiamond", + "gpt5", + "azure", + "vllm", + "mirt_bert", + "nirt_bert", + "routellm", + "routerbench_knn", + "routerbench_mlp", + "RouterDC", + ] all_router_data = {} diff --git a/tools/mypy.sh b/tools/mypy.sh index 88cc75d6..ce6eb97d 100755 --- a/tools/mypy.sh +++ b/tools/mypy.sh @@ -24,12 +24,20 @@ run_mypy() { echo "Running mypy on $target" + # Build mypy arguments, adding --exclude only when non-empty. + local mypy_args=(--python-version "${PYTHON_VERSION}" --namespace-packages) + if [[ -n "${EXCLUDE_PATTERN}" ]]; then + mypy_args+=(--exclude "${EXCLUDE_PATTERN}") + fi + if [[ "$CI" -eq 1 ]]; then - # In CI, run mypy with full strictness. - mypy --python-version "${PYTHON_VERSION}" --namespace-packages --exclude "${EXCLUDE_PATTERN}" "$@" "$target" + # In CI, skip import checking to avoid heavy env deps + mypy_args+=(--ignore-missing-imports) + mypy "${mypy_args[@]}" "$@" "$target" else # Local runs are a bit more lenient and skip heavy import following. - mypy --follow-imports skip --python-version "${PYTHON_VERSION}" --namespace-packages --exclude "${EXCLUDE_PATTERN}" "$@" "$target" + mypy_args+=(--follow-imports skip) + mypy "${mypy_args[@]}" "$@" "$target" fi }