From 1e1cd9c1e778425395c464e7991dd661c8a60729 Mon Sep 17 00:00:00 2001 From: jiarong0907 Date: Sun, 26 Oct 2025 22:45:25 -0500 Subject: [PATCH 1/7] fix white space --- {.gemini => .gemini}/config.yaml | 0 .github/{workflows => workflows}/pre-commit.yml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {.gemini => .gemini}/config.yaml (100%) rename .github/{workflows => workflows}/pre-commit.yml (100%) diff --git a/.gemini /config.yaml b/.gemini/config.yaml similarity index 100% rename from .gemini /config.yaml rename to .gemini/config.yaml diff --git a/.github/workflows /pre-commit.yml b/.github/workflows/pre-commit.yml similarity index 100% rename from .github/workflows /pre-commit.yml rename to .github/workflows/pre-commit.yml From d8d9ff2fe52c79a2645efc4d645193c1daf772c8 Mon Sep 17 00:00:00 2001 From: jiarong0907 Date: Mon, 27 Oct 2025 10:56:06 -0500 Subject: [PATCH 2/7] try fixing CI errors --- .pre-commit-config.yaml | 4 ++++ tools/mypy.sh | 11 +++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7de68a15..ae2c1891 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -81,6 +81,7 @@ repos: entry: tools/mypy.sh 1 "3.9" language: python types: [python] + pass_filenames: false additional_dependencies: *mypy_deps stages: [manual] # Only run in CI - id: mypy-3.10 @@ -88,6 +89,7 @@ repos: entry: tools/mypy.sh 1 "3.10" language: python types: [python] + pass_filenames: false additional_dependencies: *mypy_deps stages: [manual] # Only run in CI - id: mypy-3.11 @@ -95,6 +97,7 @@ repos: entry: tools/mypy.sh 1 "3.11" language: python types: [python] + pass_filenames: false additional_dependencies: *mypy_deps stages: [manual] # Only run in CI - id: mypy-3.12 @@ -102,5 +105,6 @@ repos: entry: tools/mypy.sh 1 "3.12" language: python types: [python] + pass_filenames: false additional_dependencies: *mypy_deps stages: [manual] # Only run in CI diff --git a/tools/mypy.sh b/tools/mypy.sh index 88cc75d6..36ec1d3b 100755 --- a/tools/mypy.sh +++ b/tools/mypy.sh @@ -24,12 +24,19 @@ run_mypy() { echo "Running mypy on $target" + # Build mypy arguments, adding --exclude only when non-empty. + local mypy_args=(--python-version "${PYTHON_VERSION}" --namespace-packages) + if [[ -n "${EXCLUDE_PATTERN}" ]]; then + mypy_args+=(--exclude "${EXCLUDE_PATTERN}") + fi + if [[ "$CI" -eq 1 ]]; then # In CI, run mypy with full strictness. - mypy --python-version "${PYTHON_VERSION}" --namespace-packages --exclude "${EXCLUDE_PATTERN}" "$@" "$target" + mypy "${mypy_args[@]}" "$@" "$target" else # Local runs are a bit more lenient and skip heavy import following. - mypy --follow-imports skip --python-version "${PYTHON_VERSION}" --namespace-packages --exclude "${EXCLUDE_PATTERN}" "$@" "$target" + mypy_args+=(--follow-imports skip) + mypy "${mypy_args[@]}" "$@" "$target" fi } From cbc8ead88fd8d942da6e3fae05429cdabe087bf6 Mon Sep 17 00:00:00 2001 From: yl231 Date: Wed, 29 Oct 2025 16:00:46 -0500 Subject: [PATCH 3/7] types: mypy clean across repo --- llm_evaluation/enhanced_extractor.py | 6 +- llm_evaluation/eval_reasoning.py | 2 +- llm_evaluation/evaluate_models.py | 23 ++-- llm_evaluation/livecodebench_util.py | 78 ++++++------ llm_evaluation/metric_utils.py | 67 +++++----- llm_evaluation/metrics.py | 8 +- llm_evaluation/utils.py | 4 +- llm_inference/model_inference.py | 134 ++++++++++++++------ llm_inference/pipeline.py | 2 +- llm_inference/shared_utils.py | 17 +-- router_inference/compare_router_accuracy.py | 52 ++++++-- scripts/process_datasets/prep_datasets.py | 2 +- 12 files changed, 241 insertions(+), 154 deletions(-) diff --git a/llm_evaluation/enhanced_extractor.py b/llm_evaluation/enhanced_extractor.py index 0c292c1a..cb9fb77a 100644 --- a/llm_evaluation/enhanced_extractor.py +++ b/llm_evaluation/enhanced_extractor.py @@ -79,7 +79,7 @@ def has_boxed_pattern(self, text: str) -> bool: return False return bool(re.search(r"\\boxed\{", text)) - def extract_boxed_answer(self, text: str, dataset: str = None) -> str: + def extract_boxed_answer(self, text: str, dataset: Optional[str] = None) -> str: """ Enhanced version of extract_boxed_answer that handles multiple patterns. @@ -221,7 +221,7 @@ def _extract_standard_math_boxed(self, text: str) -> Optional[str]: return None - def _extract_enhanced_answer(self, text: str, dataset: str = None) -> Optional[str]: + def _extract_enhanced_answer(self, text: str, dataset: Optional[str] = None) -> Optional[str]: """Extract answer using enhanced patterns""" # Special handling for LiveCodeBench if dataset == "LiveCodeBench": @@ -430,7 +430,7 @@ def _normalize_chess_move(self, move: str) -> str: # Drop-in replacement functions that maintain compatibility -def extract_boxed_answer(text: str, dataset: str = None) -> str: +def extract_boxed_answer(text: str, dataset: Optional[str] = None) -> str: """ Enhanced extract_boxed_answer function that automatically uses improved extraction for models with low \boxed{} usage. diff --git a/llm_evaluation/eval_reasoning.py b/llm_evaluation/eval_reasoning.py index e6fc7b82..74754f7a 100644 --- a/llm_evaluation/eval_reasoning.py +++ b/llm_evaluation/eval_reasoning.py @@ -14,7 +14,7 @@ superglue_exact_match, superglue_clozetest, ) -from datasets import load_from_disk +from datasets import load_from_disk # type: ignore[import-untyped] # Dataset to metric mapping dataset2metric = { diff --git a/llm_evaluation/evaluate_models.py b/llm_evaluation/evaluate_models.py index 4f13ed64..6af74ed8 100644 --- a/llm_evaluation/evaluate_models.py +++ b/llm_evaluation/evaluate_models.py @@ -18,7 +18,7 @@ import glob from typing import Dict, List, Any, Optional import sys -from tqdm import tqdm +from tqdm import tqdm # type: ignore[import-untyped] # Add the current directory to Python path to import eval modules sys.path.append(os.path.dirname(os.path.abspath(__file__))) @@ -31,6 +31,7 @@ try: from universal_model_names import ModelNameManager + model_name_manager: Optional[ModelNameManager] model_name_manager = ModelNameManager() except ImportError: print("Warning: Could not import ModelNameManager. Model name validation disabled.") @@ -79,10 +80,10 @@ class ModelEvaluator: def __init__(self, cached_results_dir: str = "../cached_results/"): self.cached_results_dir = cached_results_dir - self.all_data = None - self.dataset_configs = {} - self.existing_results = {} # Store existing results for incremental evaluation - self.cost_config = {} # Store cost configuration + self.all_data: Optional[List[Dict[str, Any]]] = None + self.dataset_configs: Dict[str, Dict[str, Any]] = {} + self.existing_results: Dict[str, Any] = {} # Store existing results for incremental evaluation + self.cost_config: Dict[str, Any] = {} # Store cost configuration # Load dataset configurations self.load_dataset_configs() @@ -95,8 +96,8 @@ def load_all_data(self): print("Loading ground truth data...") try: # Load data directly without LiveCodeBench dependency - from datasets import load_dataset - import pandas as pd + from datasets import load_dataset # type: ignore[import-untyped] + import pandas as pd # type: ignore[import-untyped] # Load the router eval benchmark dataset router_eval_bench = load_dataset("louielu02/RouterEvalBenchmark")["full"] @@ -238,7 +239,7 @@ def group_cached_results_by_dataset( self, cached_results: List[Dict] ) -> Dict[str, List[Dict]]: """Group cached results by dataset based on global_index.""" - dataset_groups = {} + dataset_groups: Dict[str, List[Dict[str, Any]]] = {} for entry in cached_results: global_index = entry.get("global_index", "") @@ -318,7 +319,7 @@ def evaluate_model(self, model_name: str, rerun=False) -> Dict[str, Any]: # Evaluate each dataset group evaluated_count = 0 - dataset_scores = {} + dataset_scores: Dict[str, int] = {} # Create progress bar for datasets dataset_progress = tqdm( @@ -439,6 +440,8 @@ def _get_ground_truth(self, global_index: str, dataset_name: str) -> Optional[st return None # For other datasets, find the entry with matching global_index + if self.all_data is None: + return None for item in self.all_data: if ( item.get("global index") == global_index @@ -496,7 +499,7 @@ def _compile_final_results( avg_cost = total_cost / cost_count if cost_count > 0 else 0.0 # Group results by dataset for detailed reporting - dataset_results = {} + dataset_results: Dict[str, List[Dict[str, Any]]] = {} for entry in cached_results: if not entry.get("evaluation_result"): continue diff --git a/llm_evaluation/livecodebench_util.py b/llm_evaluation/livecodebench_util.py index 3408e0e3..82d07912 100644 --- a/llm_evaluation/livecodebench_util.py +++ b/llm_evaluation/livecodebench_util.py @@ -479,60 +479,62 @@ def reliability_guard(maximum_memory_bytes: Optional[int] = None): import builtins - builtins.exit = None - builtins.quit = None + from typing import Any, cast + builtins.exit = cast(Any, None) # type: ignore[assignment] + builtins.quit = cast(Any, None) # type: ignore[assignment] import os os.environ["OMP_NUM_THREADS"] = "1" - os.kill = None - os.system = None - os.putenv = None - os.remove = None - os.removedirs = None - os.rmdir = None - os.fchdir = None - os.setuid = None - os.fork = None - os.forkpty = None - os.killpg = None - os.rename = None - os.renames = None - os.truncate = None - os.replace = None - os.unlink = None - os.fchmod = None - os.fchown = None - os.chmod = None - os.chown = None - os.chroot = None - os.fchdir = None - os.lchflags = None - os.lchmod = None - os.lchown = None - os.getcwd = None - os.chdir = None + os.kill = cast(Any, None) # type: ignore[assignment] + os.system = cast(Any, None) # type: ignore[assignment] + os.putenv = cast(Any, None) # type: ignore[assignment] + os.remove = cast(Any, None) # type: ignore[assignment] + os.removedirs = cast(Any, None) # type: ignore[assignment] + os.rmdir = cast(Any, None) # type: ignore[assignment] + os.fchdir = cast(Any, None) # type: ignore[assignment] + os.setuid = cast(Any, None) # type: ignore[assignment] + os.fork = cast(Any, None) # type: ignore[assignment] + os.forkpty = cast(Any, None) # type: ignore[assignment] + os.killpg = cast(Any, None) # type: ignore[assignment] + os.rename = cast(Any, None) # type: ignore[assignment] + os.renames = cast(Any, None) # type: ignore[assignment] + os.truncate = cast(Any, None) # type: ignore[assignment] + os.replace = cast(Any, None) # type: ignore[assignment] + os.unlink = cast(Any, None) # type: ignore[assignment] + os.fchmod = cast(Any, None) # type: ignore[assignment] + os.fchown = cast(Any, None) # type: ignore[assignment] + os.chmod = cast(Any, None) # type: ignore[assignment] + os.chown = cast(Any, None) # type: ignore[assignment] + os.chroot = cast(Any, None) # type: ignore[assignment] + os.fchdir = cast(Any, None) # type: ignore[assignment] + os.lchflags = cast(Any, None) # type: ignore[attr-defined,assignment] + os.lchmod = cast(Any, None) # type: ignore[attr-defined,assignment] + os.lchown = cast(Any, None) # type: ignore[assignment] + os.getcwd = cast(Any, None) # type: ignore[assignment] + os.chdir = cast(Any, None) # type: ignore[assignment] import shutil - shutil.rmtree = None - shutil.move = None - shutil.chown = None + shutil.rmtree = cast(Any, None) # type: ignore[assignment] + shutil.move = cast(Any, None) # type: ignore[assignment] + shutil.chown = cast(Any, None) # type: ignore[assignment] import subprocess - subprocess.Popen = None # type: ignore + import builtins as _builtins # local alias to avoid mypy confusion + setattr(subprocess, "Popen", cast(Any, None)) # type: ignore[misc] # __builtins__["help"] = None # this line is commented out as it results into error import sys - sys.modules["ipdb"] = None - sys.modules["joblib"] = None - sys.modules["resource"] = None - sys.modules["psutil"] = None - sys.modules["tkinter"] = None + sys.modules["ipdb"] = None # type: ignore[assignment] + sys.modules["joblib"] = None # type: ignore[assignment] + sys.modules["resource"] = None # type: ignore[assignment] + sys.modules["psutil"] = None # type: ignore[assignment] + sys.modules["tkinter"] = None # type: ignore[assignment] def save_original_references(): diff --git a/llm_evaluation/metric_utils.py b/llm_evaluation/metric_utils.py index ca48f309..c654995d 100644 --- a/llm_evaluation/metric_utils.py +++ b/llm_evaluation/metric_utils.py @@ -2,70 +2,69 @@ # SPDX-License-Identifier: Apache-2.0 import re -import regex +import regex # type: ignore[import-untyped] from math import isclose -from latex2sympy2 import latex2sympy -from sympy import N, simplify -from sympy.parsing.latex import parse_latex -from sympy.parsing.sympy_parser import parse_expr +from typing import Any, Optional, List +from latex2sympy2 import latex2sympy # type: ignore[import-untyped] +from sympy import N, simplify # type: ignore[import-untyped] +from sympy.parsing.latex import parse_latex # type: ignore[import-untyped] +from sympy.parsing.sympy_parser import parse_expr # type: ignore[import-untyped] -def choice_answer_clean(pred: str): + +def choice_answer_clean(pred: str) -> str: """Helper function for standardizing multiple choice answers""" - pred = pred.strip("\n").rstrip(".").rstrip("/").strip(" ").lstrip(":") - # Clean the answer based on the dataset - tmp = re.findall(r"\b(A|B|C|D|E)\b", pred.upper()) - if tmp: - pred = tmp + cleaned = pred.strip("\n").rstrip(".").rstrip("/").strip(" ").lstrip(":") + matches: List[str] = re.findall(r"\b(A|B|C|D|E)\b", cleaned.upper()) + if matches: + result = matches[-1] else: - pred = [pred.strip().strip(".")] - pred = pred[-1] - # Remove the period at the end, again! - pred = pred.rstrip(".").rstrip("/") - return pred + result = cleaned.strip().strip(".") + result = result.rstrip(".").rstrip("/") + return result -def parse_digits(num): - num = regex.sub(",", "", str(num)) +def parse_digits(num: Any) -> Optional[float]: + normalized = regex.sub(",", "", str(num)) try: - return float(num) + return float(normalized) except Exception: - if num.endswith("%"): - num = num[:-1] - if num.endswith("\\"): - num = num[:-1] + if normalized.endswith("%"): + normalized = normalized[:-1] + if normalized.endswith("\\"): + normalized = normalized[:-1] try: - return float(num) / 100 + return float(normalized) / 100 except Exception: pass return None -def is_digit(num): +def is_digit(num: Any) -> bool: # paired with parse_digits return parse_digits(num) is not None -def numeric_equal(prediction: float, reference: float): +def numeric_equal(prediction: float, reference: float) -> bool: return isclose(reference, prediction, rel_tol=1e-4) -def str_to_pmatrix(input_str): - input_str = input_str.strip() - matrix_str = re.findall(r"\{.*,.*\}", input_str) - pmatrix_list = [] +def str_to_pmatrix(input_str: str) -> str: + stripped = input_str.strip() + matrix_str = re.findall(r"\{.*,.*\}", stripped) + pmatrix_list: List[str] = [] for m in matrix_str: - m = m.strip("{}") - pmatrix = r"\begin{pmatrix}" + m.replace(",", "\\") + r"\end{pmatrix}" + content = m.strip("{}") + pmatrix = r"\begin{pmatrix}" + content.replace(",", "\\") + r"\end{pmatrix}" pmatrix_list.append(pmatrix) return ", ".join(pmatrix_list) -def symbolic_equal(a, b): - def _parse(s): +def symbolic_equal(a: Any, b: Any) -> bool: + def _parse(s: Any) -> Any: for f in [parse_latex, parse_expr, latex2sympy]: try: return f(s.replace("\\\\", "\\")) diff --git a/llm_evaluation/metrics.py b/llm_evaluation/metrics.py index 82867f6b..faf29c7b 100644 --- a/llm_evaluation/metrics.py +++ b/llm_evaluation/metrics.py @@ -6,13 +6,13 @@ import json import copy -import jieba -from fuzzywuzzy import fuzz +import jieba # type: ignore[import-untyped] +from fuzzywuzzy import fuzz # type: ignore[import-untyped] import difflib from collections import Counter -from rouge import Rouge -import regex +from rouge import Rouge # type: ignore[import-untyped] +import regex # type: ignore[import-untyped] from metric_utils import ( choice_answer_clean, diff --git a/llm_evaluation/utils.py b/llm_evaluation/utils.py index 8d336a19..ca6f028f 100644 --- a/llm_evaluation/utils.py +++ b/llm_evaluation/utils.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 import json -import pandas as pd -from datasets import load_from_disk, load_dataset +import pandas as pd # type: ignore[import-untyped] +from datasets import load_from_disk, load_dataset # type: ignore[import-untyped] def escape_format_braces(text): diff --git a/llm_inference/model_inference.py b/llm_inference/model_inference.py index d0be9d2d..1f2cdfbe 100644 --- a/llm_inference/model_inference.py +++ b/llm_inference/model_inference.py @@ -10,7 +10,7 @@ import json import time import logging -from typing import Dict, Any +from typing import Dict, Any, Optional from openai import OpenAI import tiktoken @@ -104,6 +104,16 @@ def infer( backoff_time = 2**attempt time.sleep(backoff_time) # Exponential backoff + # Fallback failure result if all retries somehow did not return + return { + "response": "", + "error": "Inference did not return a result", + "success": False, + "token_usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + "provider": provider if 'provider' in locals() else "unknown", + "model_used": model_name, + } + def _get_provider(self, model_name: str) -> str: """Determine the API provider based on model name.""" @@ -201,8 +211,8 @@ def _get_provider(self, model_name: str) -> str: def _call_xai(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call XAI API.""" - from xai_sdk import Client - from xai_sdk.chat import user, system + from xai_sdk import Client # type: ignore[import-untyped] + from xai_sdk.chat import user, system # type: ignore[import-untyped] client = Client( api_key=os.getenv("XAI_API_KEY"), @@ -229,7 +239,7 @@ def _call_xai(self, model_name: str, prompt: str) -> Dict[str, Any]: } def _call_zhipu(self, model_name: str, prompt: str) -> Dict[str, Any]: - from zhipuai import ZhipuAI + from zhipuai import ZhipuAI # type: ignore[import-untyped] client = ZhipuAI(api_key=os.getenv("ZHIPU_API_KEY")) @@ -254,7 +264,7 @@ def _call_zhipu(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_replicate(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call Replicate API.""" - import replicate + import replicate # type: ignore[import-not-found] client = replicate.Client(api_token=self.replicate_api_key) @@ -298,13 +308,18 @@ def _call_openrouter(self, model_name: str, prompt: str) -> Dict[str, Any]: model=model_name, messages=[{"role": "user", "content": prompt}] ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 + total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "openrouter", @@ -312,7 +327,7 @@ def _call_openrouter(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_openai(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call OpenAI API.""" - import openai + import openai # type: ignore[import-untyped] client = openai.OpenAI(api_key=self.openai_api_key) @@ -321,13 +336,18 @@ def _call_openai(self, model_name: str, prompt: str) -> Dict[str, Any]: messages=[{"role": "user", "content": prompt}], ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 + total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "openai", @@ -335,7 +355,7 @@ def _call_openai(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_together(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call Together AI API.""" - import together + import together # type: ignore[import-untyped] client = together.Together(api_key=self.together_api_key) @@ -348,13 +368,18 @@ def _call_together(self, model_name: str, prompt: str) -> Dict[str, Any]: model=clean_model_name, messages=[{"role": "user", "content": prompt}] ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 + total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "together", @@ -362,7 +387,7 @@ def _call_together(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_anthropic(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call Anthropic API.""" - import anthropic + import anthropic # type: ignore[import-untyped] client = anthropic.Anthropic(api_key=self.anthropic_api_key) @@ -374,14 +399,20 @@ def _call_anthropic(self, model_name: str, prompt: str) -> Dict[str, Any]: messages=[{"role": "user", "content": prompt}], ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "input_tokens", 0) if usage is not None else 0 + output_tokens = getattr(usage, "output_tokens", 0) if usage is not None else 0 + total_tokens = input_tokens + output_tokens + + content0 = response.content[0] + text = getattr(content0, "text", str(content0)) return { - "response": response.content[0].text, + "response": text, "success": True, "token_usage": { - "input_tokens": response.usage.input_tokens, - "output_tokens": response.usage.output_tokens, - "total_tokens": response.usage.input_tokens - + response.usage.output_tokens, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "anthropic", @@ -389,7 +420,7 @@ def _call_anthropic(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_google(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call Google AI API.""" - import google.generativeai as genai + import google.generativeai as genai # type: ignore[import-untyped] genai.configure(api_key=self.google_api_key) @@ -417,31 +448,37 @@ def _call_google(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_mistral(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call Mistral AI API.""" - from mistralai import Mistral + from mistralai import Mistral # type: ignore[import-untyped] client = Mistral(api_key=self.mistral_api_key) clean_model_name = model_name.replace("mistral/", "") + from typing import Any, cast response = client.chat.complete( model=clean_model_name, - messages=[ + messages=cast(Any, [ { "role": "user", "content": prompt, } - ], + ]), max_tokens=2048, temperature=0.7, ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 + total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "mistral", @@ -449,7 +486,7 @@ def _call_mistral(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_azure(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call Azure OpenAI API.""" - import openai + import openai # type: ignore[import-untyped] client = openai.AzureOpenAI( api_key=self.azure_api_key, @@ -467,13 +504,18 @@ def _call_azure(self, model_name: str, prompt: str) -> Dict[str, Any]: temperature=0.7, ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 + total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "azure", @@ -497,13 +539,18 @@ def _call_deepseek(self, model_name: str, prompt: str) -> Dict[str, Any]: temperature=0.7, ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 + total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "deepseek", @@ -527,13 +574,18 @@ def _call_perplexity(self, model_name: str, prompt: str) -> Dict[str, Any]: temperature=0.7, ) + usage = getattr(response, "usage", None) + input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 + completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 + total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + return { "response": response.choices[0].message.content, "success": True, "token_usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "input_tokens": input_tokens, + "output_tokens": completion_tokens, + "total_tokens": total_tokens, }, "model_used": model_name, "provider": "perplexity", @@ -541,8 +593,8 @@ def _call_perplexity(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_aws(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call AWS Bedrock API.""" - import boto3 - from botocore.exceptions import ClientError + import boto3 # type: ignore[import-untyped] + from botocore.exceptions import ClientError # type: ignore[import-untyped] # Map model names to their inference profile ARNs model_arn_mapping = { diff --git a/llm_inference/pipeline.py b/llm_inference/pipeline.py index 1faab28e..4abbfc62 100644 --- a/llm_inference/pipeline.py +++ b/llm_inference/pipeline.py @@ -23,7 +23,7 @@ def load_jsonl_file(file_path: str) -> List[Dict[str, Any]]: Returns: List of dictionaries parsed from the JSONL file """ - results = [] + results: List[Dict[str, Any]] = [] if not os.path.exists(file_path): return results diff --git a/llm_inference/shared_utils.py b/llm_inference/shared_utils.py index aa138f27..faf15da1 100644 --- a/llm_inference/shared_utils.py +++ b/llm_inference/shared_utils.py @@ -12,6 +12,7 @@ from zoneinfo import ZoneInfo import datetime import logging +from typing import Any, Dict, List, Optional, Tuple from main_utils import set_logger, lock_seed # Suppress the specific langchain warning @@ -20,7 +21,7 @@ ) -def setup_environment(): +def setup_environment() -> tuple[str, str]: """Set up the environment for LLM inference.""" current_dir = os.path.dirname(os.path.abspath(__file__)) base_dir = os.path.abspath(os.path.join(current_dir, "../")) @@ -39,7 +40,7 @@ def setup_environment(): return current_dir, base_dir -def setup_logging_and_seed(current_dir: str, model_name: str): +def setup_logging_and_seed(current_dir: str, model_name: str) -> tuple[logging.Logger, int]: """Set up logging and seed for reproducible results.""" SEED = 42 lock_seed(SEED) @@ -50,7 +51,7 @@ def setup_logging_and_seed(current_dir: str, model_name: str): return logger, SEED -def get_timezone_info(): +def get_timezone_info() -> tuple[ZoneInfo, datetime.datetime]: """Get timezone information for logging.""" ct_timezone = ZoneInfo("America/Chicago") start_time = datetime.datetime.now(ct_timezone) @@ -58,7 +59,7 @@ def get_timezone_info(): def log_pipeline_start( - logger, + logger: logging.Logger, pipeline_type: str, model_name: str, start_time: datetime.datetime, @@ -74,12 +75,12 @@ def log_pipeline_start( def log_pipeline_completion( - logger, + logger: logging.Logger, pipeline_type: str, model_name: str, start_time: datetime.datetime, end_time: datetime.datetime, - results: list = None, + results: Optional[List[Dict[str, Any]]] = None, ): """Log pipeline completion information.""" duration_minutes = (end_time - start_time).total_seconds() / 60 @@ -115,7 +116,7 @@ def create_result_entry( question: str, model_name: str, inference_result: dict, - additional_fields: dict = None, + additional_fields: Optional[Dict[str, Any]] = None, ) -> dict: """Create a standardized result entry.""" result_entry = { @@ -172,7 +173,7 @@ def build_chat(tokenizer, prompt, chat_template): messages, tokenize=False, add_generation_prompt=True ) elif "longchat" in chat_template or "vicuna" in chat_template: - from fastchat.model import get_conversation_template + from fastchat.model import get_conversation_template # type: ignore[import-not-found] conv = get_conversation_template("vicuna") conv.append_message(conv.roles[0], prompt) diff --git a/router_inference/compare_router_accuracy.py b/router_inference/compare_router_accuracy.py index 3924f06f..ce27bc6f 100644 --- a/router_inference/compare_router_accuracy.py +++ b/router_inference/compare_router_accuracy.py @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the RouterArena project # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, Tuple +from calendar import c +from typing import Dict, Tuple, DefaultDict, Any import json import os from collections import defaultdict @@ -31,10 +32,10 @@ def build_complete_evaluation_dictionary() -> Dict[str, Dict[str, Tuple[float, f """ print("Building complete evaluation dictionary...") - results_dir = "./cached_results" - # cost_data = load_cost_data() + results_dir = "./cached_results2" + cost_data = load_cost_data() - evaluation_dict = defaultdict(dict) + evaluation_dict: DefaultDict[str, Dict[Any, Tuple[float, float]]] = defaultdict(dict) # Get all evaluation result files if not os.path.exists(results_dir): @@ -45,6 +46,7 @@ def build_complete_evaluation_dictionary() -> Dict[str, Dict[str, Tuple[float, f for file_name in result_files: file_path = os.path.join(results_dir, file_name) + print(f"Processing file: {file_path}") # Load JSONL file (JSON Lines format) results = [] @@ -75,6 +77,21 @@ def build_complete_evaluation_dictionary() -> Dict[str, Dict[str, Tuple[float, f # Get cost for this inference inference_cost = result["evaluation_result"]["inference_cost"] + if inference_cost == 0.0: + # Handle different token naming conventions + # Token info is in result["token_usage"], not in evaluation_result + token_usage = result["token_usage"] + if token_usage is None: + continue + input_tokens = token_usage.get("input_tokens", token_usage.get("prompt_tokens", 0)) + output_tokens = token_usage.get("output_tokens", token_usage.get("completion_tokens", 0)) + + if input_tokens and output_tokens: + input_cost_per_million = cost_data[model_name]["input_token_price_per_million"] + output_cost_per_million = cost_data[model_name]["output_token_price_per_million"] + + inference_cost = (input_tokens / 1_000_000) * input_cost_per_million + (output_tokens / 1_000_000) * output_cost_per_million + # Get accuracy (score) accuracy = result["evaluation_result"]["score"] @@ -87,16 +104,16 @@ def build_complete_evaluation_dictionary() -> Dict[str, Dict[str, Tuple[float, f evaluation_dict[model_name][global_index] = (accuracy, inference_cost) # print(f" Added {len(extracted_results)} results for {model_name}") - print( - f" Added {len(extracted_results)} results for {model_name} (unvalidated: {unvalidated_count})" - ) + # print( + # f" Added {len(extracted_results)} results for {model_name} (unvalidated: {unvalidated_count})" + # ) total_pairs = sum(len(queries) for queries in evaluation_dict.values()) print( f"\nCompleted building evaluation dictionary with {total_pairs} total (model, global_index) pairs across {len(evaluation_dict)} models" ) - with open("./llm_evaluation_dict.json", "w") as f: + with open("./router_inference/llm_evaluation_dict.json", "w") as f: json.dump(dict(evaluation_dict), f) return dict(evaluation_dict) @@ -106,7 +123,7 @@ def load_predictions(router_name: str, config: Dict): print(f"Loading predictions for {router_name}.......") model_name_manager = ModelNameManager() - predictions_path = f"./router_inference/predictions/{router_name}.json" + predictions_path = f"./router_inference/predictions2/{router_name}.json" if not os.path.exists(predictions_path): print(f"Warning: Predictions file {predictions_path} not found") return {} @@ -169,7 +186,7 @@ def main(): evaluation_dict = build_complete_evaluation_dictionary() # Load RouterEvalBench dataset and create global_index to bloom_level mapping - from datasets import load_dataset + from datasets import load_dataset # type: ignore[import-untyped] # Load the routerevalbench dataset from local path dataset_path = "./dataset/routerevalbench" @@ -187,7 +204,20 @@ def main(): print(f"Created mapping for {len(global_index_to_bloom_level)} entries") print(f"Sample mapping: {dict(list(global_index_to_bloom_level.items())[:5])}") - router_names = ["carrot"] + router_names = [ + "carrot", + "graphrouter", + "notdiamond", + "gpt5", + "azure", + "vllm", + "mirt_bert", + "nirt_bert", + "routellm", + "routerbench_knn", + "routerbench_mlp", + "RouterDC" + ] all_router_data = {} diff --git a/scripts/process_datasets/prep_datasets.py b/scripts/process_datasets/prep_datasets.py index 5eea721c..ea1a53d2 100644 --- a/scripts/process_datasets/prep_datasets.py +++ b/scripts/process_datasets/prep_datasets.py @@ -9,7 +9,7 @@ import zlib sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))) -from datasets import load_dataset +from datasets import load_dataset # type: ignore[import-untyped] save_dir = "./dataset/" From 4bfa15e2f916230dadd7cce0232a40267f544775 Mon Sep 17 00:00:00 2001 From: yl231 Date: Wed, 29 Oct 2025 21:26:50 -0500 Subject: [PATCH 4/7] style: apply ruff/ruff-format fixes --- .gitignore | 1 + llm_evaluation/enhanced_extractor.py | 4 +- llm_evaluation/eval_reasoning.py | 2 +- llm_evaluation/evaluate_models.py | 6 +- llm_evaluation/livecodebench_util.py | 2 +- llm_evaluation/metric_utils.py | 8 +- llm_evaluation/metrics.py | 6 +- llm_evaluation/utils.py | 2 +- llm_inference/model_inference.py | 90 ++++++++++++++++----- llm_inference/shared_utils.py | 6 +- router_inference/compare_router_accuracy.py | 33 +++++--- tools/mypy.sh | 3 +- 12 files changed, 115 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index c148ca3a..77b8d481 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ results/ # Logs *.log logs/ +/cached_results2/ # Environment files .venv diff --git a/llm_evaluation/enhanced_extractor.py b/llm_evaluation/enhanced_extractor.py index cb9fb77a..d7439159 100644 --- a/llm_evaluation/enhanced_extractor.py +++ b/llm_evaluation/enhanced_extractor.py @@ -221,7 +221,9 @@ def _extract_standard_math_boxed(self, text: str) -> Optional[str]: return None - def _extract_enhanced_answer(self, text: str, dataset: Optional[str] = None) -> Optional[str]: + def _extract_enhanced_answer( + self, text: str, dataset: Optional[str] = None + ) -> Optional[str]: """Extract answer using enhanced patterns""" # Special handling for LiveCodeBench if dataset == "LiveCodeBench": diff --git a/llm_evaluation/eval_reasoning.py b/llm_evaluation/eval_reasoning.py index 74754f7a..2cce294f 100644 --- a/llm_evaluation/eval_reasoning.py +++ b/llm_evaluation/eval_reasoning.py @@ -14,7 +14,7 @@ superglue_exact_match, superglue_clozetest, ) -from datasets import load_from_disk # type: ignore[import-untyped] +from datasets import load_from_disk # type: ignore[import-not-found,import-untyped] # Dataset to metric mapping dataset2metric = { diff --git a/llm_evaluation/evaluate_models.py b/llm_evaluation/evaluate_models.py index 6af74ed8..ba84311f 100644 --- a/llm_evaluation/evaluate_models.py +++ b/llm_evaluation/evaluate_models.py @@ -82,7 +82,9 @@ def __init__(self, cached_results_dir: str = "../cached_results/"): self.cached_results_dir = cached_results_dir self.all_data: Optional[List[Dict[str, Any]]] = None self.dataset_configs: Dict[str, Dict[str, Any]] = {} - self.existing_results: Dict[str, Any] = {} # Store existing results for incremental evaluation + self.existing_results: Dict[ + str, Any + ] = {} # Store existing results for incremental evaluation self.cost_config: Dict[str, Any] = {} # Store cost configuration # Load dataset configurations @@ -96,7 +98,7 @@ def load_all_data(self): print("Loading ground truth data...") try: # Load data directly without LiveCodeBench dependency - from datasets import load_dataset # type: ignore[import-untyped] + from datasets import load_dataset # type: ignore[import-not-found,import-untyped] import pandas as pd # type: ignore[import-untyped] # Load the router eval benchmark dataset diff --git a/llm_evaluation/livecodebench_util.py b/llm_evaluation/livecodebench_util.py index 82d07912..651bc38b 100644 --- a/llm_evaluation/livecodebench_util.py +++ b/llm_evaluation/livecodebench_util.py @@ -480,6 +480,7 @@ def reliability_guard(maximum_memory_bytes: Optional[int] = None): import builtins from typing import Any, cast + builtins.exit = cast(Any, None) # type: ignore[assignment] builtins.quit = cast(Any, None) # type: ignore[assignment] @@ -523,7 +524,6 @@ def reliability_guard(maximum_memory_bytes: Optional[int] = None): import subprocess - import builtins as _builtins # local alias to avoid mypy confusion setattr(subprocess, "Popen", cast(Any, None)) # type: ignore[misc] # __builtins__["help"] = None # this line is commented out as it results into error diff --git a/llm_evaluation/metric_utils.py b/llm_evaluation/metric_utils.py index c654995d..f54d822b 100644 --- a/llm_evaluation/metric_utils.py +++ b/llm_evaluation/metric_utils.py @@ -7,10 +7,10 @@ from typing import Any, Optional, List -from latex2sympy2 import latex2sympy # type: ignore[import-untyped] -from sympy import N, simplify # type: ignore[import-untyped] -from sympy.parsing.latex import parse_latex # type: ignore[import-untyped] -from sympy.parsing.sympy_parser import parse_expr # type: ignore[import-untyped] +from latex2sympy2 import latex2sympy # type: ignore[import-not-found,import-untyped] +from sympy import N, simplify # type: ignore[import-not-found,import-untyped] +from sympy.parsing.latex import parse_latex # type: ignore[import-not-found,import-untyped] +from sympy.parsing.sympy_parser import parse_expr # type: ignore[import-not-found,import-untyped] def choice_answer_clean(pred: str) -> str: diff --git a/llm_evaluation/metrics.py b/llm_evaluation/metrics.py index faf29c7b..3b52fac7 100644 --- a/llm_evaluation/metrics.py +++ b/llm_evaluation/metrics.py @@ -6,12 +6,12 @@ import json import copy -import jieba # type: ignore[import-untyped] -from fuzzywuzzy import fuzz # type: ignore[import-untyped] +import jieba # type: ignore[import-not-found,import-untyped] +from fuzzywuzzy import fuzz # type: ignore[import-not-found,import-untyped] import difflib from collections import Counter -from rouge import Rouge # type: ignore[import-untyped] +from rouge import Rouge # type: ignore[import-not-found,import-untyped] import regex # type: ignore[import-untyped] from metric_utils import ( diff --git a/llm_evaluation/utils.py b/llm_evaluation/utils.py index ca6f028f..cb112aac 100644 --- a/llm_evaluation/utils.py +++ b/llm_evaluation/utils.py @@ -3,7 +3,7 @@ import json import pandas as pd # type: ignore[import-untyped] -from datasets import load_from_disk, load_dataset # type: ignore[import-untyped] +from datasets import load_from_disk, load_dataset # type: ignore[import-not-found,import-untyped] def escape_format_braces(text): diff --git a/llm_inference/model_inference.py b/llm_inference/model_inference.py index 1f2cdfbe..1dccc157 100644 --- a/llm_inference/model_inference.py +++ b/llm_inference/model_inference.py @@ -10,7 +10,7 @@ import json import time import logging -from typing import Dict, Any, Optional +from typing import Dict, Any from openai import OpenAI import tiktoken @@ -110,7 +110,7 @@ def infer( "error": "Inference did not return a result", "success": False, "token_usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, - "provider": provider if 'provider' in locals() else "unknown", + "provider": provider if "provider" in locals() else "unknown", "model_used": model_name, } @@ -310,8 +310,14 @@ def _call_openrouter(self, model_name: str, prompt: str) -> Dict[str, Any]: usage = getattr(response, "usage", None) input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 - completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 - total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) return { "response": response.choices[0].message.content, @@ -338,8 +344,14 @@ def _call_openai(self, model_name: str, prompt: str) -> Dict[str, Any]: usage = getattr(response, "usage", None) input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 - completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 - total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) return { "response": response.choices[0].message.content, @@ -370,8 +382,14 @@ def _call_together(self, model_name: str, prompt: str) -> Dict[str, Any]: usage = getattr(response, "usage", None) input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 - completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 - total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) return { "response": response.choices[0].message.content, @@ -455,22 +473,32 @@ def _call_mistral(self, model_name: str, prompt: str) -> Dict[str, Any]: clean_model_name = model_name.replace("mistral/", "") from typing import Any, cast + response = client.chat.complete( model=clean_model_name, - messages=cast(Any, [ - { - "role": "user", - "content": prompt, - } - ]), + messages=cast( + Any, + [ + { + "role": "user", + "content": prompt, + } + ], + ), max_tokens=2048, temperature=0.7, ) usage = getattr(response, "usage", None) input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 - completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 - total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) return { "response": response.choices[0].message.content, @@ -506,8 +534,14 @@ def _call_azure(self, model_name: str, prompt: str) -> Dict[str, Any]: usage = getattr(response, "usage", None) input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 - completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 - total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) return { "response": response.choices[0].message.content, @@ -541,8 +575,14 @@ def _call_deepseek(self, model_name: str, prompt: str) -> Dict[str, Any]: usage = getattr(response, "usage", None) input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 - completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 - total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) return { "response": response.choices[0].message.content, @@ -576,8 +616,14 @@ def _call_perplexity(self, model_name: str, prompt: str) -> Dict[str, Any]: usage = getattr(response, "usage", None) input_tokens = getattr(usage, "prompt_tokens", 0) if usage is not None else 0 - completion_tokens = getattr(usage, "completion_tokens", 0) if usage is not None else 0 - total_tokens = getattr(usage, "total_tokens", 0) if usage is not None else input_tokens + completion_tokens + completion_tokens = ( + getattr(usage, "completion_tokens", 0) if usage is not None else 0 + ) + total_tokens = ( + getattr(usage, "total_tokens", 0) + if usage is not None + else input_tokens + completion_tokens + ) return { "response": response.choices[0].message.content, diff --git a/llm_inference/shared_utils.py b/llm_inference/shared_utils.py index faf15da1..4831f336 100644 --- a/llm_inference/shared_utils.py +++ b/llm_inference/shared_utils.py @@ -12,7 +12,7 @@ from zoneinfo import ZoneInfo import datetime import logging -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional from main_utils import set_logger, lock_seed # Suppress the specific langchain warning @@ -40,7 +40,9 @@ def setup_environment() -> tuple[str, str]: return current_dir, base_dir -def setup_logging_and_seed(current_dir: str, model_name: str) -> tuple[logging.Logger, int]: +def setup_logging_and_seed( + current_dir: str, model_name: str +) -> tuple[logging.Logger, int]: """Set up logging and seed for reproducible results.""" SEED = 42 lock_seed(SEED) diff --git a/router_inference/compare_router_accuracy.py b/router_inference/compare_router_accuracy.py index ce27bc6f..804a0602 100644 --- a/router_inference/compare_router_accuracy.py +++ b/router_inference/compare_router_accuracy.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the RouterArena project # SPDX-License-Identifier: Apache-2.0 -from calendar import c from typing import Dict, Tuple, DefaultDict, Any import json import os @@ -35,7 +34,9 @@ def build_complete_evaluation_dictionary() -> Dict[str, Dict[str, Tuple[float, f results_dir = "./cached_results2" cost_data = load_cost_data() - evaluation_dict: DefaultDict[str, Dict[Any, Tuple[float, float]]] = defaultdict(dict) + evaluation_dict: DefaultDict[str, Dict[Any, Tuple[float, float]]] = defaultdict( + dict + ) # Get all evaluation result files if not os.path.exists(results_dir): @@ -83,14 +84,26 @@ def build_complete_evaluation_dictionary() -> Dict[str, Dict[str, Tuple[float, f token_usage = result["token_usage"] if token_usage is None: continue - input_tokens = token_usage.get("input_tokens", token_usage.get("prompt_tokens", 0)) - output_tokens = token_usage.get("output_tokens", token_usage.get("completion_tokens", 0)) - + input_tokens = token_usage.get( + "input_tokens", token_usage.get("prompt_tokens", 0) + ) + output_tokens = token_usage.get( + "output_tokens", token_usage.get("completion_tokens", 0) + ) + if input_tokens and output_tokens: - input_cost_per_million = cost_data[model_name]["input_token_price_per_million"] - output_cost_per_million = cost_data[model_name]["output_token_price_per_million"] - - inference_cost = (input_tokens / 1_000_000) * input_cost_per_million + (output_tokens / 1_000_000) * output_cost_per_million + input_cost_per_million = cost_data[model_name][ + "input_token_price_per_million" + ] + output_cost_per_million = cost_data[model_name][ + "output_token_price_per_million" + ] + + inference_cost = ( + input_tokens / 1_000_000 + ) * input_cost_per_million + ( + output_tokens / 1_000_000 + ) * output_cost_per_million # Get accuracy (score) accuracy = result["evaluation_result"]["score"] @@ -216,7 +229,7 @@ def main(): "routellm", "routerbench_knn", "routerbench_mlp", - "RouterDC" + "RouterDC", ] all_router_data = {} diff --git a/tools/mypy.sh b/tools/mypy.sh index 36ec1d3b..ce6eb97d 100755 --- a/tools/mypy.sh +++ b/tools/mypy.sh @@ -31,7 +31,8 @@ run_mypy() { fi if [[ "$CI" -eq 1 ]]; then - # In CI, run mypy with full strictness. + # In CI, skip import checking to avoid heavy env deps + mypy_args+=(--ignore-missing-imports) mypy "${mypy_args[@]}" "$@" "$target" else # Local runs are a bit more lenient and skip heavy import following. From 316738f2c890d5a116d5e79250ff8c24b5830727 Mon Sep 17 00:00:00 2001 From: yl231 Date: Wed, 29 Oct 2025 21:35:52 -0500 Subject: [PATCH 5/7] ci: retrigger checks From 9237fc2ca7f09075a8279ddf34227da1c885197f Mon Sep 17 00:00:00 2001 From: yl231 Date: Wed, 29 Oct 2025 21:46:49 -0500 Subject: [PATCH 6/7] chore: sync README.md, pyproject.toml, uv.lock, .python-version with main; apply pre-commit fixes --- README.md | 2 ++ pyproject.toml | 26 +++++++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c512cd21..7b8e39f6 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,13 @@ - Internet connection ### Step 1: Install uv (if you don't have it) + ```bash curl -LsSf https://astral.sh/uv/install.sh | sh ``` ### Step 2: Install RouterArena + ```bash cd RouterArena uv sync diff --git a/pyproject.toml b/pyproject.toml index 3de16e82..9d2cc8bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,37 +14,37 @@ dependencies = [ "google-generativeai>=0.8.5", "mistralai>=1.9.10", "together>=1.5.25", - + # Cloud Providers "boto3>=1.40.31", "botocore>=1.40.31", - + # Additional LLM Providers "xai-sdk>=1.2.0", "zhipuai>=2.1.5", "openrouter>=1.0", - + # Data Processing "datasets>=4.0.0", "pandas>=2.3.2", "numpy>=2.2.6", "pyarrow>=21.0.0", - + # Utilities "tqdm>=4.67.1", "tiktoken>=0.8.0", "requests>=2.32.5", "python-dotenv>=1.1.1", - + # Text Processing "regex>=2025.9.1", "jieba>=0.42.1", "fuzzywuzzy>=0.18.0", "python-Levenshtein>=0.21.0", - + # Evaluation Metrics "rouge>=1.0.1", - + # Hugging Face "huggingface-hub>=0.34.4", "transformers>=4.56.1", @@ -52,17 +52,17 @@ dependencies = [ "accelerate>=1.10.1", "safetensors>=0.6.2", "peft>=0.17.1", - + # Math and Symbolic Computation "sympy>=1.13.1", "latex2sympy2>=1.9.1", "mpmath>=1.3.0", - + # PyTorch (with CUDA support) "torch>=2.5.1", "torchvision>=0.19.1", "torchaudio>=2.4.1", - + # Other Dependencies "aiohttp>=3.12.4", "pydantic>=2.11.7", @@ -74,18 +74,18 @@ dependencies = [ "fsspec>=2025.3.0", "packaging>=25.0", "typing-extensions>=4.15.0", - + # Additional Utilities "rich>=14.1.0", "click>=8.1.8", "typer>=0.15.4", "python-multipart>=0.0.18", "sortedcontainers>=2.4.0", - + # Optimizations "optimum>=1.27.0", "sentencepiece>=0.2.1", - + # Special Dependencies "notdiamond>=0.4.5", "lxml>=6.0.1", From e893355c294b13c6646b4bf2268f646386c86bb6 Mon Sep 17 00:00:00 2001 From: jiarong0907 Date: Thu, 30 Oct 2025 02:48:07 -0500 Subject: [PATCH 7/7] fix type check --- llm_evaluation/eval_reasoning.py | 7 +- llm_evaluation/evaluate_models.py | 26 ++-- llm_evaluation/livecodebench_util.py | 145 +++++++++++--------- llm_evaluation/metric_utils.py | 10 +- llm_evaluation/metrics.py | 47 +++---- llm_evaluation/utils.py | 4 +- llm_inference/model_inference.py | 24 ++-- llm_inference/shared_utils.py | 2 +- pyproject.toml | 6 + router_inference/compare_router_accuracy.py | 2 +- scripts/process_datasets/prep_datasets.py | 2 +- 11 files changed, 154 insertions(+), 121 deletions(-) diff --git a/llm_evaluation/eval_reasoning.py b/llm_evaluation/eval_reasoning.py index 2cce294f..73f3b312 100644 --- a/llm_evaluation/eval_reasoning.py +++ b/llm_evaluation/eval_reasoning.py @@ -3,6 +3,7 @@ import os import json +from typing import Any, Dict from metrics import ( mcq_exact_match, mcq_accuracy, @@ -14,7 +15,7 @@ superglue_exact_match, superglue_clozetest, ) -from datasets import load_from_disk # type: ignore[import-not-found,import-untyped] +from datasets import load_from_disk # Dataset to metric mapping dataset2metric = { @@ -201,8 +202,8 @@ def eval(pred_dir, eval_params, pipeline_config, all_data): scores: Dictionary of evaluation scores raw_results: Detailed results for each prediction """ - scores = dict() - all_raw_results = dict() + scores: Dict[str, float] = {} + all_raw_results: Dict[str, Dict[str, Any]] = {} # Get the appropriate scorers for this dataset and metrics dataset_name = eval_params["dataset"] diff --git a/llm_evaluation/evaluate_models.py b/llm_evaluation/evaluate_models.py index ba84311f..c132e0c2 100644 --- a/llm_evaluation/evaluate_models.py +++ b/llm_evaluation/evaluate_models.py @@ -18,7 +18,7 @@ import glob from typing import Dict, List, Any, Optional import sys -from tqdm import tqdm # type: ignore[import-untyped] +from tqdm import tqdm # Add the current directory to Python path to import eval modules sys.path.append(os.path.dirname(os.path.abspath(__file__))) @@ -98,8 +98,8 @@ def load_all_data(self): print("Loading ground truth data...") try: # Load data directly without LiveCodeBench dependency - from datasets import load_dataset # type: ignore[import-not-found,import-untyped] - import pandas as pd # type: ignore[import-untyped] + from datasets import load_dataset + import pandas as pd # Load the router eval benchmark dataset router_eval_bench = load_dataset("louielu02/RouterEvalBenchmark")["full"] @@ -347,14 +347,21 @@ def evaluate_model(self, model_name: str, rerun=False) -> Dict[str, Any]: # Evaluate each entry in this dataset for entry in dataset_entries: - global_index = entry.get("global_index") + global_index_val = entry.get("global_index") generated_answer = entry.get("generated_answer", "") try: # Get ground truth for this entry - ground_truth = self._get_ground_truth(global_index, dataset_name) + if not isinstance(global_index_val, str): + print( + f"Warning: Invalid global_index {global_index_val} for dataset {dataset_name}" + ) + continue + ground_truth = self._get_ground_truth( + global_index_val, dataset_name + ) if ground_truth is None: - print(f"Warning: No ground truth found for {global_index}") + print(f"Warning: No ground truth found for {global_index_val}") continue # Evaluate using the appropriate scorer @@ -394,7 +401,7 @@ def evaluate_model(self, model_name: str, rerun=False) -> Dict[str, Any]: "metric": "error", "inference_cost": 0.0, } - print(f"Error evaluating {global_index}: {e}") + print(f"Error evaluating {global_index_val}: {e}") continue dataset_scores[dataset_name] = len(dataset_entries) @@ -572,7 +579,10 @@ def main(): args = parser.parse_args() - universal_name = model_name_manager.get_universal_name(args.model_name) + if model_name_manager is not None: + universal_name = model_name_manager.get_universal_name(args.model_name) + else: + universal_name = args.model_name print(f"Input model name: {args.model_name}") print(f"Universal model name: {universal_name}") diff --git a/llm_evaluation/livecodebench_util.py b/llm_evaluation/livecodebench_util.py index 651bc38b..304db062 100644 --- a/llm_evaluation/livecodebench_util.py +++ b/llm_evaluation/livecodebench_util.py @@ -20,7 +20,10 @@ import time import zlib from io import StringIO -from typing import Optional +from typing import Optional, Any, Dict + +# Global storage for original references used by reliability_guard +originals: Dict[str, Any] = {} def has_code(response): @@ -126,8 +129,7 @@ def post_process_tests_inputs(raw_text, is_stdin): # If no matches are found, fall back to line-by-line parsing cleaned_lines = cleaned_string.split("\n") - if test_cases is None: - test_cases = [] + test_cases = [] for line in cleaned_lines: try: test_case = json.loads(line) @@ -229,10 +231,9 @@ def prepare_test_input_output_std(test_case): def run_test_func(completion, is_extracted, test_input, test_output): # print(f"inside: {completion}") + # Define the namespace in which to execute the completion code + namespace: Dict[str, Any] = {} if not is_extracted: - # Define the namespace in which to execute the completion code - namespace = {} - # Execute the generated code in the namespace exec(completion, namespace) @@ -273,8 +274,6 @@ def run_test_func(completion, is_extracted, test_input, test_output): return True, result_output else: - namespace = {} - # Execute the generated code in the namespace exec(completion, namespace) @@ -313,7 +312,7 @@ def run_test_std(completion, test_input, test_output): # Simulate that the code is being run as the main script completion = '__name__ = "__main__"\n' + completion - namespace = {} + namespace: Dict[str, Any] = {} exec(completion, namespace) output_value = output.getvalue().strip() @@ -409,7 +408,7 @@ def swallow_io(redirect_input=True): with contextlib.redirect_stdout(stream), contextlib.redirect_stderr(stream): if redirect_input: - with contextlib.redirect_stdin(StringIO()): # Redirect stdin if enabled + with redirect_stdin(StringIO()): # Redirect stdin if enabled yield stream else: yield stream # Do not redirect stdin @@ -443,8 +442,18 @@ def readable(self, *args, **kwargs): return False -class redirect_stdin(contextlib._RedirectStream): # type: ignore - _stream = "stdin" +class redirect_stdin: + def __init__(self, new_target: Any): + self._new_target = new_target + self._old_target: Any = None + + def __enter__(self) -> Any: + self._old_target = sys.stdin + sys.stdin = self._new_target + return self._new_target + + def __exit__(self, exc_type, exc, tb) -> None: + sys.stdin = self._old_target @contextlib.contextmanager @@ -479,62 +488,68 @@ def reliability_guard(maximum_memory_bytes: Optional[int] = None): import builtins - from typing import Any, cast - - builtins.exit = cast(Any, None) # type: ignore[assignment] - builtins.quit = cast(Any, None) # type: ignore[assignment] + from typing import Any - import os + # Prepare Any-typed aliases to avoid mypy assignment errors + builtins_mod: Any = builtins + os_mod: Any = os + shutil_mod: Any = shutil + subprocess_mod: Any = subprocess + modules_any: Any = sys.modules os.environ["OMP_NUM_THREADS"] = "1" - os.kill = cast(Any, None) # type: ignore[assignment] - os.system = cast(Any, None) # type: ignore[assignment] - os.putenv = cast(Any, None) # type: ignore[assignment] - os.remove = cast(Any, None) # type: ignore[assignment] - os.removedirs = cast(Any, None) # type: ignore[assignment] - os.rmdir = cast(Any, None) # type: ignore[assignment] - os.fchdir = cast(Any, None) # type: ignore[assignment] - os.setuid = cast(Any, None) # type: ignore[assignment] - os.fork = cast(Any, None) # type: ignore[assignment] - os.forkpty = cast(Any, None) # type: ignore[assignment] - os.killpg = cast(Any, None) # type: ignore[assignment] - os.rename = cast(Any, None) # type: ignore[assignment] - os.renames = cast(Any, None) # type: ignore[assignment] - os.truncate = cast(Any, None) # type: ignore[assignment] - os.replace = cast(Any, None) # type: ignore[assignment] - os.unlink = cast(Any, None) # type: ignore[assignment] - os.fchmod = cast(Any, None) # type: ignore[assignment] - os.fchown = cast(Any, None) # type: ignore[assignment] - os.chmod = cast(Any, None) # type: ignore[assignment] - os.chown = cast(Any, None) # type: ignore[assignment] - os.chroot = cast(Any, None) # type: ignore[assignment] - os.fchdir = cast(Any, None) # type: ignore[assignment] - os.lchflags = cast(Any, None) # type: ignore[attr-defined,assignment] - os.lchmod = cast(Any, None) # type: ignore[attr-defined,assignment] - os.lchown = cast(Any, None) # type: ignore[assignment] - os.getcwd = cast(Any, None) # type: ignore[assignment] - os.chdir = cast(Any, None) # type: ignore[assignment] - - import shutil - - shutil.rmtree = cast(Any, None) # type: ignore[assignment] - shutil.move = cast(Any, None) # type: ignore[assignment] - shutil.chown = cast(Any, None) # type: ignore[assignment] - - import subprocess - - setattr(subprocess, "Popen", cast(Any, None)) # type: ignore[misc] - - # __builtins__["help"] = None # this line is commented out as it results into error - - import sys - - sys.modules["ipdb"] = None # type: ignore[assignment] - sys.modules["joblib"] = None # type: ignore[assignment] - sys.modules["resource"] = None # type: ignore[assignment] - sys.modules["psutil"] = None # type: ignore[assignment] - sys.modules["tkinter"] = None # type: ignore[assignment] + # Disable selected builtins + setattr(builtins_mod, "exit", None) + setattr(builtins_mod, "quit", None) + + # Disable destructive os functions (guard where platform-specific) + for name in [ + "kill", + "system", + "putenv", + "remove", + "removedirs", + "rmdir", + "fchdir", + "setuid", + "fork", + "forkpty", + "killpg", + "rename", + "renames", + "truncate", + "replace", + "unlink", + "fchmod", + "fchown", + "chmod", + "chown", + "chroot", + "getcwd", + "chdir", + "lchflags", + "lchmod", + "lchown", + ]: + try: + setattr(os_mod, name, None) + except Exception: + pass + + # Disable dangerous shutil functions + for name in ["rmtree", "move", "chown"]: + try: + setattr(shutil_mod, name, None) + except Exception: + pass + + # Disable subprocess.Popen + setattr(subprocess_mod, "Popen", None) + + # Hide selected modules + for name in ["ipdb", "joblib", "resource", "psutil", "tkinter"]: + modules_any[name] = None def save_original_references(): @@ -600,7 +615,7 @@ def restore_original_references(): setattr(shutil, func_name, original_func) # Restore 'subprocess' functions - subprocess.Popen = originals["subprocess"]["Popen"] + setattr(subprocess, "Popen", originals["subprocess"]["Popen"]) # type: ignore[misc] # Restore sys modules for module_name, original_module in originals["sys_modules"].items(): diff --git a/llm_evaluation/metric_utils.py b/llm_evaluation/metric_utils.py index f54d822b..22e0784e 100644 --- a/llm_evaluation/metric_utils.py +++ b/llm_evaluation/metric_utils.py @@ -2,15 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 import re -import regex # type: ignore[import-untyped] +import regex from math import isclose from typing import Any, Optional, List -from latex2sympy2 import latex2sympy # type: ignore[import-not-found,import-untyped] -from sympy import N, simplify # type: ignore[import-not-found,import-untyped] -from sympy.parsing.latex import parse_latex # type: ignore[import-not-found,import-untyped] -from sympy.parsing.sympy_parser import parse_expr # type: ignore[import-not-found,import-untyped] +from latex2sympy2 import latex2sympy +from sympy import N, simplify +from sympy.parsing.latex import parse_latex +from sympy.parsing.sympy_parser import parse_expr def choice_answer_clean(pred: str) -> str: diff --git a/llm_evaluation/metrics.py b/llm_evaluation/metrics.py index 3b52fac7..014ddc76 100644 --- a/llm_evaluation/metrics.py +++ b/llm_evaluation/metrics.py @@ -6,13 +6,13 @@ import json import copy -import jieba # type: ignore[import-not-found,import-untyped] -from fuzzywuzzy import fuzz # type: ignore[import-not-found,import-untyped] +import jieba +from fuzzywuzzy import fuzz import difflib from collections import Counter -from rouge import Rouge # type: ignore[import-not-found,import-untyped] -import regex # type: ignore[import-untyped] +from rouge import Rouge +import regex from metric_utils import ( choice_answer_clean, @@ -130,7 +130,7 @@ def classification_score(prediction, ground_truth, **kwargs): score = 0.0 else: best_match = None - highest_similarity = 0 + highest_similarity = 0.0 for string in all_classes: similarity = difflib.SequenceMatcher(None, string, prediction).ratio() if similarity > highest_similarity: @@ -528,24 +528,25 @@ def math_equal( try: # 1. numerical equal if is_digit(prediction) and is_digit(reference): - prediction = parse_digits(prediction) - reference = parse_digits(reference) - # number questions - if include_percentage: - gt_result = [reference / 100, reference, reference * 100] - else: - gt_result = [reference] - for item in gt_result: - try: - if is_close: - if numeric_equal(prediction, item): - return True - else: - if item == prediction: - return True - except Exception: - continue - return False + pred_val = parse_digits(prediction) + ref_val = parse_digits(reference) + if pred_val is not None and ref_val is not None: + # number questions + if include_percentage: + gt_result: list[float] = [ref_val / 100, ref_val, ref_val * 100] + else: + gt_result = [ref_val] + for item in gt_result: + try: + if is_close: + if numeric_equal(pred_val, item): + return True + else: + if item == pred_val: + return True + except Exception: + continue + return False except Exception: pass diff --git a/llm_evaluation/utils.py b/llm_evaluation/utils.py index cb112aac..8d336a19 100644 --- a/llm_evaluation/utils.py +++ b/llm_evaluation/utils.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 import json -import pandas as pd # type: ignore[import-untyped] -from datasets import load_from_disk, load_dataset # type: ignore[import-not-found,import-untyped] +import pandas as pd +from datasets import load_from_disk, load_dataset def escape_format_braces(text): diff --git a/llm_inference/model_inference.py b/llm_inference/model_inference.py index 1dccc157..435529ed 100644 --- a/llm_inference/model_inference.py +++ b/llm_inference/model_inference.py @@ -211,8 +211,8 @@ def _get_provider(self, model_name: str) -> str: def _call_xai(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call XAI API.""" - from xai_sdk import Client # type: ignore[import-untyped] - from xai_sdk.chat import user, system # type: ignore[import-untyped] + from xai_sdk import Client + from xai_sdk.chat import user, system client = Client( api_key=os.getenv("XAI_API_KEY"), @@ -239,7 +239,7 @@ def _call_xai(self, model_name: str, prompt: str) -> Dict[str, Any]: } def _call_zhipu(self, model_name: str, prompt: str) -> Dict[str, Any]: - from zhipuai import ZhipuAI # type: ignore[import-untyped] + from zhipuai import ZhipuAI client = ZhipuAI(api_key=os.getenv("ZHIPU_API_KEY")) @@ -264,7 +264,7 @@ def _call_zhipu(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_replicate(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call Replicate API.""" - import replicate # type: ignore[import-not-found] + import replicate client = replicate.Client(api_token=self.replicate_api_key) @@ -333,7 +333,7 @@ def _call_openrouter(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_openai(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call OpenAI API.""" - import openai # type: ignore[import-untyped] + import openai client = openai.OpenAI(api_key=self.openai_api_key) @@ -367,7 +367,7 @@ def _call_openai(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_together(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call Together AI API.""" - import together # type: ignore[import-untyped] + import together client = together.Together(api_key=self.together_api_key) @@ -405,7 +405,7 @@ def _call_together(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_anthropic(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call Anthropic API.""" - import anthropic # type: ignore[import-untyped] + import anthropic client = anthropic.Anthropic(api_key=self.anthropic_api_key) @@ -438,7 +438,7 @@ def _call_anthropic(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_google(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call Google AI API.""" - import google.generativeai as genai # type: ignore[import-untyped] + import google.generativeai as genai genai.configure(api_key=self.google_api_key) @@ -466,7 +466,7 @@ def _call_google(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_mistral(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call Mistral AI API.""" - from mistralai import Mistral # type: ignore[import-untyped] + from mistralai import Mistral client = Mistral(api_key=self.mistral_api_key) @@ -514,7 +514,7 @@ def _call_mistral(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_azure(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call Azure OpenAI API.""" - import openai # type: ignore[import-untyped] + import openai client = openai.AzureOpenAI( api_key=self.azure_api_key, @@ -639,8 +639,8 @@ def _call_perplexity(self, model_name: str, prompt: str) -> Dict[str, Any]: def _call_aws(self, model_name: str, prompt: str) -> Dict[str, Any]: """Call AWS Bedrock API.""" - import boto3 # type: ignore[import-untyped] - from botocore.exceptions import ClientError # type: ignore[import-untyped] + import boto3 + from botocore.exceptions import ClientError # Map model names to their inference profile ARNs model_arn_mapping = { diff --git a/llm_inference/shared_utils.py b/llm_inference/shared_utils.py index 4831f336..3c74569b 100644 --- a/llm_inference/shared_utils.py +++ b/llm_inference/shared_utils.py @@ -175,7 +175,7 @@ def build_chat(tokenizer, prompt, chat_template): messages, tokenize=False, add_generation_prompt=True ) elif "longchat" in chat_template or "vicuna" in chat_template: - from fastchat.model import get_conversation_template # type: ignore[import-not-found] + from fastchat.model import get_conversation_template conv = get_conversation_template("vicuna") conv.append_message(conv.roles[0], prompt) diff --git a/pyproject.toml b/pyproject.toml index 9d2cc8bd..2806f0d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,3 +99,9 @@ packages = ["."] [tool.uv] dev-dependencies = [] + +[tool.mypy] +plugins = ['pydantic.mypy'] +ignore_missing_imports = true +check_untyped_defs = true +follow_imports = "silent" diff --git a/router_inference/compare_router_accuracy.py b/router_inference/compare_router_accuracy.py index 804a0602..bf05f311 100644 --- a/router_inference/compare_router_accuracy.py +++ b/router_inference/compare_router_accuracy.py @@ -199,7 +199,7 @@ def main(): evaluation_dict = build_complete_evaluation_dictionary() # Load RouterEvalBench dataset and create global_index to bloom_level mapping - from datasets import load_dataset # type: ignore[import-untyped] + from datasets import load_dataset # Load the routerevalbench dataset from local path dataset_path = "./dataset/routerevalbench" diff --git a/scripts/process_datasets/prep_datasets.py b/scripts/process_datasets/prep_datasets.py index ea1a53d2..5eea721c 100644 --- a/scripts/process_datasets/prep_datasets.py +++ b/scripts/process_datasets/prep_datasets.py @@ -9,7 +9,7 @@ import zlib sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))) -from datasets import load_dataset # type: ignore[import-untyped] +from datasets import load_dataset save_dir = "./dataset/"