From 1410d03cc9b4e4e0103dc58a5d8c067e46a6b4a5 Mon Sep 17 00:00:00 2001 From: "yijiang.li" Date: Wed, 22 Apr 2026 10:23:59 +0000 Subject: [PATCH 01/12] fix tsv multi image placeholder bug --- mmeval/data/tsv.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/mmeval/data/tsv.py b/mmeval/data/tsv.py index 7265caf..e2596ce 100644 --- a/mmeval/data/tsv.py +++ b/mmeval/data/tsv.py @@ -19,6 +19,7 @@ """, re.IGNORECASE | re.VERBOSE, ) +MEDIA_PLACEHOLDER_RE = re.compile(r"<(?:video|image)>") class TSVDataset(BaseDataset): """Dataset class for loading TSV files. @@ -125,6 +126,26 @@ def _process_sample(self, index: int) -> Dict[str, Any]: question = IMG_PLACEHOLDER_RE.sub("", question) elif media_list: question = f'{"" * len(media_list)} {question}'.strip() + + placeholder_count = len(MEDIA_PLACEHOLDER_RE.findall(question)) + media_count = len(media_list) + if placeholder_count > media_count: + # If prompt has more placeholders than media, keep only media_count + # placeholders and move them to the prompt prefix. + question_without_placeholders = MEDIA_PLACEHOLDER_RE.sub(" ", question) + question_without_placeholders = re.sub(r"\s+", " ", question_without_placeholders).strip() + placeholder_prefix = "" * media_count + question = f"{placeholder_prefix} {question_without_placeholders}".strip() if question_without_placeholders else placeholder_prefix + placeholder_count = media_count + + if placeholder_count < media_count: + eval_id = sample.get("eval-id", index) + question_preview = question.replace("\n", " ")[:200] + raise ValueError( + "Prompt/media mismatch for eval-id " + f"{eval_id}: placeholders={placeholder_count}, media={media_count}, " + f"question='{question_preview}'" + ) # Build options dict and choices list options = { From 53409bf5f228594fcccbf0915acddcaa0599f52c Mon Sep 17 00:00:00 2001 From: "yijiang.li" Date: Wed, 22 Apr 2026 10:28:53 +0000 Subject: [PATCH 02/12] update inference launch options and media placeholder checks Made-with: Cursor --- mmeval/data/base.py | 16 +++++++++++++++- mmeval/run.py | 12 ++++++++---- mmeval/utils/argparser.py | 3 ++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/mmeval/data/base.py b/mmeval/data/base.py index b1ad971..8b2bc45 100644 --- a/mmeval/data/base.py +++ b/mmeval/data/base.py @@ -208,8 +208,16 @@ def _process_messages(self, message_list: List[Dict], media_list: List = None) - except Exception as e: raise ValueError(f"{source} template rendering failed: {e}") - # Load media for this message's placeholders (zip auto-stops at shorter list) + # Validate and load media for this message's placeholders. placeholder_list = re.findall(r"<(video|image)>", prompt) + remaining_media = len(media_list) - media_idx + placeholder_count = len(placeholder_list) + if placeholder_count > remaining_media: + raise ValueError( + "Prompt/media mismatch while processing messages: " + f"message_index={len(processed_message_list)}, " + f"placeholders={placeholder_count}, remaining_media={remaining_media}" + ) processed_media_list = [] for placeholder, media in zip(placeholder_list, media_list[media_idx:]): if placeholder == "image": @@ -225,6 +233,12 @@ def _process_messages(self, message_list: List[Dict], media_list: List = None) - message["media"] = processed_media_list processed_message_list.append(message) + if media_idx != len(media_list): + raise ValueError( + "Prompt/media mismatch while processing messages: " + f"used_media={media_idx}, total_media={len(media_list)}" + ) + return processed_message_list def convert_circular(self, **kwargs) -> Any: diff --git a/mmeval/run.py b/mmeval/run.py index d7b6c1b..5eb809b 100644 --- a/mmeval/run.py +++ b/mmeval/run.py @@ -123,17 +123,21 @@ def append_args(ns): cmd.append(f"--no_{key}") else: cmd.extend([f"--{key}", str(val)]) - # Build the conda-run command + # Build the command + if args.no_conda: + cmd = [ + "python", os.path.join("mmeval/infer", infer_file), + ] # if infer_env is a directory, then use -p - if os.path.isdir(infer_env): + elif os.path.isdir(infer_env): cmd = [ - "conda", "run", "--no-capture-output", "-p", infer_env, + "conda", "run", "--no-capture-output", "-p", infer_env, "python", os.path.join("mmeval/infer", infer_file), ] # if infer_env is env name, then use -n else: cmd = [ - "conda", "run", "--no-capture-output", "-n", infer_env, + "conda", "run", "--no-capture-output", "-n", infer_env, "python", os.path.join("mmeval/infer", infer_file), ] diff --git a/mmeval/utils/argparser.py b/mmeval/utils/argparser.py index 60dcd91..b3efd36 100644 --- a/mmeval/utils/argparser.py +++ b/mmeval/utils/argparser.py @@ -72,7 +72,8 @@ class InferenceArguments: class ExperimentArguments: gpu_per_parallel: int = field(default=1, metadata={"help": "number of gpus per task"}) parallel_per_task: int = field(default=4, metadata={"help": "number of parallel tasks."}) - rank: int = field(default=-1, metadata={"help": "rank for parallel inference"}) + rank: int = field(default=-1, metadata={"help": "rank for parallel inference"}) + no_conda: bool = field(default=False, metadata={"help": "use current python env instead of conda"}) ARGUMENT_DATACLASSES = (ModelArguments, DataArguments, InferenceArguments, ExperimentArguments) From be4bd680a6e34f6ec0d1b70b6d375c5989603df1 Mon Sep 17 00:00:00 2001 From: "yijiang.li" Date: Tue, 28 Apr 2026 06:54:05 +0000 Subject: [PATCH 03/12] add scoring --- .gitignore | 5 +- mmeval/infer/qwenvl2d5.py | 12 + mmeval/registry.py | 2 +- mmeval/score.py | 38 +++ mmeval/scoring/__init__.py | 6 + mmeval/scoring/match/__init__.py | 18 ++ mmeval/scoring/match/base.py | 18 ++ mmeval/scoring/match/exact.py | 29 +++ mmeval/scoring/match/llm.py | 144 +++++++++++ mmeval/scoring/match/template.py | 82 +++++++ mmeval/scoring/pipeline.py | 277 ++++++++++++++++++++++ mmeval/scoring/report.py | 31 +++ mmeval/scoring/schema.py | 183 ++++++++++++++ mmeval/utils/argparser.py | 61 ++++- scripts/examples/run_evalkit.sh | 168 +++++++++++++ scripts/examples/run_score_evalkit_llm.sh | 79 ++++++ 16 files changed, 1146 insertions(+), 7 deletions(-) create mode 100644 mmeval/score.py create mode 100644 mmeval/scoring/__init__.py create mode 100644 mmeval/scoring/match/__init__.py create mode 100644 mmeval/scoring/match/base.py create mode 100644 mmeval/scoring/match/exact.py create mode 100644 mmeval/scoring/match/llm.py create mode 100644 mmeval/scoring/match/template.py create mode 100644 mmeval/scoring/pipeline.py create mode 100644 mmeval/scoring/report.py create mode 100644 mmeval/scoring/schema.py create mode 100644 scripts/examples/run_evalkit.sh create mode 100755 scripts/examples/run_score_evalkit_llm.sh diff --git a/.gitignore b/.gitignore index 7813c1d..7e2b518 100644 --- a/.gitignore +++ b/.gitignore @@ -172,4 +172,7 @@ cython_debug/ # PyPI configuration file .pypirc -mydata \ No newline at end of file +mydata +datasets +.claude +work_dirs \ No newline at end of file diff --git a/mmeval/infer/qwenvl2d5.py b/mmeval/infer/qwenvl2d5.py index 0c6ee5f..849d35a 100644 --- a/mmeval/infer/qwenvl2d5.py +++ b/mmeval/infer/qwenvl2d5.py @@ -11,6 +11,12 @@ from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs from mmeval.utils.scorer import IncrementalLMScorer, target_tokens +SYSTEM_PROMPT = ( + "You are a helpful assistant. When the user asks a question, your response must include two parts: " + "first, the reasoning process enclosed in ... tags, then the final answer enclosed in ... tags." + "Please provide a clear, concise response within ... tags that directly addresses the question." + "Example:\nThis is my reasoning.\n\n\nThis is my answer.\n.\n" +) class TaskRunner(Task): def __init__(self, args): @@ -43,6 +49,12 @@ def parse_input(self, message): media_list = message.get('media', []) messages = [ + { + "role": "system", + "content": [ + {"type": "text", "text": SYSTEM_PROMPT} + ], + }, { "role": "user", "content": [] diff --git a/mmeval/registry.py b/mmeval/registry.py index dfbf0b9..fa71b92 100644 --- a/mmeval/registry.py +++ b/mmeval/registry.py @@ -54,7 +54,7 @@ "qwenvl2": ["Qwen2-VL-2B-Instruct", "Qwen2-VL-7B-Instruct", "Qwen2-VL-72B-Instruct", "Qwen2-VL-2B-Instruct-AWQ", "Qwen2-VL-7B-Instruct-AWQ", "Qwen2-VL-72B-Instruct-AWQ", "Qwen2-VL-2B-Instruct-GPTQ-Int4", "Qwen2-VL-7B-Instruct-GPTQ-Int4", "Qwen2-VL-72B-Instruct-GPTQ-Int4"], - "qwenvl2d5": ["Qwen2.5-VL-3B-Instruct", "Qwen2.5-VL-7B-Instruct", "Qwen2.5-VL-32B-Instruct", "Qwen2.5-VL-72B-Instruct", + "qwenvl2d5": ["Qwen2.5-VL-3B-Instruct", "Qwen2.5-VL-7B-Instruct", "Qwen2.5-VL-32B-Instruct", "Qwen2.5-VL-72B-Instruct", "qwen25vl3b_2epochs_lr1e5_bs4_acc2_numgen4-merged", "qwen25vl3b_opsd_vlm-merged", "qwen25vl3b_opsd_vlm_swa_unsup-merged", "qwen25vl3b_opsd_vlm_unsup-merged", "Qwen2.5-VL-3B-Instruct-AWQ", "Qwen2.5-VL-7B-Instruct-AWQ", "Qwen2.5-VL-32B-Instruct-AWQ", "Qwen2.5-VL-72B-Instruct-AWQ"], "qwenvl2d5_omni": ["Qwen2.5-Omni-3B", "Qwen2.5-Omni-7B", "Qwen2.5-Omni-7B-GPTQ-Int4"], "qwen3_vl": ["Qwen3-VL-2B-Instruct", "Qwen3-VL-4B-Instruct", "Qwen3-VL-8B-Instruct", "Qwen3-VL-32B-Instruct", "Qwen3-VL-30B-A3B-Instruct", "Qwen3-VL-235B-A22B-Instruct", diff --git a/mmeval/score.py b/mmeval/score.py new file mode 100644 index 0000000..44aa040 --- /dev/null +++ b/mmeval/score.py @@ -0,0 +1,38 @@ +from mmeval.scoring.pipeline import discover_result_files, score_result_file +from mmeval.utils.argparser import parse_args + + +def run_score(args): + files = discover_result_files( + out_dir=args.out_dir, + pattern=args.score_result_glob, + recursive=args.score_scan_recursive, + ) + if not files: + raise RuntimeError(f"No result files found under {args.out_dir} with pattern {args.score_result_glob}") + + done = [] + failed = [] + for result_file in files: + try: + done.append(score_result_file(result_file, args)) + except Exception as exc: + failed.append({"result_file": result_file, "error": str(exc)}) + + done.sort(key=lambda x: x["result_file"]) + failed.sort(key=lambda x: x["result_file"]) + + for item in done: + print( + f"✅ [score] {item['result_file']} -> {item['score_file']} " + f"(acc={item['summary']['accuracy']:.4f}, resumed={item.get('resumed_count', 0)})" + ) + if failed: + for item in failed: + print(f"❌ [score] {item['result_file']}: {item['error']}") + raise SystemExit(1) + + +if __name__ == "__main__": + args = parse_args() + run_score(args) diff --git a/mmeval/scoring/__init__.py b/mmeval/scoring/__init__.py new file mode 100644 index 0000000..0a22a81 --- /dev/null +++ b/mmeval/scoring/__init__.py @@ -0,0 +1,6 @@ +from mmeval.scoring.pipeline import score_result_file, discover_result_files + +__all__ = [ + "score_result_file", + "discover_result_files", +] diff --git a/mmeval/scoring/match/__init__.py b/mmeval/scoring/match/__init__.py new file mode 100644 index 0000000..5e74a05 --- /dev/null +++ b/mmeval/scoring/match/__init__.py @@ -0,0 +1,18 @@ +from mmeval.scoring.match.base import MatchResult +from mmeval.scoring.match.exact import ExactMatcher +from mmeval.scoring.match.template import TemplateMatcher +from mmeval.scoring.match.llm import LLMJudgeMatcher + +MATCHER_REGISTRY = { + "exact": ExactMatcher, + "template": TemplateMatcher, + "llm": LLMJudgeMatcher, +} + +__all__ = [ + "MatchResult", + "ExactMatcher", + "TemplateMatcher", + "LLMJudgeMatcher", + "MATCHER_REGISTRY", +] diff --git a/mmeval/scoring/match/base.py b/mmeval/scoring/match/base.py new file mode 100644 index 0000000..46bdcf0 --- /dev/null +++ b/mmeval/scoring/match/base.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + + +@dataclass +class MatchResult: + is_match: bool + matched: Optional[str] = None + reason: Optional[str] = None + meta: Dict[str, Any] = field(default_factory=dict) + + +class BaseMatcher: + name = "base" + + def match(self, sample: Dict[str, Any], context: Dict[str, Any]) -> MatchResult: + raise NotImplementedError + diff --git a/mmeval/scoring/match/exact.py b/mmeval/scoring/match/exact.py new file mode 100644 index 0000000..8802524 --- /dev/null +++ b/mmeval/scoring/match/exact.py @@ -0,0 +1,29 @@ +from mmeval.scoring.match.base import BaseMatcher, MatchResult +from mmeval.scoring.schema import extract_mcq_option, normalize_for_exact, normalize_open_answer + + +class ExactMatcher(BaseMatcher): + name = "exact" + + def match(self, sample, context): + question_type = context["question_type"] + pred_raw = context["pred"] + gt_raw = context["gt"] + + if gt_raw is None or pred_raw is None: + return MatchResult(is_match=False) + + if question_type == "mcq": + candidates = context["options"] + pred_option = extract_mcq_option(pred_raw, candidates) + gt_option = extract_mcq_option(gt_raw, candidates) or normalize_for_exact(gt_raw).upper() + if pred_option and gt_option and pred_option == gt_option: + return MatchResult(is_match=True, matched=gt_option) + return MatchResult(is_match=False) + + pred_norm = normalize_open_answer(pred_raw) + gt_norm = normalize_open_answer(gt_raw) + if pred_norm and gt_norm and pred_norm == gt_norm: + return MatchResult(is_match=True, matched=str(gt_raw)) + return MatchResult(is_match=False) + diff --git a/mmeval/scoring/match/llm.py b/mmeval/scoring/match/llm.py new file mode 100644 index 0000000..410a317 --- /dev/null +++ b/mmeval/scoring/match/llm.py @@ -0,0 +1,144 @@ +import json +import os +import re +import time +from typing import Any, Dict + +from mmeval.scoring.match.base import BaseMatcher, MatchResult +from mmeval.scoring.schema import normalize_text + + +class LLMJudgeMatcher(BaseMatcher): + name = "llm" + + def __init__(self, args): + self.provider = (args.judge_provider or "openai").strip().lower() + self.model = args.judge_model + self.max_retry = max(1, int(args.judge_max_retry)) + self.temperature = float(args.judge_temperature) + self.include_reason = bool(args.judge_include_reason) + self.client = self._build_client() + if not self.model: + raise ValueError("`--judge_model` is required when llm matcher is enabled.") + + def _build_client(self): + try: + from openai import AzureOpenAI, OpenAI + except Exception as exc: + raise RuntimeError("openai package is required for llm-as-judge matcher.") from exc + + if self.provider == "azure_openai": + key = os.getenv("AZURE_OPENAI_KEY") + endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") + if not key or not endpoint: + raise ValueError("AZURE_OPENAI_KEY and AZURE_OPENAI_ENDPOINT are required for azure_openai judge.") + self.model = os.getenv("AZURE_OPENAI_DEPLOYNAME", self.model) + return AzureOpenAI( + api_key=key, + azure_endpoint=endpoint, + api_version="2024-02-15-preview", + ) + + key = os.getenv("OPENAI_API_KEY") + base_url = os.getenv("OPENAI_BASE_URL") + if not key: + raise ValueError("OPENAI_API_KEY is required for openai judge.") + kwargs = {"api_key": key} + if base_url: + kwargs["base_url"] = base_url + return OpenAI(**kwargs) + + def match(self, sample: Dict[str, Any], context: Dict[str, Any]) -> MatchResult: + # Keep more source detail for judge prompts: do light cleanup only. + pred = normalize_text(context["pred"], extract_boxed=False, strip_latex_commands=False) + gt = normalize_text(context["gt"], extract_boxed=False, strip_latex_commands=False) + question = normalize_text(context["question"], extract_boxed=False, strip_latex_commands=False) + question_type = context["question_type"] + + prompt = self._build_prompt(question_type, question, pred, gt, context) + messages = [ + {"role": "system", "content": "You are a strict evaluator for VLM outputs."}, + {"role": "user", "content": prompt}, + ] + + parsed = self._call_with_retry(messages) + if parsed is None: + return MatchResult(is_match=False, reason="llm_judge_failed") + + is_correct = int(parsed.get("is_correct", 0)) == 1 + reason = parsed.get("reason") if self.include_reason else None + if is_correct: + return MatchResult(is_match=True, matched=str(context["gt"]), reason=reason) + return MatchResult(is_match=False, matched=None, reason=reason) + + def _build_prompt(self, question_type, question, pred, gt, context): + options = context.get("option_text_map", {}) + options_text = "" + if question_type == "mcq" and options: + rendered = [f"{k}. {v}" for k, v in options.items()] + options_text = "\nOptions:\n" + "\n".join(rendered) + + reason_instruction = "Include a concise reason." if self.include_reason else "Set reason to an empty string." + return ( + "Judge whether MODEL_RESPONSE is correct given QUESTION and GROUND_TRUTH.\n" + f"QuestionType: {question_type}\n" + f"QUESTION: {question}\n" + f"{options_text}\n" + f"MODEL_RESPONSE: {pred}\n" + f"GROUND_TRUTH: {gt}\n\n" + "Return JSON only with schema:\n" + '{"is_correct": 0_or_1, "reason": "string"}\n' + f"{reason_instruction}\n" + "No markdown, no extra fields." + ) + + def _call_with_retry(self, messages): + for _ in range(self.max_retry): + try: + resp = self.client.chat.completions.create( + model=self.model, + messages=messages, + temperature=self.temperature, + ) + text = resp.choices[0].message.content or "" + parsed = self._parse_response(text) + if parsed is not None: + return parsed + except Exception: + time.sleep(1.5) + return None + + def _parse_response(self, text): + text = text.strip() + if not text: + return None + try: + obj = json.loads(text) + return self._normalize_judge_obj(obj) + except Exception: + pass + + match = re.search(r"\{.*\}", text, flags=re.DOTALL) + if match: + try: + obj = json.loads(match.group(0)) + return self._normalize_judge_obj(obj) + except Exception: + return None + return None + + @staticmethod + def _normalize_judge_obj(obj): + if not isinstance(obj, dict): + return None + raw = obj.get("is_correct", 0) + if isinstance(raw, bool): + value = 1 if raw else 0 + else: + try: + value = int(raw) + except Exception: + value = 0 + reason = obj.get("reason", "") + return {"is_correct": 1 if value == 1 else 0, "reason": str(reason)} + diff --git a/mmeval/scoring/match/template.py b/mmeval/scoring/match/template.py new file mode 100644 index 0000000..05748b0 --- /dev/null +++ b/mmeval/scoring/match/template.py @@ -0,0 +1,82 @@ +import re + +from mmeval.scoring.match.base import BaseMatcher, MatchResult +from mmeval.scoring.schema import ( + extract_mcq_option, + normalize_for_exact, + normalize_open_answer, + normalize_text, +) + + +class TemplateMatcher(BaseMatcher): + name = "template" + + def match(self, sample, context): + question_type = context["question_type"] + pred_raw = context["pred"] + gt_raw = context["gt"] + if gt_raw is None or pred_raw is None: + return MatchResult(is_match=False) + + if question_type == "mcq": + return self._match_mcq(context) + return self._match_open(context) + + def _match_mcq(self, context): + pred_raw = context["pred"] + gt_raw = context["gt"] + candidates = context["options"] + + option = extract_mcq_option(pred_raw, candidates) + gt_option = extract_mcq_option(gt_raw, candidates) or normalize_for_exact(gt_raw).upper() + if option and gt_option and option == gt_option: + return MatchResult(is_match=True, matched=gt_option) + + # fallback: match option text in prediction body + option_text_map = context.get("option_text_map", {}) + pred_norm = normalize_for_exact(pred_raw) + for label, text in option_text_map.items(): + if not text: + continue + text_norm = normalize_for_exact(text) + if text_norm and text_norm in pred_norm: + if label == gt_option: + return MatchResult(is_match=True, matched=gt_option) + return MatchResult(is_match=False) + return MatchResult(is_match=False) + + def _match_open(self, context): + pred_raw = normalize_text(context["pred"]) + gt_raw = context["gt"] + gt_norm = normalize_open_answer(gt_raw) + pred_norm = normalize_open_answer(pred_raw) + + # yes/no templates + if gt_norm in {"yes", "no"}: + m = re.search(r"\b(yes|no)\b", pred_norm) + if m and m.group(1) == gt_norm: + return MatchResult(is_match=True, matched=str(gt_raw)) + return MatchResult(is_match=False) + + # number templates + if re.fullmatch(r"-?\d+(\.\d+)?", gt_norm or ""): + nums = re.findall(r"-?\d+(?:\.\d+)?", pred_norm) + if nums and nums[0] == gt_norm: + return MatchResult(is_match=True, matched=str(gt_raw)) + return MatchResult(is_match=False) + + # generic "final answer is ..." + patterns = [ + r"(?:final answer|answer is|the answer is)\s*[:\-]?\s*(.+)$", + r"^\s*(.+?)\s*$", + ] + for pattern in patterns: + m = re.search(pattern, pred_raw, flags=re.IGNORECASE) + if not m: + continue + candidate = normalize_open_answer(m.group(1)) + if candidate == gt_norm and candidate: + return MatchResult(is_match=True, matched=str(gt_raw)) + return MatchResult(is_match=False) + diff --git a/mmeval/scoring/pipeline.py b/mmeval/scoring/pipeline.py new file mode 100644 index 0000000..08fb761 --- /dev/null +++ b/mmeval/scoring/pipeline.py @@ -0,0 +1,277 @@ +import glob +import json +import os +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Dict, List + +from mmeval.scoring.match import MATCHER_REGISTRY +from mmeval.scoring.report import build_summary +from mmeval.scoring.schema import ( + extract_options, + get_field, + get_question, + infer_question_type, +) + +try: + from tqdm import tqdm +except Exception: # pragma: no cover - fallback when tqdm is unavailable + tqdm = None + + +def discover_result_files(out_dir: str, pattern: str = "**/result.json", recursive: bool = True) -> List[str]: + if not out_dir: + return [] + if recursive: + search_pattern = os.path.join(out_dir, pattern) + return sorted([x for x in glob.glob(search_pattern, recursive=True) if os.path.isfile(x)]) + direct = os.path.join(out_dir, "result.json") + return [direct] if os.path.exists(direct) else [] + + +def _build_option_text_map(sample: Dict[str, Any], options: List[str]) -> Dict[str, str]: + messages = sample.get("messages") or [] + if messages and isinstance(messages[0], dict): + msg_options = messages[0].get("options") + if isinstance(msg_options, dict): + return {str(k).strip().upper(): str(v) for k, v in msg_options.items()} + + option_text_map = {} + for label in options: + if label in sample: + option_text_map[label] = str(sample[label]) + return option_text_map + + +def build_matchers(args): + matcher_names = [x.strip().lower() for x in args.matching_order.split(",") if x.strip()] + matchers = [] + for name in matcher_names: + if name not in MATCHER_REGISTRY: + raise ValueError(f"Unknown matcher `{name}` in --matching_order") + matcher_cls = MATCHER_REGISTRY[name] + if name == "llm": + matchers.append(matcher_cls(args)) + else: + matchers.append(matcher_cls()) + return matchers + + +def _normalize_eval_id(value: Any) -> str: + return str(value) + + +def _load_cached_results(out_path: str, resume_enabled: bool) -> Dict[str, Dict[str, Any]]: + if not resume_enabled: + return {} + + cache_by_id: Dict[str, Dict[str, Any]] = {} + tmp_path = f"{out_path}.tmp" + candidates = [tmp_path, out_path] + for path in candidates: + if not os.path.exists(path): + continue + try: + with open(path, "r") as f: + payload = json.load(f) + rows = payload.get("samples") if isinstance(payload, dict) else payload + if not isinstance(rows, list): + continue + for row in rows: + if not isinstance(row, dict): + continue + if "eval-id" not in row: + continue + cache_by_id[_normalize_eval_id(row["eval-id"])] = row + # Prefer tmp (fresher) when available. + if path.endswith(".tmp"): + return cache_by_id + except Exception: + continue + return cache_by_id + + +def _flush_resume_tmp(tmp_path: str, samples: List[Dict[str, Any]]) -> None: + with open(tmp_path, "w") as f: + json.dump(samples, f, indent=2) + + +def _score_single_sample(sample: Dict[str, Any], args, matchers) -> Dict[str, Any]: + gt = get_field(sample, args.score_gt_field, None) + pred = get_field(sample, args.score_pred_field, None) + question = get_question(sample) + question_type = infer_question_type( + sample=sample, + score_force_question_type=args.score_force_question_type, + score_type_field=args.score_type_field, + ) + options = extract_options(sample) if question_type == "mcq" else [] + option_text_map = _build_option_text_map(sample, options) + + eval_id = sample.get("eval-id", sample.get("id")) + output = { + "eval-id": eval_id, + "question_type": question_type, + "gt": gt, + "pred": pred, + "matcher_used": None, + "matched": None, + "is_correct": 0, + "status": "ok", + } + + if gt is None or pred is None: + output["status"] = "invalid" + output["reason"] = "missing_gt_or_pred" + return output + + context = { + "question_type": question_type, + "question": question, + "pred": pred, + "gt": gt, + "options": options, + "option_text_map": option_text_map, + } + + trace = [] + for matcher in matchers: + result = matcher.match(sample, context) + trace.append( + { + "matcher": matcher.name, + "is_match": result.is_match, + "matched": result.matched, + "reason": result.reason, + "meta": result.meta, + } + ) + if result.is_match: + output["matcher_used"] = matcher.name + output["matched"] = result.matched + output["is_correct"] = 1 + if result.reason is not None and args.judge_include_reason: + output["reason"] = result.reason + break + if matcher.name == "llm" and result.reason is not None and args.judge_include_reason: + output["reason"] = result.reason + if args.matching_stop_on_first and result.is_match: + break + + if args.score_debug: + output["trace"] = trace + return output + + +def _score_single_sample_with_index(idx: int, sample: Dict[str, Any], args, matchers): + scored = _score_single_sample(sample, args, matchers) + if "eval-id" not in scored or scored["eval-id"] is None: + scored["eval-id"] = sample.get("eval-id", sample.get("id", idx)) + return idx, scored + + +def score_result_file(result_file: str, args) -> Dict[str, Any]: + with open(result_file, "r") as f: + samples = json.load(f) + if not isinstance(samples, list): + raise ValueError(f"{result_file} must be a JSON list.") + + out_path = os.path.join(os.path.dirname(result_file), args.score_output_name) + tmp_path = f"{out_path}.tmp" + cached_by_id = _load_cached_results(out_path=out_path, resume_enabled=args.score_resume) + matchers = build_matchers(args) + + scored_samples: List[Dict[str, Any]] = [None] * len(samples) + resumed_count = 0 + pbar = None + if args.score_progress_bar and tqdm is not None: + pbar = tqdm( + total=len(samples), + desc=f"Scoring {os.path.basename(os.path.dirname(result_file))}", + leave=False, + ) + + uncached_samples = [] + for idx, sample in enumerate(samples): + eval_id = _normalize_eval_id(sample.get("eval-id", sample.get("id", idx))) + cached = cached_by_id.get(eval_id) + if cached is not None: + scored_samples[idx] = cached + resumed_count += 1 + if pbar is not None: + pbar.update(1) + continue + uncached_samples.append((idx, sample)) + + sample_workers = max(1, int(getattr(args, "parallel_per_task", 1))) + processed_new = 0 + + if sample_workers <= 1 or len(uncached_samples) <= 1: + for idx, sample in uncached_samples: + i, scored = _score_single_sample_with_index(idx, sample, args, matchers) + scored_samples[i] = scored + processed_new += 1 + if pbar is not None: + pbar.update(1) + if args.score_resume and processed_new % max(1, int(args.score_save_freq)) == 0: + _flush_resume_tmp(tmp_path, [x for x in scored_samples if x is not None]) + else: + with ThreadPoolExecutor(max_workers=sample_workers) as executor: + futures = [ + executor.submit(_score_single_sample_with_index, idx, sample, args, matchers) + for idx, sample in uncached_samples + ] + for future in as_completed(futures): + i, scored = future.result() + scored_samples[i] = scored + processed_new += 1 + if pbar is not None: + pbar.update(1) + if args.score_resume and processed_new % max(1, int(args.score_save_freq)) == 0: + _flush_resume_tmp(tmp_path, [x for x in scored_samples if x is not None]) + + if pbar is not None: + pbar.close() + + if any(x is None for x in scored_samples): + raise RuntimeError(f"Incomplete scoring results found for {result_file}") + + summary = build_summary(scored_samples) + + payload = { + "summary": summary, + "config": { + "matching_order": args.matching_order, + "matching_stop_on_first": args.matching_stop_on_first, + "score_gt_field": args.score_gt_field, + "score_pred_field": args.score_pred_field, + "score_type_field": args.score_type_field, + "score_force_question_type": args.score_force_question_type, + "judge_include_reason": args.judge_include_reason, + "score_progress_bar": args.score_progress_bar, + "score_resume": args.score_resume, + "score_save_freq": args.score_save_freq, + "parallel_per_task": sample_workers, + "resumed_count": resumed_count, + }, + "samples": scored_samples, + } + + with open(out_path, "w") as f: + json.dump(payload, f, indent=2) + if args.score_resume and os.path.exists(tmp_path): + os.remove(tmp_path) + + if args.score_dump_failures: + fail_path = os.path.join(os.path.dirname(result_file), "score_failures.json") + failed = [x for x in scored_samples if x.get("is_correct") != 1] + with open(fail_path, "w") as f: + json.dump(failed, f, indent=2) + + return { + "result_file": result_file, + "score_file": out_path, + "summary": summary, + "resumed_count": resumed_count, + } + diff --git a/mmeval/scoring/report.py b/mmeval/scoring/report.py new file mode 100644 index 0000000..da83d35 --- /dev/null +++ b/mmeval/scoring/report.py @@ -0,0 +1,31 @@ +from collections import defaultdict +from typing import Any, Dict, List + + +def build_summary(sample_results: List[Dict[str, Any]]) -> Dict[str, Any]: + total = len(sample_results) + correct = sum(1 for x in sample_results if x.get("is_correct") == 1) + accuracy = (correct / total) if total else 0.0 + + by_matcher = defaultdict(lambda: {"total": 0, "correct": 0}) + by_question_type = defaultdict(lambda: {"total": 0, "correct": 0}) + invalid = 0 + for item in sample_results: + matcher = item.get("matcher_used") or "unmatched" + qtype = item.get("question_type") or "unknown" + by_matcher[matcher]["total"] += 1 + by_matcher[matcher]["correct"] += int(item.get("is_correct") == 1) + by_question_type[qtype]["total"] += 1 + by_question_type[qtype]["correct"] += int(item.get("is_correct") == 1) + if item.get("status") == "invalid": + invalid += 1 + + return { + "total": total, + "correct": correct, + "accuracy": accuracy, + "invalid": invalid, + "by_matcher": dict(by_matcher), + "by_question_type": dict(by_question_type), + } + diff --git a/mmeval/scoring/schema.py b/mmeval/scoring/schema.py new file mode 100644 index 0000000..0f81cbf --- /dev/null +++ b/mmeval/scoring/schema.py @@ -0,0 +1,183 @@ +import re +import unicodedata +from typing import Any, Dict, List, Optional + + +MCQ_OPTIONS_DEFAULT = list("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + +def _to_text(value: Any) -> str: + if isinstance(value, list): + if not value: + return "" + value = value[0] + if value is None: + return "" + return str(value) + + +def _extract_boxed_content(text: str) -> str: + marker = "\\boxed{" + start = text.find(marker) + if start < 0: + return text + i = start + len(marker) + depth = 1 + content = [] + while i < len(text): + ch = text[i] + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return "".join(content).strip() + content.append(ch) + i += 1 + return text + + +def normalize_text( + value: Any, + lower: bool = False, + extract_boxed: bool = True, + strip_latex_commands: bool = True, +) -> str: + text = _to_text(value) + + # Keep normalization mostly syntax-level and semantics-preserving. + text = unicodedata.normalize("NFKC", text) + if extract_boxed: + text = _extract_boxed_content(text) + + # Remove CoT / answer wrappers if models output tagged sections. + text = re.sub(r"", " ", text, flags=re.IGNORECASE) + + # Remove markdown code fences while preserving body text. + text = text.replace("```", " ") + + # Common inference artifacts. + text = text.replace("<|end_of_sentence|>", "") + text = text.replace("<|end▁of▁sentence|>", "") + text = text.replace("", "") + text = text.replace("", "") + text = text.replace("", "") + text = text.replace("Falcon: ", "") + + # Light LaTeX cleanup inspired by math-verify style normalization. + if strip_latex_commands: + text = re.sub(r"\\(?:left|right|displaystyle|mathrm|textbf|textit|text)\b", " ", text) + text = re.sub(r"\\(?:,|;|!|\:)", " ", text) # spacing commands + text = text.replace("\\%", "%") + + # Normalize common unicode/operator variants. + text = text.replace("−", "-").replace("–", "-").replace("—", "-") + text = text.replace("×", "*").replace("·", "*") + text = text.replace("÷", "/") + + text = text.strip() + text = re.sub(r"\s+", " ", text) + if lower: + text = text.lower() + return text + + +def normalize_for_exact(value: Any) -> str: + return normalize_text(value, lower=True).strip() + + +def normalize_open_answer(value: Any) -> str: + text = normalize_for_exact(value) + text = re.sub(r"^[\"'`]+|[\"'`]+$", "", text) + text = re.sub(r"\s+", " ", text) + return text + + +def get_field(sample: Dict[str, Any], field: str, default: Any = None) -> Any: + if not field: + return default + if field == "messages[-1].response": + messages = sample.get("messages") or [] + if not messages: + return default + last = messages[-1] + if not isinstance(last, dict): + return default + return last.get("response", default) + return sample.get(field, default) + + +def get_question(sample: Dict[str, Any]) -> str: + messages = sample.get("messages") or [] + if messages and isinstance(messages[0], dict): + q = messages[0].get("question") + if q is not None: + return str(q) + return str(sample.get("question", "")) + + +def extract_options(sample: Dict[str, Any]) -> List[str]: + messages = sample.get("messages") or [] + if messages and isinstance(messages[0], dict): + msg = messages[0] + choices = msg.get("choices") + if isinstance(choices, list) and choices: + return [str(c).strip().upper() for c in choices] + options = msg.get("options") + if isinstance(options, dict) and options: + return [str(k).strip().upper() for k in options.keys()] + + if isinstance(sample.get("choices"), list) and sample["choices"]: + return [str(c).strip().upper() for c in sample["choices"]] + if isinstance(sample.get("options"), dict) and sample["options"]: + return [str(k).strip().upper() for k in sample["options"].keys()] + + dynamic = [x for x in MCQ_OPTIONS_DEFAULT if x in sample] + return dynamic or MCQ_OPTIONS_DEFAULT[:6] + + +def infer_question_type( + sample: Dict[str, Any], + score_force_question_type: str = "auto", + score_type_field: Optional[str] = None, +) -> str: + forced = (score_force_question_type or "auto").strip().lower() + if forced in {"mcq", "open"}: + return forced + + if score_type_field: + raw = str(sample.get(score_type_field, "")).strip().lower() + if raw in {"mcq", "mc", "multi-choice", "multiple_choice"}: + return "mcq" + if raw in {"open", "open-ended", "open_ended", "free"}: + return "open" + + options = extract_options(sample) + if options: + return "mcq" + return "open" + + +def extract_mcq_option(text: Any, candidates: List[str]) -> Optional[str]: + clean = normalize_text(text) + if not clean: + return None + + uppercase = clean.strip().upper() + if uppercase in candidates: + return uppercase + if len(uppercase) >= 2 and uppercase[0] in candidates and uppercase[1] in {".", ")", ":"}: + return uppercase[0] + + patterns = [ + r"^(?:answer|option|choice)?\s*[:\-]?\s*([A-Z])(?:[\s\.\),:;]|$)", + r"(?:final answer|the answer is|answer is|choice is|option is)\s*[:\-]?\s*([A-Z])(?:[\s\.\),:;]|$)", + ] + for pattern in patterns: + match = re.search(pattern, uppercase, re.IGNORECASE) + if match: + candidate = match.group(1).upper() + if candidate in candidates: + return candidate + return None + diff --git a/mmeval/utils/argparser.py b/mmeval/utils/argparser.py index b3efd36..4cc2d49 100644 --- a/mmeval/utils/argparser.py +++ b/mmeval/utils/argparser.py @@ -1,5 +1,7 @@ import argparse import inspect +import os +import sys import warnings from dataclasses import dataclass, field, fields from typing import Dict, Optional, Sequence, get_args @@ -76,7 +78,41 @@ class ExperimentArguments: no_conda: bool = field(default=False, metadata={"help": "use current python env instead of conda"}) -ARGUMENT_DATACLASSES = (ModelArguments, DataArguments, InferenceArguments, ExperimentArguments) +@dataclass +class ScoreRuntimeArguments: + out_dir: Optional[str] = field(default=None, metadata={"help": "output directory containing result.json files"}) + parallel_per_task: int = field(default=4, metadata={"help": "number of sample workers inside each result.json"}) + + +@dataclass +class ScoreArguments: + score_scan_recursive: bool = field(default=True, metadata={"help": "recursively scan out_dir for result.json"}) + score_result_glob: str = field(default="**/result.json", metadata={"help": "glob pattern for result files in score mode"}) + score_output_name: str = field(default="score.json", metadata={"help": "output score json file name"}) + score_progress_bar: bool = field(default=True, metadata={"help": "show progress bar while scoring samples"}) + score_resume: bool = field(default=True, metadata={"help": "resume scoring from score tmp/final files when available"}) + score_save_freq: int = field(default=20, metadata={"help": "flush frequency for score resume tmp file"}) + score_dump_failures: bool = field(default=False, metadata={"help": "dump failed/unmatched samples into score_failures.json"}) + score_debug: bool = field(default=False, metadata={"help": "write matcher trace into score outputs"}) + matching_order: str = field(default="exact,template", metadata={"help": "matcher chain order, comma-separated"}) + matching_stop_on_first: bool = field(default=True, metadata={"help": "stop matcher chain once one matcher succeeds"}) + + score_gt_field: str = field(default="answer", metadata={"help": "field name for ground-truth in result sample"}) + score_pred_field: str = field(default="messages[-1].response", metadata={"help": "field path for prediction in result sample"}) + score_type_field: Optional[str] = field(default=None, metadata={"help": "optional field name for question type"}) + score_force_question_type: str = field(default="auto", metadata={"help": "force question type: auto|mcq|open"}) + + judge_provider: Optional[str] = field(default=None, metadata={"help": "llm judge provider: openai|azure_openai"}) + judge_model: Optional[str] = field(default=None, metadata={"help": "llm judge model/deployment name"}) + judge_max_retry: int = field(default=2, metadata={"help": "llm judge max retry count"}) + judge_temperature: float = field(default=0.0, metadata={"help": "llm judge generation temperature"}) + judge_concurrency: int = field(default=1, metadata={"help": "reserved for llm judge concurrency control"}) + judge_include_reason: bool = field(default=False, metadata={"help": "include llm judge reason in output"}) + + +ARGUMENT_DATACLASSES = (ModelArguments, DataArguments, InferenceArguments, ExperimentArguments, ScoreArguments) +INFER_ARGUMENT_DATACLASSES = (ModelArguments, DataArguments, InferenceArguments, ExperimentArguments) +SCORE_ARGUMENT_DATACLASSES = (ScoreRuntimeArguments, ScoreArguments) BOOL_DEFAULTS = { @@ -92,9 +128,8 @@ class ExperimentArguments: ) } -def parse_args(): - parser = argparse.ArgumentParser() - for dc in ARGUMENT_DATACLASSES: +def _add_dataclass_arguments(parser, dataclasses): + for dc in dataclasses: for f in fields(dc): tp = f.type type_args = getattr(tp, '__args__', None) @@ -121,8 +156,24 @@ def parse_args(): parser.add_argument(f"--{cli_name}", f"--{f.name}", dest=f.name, type=tp, default=f.default, help=help_text) - return parser.parse_args() + +def parse_args(mode: str = "auto"): + mode = (mode or "auto").strip().lower() + if mode == "auto": + script_name = os.path.basename(sys.argv[0]) + mode = "score" if script_name == "score.py" else "infer" + + if mode == "infer": + dataclasses = INFER_ARGUMENT_DATACLASSES + elif mode == "score": + dataclasses = SCORE_ARGUMENT_DATACLASSES + else: + raise ValueError(f"Unsupported parse mode: {mode}. Expected infer|score|auto.") + + parser = argparse.ArgumentParser() + _add_dataclass_arguments(parser, dataclasses) + return parser.parse_args() def parse_model_kwargs(args, default_kwargs=None): default_kwargs = default_kwargs or {} model_kwargs = {} diff --git a/scripts/examples/run_evalkit.sh b/scripts/examples/run_evalkit.sh new file mode 100644 index 0000000..95e62d5 --- /dev/null +++ b/scripts/examples/run_evalkit.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash + +set -u + +if [ "$#" -lt 1 ]; then + echo "Usage: $0 " + echo "Example: $0 4" + exit 2 +fi + +PARALLEL_DATASETS="$1" +if ! [[ "${PARALLEL_DATASETS}" =~ ^[1-9][0-9]*$ ]]; then + echo "[ERROR] parallel_datasets must be a positive integer, got: ${PARALLEL_DATASETS}" + exit 2 +fi + +# Run Qwen on VLMEvalKit datasets that are compatible with simple-mmeval TSVDataset flow. +# You can override these with environment variables before running: +# MODEL_NAME_OR_PATH, OUT_DIR, GPU_PER_PARALLEL, PARALLEL_PER_TASK +export PYTHONPATH=./:${PYTHONPATH:-} +export DATASET_DIR=${DATASET_DIR:-./datasets} + +MODEL_NAME_OR_PATH="${MODEL_NAME_OR_PATH:-/home/tiger/yijiangli/project/OPSD/work_dirs/grpo_vlm/OpenMMReasoner-RL-74K_multimodal-open-r1-8k-verified/multimodal-open-r1-8k-verified/qwen25vl3b_2epochs_lr1e5_bs4_acc2_numgen4-merged}" +OUT_DIR="${OUT_DIR:-work_dirs/qwen25vl3b_2epochs_lr1e5_bs4_acc2_numgen4-merged}" +GPU_PER_PARALLEL="${GPU_PER_PARALLEL:-1}" +PARALLEL_PER_TASK="${PARALLEL_PER_TASK:-8}" + +DATASETS=( + # Standard ImageMCQDataset + # "3DSRBench" + # "A-Bench_TEST" + # "A-Bench_VAL" + # "A-OKVQA" + # "AI2D_TEST" + # "AI2D_TEST_NO_MASK" + # "AesBench_TEST" + # "AesBench_VAL" + "BLINK" + # "CMMU_MCQ" + # "HRBench4K" + # "HRBench8K" + # "MedXpertQA_MM_test" + # "MicroVQA" + # "MLLMGuard_DS" + # "MMBench_dev_ar" + # "MMBench_dev_cn" + "MMBench_dev_en" + # "MMBench_dev_en_test" + # "MMBench_dev_pt" + # "MMBench_dev_ru" + # "MMBench_dev_tr" + # "MMMB_ar" + # "MMMB_cn" + # "MMMB_en" + # "MMMB_pt" + # "MMMB_ru" + # "MMMB_tr" + # "MMSci_DEV_MCQ" + "MMStar" + # "MMT-Bench_ALL" + # "MMT-Bench_VAL" + # "MMVP" + # "PathMMU_TEST" + # "PathMMU_VAL" + # "Q-Bench1_TEST" + # "Q-Bench1_VAL" + # "R-Bench-Dis" + # "R-Bench-Ref" + "RealWorldQA" + "SEEDBench2" + # "SEEDBench2_Plus" + "SEEDBench_IMG" + # "ScienceQA_TEST" + "ScienceQA_VAL" + # "TaskMeAnything_v1_imageqa_random" + # "VisOnlyQA-VLMEvalKit" + "VStarBench" + # "WorldMedQA-V" + + # Multipart / Concat (supported in simple-mmeval evalkit loader) + # "MicroBench" + # "OmniMedVQA" + # "MMMB" + # "MTL_MMBench_DEV" + # Standard ImageYORNDataset (POPE only) + # "POPE" + + # VCRDataset + # "VCR_EN_EASY_ALL" + # "VCR_EN_HARD_ALL" + # "VCR_ZH_EASY_ALL" + # "VCR_ZH_HARD_ALL" +) + +echo "[INFO] model: ${MODEL_NAME_OR_PATH}" +echo "[INFO] out_dir: ${OUT_DIR}" +echo "[INFO] gpu_per_parallel: ${GPU_PER_PARALLEL}" +echo "[INFO] parallel_per_task: ${PARALLEL_PER_TASK}" +echo "[INFO] parallel_datasets: ${PARALLEL_DATASETS}" +echo "[INFO] dataset_dir: ${DATASET_DIR}" +echo "[INFO] total datasets: ${#DATASETS[@]}" + +mkdir -p "${OUT_DIR}" + +idx=0 +total=${#DATASETS[@]} +status_dir="${OUT_DIR}/.dataset_status.$$" +mkdir -p "${status_dir}" + +for dataset in "${DATASETS[@]}"; do + idx=$((idx + 1)) + ( + start_ts="$(date '+%Y-%m-%d %H:%M:%S')" + echo "============================================================" + echo "[${start_ts}] [${idx}/${total}] START evalkit@${dataset}" + + python mmeval/run.py \ + --model_name_or_path "${MODEL_NAME_OR_PATH}" \ + --dataset "evalkit@${dataset}" \ + --out_dir "${OUT_DIR}/${dataset}" \ + --no_conda \ + --gpu_per_parallel "${GPU_PER_PARALLEL}" \ + --parallel_per_task "${PARALLEL_PER_TASK}" + exit_code=$? + + end_ts="$(date '+%Y-%m-%d %H:%M:%S')" + if [ "${exit_code}" -eq 0 ]; then + echo "[${end_ts}] [${idx}/${total}] DONE evalkit@${dataset}" + echo 0 > "${status_dir}/${dataset}.status" + else + echo "[${end_ts}] [${idx}/${total}] FAIL evalkit@${dataset} (exit=${exit_code})" + echo "${exit_code}" > "${status_dir}/${dataset}.status" + fi + ) & + + while [ "$(jobs -rp | wc -l)" -ge "${PARALLEL_DATASETS}" ]; do + wait -n + done +done + +wait + +failed=() +for dataset in "${DATASETS[@]}"; do + status_file="${status_dir}/${dataset}.status" + if [ ! -f "${status_file}" ]; then + failed+=("${dataset}") + continue + fi + status="$(< "${status_file}")" + if [ "${status}" != "0" ]; then + failed+=("${dataset}") + fi +done + +rm -rf "${status_dir}" + +echo "============================================================" +echo "[SUMMARY] total=${total}, failed=${#failed[@]}" +if [ "${#failed[@]}" -gt 0 ]; then + echo "[SUMMARY] failed datasets:" + for dataset in "${failed[@]}"; do + echo " - ${dataset}" + done + exit 1 +fi + +echo "[SUMMARY] all datasets finished successfully." diff --git a/scripts/examples/run_score_evalkit_llm.sh b/scripts/examples/run_score_evalkit_llm.sh new file mode 100755 index 0000000..fecbb11 --- /dev/null +++ b/scripts/examples/run_score_evalkit_llm.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Example script: run scoring (exact -> template -> llm) on evalkit outputs. +# Defaults are intentionally conservative for cost/time. +# +# Usage: +# OPENAI_API_KEY=xxx bash scripts/examples/run_score_evalkit_llm.sh +# +# Optional env vars: +# OUT_DIR=work_dirs/evalkit_all_qwen +# SCORE_RESULT_GLOB='VStarBench/result.json' # or '*/result.json' +# SCORE_OUTPUT_NAME=score_llm.json +# PARALLEL_PER_TASK=1 # sample workers inside each result.json +# MATCHING_ORDER='exact,template,llm' +# JUDGE_PROVIDER='openai' +# JUDGE_MODEL='gpt-5' +# JUDGE_INCLUDE_REASON='false' +# SCORE_RESUME='true' +# SCORE_SAVE_FREQ=20 + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)" +cd "${ROOT_DIR}" + +export PYTHONPATH="./:${PYTHONPATH:-}" + +OUT_DIR="${OUT_DIR:-work_dirs/evalkit_all_qwen}" +SCORE_RESULT_GLOB="${SCORE_RESULT_GLOB:-VStarBench/result.json}" +SCORE_OUTPUT_NAME="${SCORE_OUTPUT_NAME:-score_llm.json}" +PARALLEL_PER_TASK="${PARALLEL_PER_TASK:-1}" +MATCHING_ORDER="${MATCHING_ORDER:-exact,template,llm}" +JUDGE_PROVIDER="${JUDGE_PROVIDER:-openai}" +JUDGE_MODEL="${JUDGE_MODEL:-gpt-5}" +JUDGE_INCLUDE_REASON="${JUDGE_INCLUDE_REASON:-false}" +SCORE_RESUME="${SCORE_RESUME:-true}" +SCORE_SAVE_FREQ="${SCORE_SAVE_FREQ:-20}" + +if [[ "${MATCHING_ORDER}" == *"llm"* ]]; then + if [[ -z "${OPENAI_API_KEY:-}" && "${JUDGE_PROVIDER}" == "openai" ]]; then + echo "ERROR: OPENAI_API_KEY is required when using llm matcher with openai provider." + exit 1 + fi +fi + +echo "Scoring config:" +echo " OUT_DIR=${OUT_DIR}" +echo " SCORE_RESULT_GLOB=${SCORE_RESULT_GLOB}" +echo " SCORE_OUTPUT_NAME=${SCORE_OUTPUT_NAME}" +echo " PARALLEL_PER_TASK=${PARALLEL_PER_TASK} (sample workers per result.json)" +echo " MATCHING_ORDER=${MATCHING_ORDER}" +echo " JUDGE_PROVIDER=${JUDGE_PROVIDER}" +echo " JUDGE_MODEL=${JUDGE_MODEL}" +echo " JUDGE_INCLUDE_REASON=${JUDGE_INCLUDE_REASON}" +echo " SCORE_RESUME=${SCORE_RESUME}" +echo " SCORE_SAVE_FREQ=${SCORE_SAVE_FREQ}" +echo + +CMD=( + python mmeval/score.py + --out_dir "${OUT_DIR}" + --score_result_glob "${SCORE_RESULT_GLOB}" + --parallel_per_task "${PARALLEL_PER_TASK}" + --matching_order "${MATCHING_ORDER}" + --score_output_name "${SCORE_OUTPUT_NAME}" + --judge_provider "${JUDGE_PROVIDER}" + --judge_model "${JUDGE_MODEL}" + --score_save_freq "${SCORE_SAVE_FREQ}" +) + +if [[ "${JUDGE_INCLUDE_REASON}" == "true" ]]; then + CMD+=(--judge_include_reason) +fi + +if [[ "${SCORE_RESUME}" == "false" ]]; then + CMD+=(--no_score_resume) +fi + +"${CMD[@]}" From e34903869908a8eda5e034045d31922eb52c18fa Mon Sep 17 00:00:00 2001 From: "yijiang.li" Date: Tue, 28 Apr 2026 08:35:23 +0000 Subject: [PATCH 04/12] add series arg for better testing local models --- mmeval/run.py | 2 +- mmeval/utils/argparser.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mmeval/run.py b/mmeval/run.py index 5eb809b..0c21121 100644 --- a/mmeval/run.py +++ b/mmeval/run.py @@ -19,7 +19,7 @@ def get_series(model_name: str): args = parse_args() model_name_or_path = args.model_name_or_path - series = get_series(model_name_or_path.split("/")[-1]) + series = args.model_series or get_series(model_name_or_path.split("/")[-1]) infer_file = series_infer_env_mapping[series]["infer_file"] infer_env = series_infer_env_mapping[series]["env"] diff --git a/mmeval/utils/argparser.py b/mmeval/utils/argparser.py index 4cc2d49..d215fa2 100644 --- a/mmeval/utils/argparser.py +++ b/mmeval/utils/argparser.py @@ -9,6 +9,7 @@ @dataclass class ModelArguments: model_name_or_path: Optional[str] = field(default=None) + model_series: Optional[str] = field(default=None) # parameters for model low_cpu_mem_usage: Optional[bool] = field(default=None, metadata={"help": "Tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model."}) From 2c255f8c793cf6056d4b2382bd038b4239e9bb44 Mon Sep 17 00:00:00 2001 From: "yijiang.li" Date: Sat, 13 Jun 2026 02:36:15 +0000 Subject: [PATCH 05/12] revise grading and try to match results with official --- mmeval/data/mmeval_hf.py | 95 +++++- mmeval/infer/qwen3_vl.py | 9 +- mmeval/infer/qwenvl2d5.py | 21 +- mmeval/infer/qwenvl2d5_my.py | 176 +++++++++++ mmeval/registry.py | 3 +- mmeval/scoring/match/__init__.py | 12 +- mmeval/scoring/match/llm.py | 229 ++++++++++---- mmeval/scoring/match/template.py | 27 +- mmeval/scoring/pipeline.py | 29 +- mmeval/scoring/schema.py | 208 ++++++++++++- scripts/examples/_finish_7b_mcq.sh | 46 +++ scripts/examples/_stacked_density_watch.sh | 34 +++ scripts/examples/chain_all.sh | 18 ++ scripts/examples/chain_tail2.sh | 28 ++ scripts/examples/hf_dataset.sh | 38 ++- scripts/examples/llm_judge_api_reference.md | 44 +++ .../examples/resume_hrbench_then_qwen25.sh | 40 +++ scripts/examples/run_evalkit.sh | 6 + scripts/examples/run_hrbench_final.sh | 54 ++++ scripts/examples/run_hrbench_spg1.sh | 71 +++++ scripts/examples/run_qwen25vl7b_all.sh | 37 +++ .../examples/run_qwen3vl4b_blink_mmstar.sh | 288 ++++++++++++++++++ scripts/examples/run_qwen3vl4b_mcq_suite.sh | 106 +++++++ scripts/examples/run_vision_opd_step65.sh | 22 ++ scripts/examples/run_vopd_othernode.sh | 92 ++++++ scripts/examples/verify_judge.py | 47 +++ scripts/templates/vlmevalkit_mcq.txt | 5 + 27 files changed, 1662 insertions(+), 123 deletions(-) create mode 100644 mmeval/infer/qwenvl2d5_my.py create mode 100644 scripts/examples/_finish_7b_mcq.sh create mode 100644 scripts/examples/_stacked_density_watch.sh create mode 100644 scripts/examples/chain_all.sh create mode 100755 scripts/examples/chain_tail2.sh create mode 100644 scripts/examples/llm_judge_api_reference.md create mode 100644 scripts/examples/resume_hrbench_then_qwen25.sh create mode 100644 scripts/examples/run_hrbench_final.sh create mode 100644 scripts/examples/run_hrbench_spg1.sh create mode 100644 scripts/examples/run_qwen25vl7b_all.sh create mode 100755 scripts/examples/run_qwen3vl4b_blink_mmstar.sh create mode 100644 scripts/examples/run_qwen3vl4b_mcq_suite.sh create mode 100644 scripts/examples/run_vision_opd_step65.sh create mode 100755 scripts/examples/run_vopd_othernode.sh create mode 100644 scripts/examples/verify_judge.py create mode 100644 scripts/templates/vlmevalkit_mcq.txt diff --git a/mmeval/data/mmeval_hf.py b/mmeval/data/mmeval_hf.py index 1c043f6..ef1a805 100644 --- a/mmeval/data/mmeval_hf.py +++ b/mmeval/data/mmeval_hf.py @@ -1,5 +1,5 @@ import json -from datasets import load_dataset +from datasets import load_dataset, get_dataset_config_names from mmeval.data.base import BaseDataset @@ -7,7 +7,13 @@ class MMEvalHFDataset(BaseDataset): """Dataset loader for HuggingFace datasets in mm-eval format.""" def __init__(self, args): - self.dataset_name = args.dataset.split("@")[1] if "@" in args.dataset else args.dataset + dataset_str = args.dataset.split("@")[1] if "@" in args.dataset else args.dataset + # Support optional config suffix: "mm-eval/MMMU_Pro:vision" + if ":" in dataset_str: + self.dataset_name, self.dataset_config = dataset_str.rsplit(":", 1) + else: + self.dataset_name = dataset_str + self.dataset_config = None self.split = args.split self.circular = args.circular self.resize = args.resize # Used by base._process_messages for image resizing @@ -16,12 +22,52 @@ def __init__(self, args): super().__init__(args) def _load_raw_data(self, args) -> tuple: - # Load metadata subset to get jinja_template for the current split - metadata_ds = load_dataset(self.dataset_name, name="metadata", split=self.split) - dataset_template = metadata_ds[0]["jinja_template"] if len(metadata_ds) > 0 else None - - # Load default subset with the current split for data - ds = load_dataset(self.dataset_name, name="default", split=self.split) + all_configs = get_dataset_config_names(self.dataset_name) + if self.dataset_config: + data_config = self.dataset_config + metadata_config = f"{data_config}_metadata" + else: + # Auto-detect: older datasets use "metadata"/"default"; newer ones use "_metadata" + if "metadata" in all_configs: + data_config = "default" + metadata_config = "metadata" + else: + data_configs = [c for c in all_configs if not c.endswith("_metadata")] + if not data_configs: + raise ValueError( + f"No data configs found for {self.dataset_name}. Available: {all_configs}\n" + f"Specify a config explicitly, e.g. mmeval_hf@{self.dataset_name}:" + ) + data_config = data_configs[0] + metadata_config = f"{data_config}_metadata" + print(f"Auto-selected config '{data_config}' (available: {all_configs}). " + f"Use mmeval_hf@{self.dataset_name}: to pick a specific one.") + + # Some upstream HF datasets (e.g. mm-eval/MMMU, mm-eval/MMMU-pro) ship only + # the data config without a paired '_metadata'. Others (e.g. + # mm-eval/DynaMath) keep the metadata config name but ship it empty for + # the requested split (load_dataset raises 'Instruction "" + # corresponds to no data!'). Treat the metadata split as optional. + # verification_mode='no_checks' tolerates upstream split-size drift + # (e.g. mm-eval/DynaMath data config's dataset_infos.json lists 1 example + # while the parquet actually has 501). + dataset_template = None + if metadata_config in all_configs: + try: + metadata_ds = load_dataset( + self.dataset_name, name=metadata_config, split=self.split, + verification_mode="no_checks", + ) + if len(metadata_ds) > 0: + dataset_template = metadata_ds[0]["jinja_template"] + except Exception as e: + print(f"[mmeval_hf] metadata config '{metadata_config}' present but unusable " + f"({type(e).__name__}: {str(e)[:120]}); proceeding with dataset_template=None") + + ds = load_dataset( + self.dataset_name, name=data_config, split=self.split, + verification_mode="no_checks", + ) return ds, dataset_template # Return HF's jinja_template as dataset template def convert_circular(self, **kwargs) -> any: @@ -50,6 +96,39 @@ def _process_sample(self, idx: int): # Ensure sample["media"] is always a list sample["media"] = media_list + # If the default template will be used (no per-dataset jinja template) + # and the user message's question text does not contain media placeholders + # while media is present (e.g. mm-eval/DynaMath, whose 'metadata' config + # is unusable upstream), prepend "" tokens to the question so the + # default template renders them into the prompt. Without this the + # framework raises "Prompt/media mismatch: used_media=0, total_media=N". + if self._dataset_template is None and media_list: + for msg in message_list: + if not isinstance(msg, dict) or msg.get("role") != "user": + continue + q = msg.get("question") + if not isinstance(q, str): + continue + # Check for existing placeholders across ALL fields the default + # template renders (question + options values + hint), not just the + # question. Some datasets (e.g. mm-eval/MMMU) reference images only + # in the options; checking the question alone would wrongly conclude + # "no placeholder" and over-inject, producing placeholders > media. + rendered_fields = [q] + opts = msg.get("options") + if isinstance(opts, dict): + rendered_fields += [str(v) for v in opts.values()] + elif isinstance(opts, (list, tuple)): + rendered_fields += [str(v) for v in opts] + hint = msg.get("hint") + if isinstance(hint, str): + rendered_fields.append(hint) + if any(("" in f or "