Dev eval opsd

.gitignore

@@ -173,4 +173,7 @@ cython_debug/
 # PyPI configuration file
 .pypirc
 mydata
-.tmp/
+datasets
+.claude
+work_dirs
+.tmp/

docs/en/SUPPORTED.md

@@ -163,7 +163,7 @@ Currently available mm-eval HuggingFace datasets:
 | Dataset | Splits | Usage |
 |---------|--------|-------|
 | [MMBench-en](https://huggingface.co/datasets/mm-eval/MMBench-en) | dev, test | `mmeval_hf@mm-eval/MMBench-en` |
-| [MMBench-en-V11](https://huggingface.co/datasets/mm-eval/MMBench-en-V11) | dev, test | `mmeval_hf@mm-eval/MMBench-en-V11` |
+| [MMBench-V11](https://huggingface.co/datasets/mm-eval/MMBench-V11) | dev, test | `mmeval_hf@mm-eval/MMBench-V11 --subset en` (subsets: en, cn) |
 
 ### 3. VLMEvalKit Datasets (`evalkit@`)
 

docs/en/USAGE.md

@@ -42,7 +42,8 @@ Run evaluation on a HuggingFace-hosted dataset with built-in prompt templates:
 ```bash
 python mmeval/run.py \
     --model_name_or_path Qwen/Qwen2.5-VL-3B-Instruct \
-    --dataset mmeval_hf@mm-eval/MMBench-en-V11 \
+    --dataset mmeval_hf@mm-eval/MMBench-V11 \
+    --subset en \
     --split test \
     --out_dir work_dirs/mmbench_test \
     --gpu_per_parallel 1 \
@@ -98,7 +99,8 @@ Distribute inference across multiple GPUs with automatic data sharding:
 ```bash
 python mmeval/run.py \
     --model_name_or_path Qwen/Qwen2.5-VL-72B-Instruct \
-    --dataset mmeval_hf@mm-eval/MMBench-en-V11 \
+    --dataset mmeval_hf@mm-eval/MMBench-V11 \
+    --subset en \
     --split test \
     --out_dir work_dirs/mmbench_72b \
     --gpu_per_parallel 4 \
@@ -114,7 +116,8 @@ Run a quick deterministic smoke test on 20 randomly selected samples:
 ```bash
 python mmeval/run.py \
     --model_name_or_path Qwen/Qwen2.5-VL-3B-Instruct \
-    --dataset mmeval_hf@mm-eval/MMBench-en-V11 \
+    --dataset mmeval_hf@mm-eval/MMBench-V11 \
+    --subset en \
     --split test \
     --out_dir work_dirs/mmbench_smoke \
     --gpu_per_parallel 1 \

mmeval/data/base.py

@@ -9,7 +9,6 @@
 from jinja2 import Environment
 
 
-
 class BaseDataset(ABC):
     """Dataset base class for loading and processing datasets.
 
@@ -209,7 +208,7 @@ def build_prompt(self, prompt_template: str, sample: Dict[str, Any]) -> str:
             Rendered template string
         """
         env = Environment()
-        env.globals.update({'zip': zip, 'enumerate': enumerate, 'len': len, 'range': range, 'list': list, 
+        env.globals.update({'zip': zip, 'enumerate': enumerate, 'len': len, 'range': range, 'list': list,
         'dict': dict, 'str': str, 'int': int, 'float': float, 'bool': bool, 'sum': sum, 'max': max, 'min': min})
         template = env.from_string(prompt_template)
         return template.render(**sample)

mmeval/data/mmeval_hf.py

@@ -7,7 +7,15 @@
 
 
 class MMEvalHFDataset(BaseDataset):
-    """Dataset loader for HuggingFace datasets in mm-eval format."""
+    """Dataset loader for HuggingFace datasets in mm-eval format.
+
+    Reads the subset manifest from the dataset repo's metadata.json and injects
+    the scoring-relevant metadata as the flat top-level `dataset_meta` sample
+    field. Rows are expected in the flat layout (grading fields as top-level
+    columns; messages[0] carries render inputs only) — non-conforming rows are
+    passed through unchanged and surface at scoring time via the scorer's
+    explicit layout rejection.
+    """
 
     def __init__(self, args):
         self.dataset_name = args.dataset.split("@", 1)[1]
@@ -40,9 +48,31 @@ def _load_raw_data(self, args) -> tuple:
             raise ValueError(
                 f"{self.dataset_name}: subset {self.subset!r} not in {list(subsets)}"
             )
-        dataset_template = subsets[self.subset].get("prompt_template")
+        subset_block = subsets[self.subset]
+        dataset_template = subset_block.get("prompt_template")
+
+        # Scoring data-flow contract: inject the subset's scoring-relevant
+        # metadata into every sample as the flat top-level `dataset_meta`
+        # field, so it lands in result.json and the scorer can resolve the
+        # official grading protocol without access to the HF manifest. CLI
+        # flags on the scorer always override these values.
+        meta_block = {}
+        for key in ("task_type", "score_type", "score_params"):
+            value = subset_block.get(key)
+            if value:
+                meta_block[key] = value
+        note = ((subset_block.get("score_protocol") or {}).get("note") or "").strip()
+        if note:
+            meta_block["score_note"] = note
+        self._dataset_meta = meta_block or None
 
-        ds = load_dataset(self.dataset_name, name="default", split=self.split)
+        # Prefer multi-config layout: HF config name == mm-eval subset name.
+        # Fall back to the single-`default` config layout, where one config
+        # holds all subsets' splits (named `<subset>_<split>`).
+        try:
+            ds = load_dataset(self.dataset_name, name=self.subset, split=self.split)
+        except (ValueError, FileNotFoundError):
+            ds = load_dataset(self.dataset_name, name="default", split=self.split)
         return ds, dataset_template
 
     def convert_circular(self, **kwargs) -> any:
@@ -52,14 +82,12 @@ def convert_circular(self, **kwargs) -> any:
     def _process_sample(self, idx: int):
         sample = dict(self._raw_dataset[idx])
 
-        # Add eval-id if not present
         if "eval-id" not in sample:
             sample["eval-id"] = idx
 
         messages = sample["messages"]
         message_list = json.loads(messages) if isinstance(messages, str) else messages
 
-        # Get media from sample level (HF datasets may store as single image or list)
         media = sample.get("media")
         if media is None:
             media_list = []
@@ -68,10 +96,10 @@ def _process_sample(self, idx: int):
         else:
             media_list = [media]
 
-        # Ensure sample["media"] is always a list
         sample["media"] = media_list
-
-        # Process all messages with sample-level media indexed by placeholder order
         sample["messages"] = self._process_messages(message_list, media_list)
 
+        if self._dataset_meta:
+            sample.setdefault("dataset_meta", dict(self._dataset_meta))
+
         return sample

mmeval/data/tsv.py

@@ -156,7 +156,7 @@ def _process_sample(self, index: int) -> Dict[str, Any]:
         if options:
             message["options"] = options
             message["choices"] = list(options.keys())
-        
+
         hint = sample.get("hint", None)
         if hint and pd.notna(hint):
             message["hint"] = hint

mmeval/infer/qwen3_vl.py

@@ -1,3 +1,4 @@
+import os
 import re
 import copy
 
@@ -14,8 +15,16 @@ class TaskRunner(Task):
     def __init__(self, args):
         self.args = args
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        self.dtype = getattr(args, "dtype") or "auto"
-        self.default_model_kwargs = {"device_map": "auto"}
+        # Qwen3-VL is a bf16-native model; "auto" mis-resolves to fp32 on this merged ckpt,
+        # which both doubles attention memory AND breaks flash_attention_2 (needs fp16/bf16).
+        # Default to bf16 unless the user explicitly passes --dtype.
+        _dt = getattr(args, "dtype", None)
+        self.dtype = _dt if _dt and _dt != "auto" else torch.bfloat16
+        # flash_attention_2 is EXACT attention (same math as sdpa, just memory-efficient):
+        # avoids materializing the O(N^2) attention matrix that OOMs on no-cap full-res images.
+        # Overridable via --attn_implementation; default on since flash-attn is installed.
+        _attn = getattr(args, "attn_implementation", None) or os.environ.get("MMEVAL_ATTN", "flash_attention_2")
+        self.default_model_kwargs = {"device_map": "auto", "attn_implementation": _attn}
         self.default_gen_kwargs = {"max_new_tokens": 128}
         self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs)
         self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs)
@@ -53,10 +62,13 @@ def parse_input(self, message):
                     {
                         "type": "image",
                         "image": media,
-                        "min_pixels": 4 * 32 * 32,
-                        "max_pixels": 256 * 32 * 32,
+                        # Aligned with VLMEvalKit: no hardcoded min/max_pixels, so the
+                        # processor's defaults apply (shortest_edge=65536,
+                        # longest_edge=16777216). The previous cap of max_pixels=256*32*32
+                        # (262144 ≈ 512x512) downscaled ~17% of MMStar images and hurt
+                        # fine-perception accuracy.
                     }
-                )       
+                )
             elif chunk == constants.video:
                 media = media_list[media_idx]
                 media_idx += 1
@@ -94,9 +106,9 @@ def _generate_response(self, inputs):
     def run_sample(self, sample: dict):
         ori_sample = copy.deepcopy(sample)
         message = sample["messages"][0]
-        
+
         user_message = self.parse_input(message)
-        
+
         text = self.processor.apply_chat_template(
             user_message, tokenize=False, add_generation_prompt=True
         )
@@ -112,12 +124,12 @@ def run_sample(self, sample: dict):
             video_metadatas = None
 
         inputs = self.processor(
-            text=text, 
-            images=images, 
-            videos=videos, 
+            text=text,
+            images=images,
+            videos=videos,
             video_metadata=video_metadatas,
-            return_tensors="pt", 
-            do_resize=False, 
+            return_tensors="pt",
+            do_resize=False,
             **video_kwargs
         )
         inputs = inputs.to(self.model.device)

mmeval/infer/qwenvl2d5.py

@@ -32,9 +32,10 @@ def __init__(self, args):
     def load_model(self, args):
         self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained(args.model_name_or_path, torch_dtype=self.dtype, **self.model_kwargs)
         self.tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
-        min_pixels = 256 * 28 * 28
-        max_pixels = 1280 * 28 * 28
-        self.processor = AutoProcessor.from_pretrained(args.model_name_or_path, min_pixels=min_pixels, max_pixels=max_pixels)
+        # Aligned with VLMEvalKit: do NOT pass min/max_pixels, so the processor's
+        # defaults apply (min_pixels=3136, max_pixels=12845056 ≈ 12.8M). The previous
+        # cap of max_pixels=1280*28*28 (~1MP) was ~12.8x lower and downscaled images.
+        self.processor = AutoProcessor.from_pretrained(args.model_name_or_path)
 
     def parse_input(self, message):
         question = message["prompt"]
@@ -145,4 +146,4 @@ def run_sample(self, sample: dict):
 if __name__ == "__main__":
     args = parse_args()
     model_evaluator = TaskRunner(args)
-    model_evaluator.inference_dataset()
+    model_evaluator.inference_dataset()

mmeval/run.py

@@ -19,7 +19,7 @@ def get_series(model_name: str):
     args = parse_args()
 
     model_name_or_path = args.model_name_or_path
-    series = get_series(model_name_or_path.split("/")[-1])
+    series = args.model_series or get_series(model_name_or_path.split("/")[-1])
 
     infer_file = series_infer_env_mapping[series]["infer_file"]
     infer_env = series_infer_env_mapping[series]["env"]

mmeval/score.py

@@ -0,0 +1,31 @@
+from mmeval.scoring.pipeline import discover_result_files, score_result_file
+from mmeval.utils.argparser import parse_args
+
+
+def run_score(args):
+    files = discover_result_files(out_dir=args.out_dir, pattern=args.score_result_glob)
+    if not files:
+        raise RuntimeError(f"No result files found under {args.out_dir} with pattern {args.score_result_glob}")
+
+    done = []
+    failed = []
+    for result_file in files:
+        try:
+            done.append(score_result_file(result_file, args))
+        except Exception as exc:
+            failed.append({"result_file": result_file, "error": str(exc)})
+
+    for item in done:
+        print(
+            f"✅ [score] {item['result_file']} -> {item['score_file']} "
+            f"(acc={item['summary']['accuracy']:.4f}, resumed={item.get('resumed_count', 0)})"
+        )
+    if failed:
+        for item in failed:
+            print(f"❌ [score] {item['result_file']}: {item['error']}")
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    run_score(args)

mmeval/scoring/__init__.py

@@ -0,0 +1,6 @@
+from mmeval.scoring.pipeline import score_result_file, discover_result_files
+
+__all__ = [
+    "score_result_file",
+    "discover_result_files",
+]