Add kimi_vl model integration (Kimi-VL-A3B)

env_files/kimi_vl_requirements.txt

@@ -0,0 +1,14 @@
+torch==2.4.0
+torchvision==0.19.0
+transformers==4.48.2
+accelerate
+datasets
+pandas
+numpy==1.26.4
+Pillow
+sentencepiece
+protobuf
+einops
+timm
+tiktoken
+blobfile

mmeval/infer/kimi_vl.py

@@ -0,0 +1,110 @@
+"""kimi_vl — Kimi-VL-A3B-{Instruct,Thinking}.
+
+HF:  https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct
+GH:  https://github.com/MoonshotAI/Kimi-VL
+Paper: https://arxiv.org/abs/2504.07491
+"""
+import re
+import copy
+
+import torch
+from PIL import Image
+from transformers import AutoModelForCausalLM, AutoProcessor
+
+from mmeval.infer.task import Task
+from mmeval.utils import constants
+from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs
+
+
+class TaskRunner(Task):
+    def __init__(self, args):
+        self.args = args
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.dtype = getattr(args, "dtype") or "auto"
+        self.default_model_kwargs = {"device_map": "auto"}
+        self.default_gen_kwargs = {"max_new_tokens": 512}
+        self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs)
+        self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs)
+
+        super().__init__(args)
+
+    def load_model(self, args):
+        self.model = AutoModelForCausalLM.from_pretrained(
+            args.model_name_or_path,
+            torch_dtype=self.dtype,
+            trust_remote_code=True,
+            **self.model_kwargs,
+        ).eval()
+        self.processor = AutoProcessor.from_pretrained(
+            args.model_name_or_path, trust_remote_code=True,
+        )
+
+    def parse_input(self, message):
+        question = message["prompt"]
+        q_chunks = re.split(r'(<(?:image|video)>)', question)
+        media_list = message.get('media', [])
+
+        content = []
+        media_idx = 0
+        for chunk in q_chunks:
+            if not chunk.strip():
+                continue
+            if chunk == constants.image:
+                content.append({"type": "image", "image": media_list[media_idx]})
+                media_idx += 1
+            elif chunk == constants.video:
+                raise NotImplementedError("kimi_vl video input not implemented")
+            else:
+                content.append({"type": "text", "text": chunk})
+        return [{"role": "user", "content": content}]
+
+    def _materialize_images(self, user_message):
+        images = []
+        for turn in user_message:
+            for item in turn["content"]:
+                if item.get("type") == "image":
+                    img = item["image"]
+                    if isinstance(img, str):
+                        img = Image.open(img).convert("RGB")
+                    elif hasattr(img, "convert"):
+                        img = img.convert("RGB")
+                    images.append(img)
+        return images
+
+    def _generate_response(self, inputs):
+        generated_ids = self.model.generate(**inputs, **self.gen_kwargs)
+        trimmed = [
+            out[len(inp):] for inp, out in zip(inputs.input_ids, generated_ids)
+        ]
+        return self.processor.batch_decode(
+            trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False,
+        )[0]
+
+    def run_sample(self, sample: dict):
+        if self.args.score_target:
+            raise NotImplementedError(
+                "kimi_vl: score_target is not implemented yet"
+            )
+
+        ori_sample = copy.deepcopy(sample)
+        message = sample["messages"][0]
+        user_message = self.parse_input(message)
+        images = self._materialize_images(user_message)
+
+        text = self.processor.apply_chat_template(
+            user_message, add_generation_prompt=True, return_tensors="pt",
+        )
+        proc_kwargs = {"text": text, "return_tensors": "pt", "padding": True, "truncation": True}
+        if images:
+            proc_kwargs["images"] = images
+        inputs = self.processor(**proc_kwargs).to(self.model.device)
+
+        response = self._generate_response(inputs)
+        ori_sample["messages"].append({"role": "assistant", "response": response})
+        return ori_sample
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    model_evaluator = TaskRunner(args)
+    model_evaluator.inference_dataset()

mmeval/registry.py

@@ -77,6 +77,7 @@
                    "doubao-seed-2-0-mini-260215", "doubao-seed-2-0-lite-260215", "doubao-seed-2-0-code-preview-260215", "doubao-seed-2-0-pro-260215"],
     "hunyuan_vision": ["hunyuan-vision", "hunyuan-vision-1.5-instruct", "hunyuan-t1-vision", "hunyuan-turbos-vision", "hunyuan-large-vision"],
     "cosmos_reason2": ["Cosmos-Reason2-2B", "Cosmos-Reason2-8B"],
+    "kimi_vl": ["Kimi-VL-A3B-Instruct", "Kimi-VL-A3B-Thinking"],
 }
 
 series_infer_env_mapping = {
@@ -316,4 +317,8 @@
         "env": os.path.join(env_dir, "cosmos_reason2"),
         "infer_file": "cosmos_reason2.py",
     },
+    "kimi_vl": {
+        "env": os.path.join(env_dir, "kimi_vl"),
+        "infer_file": "kimi_vl.py",
+    },
 }

test_results/kimi_vl_2026-06-10/no_media/result.json

@@ -0,0 +1,44 @@
+[
+    {
+        "id": 0,
+        "media": [],
+        "messages": [
+            {
+                "role": "user",
+                "question": "What is the result of 1 plus 1?",
+                "answer": "",
+                "options": {},
+                "choices": [],
+                "prompt": "What is the result of 1 plus 1?",
+                "hint": ""
+            },
+            {
+                "role": "assistant",
+                "response": "The result of 1 plus 1 is 2."
+            }
+        ],
+        "comment": "Test 'no media' modality.",
+        "eval-id": 0
+    },
+    {
+        "id": 1,
+        "media": [],
+        "messages": [
+            {
+                "role": "user",
+                "question": "What is the capital city of China?",
+                "answer": "",
+                "options": {},
+                "choices": [],
+                "prompt": "What is the capital city of China?",
+                "hint": ""
+            },
+            {
+                "role": "assistant",
+                "response": "The capital city of China is Beijing."
+            }
+        ],
+        "comment": "Test 'no media' modality.",
+        "eval-id": 1
+    }
+]

test_results/kimi_vl_2026-06-10/single_image_start/result.json

@@ -0,0 +1,48 @@
+[
+    {
+        "id": 0,
+        "media": [
+            "truck.png"
+        ],
+        "messages": [
+            {
+                "role": "user",
+                "question": "<image> Please provide a detailed description of the contents shown in the image.",
+                "answer": "",
+                "options": {},
+                "choices": [],
+                "prompt": "<image> Please provide a detailed description of the contents shown in the image.",
+                "hint": ""
+            },
+            {
+                "role": "assistant",
+                "response": "The image showcases a GMC Sierra 1500 AT4 truck in action, driving through a rugged, desert-like terrain. The truck is painted in a sleek, dark gray color and is equipped with black wheels and off-road tires, emphasizing its robust and adventurous design. The front grille prominently displays the GMC logo, and the"
+            }
+        ],
+        "comment": "Test 'single image start' modality.",
+        "eval-id": 0
+    },
+    {
+        "id": 1,
+        "media": [
+            "boat.png"
+        ],
+        "messages": [
+            {
+                "role": "user",
+                "question": "<image> Please provide a detailed description of the contents shown in the image.",
+                "answer": "",
+                "options": {},
+                "choices": [],
+                "prompt": "<image> Please provide a detailed description of the contents shown in the image.",
+                "hint": ""
+            },
+            {
+                "role": "assistant",
+                "response": "The image features a sleek, white luxury yacht cruising through calm blue waters. The yacht is designed with a streamlined shape, enhancing its speed and stability. It has multiple decks, with the upper deck housing what appears to be an open seating area, possibly for relaxation or social gatherings. The lower deck includes a shaded area,"
+            }
+        ],
+        "comment": "Test 'single image start' modality.",
+        "eval-id": 1
+    }
+]

test_results/kimi_vl_2026-06-10/test_summary.json

@@ -0,0 +1,20 @@
+{
+  "model_series": "kimi_vl",
+  "model_name": "moonshotai/Kimi-VL-A3B-Instruct",
+  "test_date": "2026-06-10",
+  "conda_env": "/raid/ztw/envs/kimi_vl",
+  "env_summary": "transformers==4.48.2",
+  "status": "PASS",
+  "modalities_tested": {
+    "no_media": {
+      "status": "pass",
+      "rows": 2,
+      "result_path": "/raid/ztw/simple-mmeval-test-result/work_dirs/kimi_vl/2026-06-10/no_media/result.json"
+    },
+    "single_image_start": {
+      "status": "pass",
+      "rows": 2,
+      "result_path": "/raid/ztw/simple-mmeval-test-result/work_dirs/kimi_vl/2026-06-10/single_image_start/result.json"
+    }
+  }
+}

test_results/test_kimi_vl.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Smoke test for kimi_vl (model: moonshotai/Kimi-VL-A3B-Instruct).
+set -euo pipefail
+
+REPO_DIR=${REPO_DIR:-/raid/ztw/simple-mmeval-skills-dev}
+OUT_ROOT=${OUT_ROOT:-/raid/ztw/simple-mmeval-test-result/work_dirs/kimi_vl/$(date +%Y-%m-%d)}
+GPU=${GPU:-0}
+MODEL_ID=moonshotai/Kimi-VL-A3B-Instruct
+
+cd "$REPO_DIR"
+mkdir -p "$OUT_ROOT"
+
+for spec in "no_media|32|" "single_image_start|64|tests/media/448"; do
+  IFS='|' read -r sample maxtok imgdir <<< "$spec"
+  out="$OUT_ROOT/$sample"
+  extra=()
+  [[ -n "$imgdir" ]] && extra=(--img_dir "$imgdir")
+  PYTHONPATH="$REPO_DIR" ENV_DIR=/raid/ztw/envs CUDA_VISIBLE_DEVICES="$GPU" \
+    /raid/ztw/envs/kimi_vl/bin/python mmeval/run.py \
+      --model_name_or_path "$MODEL_ID" \
+      --dataset local@json --infile tests/samples/$sample.json \
+      --out_dir "$out" \
+      --gpu_per_parallel 1 --parallel_per_task 1 --max_new_tokens "$maxtok" \
+      "${extra[@]}"
+done